From 80e5f64dff830b3e889ae6b60e6af9e6670e6d64 Mon Sep 17 00:00:00 2001 From: Felipe Zimmerle Date: Thu, 24 Sep 2020 20:39:49 -0300 Subject: [PATCH 1/7] Import stuff from the older repository https://github.com/libinjection/libinjection/tree/main/python --- Makefile | 52 +++++++++++ apitest.py | 58 +++++++++++++ json2python.py | 52 +++++++++++ libinjection/__init__.py | 1 + libinjection/libinjection.i | 81 +++++++++++++++++ libinjection/sqli_fingerprints.py | 4 + pytest.py | 35 ++++++++ setup.py | 49 +++++++++++ speedtest.py | 89 +++++++++++++++++++ test_driver.py | 139 ++++++++++++++++++++++++++++++ 10 files changed, 560 insertions(+) create mode 100644 Makefile create mode 100755 apitest.py create mode 100755 json2python.py create mode 100644 libinjection/__init__.py create mode 100644 libinjection/libinjection.i create mode 100644 libinjection/sqli_fingerprints.py create mode 100755 pytest.py create mode 100644 setup.py create mode 100755 speedtest.py create mode 100755 test_driver.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..dc75977 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ + +all: build +# +# + +build: copy libinjection/libinjection_wrap.c + rm -f libinjection.py libinjection.pyc + python setup.py --verbose build --force + +install: build + sudo python setup.py --verbose install + +test-unit: build words.py + python setup.py build_ext --inplace + PYTHON_PATH='.' nosetests -v --with-xunit test_driver.py + +.PHONY: test +test: test-unit + +.PHONY: speed +speed: + ./speedtest.py + + +words.py: Makefile json2python.py ../src/sqlparse_data.json + ./json2python.py < ../src/sqlparse_data.json > words.py + + +libinjection/libinjection_wrap.c: libinjection/libinjection.i libinjection/libinjection.h libinjection/libinjection_sqli.h + swig -version + swig -python -builtin -Wall -Wextra libinjection/libinjection.i + + +copy: + cp ../src/libinjection*.h ../src/libinjection*.c libinjection/ + +.PHONY: copy + +libinjection.so: copy + gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_sqli.c + gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_xss.c + gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_html5.c + gcc -dynamiclib -shared -o libinjection.so libinjection_sqli.o libinjection_xss.o libinjection_html5.o + +clean: + @rm -rf build dist + @rm -f *.pyc *~ *.so *.o + @rm -f nosetests.xml + @rm -f words.py + @rm -f libinjection/*~ libinjection/*.pyc + @rm -f libinjection/libinjection.h libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c libinjection/libinjection_sqli_data.h + @rm -f libinjection/libinjection_wrap.c libinjection/libinjection.py diff --git a/apitest.py b/apitest.py new file mode 100755 index 0000000..fd103de --- /dev/null +++ b/apitest.py @@ -0,0 +1,58 @@ +#!/usr/bin/python + +""" +Work-in-progress +""" + +from libinjection import * +from words import words + +print dir(libinjection) + +def print_token_string(tok): + """ + returns the value of token, handling opening and closing quote characters + """ + out = '' + if tok.str_open != "\0": + out += tok.str_open + out += tok.val + if tok.str_close != "\0": + out += tok.str_close + return out + +def print_token(tok): + """ + prints a token for use in unit testing + """ + out = '' + out += tok.type + out += ' ' + if tok.type == 's': + out += print_token_string(tok) + elif tok.type == 'v': + vc = tok.count; + if vc == 1: + out += '@' + elif vc == 2: + out += '@@' + out += print_token_string(tok) + else: + out += tok.val + return out + +def lookup(state, stype, keyword): + keyword = keyword.upper() + if stype == 'v': + keyword = '0' + keyword + ch = words.get(keyword, '') + return ch + +sqli = '1 union all select 1 --' + +s = sqli_state() +sqli_init(s, sqli, libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) +sqli_callback(s, lookup) + +while sqli_tokenize(s): + print print_token(s.current) diff --git a/json2python.py b/json2python.py new file mode 100755 index 0000000..ac8505e --- /dev/null +++ b/json2python.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Copyright 2012, 2013 Nick Galbreath +# nickg@client9.com +# BSD License -- see COPYING.txt for details +# + +""" +Converts a libinjection JSON data file to python dict +""" + +def toc(obj): + """ main routine """ + + print """ +import libinjection + +def lookup(state, stype, keyword): + keyword = keyword.upper() + if stype == libinjection.LOOKUP_FINGERPRINT: + if keyword in fingerprints and libinjection.sqli_not_whitelist(state): + return 'F' + else: + return chr(0) + return words.get(keyword, chr(0)) + +""" + + words = {} + keywords = obj['keywords'] + for k,v in keywords.iteritems(): + words[str(k)] = str(v) + + print 'words = {' + for k in sorted(words.keys()): + print "'{0}': '{1}',".format(k, words[k]) + print '}\n' + + + keywords = obj['fingerprints'] + print 'fingerprints = set([' + for k in sorted(keywords): + print "'{0}',".format(k.upper()) + print '])' + + return 0 + +if __name__ == '__main__': + import sys + import json + sys.exit(toc(json.load(sys.stdin))) + diff --git a/libinjection/__init__.py b/libinjection/__init__.py new file mode 100644 index 0000000..84d587a --- /dev/null +++ b/libinjection/__init__.py @@ -0,0 +1 @@ +from libinjection import * diff --git a/libinjection/libinjection.i b/libinjection/libinjection.i new file mode 100644 index 0000000..3f279da --- /dev/null +++ b/libinjection/libinjection.i @@ -0,0 +1,81 @@ +/* libinjection.i SWIG interface file */ +%module libinjection +%{ +#include "libinjection.h" +#include "libinjection_sqli.h" +#include + +/* This is the callback function that runs a python function + * + */ +static char libinjection_python_check_fingerprint(sfilter* sf, int lookuptype, const char* word, size_t len) +{ + PyObject *fp; + PyObject *arglist; + PyObject *result; + const char* strtype; + char ch; + + // get sfilter->pattern + // convert to python string + fp = SWIG_InternalNewPointerObj((void*)sf, SWIGTYPE_p_libinjection_sqli_state,0); + + arglist = Py_BuildValue("(Nis#)", fp, lookuptype, word, len); + // call pyfunct with string arg + result = PyObject_CallObject((PyObject*) sf->userdata, arglist); + Py_DECREF(arglist); + if (result == NULL) { + printf("GOT NULL\n"); + // python call has an exception + // pass it back + ch = '\0'; + } else { + // convert value of python call to a char + strtype = PyString_AsString(result); + ch = strtype[0]; + Py_DECREF(result); + } + return ch; +} + +%} +%include "typemaps.i" + +// The C functions all start with 'libinjection_' as a namespace +// We don't need this since it's in the libinjection python package +// i.e. libinjection.libinjection_is_sqli --> libinjection.is_sqli + // +%rename("%(strip:[libinjection_])s") ""; + +// SWIG doesn't natively support fixed sized arrays. +// this typemap converts the fixed size array sfilter.tokevec +// into a list of pointers to stoken_t types. In otherword this code makes this example work +// s = sfilter() +// libinjection_is_sqli(s, "a string",...) +// for i in len(s.pat): +// print s.tokevec[i].val +// + +%typemap(out) stoken_t [ANY] { +int i; +$result = PyList_New($1_dim0); +for (i = 0; i < $1_dim0; i++) { + PyObject *o = SWIG_NewPointerObj((void*)(& $1[i]), SWIGTYPE_p_stoken_t,0); + PyList_SetItem($result,i,o); +} +} + +// automatically append string length into arg array +%apply (char *STRING, size_t LENGTH) { (const char *s, size_t slen) }; + +%typemap(in) (ptr_lookup_fn fn, void* userdata) { + if ($input == Py_None) { + $1 = NULL; + $2 = NULL; + } else { + $1 = libinjection_python_check_fingerprint; + $2 = $input; + } +} +%include "libinjection.h" +%include "libinjection_sqli.h" diff --git a/libinjection/sqli_fingerprints.py b/libinjection/sqli_fingerprints.py new file mode 100644 index 0000000..8881857 --- /dev/null +++ b/libinjection/sqli_fingerprints.py @@ -0,0 +1,4 @@ + +sqli_fingerprints = set([ +'1234' +]) diff --git a/pytest.py b/pytest.py new file mode 100755 index 0000000..4c4099e --- /dev/null +++ b/pytest.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +from libinjection import * + +sqli= "1 UNION ALL SELECT * FROM FOO" + +if False: + s = sfilter() + print sqli_fingerprint(s, sqli, CHAR_NULL, COMMENTS_ANSI) + print "----" + +if False: + s = sfilter() + current = stoken_t() + sqli_init(s, sqli, CHAR_NULL, COMMENTS_ANSI) + while sqli_tokenize(s, current): + print current.type, current.val + print "----" + +def is_pattern(state): + return sqli_blacklist(state) and sqli_not_whitelist(state) + +s = sfilter() + +if is_sqli(s, sqli, None): + print "IS SQLI" + print len(s.pat) + print s.current.val + print s.current.type + vec = s.tokenvec + for i in range(len(s.pat)): + atoken = vec[i] + print atoken.type, atoken.val +else: + print "IS NOT SQLI" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..eb47024 --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +""" +libinjection module for python + + Copyright 2012, 2013, 2014 Nick Galbreath + nickg@client9.com + BSD License -- see COPYING.txt for details +""" +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension + +MODULE = Extension( + '_libinjection', [ + 'libinjection/libinjection_wrap.c', + 'libinjection/libinjection_sqli.c', + 'libinjection/libinjection_html5.c', + 'libinjection/libinjection_xss.c' + ], + swig_opts=['-Wextra', '-builtin'], + define_macros = [], + include_dirs = [], + libraries = [], + library_dirs = [], + ) + +setup ( + name = 'libinjection', + version = '3.9.1', + description = 'Wrapper around libinjection c-code to detect sqli', + author = 'Nick Galbreath', + author_email = 'nickg@client9.com', + url = 'https://libinjection.client9.com/', + ext_modules = [MODULE], + packages = ['libinjection'], + long_description = ''' +wrapper around libinjection +''', + classifiers = [ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Topic :: Database', + 'Topic :: Security', + 'Operating System :: OS Independent', + 'Development Status :: 3 - Alpha', + 'Topic :: Internet :: Log Analysis', + 'Topic :: Internet :: WWW/HTTP' + ] + ) diff --git a/speedtest.py b/speedtest.py new file mode 100755 index 0000000..46f8592 --- /dev/null +++ b/speedtest.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +from libinjection import * +from words import * +import time + +def lookup_null(state, style, keyword): + return '' + +def lookup_c(state, style, keyword): + return '' + #return sqli_lookup_word(state, style, keyword) + +def lookup_upcase(state, stype, keyword): + if stype == libinjection.LOOKUP_FINGERPRINT: + return words.get('0' + keyword.upper(), '') + else: + return words.get(keyword.upper(), '') + +def main(): + + inputs = ( + "123 LIKE -1234.5678E+2;", + "APPLE 19.123 'FOO' \"BAR\"", + "/* BAR */ UNION ALL SELECT (2,3,4)", + "1 || COS(+0X04) --FOOBAR", + "dog apple @cat banana bar", + "dog apple cat \"banana \'bar", + "102 TABLE CLOTH" + ) + imax = 100000 + + t0 = time.clock() + sfilter = sqli_state() + for i in xrange(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + is_sqli(sfilter) + t1 = time.clock() + total = imax / (t1 - t0) + print("python->c TPS = {0}".format(total)) + + t0 = time.clock() + sfilter = sqli_state() + for i in xrange(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup_null) + is_sqli(sfilter) + t1 = time.clock() + total = imax / (t1 - t0) + print("python lookup_null TPS = {0}".format(total)) + + t0 = time.clock() + sfilter = sqli_state() + for i in xrange(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup_upcase) + is_sqli(sfilter) + t1 = time.clock() + total = imax / (t1 - t0) + print("python lookup_upcase TPS = {0}".format(total)) + + t0 = time.clock() + sfilter = sqli_state() + for i in xrange(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup_c) + is_sqli(sfilter) + t1 = time.clock() + total = imax / (t1 - t0) + print("python lookup_c TPS = {0}".format(total)) + + + t0 = time.clock() + sfilter = sqli_state() + for i in xrange(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup) + is_sqli(sfilter) + t1 = time.clock() + total = imax / (t1 - t0) + print("python lookup TPS = {0}".format(total)) + + +if __name__ == '__main__': + main() diff --git a/test_driver.py b/test_driver.py new file mode 100755 index 0000000..066805f --- /dev/null +++ b/test_driver.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +""" +Test driver +Runs off plain text files, similar to how PHP's test harness works +""" +import os +import glob +from libinjection import * +from words import * + +print version() + +def print_token_string(tok): + """ + returns the value of token, handling opening and closing quote characters + """ + out = '' + if tok.str_open != "\0": + out += tok.str_open + out += tok.val + if tok.str_close != "\0": + out += tok.str_close + return out + +def print_token(tok): + """ + prints a token for use in unit testing + """ + out = '' + out += tok.type + out += ' ' + if tok.type == 's': + out += print_token_string(tok) + elif tok.type == 'v': + vc = tok.count; + if vc == 1: + out += '@' + elif vc == 2: + out += '@@' + out += print_token_string(tok) + else: + out += tok.val + return out.strip() + +def toascii(data): + """ + Converts a utf-8 string to ascii. needed since nosetests xunit is not UTF-8 safe + https://github.com/nose-devs/nose/issues/649 + https://github.com/nose-devs/nose/issues/692 + """ + return data + udata = data.decode('utf-8') + return udata.encode('ascii', 'xmlcharrefreplace') + +def readtestdata(filename): + """ + Read a test file and split into components + """ + + state = None + info = { + '--TEST--': '', + '--INPUT--': '', + '--EXPECTED--': '' + } + + for line in open(filename, 'r'): + line = line.rstrip() + if line in ('--TEST--', '--INPUT--', '--EXPECTED--'): + state = line + elif state: + info[state] += line + '\n' + + # remove last newline from input + info['--INPUT--'] = info['--INPUT--'][0:-1] + + return (info['--TEST--'], info['--INPUT--'].strip(), info['--EXPECTED--'].strip()) + +def runtest(testname, flag, sqli_flags): + """ + runs a test, optionally with valgrind + """ + data = readtestdata(os.path.join('../tests', testname)) + + sql_state = sqli_state() + sqli_init(sql_state, data[1], sqli_flags) + sqli_callback(sql_state, lookup) + actual = '' + + if flag == 'tokens': + while sqli_tokenize(sql_state): + actual += print_token(sql_state.current) + '\n'; + actual = actual.strip() + elif flag == 'folding': + num_tokens = sqli_fold(sql_state) + for i in range(num_tokens): + actual += print_token(sqli_get_token(sql_state, i)) + '\n'; + elif flag == 'fingerprints': + ok = is_sqli(sql_state) + if ok: + actual = sql_state.fingerprint + else: + raise RuntimeException("unknown flag") + + actual = actual.strip() + + if actual != data[2]: + print "INPUT: \n" + toascii(data[1]) + print + print "EXPECTED: \n" + toascii(data[2]) + print + print "GOT: \n" + toascii(actual) + assert actual == data[2] + +def test_tokens(): + for testname in sorted(glob.glob('../tests/test-tokens-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'tokens', libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) + +def test_tokens_mysql(): + for testname in sorted(glob.glob('../tests/test-tokens_mysql-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'tokens', libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_MYSQL) + +def test_folding(): + for testname in sorted(glob.glob('../tests/test-folding-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'folding', libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) + +def test_fingerprints(): + for testname in sorted(glob.glob('../tests/test-sqli-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'fingerprints', 0) + + +if __name__ == '__main__': + import sys + sys.stderr.write("run using nosetests\n") + sys.exit(1) From 6de6af0dc731c79b5826d46f7115579ba629de8c Mon Sep 17 00:00:00 2001 From: Felipe Zimmerle Date: Thu, 24 Sep 2020 20:49:09 -0300 Subject: [PATCH 2/7] speedtest.py: Make it workable with different Python versions --- speedtest.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/speedtest.py b/speedtest.py index 46f8592..e285292 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1,7 +1,12 @@ -#!/usr/bin/env python -from libinjection import * +#!/usr/bin/env python3 +from libinjection import sqli_state from words import * import time +import sys +if sys.version_info > (3, 7): + from time import process_time as clock +else: + from time import clock as clock def lookup_null(state, style, keyword): return '' @@ -29,58 +34,58 @@ def main(): ) imax = 100000 - t0 = time.clock() + t0 = clock() sfilter = sqli_state() for i in xrange(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) is_sqli(sfilter) - t1 = time.clock() + t1 = clock() total = imax / (t1 - t0) print("python->c TPS = {0}".format(total)) - t0 = time.clock() + t0 = clock() sfilter = sqli_state() for i in xrange(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup_null) is_sqli(sfilter) - t1 = time.clock() + t1 = clock() total = imax / (t1 - t0) print("python lookup_null TPS = {0}".format(total)) - t0 = time.clock() + t0 = clock() sfilter = sqli_state() for i in xrange(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup_upcase) is_sqli(sfilter) - t1 = time.clock() + t1 = clock() total = imax / (t1 - t0) print("python lookup_upcase TPS = {0}".format(total)) - t0 = time.clock() + t0 = clock() sfilter = sqli_state() for i in xrange(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup_c) is_sqli(sfilter) - t1 = time.clock() + t1 = clock() total = imax / (t1 - t0) print("python lookup_c TPS = {0}".format(total)) - t0 = time.clock() + t0 = clock() sfilter = sqli_state() for i in xrange(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup) is_sqli(sfilter) - t1 = time.clock() + t1 = clock() total = imax / (t1 - t0) print("python lookup TPS = {0}".format(total)) From 84b39d598056a01a0860c659dcfc33beeb79fff5 Mon Sep 17 00:00:00 2001 From: Jorge Pereira Date: Thu, 24 Sep 2020 21:10:08 -0300 Subject: [PATCH 3/7] Update the README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9eaf0be..d2937aa 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# libinjection-py -libInjection Python bindings +# python3-libinjection +libInjection Python3 bindings From cee7cc9f0749670f5f57a73af8e56b61b2de07b7 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sun, 8 Nov 2020 10:32:58 -0300 Subject: [PATCH 4/7] fix(build): fixes build process for split project Signed-off-by: Felipe Zipitria --- .gitignore | 9 +++++++++ Makefile | 20 +++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index b6e4761..3d30f57 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,12 @@ dmypy.json # Pyre type checker .pyre/ + +# Upstream git repo +upstream/ +libinjection/libinjection.h +libinjection/libinjection.py +libinjection/libinjection_xss.* +libinjection/libinjection_html5.* +libinjection/libinjection_sqli* +libinjection/libinjection_wrap* diff --git a/Makefile b/Makefile index dc75977..f3aeae4 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: build # # -build: copy libinjection/libinjection_wrap.c +build: upstream libinjection/libinjection_wrap.c rm -f libinjection.py libinjection.pyc python setup.py --verbose build --force @@ -21,29 +21,31 @@ test: test-unit speed: ./speedtest.py +upstream: + [ -d $@ ] || git clone --depth=1 https://github.com/libinjection/libinjection.git upstream -words.py: Makefile json2python.py ../src/sqlparse_data.json - ./json2python.py < ../src/sqlparse_data.json > words.py +libinjection/libinjection.h libinjection/libinjection_sqli.h: upstream + cp -f upstream/src/libinjection*.h upstream/src/libinjection*.c libinjection/ + +words.py: Makefile json2python.py upstream + ./json2python.py < upstream/src/sqlparse_data.json > words.py libinjection/libinjection_wrap.c: libinjection/libinjection.i libinjection/libinjection.h libinjection/libinjection_sqli.h swig -version - swig -python -builtin -Wall -Wextra libinjection/libinjection.i - + swig -py3 -python -builtin -Wall -Wextra libinjection/libinjection.i -copy: - cp ../src/libinjection*.h ../src/libinjection*.c libinjection/ .PHONY: copy -libinjection.so: copy +libinjection.so: libinjection/libinjection_wrap.c gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_sqli.c gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_xss.c gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_html5.c gcc -dynamiclib -shared -o libinjection.so libinjection_sqli.o libinjection_xss.o libinjection_html5.o clean: - @rm -rf build dist + @rm -rf build dist upstream @rm -f *.pyc *~ *.so *.o @rm -f nosetests.xml @rm -f words.py From 9455021b4d1c1d1e3c54812091fa72d44e6d8293 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sun, 8 Nov 2020 10:00:37 -0300 Subject: [PATCH 5/7] fix(py3): change to py3 syntax Signed-off-by: Felipe Zipitria --- apitest.py | 4 ++-- json2python.py | 18 +++++++++--------- pytest.py | 20 ++++++++++---------- speedtest.py | 20 ++++++++++---------- test_driver.py | 12 ++++++------ 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/apitest.py b/apitest.py index fd103de..f23f956 100755 --- a/apitest.py +++ b/apitest.py @@ -7,7 +7,7 @@ from libinjection import * from words import words -print dir(libinjection) +print(dir(libinjection)) def print_token_string(tok): """ @@ -55,4 +55,4 @@ def lookup(state, stype, keyword): sqli_callback(s, lookup) while sqli_tokenize(s): - print print_token(s.current) + print(print_token(s.current)) diff --git a/json2python.py b/json2python.py index ac8505e..83fedc6 100755 --- a/json2python.py +++ b/json2python.py @@ -12,7 +12,7 @@ def toc(obj): """ main routine """ - print """ + print(""" import libinjection def lookup(state, stype, keyword): @@ -24,24 +24,24 @@ def lookup(state, stype, keyword): return chr(0) return words.get(keyword, chr(0)) -""" +""") words = {} keywords = obj['keywords'] - for k,v in keywords.iteritems(): + for k,v in keywords.items(): words[str(k)] = str(v) - print 'words = {' + print('words = {') for k in sorted(words.keys()): - print "'{0}': '{1}',".format(k, words[k]) - print '}\n' + print("'{0}': '{1}',".format(k, words[k])) + print('}\n') keywords = obj['fingerprints'] - print 'fingerprints = set([' + print('fingerprints = set([') for k in sorted(keywords): - print "'{0}',".format(k.upper()) - print '])' + print("'{0}',".format(k.upper())) + print('])') return 0 diff --git a/pytest.py b/pytest.py index 4c4099e..fe13279 100755 --- a/pytest.py +++ b/pytest.py @@ -6,16 +6,16 @@ if False: s = sfilter() - print sqli_fingerprint(s, sqli, CHAR_NULL, COMMENTS_ANSI) - print "----" + print(sqli_fingerprint(s, sqli, CHAR_NULL, COMMENTS_ANSI)) + print("----") if False: s = sfilter() current = stoken_t() sqli_init(s, sqli, CHAR_NULL, COMMENTS_ANSI) while sqli_tokenize(s, current): - print current.type, current.val - print "----" + print(current.type, current.val) + print("----") def is_pattern(state): return sqli_blacklist(state) and sqli_not_whitelist(state) @@ -23,13 +23,13 @@ def is_pattern(state): s = sfilter() if is_sqli(s, sqli, None): - print "IS SQLI" - print len(s.pat) - print s.current.val - print s.current.type + print("IS SQLI") + print(len(s.pat)) + print(s.current.val) + print(s.current.type) vec = s.tokenvec for i in range(len(s.pat)): atoken = vec[i] - print atoken.type, atoken.val + print(atoken.type, atoken.val) else: - print "IS NOT SQLI" + print("IS NOT SQLI") diff --git a/speedtest.py b/speedtest.py index e285292..31e96e2 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,58 +36,58 @@ def main(): t0 = clock() sfilter = sqli_state() - for i in xrange(imax): + for i in range(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) is_sqli(sfilter) t1 = clock() total = imax / (t1 - t0) - print("python->c TPS = {0}".format(total)) + print(("python->c TPS = {0}".format(total))) t0 = clock() sfilter = sqli_state() - for i in xrange(imax): + for i in range(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup_null) is_sqli(sfilter) t1 = clock() total = imax / (t1 - t0) - print("python lookup_null TPS = {0}".format(total)) + print(("python lookup_null TPS = {0}".format(total))) t0 = clock() sfilter = sqli_state() - for i in xrange(imax): + for i in range(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup_upcase) is_sqli(sfilter) t1 = clock() total = imax / (t1 - t0) - print("python lookup_upcase TPS = {0}".format(total)) + print(("python lookup_upcase TPS = {0}".format(total))) t0 = clock() sfilter = sqli_state() - for i in xrange(imax): + for i in range(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup_c) is_sqli(sfilter) t1 = clock() total = imax / (t1 - t0) - print("python lookup_c TPS = {0}".format(total)) + print(("python lookup_c TPS = {0}".format(total))) t0 = clock() sfilter = sqli_state() - for i in xrange(imax): + for i in range(imax): s = inputs[i % 7] sqli_init(sfilter, s, 0) sqli_callback(sfilter, lookup) is_sqli(sfilter) t1 = clock() total = imax / (t1 - t0) - print("python lookup TPS = {0}".format(total)) + print(("python lookup TPS = {0}".format(total))) if __name__ == '__main__': diff --git a/test_driver.py b/test_driver.py index 066805f..0991c07 100755 --- a/test_driver.py +++ b/test_driver.py @@ -8,7 +8,7 @@ from libinjection import * from words import * -print version() +print(version()) def print_token_string(tok): """ @@ -105,11 +105,11 @@ def runtest(testname, flag, sqli_flags): actual = actual.strip() if actual != data[2]: - print "INPUT: \n" + toascii(data[1]) - print - print "EXPECTED: \n" + toascii(data[2]) - print - print "GOT: \n" + toascii(actual) + print("INPUT: \n" + toascii(data[1])) + print() + print("EXPECTED: \n" + toascii(data[2])) + print() + print("GOT: \n" + toascii(actual)) assert actual == data[2] def test_tokens(): From 91bd9ef1db27f897a775b01af4f422cd4d1de78f Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:07:13 -0300 Subject: [PATCH 6/7] chore: update Python3 bindings to work with upstream libinjection v4.0.0 API (#7) * Initial plan * Initial plan: update module to work with upstream libinjection Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> * Update module to work with upstream libinjection and improve Python 3 compatibility Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> * Address review feedback: fix SWIG interface, Makefile, test infrastructure, and add API tests Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> --- .gitignore | 4 ++ Makefile | 38 +++++++++++---- README.md | 95 ++++++++++++++++++++++++++++++++++++ pytest.py => example_sqli.py | 0 json2python.py | 3 ++ libinjection/__init__.py | 2 +- libinjection/libinjection.i | 93 ++++++++++++++++++++++++++++++++--- setup.py | 24 ++++++++- test_api.py | 84 +++++++++++++++++++++++++++++++ test_driver.py | 47 +++++++++++------- 10 files changed, 351 insertions(+), 39 deletions(-) rename pytest.py => example_sqli.py (100%) create mode 100644 test_api.py diff --git a/.gitignore b/.gitignore index 3d30f57..c5da1f0 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,7 @@ libinjection/libinjection_xss.* libinjection/libinjection_html5.* libinjection/libinjection_sqli* libinjection/libinjection_wrap* +libinjection/libinjection_error.h + +# Generated files +words.py diff --git a/Makefile b/Makefile index f3aeae4..ad93292 100644 --- a/Makefile +++ b/Makefile @@ -4,15 +4,13 @@ all: build # build: upstream libinjection/libinjection_wrap.c - rm -f libinjection.py libinjection.pyc - python setup.py --verbose build --force + python3 setup.py --verbose build_ext --inplace install: build - sudo python setup.py --verbose install + sudo python3 setup.py --verbose install test-unit: build words.py - python setup.py build_ext --inplace - PYTHON_PATH='.' nosetests -v --with-xunit test_driver.py + python3 -m pytest test_driver.py -v .PHONY: test test: test-unit @@ -24,16 +22,35 @@ speed: upstream: [ -d $@ ] || git clone --depth=1 https://github.com/libinjection/libinjection.git upstream -libinjection/libinjection.h libinjection/libinjection_sqli.h: upstream +libinjection/libinjection.h libinjection/libinjection_sqli.h libinjection/libinjection_error.h \ +libinjection/libinjection_xss.h libinjection/libinjection_html5.h: upstream cp -f upstream/src/libinjection*.h upstream/src/libinjection*.c libinjection/ + # Compatibility patches for SWIG wrapping: fix type mismatches and visibility. + # These sed invocations are pattern-matched to avoid breaking unrelated code. + # + # Fix return type mismatch: h5_state_data uses injection_result_t in definition but int in declaration + sed -i 's/^static int h5_state_data(/static injection_result_t h5_state_data(/' libinjection/libinjection_html5.c + # Fix return type mismatch: libinjection_is_sqli declared as injection_result_t but defined as int + sed -i 's/^int libinjection_is_sqli(/injection_result_t libinjection_is_sqli(/' libinjection/libinjection_sqli.c + # Remove static from helper functions so SWIG can wrap and expose them to Python + # (static functions in a header cannot be called from libinjection_wrap.c) + sed -i 's/^static void libinjection_sqli_reset(/void libinjection_sqli_reset(/' libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c + sed -i ':a;N;$$!ba;s/static char\nlibinjection_sqli_lookup_word/char\nlibinjection_sqli_lookup_word/g' libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c + sed -i ':a;N;$$!ba;s/static int\nlibinjection_sqli_blacklist/int\nlibinjection_sqli_blacklist/g' libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c + sed -i ':a;N;$$!ba;s/static int\nlibinjection_sqli_not_whitelist/int\nlibinjection_sqli_not_whitelist/g' libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c words.py: Makefile json2python.py upstream - ./json2python.py < upstream/src/sqlparse_data.json > words.py + python3 json2python.py < upstream/src/sqlparse_data.json > words.py -libinjection/libinjection_wrap.c: libinjection/libinjection.i libinjection/libinjection.h libinjection/libinjection_sqli.h +libinjection/libinjection_wrap.c: libinjection/libinjection.i libinjection/libinjection.h \ +libinjection/libinjection_sqli.h libinjection/libinjection_error.h \ +libinjection/libinjection_xss.h libinjection/libinjection_html5.h swig -version - swig -py3 -python -builtin -Wall -Wextra libinjection/libinjection.i + swig -python -builtin -Wall -Wextra \ + -o libinjection/libinjection_wrap.c \ + -outdir libinjection \ + libinjection/libinjection.i .PHONY: copy @@ -50,5 +67,6 @@ clean: @rm -f nosetests.xml @rm -f words.py @rm -f libinjection/*~ libinjection/*.pyc - @rm -f libinjection/libinjection.h libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c libinjection/libinjection_sqli_data.h + @rm -f libinjection/libinjection.h libinjection/libinjection_error.h libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c libinjection/libinjection_sqli_data.h + @rm -f libinjection/libinjection_html5.h libinjection/libinjection_html5.c libinjection/libinjection_xss.h libinjection/libinjection_xss.c @rm -f libinjection/libinjection_wrap.c libinjection/libinjection.py diff --git a/README.md b/README.md index d2937aa..e22005a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,97 @@ # python3-libinjection libInjection Python3 bindings + +## Overview + +Python3 bindings for [libinjection](https://github.com/libinjection/libinjection) - a SQL/SQLI tokenizer, parser and analyzer. + +## Requirements + +- Python 3.x +- SWIG 4.x +- GCC or compatible C compiler + +## Building + +### 1. Clone the repository and get upstream libinjection + +```bash +git clone https://github.com/libinjection/python3-libinjection.git +cd python3-libinjection +make upstream +``` + +### 2. Copy upstream C source files + +```bash +make libinjection/libinjection.h libinjection/libinjection_sqli.h libinjection/libinjection_error.h +``` + +### 3. Generate the SWIG wrapper + +```bash +swig -python -builtin -Wall -Wextra \ + -o libinjection/libinjection_wrap.c \ + -outdir libinjection \ + libinjection/libinjection.i +``` + +### 4. Build the Python extension + +```bash +python3 setup.py build_ext --inplace +``` + +Or using the Makefile: + +```bash +make build +``` + +### 5. Generate the word lookup table (needed for tests) + +```bash +python3 json2python.py < upstream/src/sqlparse_data.json > words.py +``` + +## Usage + +### SQLi Detection + +```python +import libinjection + +# Simple API - detect SQLi in a string +result, fingerprint = libinjection.sqli("1 UNION SELECT * FROM users") +if result: + print(f"SQLi detected! Fingerprint: {fingerprint}") + +# Advanced API with state object +state = libinjection.sqli_state() +libinjection.sqli_init(state, "1 UNION SELECT * FROM users", + libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) +libinjection.sqli_callback(state, None) +if libinjection.is_sqli(state): + print(f"SQLi detected! Fingerprint: {state.fingerprint}") +``` + +### XSS Detection + +```python +import libinjection + +# Detect XSS in a string +result = libinjection.xss("") +if result: + print("XSS detected!") +``` + +## Testing + +Run the test suite using pytest from the repository root: + +```bash +python3 -m pytest test_driver.py test_api.py -v +``` + +> **Note:** `upstream/tests/` must exist (run `make upstream` first) for `test_driver.py` to find test data. diff --git a/pytest.py b/example_sqli.py similarity index 100% rename from pytest.py rename to example_sqli.py diff --git a/json2python.py b/json2python.py index 83fedc6..381fc5e 100755 --- a/json2python.py +++ b/json2python.py @@ -16,6 +16,9 @@ def toc(obj): import libinjection def lookup(state, stype, keyword): + # keyword is passed as bytes from C; decode to str for dict lookup + if isinstance(keyword, bytes): + keyword = keyword.decode('latin-1') keyword = keyword.upper() if stype == libinjection.LOOKUP_FINGERPRINT: if keyword in fingerprints and libinjection.sqli_not_whitelist(state): diff --git a/libinjection/__init__.py b/libinjection/__init__.py index 84d587a..f16540d 100644 --- a/libinjection/__init__.py +++ b/libinjection/__init__.py @@ -1 +1 @@ -from libinjection import * +from .libinjection import * diff --git a/libinjection/libinjection.i b/libinjection/libinjection.i index 3f279da..2a9b370 100644 --- a/libinjection/libinjection.i +++ b/libinjection/libinjection.i @@ -3,7 +3,10 @@ %{ #include "libinjection.h" #include "libinjection_sqli.h" +#include "libinjection_xss.h" +#include "libinjection_error.h" #include +#include /* This is the callback function that runs a python function * @@ -13,26 +16,45 @@ static char libinjection_python_check_fingerprint(sfilter* sf, int lookuptype, c PyObject *fp; PyObject *arglist; PyObject *result; - const char* strtype; char ch; // get sfilter->pattern // convert to python string fp = SWIG_InternalNewPointerObj((void*)sf, SWIGTYPE_p_libinjection_sqli_state,0); - arglist = Py_BuildValue("(Nis#)", fp, lookuptype, word, len); + // Use y# (bytes) format instead of s# (str) to avoid UnicodeDecodeError on + // non-UTF-8 bytes (e.g. 0xA0 word separators). The Python callback will + // receive the word as a bytes object and should decode it as needed. + arglist = Py_BuildValue("(Niy#)", fp, lookuptype, word, len); + if (arglist == NULL) { + // Py_BuildValue failed (e.g., encoding error); treat as not found + return '\0'; + } // call pyfunct with string arg result = PyObject_CallObject((PyObject*) sf->userdata, arglist); Py_DECREF(arglist); if (result == NULL) { - printf("GOT NULL\n"); // python call has an exception // pass it back ch = '\0'; } else { - // convert value of python call to a char - strtype = PyString_AsString(result); - ch = strtype[0]; + // convert value of python call to a char (Python 3 compatible) + if (PyUnicode_Check(result)) { + Py_ssize_t size; + const char* str = PyUnicode_AsUTF8AndSize(result, &size); + if (str != NULL && size > 0) { + ch = str[0]; + } else { + // Clear any exception set by PyUnicode_AsUTF8AndSize on failure + PyErr_Clear(); + ch = '\0'; + } + } else if (PyBytes_Check(result)) { + const char* str = PyBytes_AsString(result); + ch = (str != NULL) ? str[0] : '\0'; + } else { + ch = '\0'; + } Py_DECREF(result); } return ch; @@ -65,8 +87,61 @@ for (i = 0; i < $1_dim0; i++) { } } -// automatically append string length into arg array -%apply (char *STRING, size_t LENGTH) { (const char *s, size_t slen) }; +// automatically append string length into arg array. +// Accept both str (encoded as UTF-8) and bytes (passed through as-is). +// Using bytes is recommended when the input may contain non-ASCII octets, +// since str will be UTF-8 encoded which changes the byte values. +%typemap(in) (const char *s, size_t slen) (Py_buffer _view, int _must_release) { + _must_release = 0; + if (PyBytes_Check($input)) { + if (PyObject_GetBuffer($input, &_view, PyBUF_SIMPLE) != 0) SWIG_fail; + $1 = (const char *)_view.buf; + $2 = (size_t)_view.len; + _must_release = 1; + } else if (PyUnicode_Check($input)) { + Py_ssize_t _len; + $1 = PyUnicode_AsUTF8AndSize($input, &_len); + if (!$1) SWIG_fail; + $2 = (size_t)_len; + } else { + PyErr_SetString(PyExc_TypeError, "expected str or bytes"); + SWIG_fail; + } +} +%typemap(freearg) (const char *s, size_t slen) { + if (_must_release$argnum) PyBuffer_Release(&_view$argnum); +} +%typemap(in) (const char *s, size_t len) (Py_buffer _view, int _must_release) { + _must_release = 0; + if (PyBytes_Check($input)) { + if (PyObject_GetBuffer($input, &_view, PyBUF_SIMPLE) != 0) SWIG_fail; + $1 = (const char *)_view.buf; + $2 = (size_t)_view.len; + _must_release = 1; + } else if (PyUnicode_Check($input)) { + Py_ssize_t _len; + $1 = PyUnicode_AsUTF8AndSize($input, &_len); + if (!$1) SWIG_fail; + $2 = (size_t)_len; + } else { + PyErr_SetString(PyExc_TypeError, "expected str or bytes"); + SWIG_fail; + } +} +%typemap(freearg) (const char *s, size_t len) { + if (_must_release$argnum) PyBuffer_Release(&_view$argnum); +} + +// Make the fingerprint output parameter in libinjection_sqli() work as an output +// The fingerprint buffer size matches libinjection's internal LIBINJECTION_SQLI_MAX_TOKENS (5) + null byte +#define LIBINJECTION_FINGERPRINT_SIZE 8 +%typemap(in, numinputs=0) char fingerprint[] (char temp[LIBINJECTION_FINGERPRINT_SIZE]) { + memset(temp, 0, sizeof(temp)); + $1 = temp; +} +%typemap(argout) char fingerprint[] { + $result = SWIG_Python_AppendOutput($result, PyUnicode_FromString($1)); +} %typemap(in) (ptr_lookup_fn fn, void* userdata) { if ($input == Py_None) { @@ -77,5 +152,7 @@ for (i = 0; i < $1_dim0; i++) { $2 = $input; } } +%include "libinjection_error.h" %include "libinjection.h" %include "libinjection_sqli.h" +%include "libinjection_xss.h" diff --git a/setup.py b/setup.py index eb47024..b4010e1 100644 --- a/setup.py +++ b/setup.py @@ -5,20 +5,40 @@ nickg@client9.com BSD License -- see COPYING.txt for details """ +import os + try: from setuptools import setup, Extension except ImportError: from distutils.core import setup, Extension + +def get_libinjection_version(): + """Read the libinjection version from the upstream source file, if available.""" + version_file = os.path.join(os.path.dirname(__file__), + 'upstream', 'src', 'libinjection_sqli.c') + if os.path.exists(version_file): + with open(version_file, encoding="utf-8") as f: + for line in f: + if '#define LIBINJECTION_VERSION' in line and '__clang_analyzer__' not in line: + # Extract version string from: #define LIBINJECTION_VERSION "x.y.z" + parts = line.strip().split('"') + if len(parts) >= 2: + return parts[1] + return 'undefined' + + +LIBINJECTION_VERSION = get_libinjection_version() + MODULE = Extension( - '_libinjection', [ + 'libinjection._libinjection', [ 'libinjection/libinjection_wrap.c', 'libinjection/libinjection_sqli.c', 'libinjection/libinjection_html5.c', 'libinjection/libinjection_xss.c' ], swig_opts=['-Wextra', '-builtin'], - define_macros = [], + define_macros = [('LIBINJECTION_VERSION', '"{}"'.format(LIBINJECTION_VERSION))], include_dirs = [], libraries = [], library_dirs = [], diff --git a/test_api.py b/test_api.py new file mode 100644 index 0000000..afaa376 --- /dev/null +++ b/test_api.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +""" +API tests for libinjection Python bindings. +Covers the simple sqli() and xss() APIs as well as the stateful sqli API. +""" +import libinjection + + +def test_sqli_returns_tuple(): + """sqli() should return a (result, fingerprint) sequence.""" + result = libinjection.sqli("1 UNION SELECT * FROM users") + assert len(result) == 2, "sqli() must return a 2-element sequence (result, fingerprint)" + + +def test_sqli_detects_injection(): + """sqli() must detect a known SQLi payload.""" + is_sqli, fingerprint = libinjection.sqli("1 UNION SELECT * FROM users") + assert is_sqli == 1, "Expected SQLi to be detected" + assert fingerprint != "", "Expected non-empty fingerprint for SQLi input" + + +def test_sqli_benign_input(): + """sqli() must not flag benign input.""" + is_sqli, fingerprint = libinjection.sqli("hello world") + assert is_sqli == 0, "Benign input should not be flagged as SQLi" + assert fingerprint == "", "Benign input should produce an empty fingerprint" + + +def test_sqli_fingerprint_content(): + """sqli() fingerprint should be a non-empty string for detected SQLi.""" + is_sqli, fingerprint = libinjection.sqli("1 UNION ALL SELECT * FROM foo") + assert is_sqli == 1 + assert isinstance(fingerprint, str) + assert len(fingerprint) > 0 + + +def test_is_sqli_stateful_api(): + """Advanced stateful API using sqli_state / sqli_init / sqli_callback / is_sqli.""" + state = libinjection.sqli_state() + libinjection.sqli_init( + state, + "1 UNION SELECT * FROM users", + libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI, + ) + libinjection.sqli_callback(state, None) + assert libinjection.is_sqli(state) == 1, "Expected SQLi detection via stateful API" + assert state.fingerprint != "", "Expected fingerprint set in state" + + +def test_is_sqli_stateful_benign(): + """Stateful API should not flag benign input.""" + state = libinjection.sqli_state() + libinjection.sqli_init( + state, + "hello world", + libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI, + ) + libinjection.sqli_callback(state, None) + assert libinjection.is_sqli(state) == 0, "Benign input should not be SQLi" + + +def test_xss_detects_script_tag(): + """xss() must detect a basic XSS payload.""" + result = libinjection.xss("") + assert result == 1, "Expected XSS detection for