Fixes #4220

* Fix sqlite regex for create table to support implicit column types

* Fix sqlite when dumping large tables

.travis.yml

@@ -11,7 +11,6 @@ jobs:
           dist: trusty
         - python: 3.9-dev
           dist: bionic
-sudo: false
 git:
     depth: 1
 script:

doc/THANKS.md

@@ -486,6 +486,9 @@ Marek Sarvas, <marek.sarvas(at)gmail.com>
 Philippe A. R. Schaeffer, <schaeff(at)compuphil.de>
 * for reporting a minor bug
 
+Henri Salo <henri(at)nerv.fi>
+* for a donation
+
 Mohd Zamiri Sanin, <zamiri.sanin(at)gmail.com>
 * for reporting a minor bug
 

extra/vulnserver/vulnserver.py

@@ -18,6 +18,7 @@ import traceback
 
 PY3 = sys.version_info >= (3, 0)
 UNICODE_ENCODING = "utf-8"
+DEBUG = False
 
 if PY3:
     from http.client import INTERNAL_SERVER_ERROR
@@ -83,7 +84,8 @@ class ThreadingServer(ThreadingMixIn, HTTPServer):
         try:
             HTTPServer.finish_request(self, *args, **kwargs)
         except Exception:
-            traceback.print_exc()
+            if DEBUG:
+                traceback.print_exc()
 
 class ReqHandler(BaseHTTPRequestHandler):
     def do_REQUEST(self):
@@ -131,7 +133,7 @@ class ReqHandler(BaseHTTPRequestHandler):
                 self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING)
                 self.send_header("Connection", "close")
                 self.end_headers()
-                self.wfile.write(b"<html><p><h3>GET:</h3><a href='/?id=1'>link</a></p><hr><p><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></p></html>")
+                self.wfile.write(b"<!DOCTYPE html><html><head><title>vulnserver</title></head><body><h3>GET:</h3><a href='/?id=1'>link</a><hr><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></body></html>")
             else:
                 code, output = OK, ""
 
@@ -147,16 +149,21 @@ class ReqHandler(BaseHTTPRequestHandler):
                             _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params["id"])
                         results = _cursor.fetchall()
 
-                    output += "<b>SQL results:</b>\n"
+                    output += "<b>SQL results:</b><br>\n"
-                    output += "<table border=\"1\">\n"
 
-                    for row in results:
+                    if results:
-                        output += "<tr>"
+                        output += "<table border=\"1\">\n"
-                        for value in row:
+
-                            output += "<td>%s</td>" % value
+                        for row in results:
-                        output += "</tr>\n"
+                            output += "<tr>"
+                            for value in row:
+                                output += "<td>%s</td>" % value
+                            output += "</tr>\n"
+
+                        output += "</table>\n"
+                    else:
+                        output += "no results found"
 
-                    output += "</table>\n"
                     output += "</body></html>"
                 except Exception as ex:
                     code = INTERNAL_SERVER_ERROR
@@ -221,7 +228,7 @@ def run(address=LISTEN_ADDRESS, port=LISTEN_PORT):
     global _server
     try:
         _server = ThreadingServer((address, port), ReqHandler)
-        print("[i] running HTTP server at '%s:%d'" % (address, port))
+        print("[i] running HTTP server at 'http://%s:%d'" % (address, port))
         _server.serve_forever()
     except KeyboardInterrupt:
         _server.socket.close()

lib/controller/checks.py

@@ -506,7 +506,7 @@ def checkSqlInjection(place, parameter, value):
                             falseRawResponse = "%s%s" % (falseHeaders, falsePage)
 
                             # Checking if there is difference between current FALSE, original and heuristics page (i.e. not used parameter)
-                            if not kb.negativeLogic:
+                            if not any((kb.negativeLogic, conf.string, conf.notString)):
                                 try:
                                     ratio = 1.0
                                     seqMatcher = getCurrentThreadData().seqMatcher
@@ -939,6 +939,9 @@ def checkFalsePositives(injection):
                 if conf.string and any(conf.string in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
                     continue
 
+                if conf.notString and any(conf.notString in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
+                    continue
+
                 if randInt3 > randInt2 > randInt1:
                     break
 

lib/core/agent.py

@@ -178,6 +178,10 @@ class Agent(object):
             _newValue = newValue
             _origValue = origValue
 
+            if newValue:
+                newValue = newValue.replace(BOUNDARY_BACKSLASH_MARKER, '\\')
+                newValue = self.adjustLateValues(newValue)
+
             # TODO: support for POST_HINT
             newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
             origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
@@ -420,7 +424,7 @@ class Agent(object):
         rootQuery = queries[Backend.getIdentifiedDbms()]
         hexField = field
 
-        if "hex" in rootQuery:
+        if "hex" in rootQuery and hasattr(rootQuery.hex, "query"):
             hexField = rootQuery.hex.query % field
         else:
             warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()

lib/core/bigarray.py

@@ -10,11 +10,11 @@ try:
 except:
     import pickle
 
-import bz2
 import itertools
 import os
 import sys
 import tempfile
+import zlib
 
 from lib.core.compat import xrange
 from lib.core.enums import MKSTEMP_PREFIX
@@ -24,17 +24,17 @@ from lib.core.settings import BIGARRAY_COMPRESS_LEVEL
 
 DEFAULT_SIZE_OF = sys.getsizeof(object())
 
-def _size_of(object_):
+def _size_of(instance):
     """
-    Returns total size of a given object_ (in bytes)
+    Returns total size of a given instance / object (in bytes)
     """
 
-    retval = sys.getsizeof(object_, DEFAULT_SIZE_OF)
+    retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)
 
-    if isinstance(object_, dict):
+    if isinstance(instance, dict):
-        retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(object_.items()))
+        retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
-    elif hasattr(object_, "__iter__"):
+    elif hasattr(instance, "__iter__"):
-        retval += sum(_size_of(_) for _ in object_ if _ != object_)
+        retval += sum(_size_of(_) for _ in instance if _ != instance)
 
     return retval
 
@@ -54,8 +54,8 @@ class BigArray(list):
 
     >>> _ = BigArray(xrange(100000))
     >>> _[20] = 0
-    >>> _[100]
+    >>> _[99999]
-    100
+    99999
     """
 
     def __init__(self, items=None):
@@ -92,7 +92,7 @@ class BigArray(list):
             self.chunks.pop()
             try:
                 with open(self.chunks[-1], "rb") as f:
-                    self.chunks[-1] = pickle.loads(bz2.decompress(f.read()))
+                    self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
             except IOError as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex
@@ -113,7 +113,7 @@ class BigArray(list):
             self.filenames.add(filename)
             os.close(handle)
             with open(filename, "w+b") as f:
-                f.write(bz2.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
+                f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
             return filename
         except (OSError, IOError) as ex:
             errMsg = "exception occurred while storing data "
@@ -131,7 +131,7 @@ class BigArray(list):
         if not (self.cache and self.cache.index == index):
             try:
                 with open(self.chunks[index], "rb") as f:
-                    self.cache = Cache(index, pickle.loads(bz2.decompress(f.read())), False)
+                    self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
             except Exception as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex

lib/core/common.py

@@ -675,17 +675,21 @@ def paramToDict(place, parameters=None):
                                     elif isinstance(current, dict):
                                         for key in current.keys():
                                             value = current[key]
-                                            if isinstance(value, (list, tuple, set, dict)):
+                                            if isinstance(value, (bool, int, float, six.string_types)) or value in (None, []):
-                                                if value:
-                                                    walk(head, value)
-                                            elif isinstance(value, (bool, int, float, six.string_types)):
                                                 original = current[key]
                                                 if isinstance(value, bool):
                                                     current[key] = "%s%s" % (getUnicode(value).lower(), BOUNDED_INJECTION_MARKER)
+                                                elif value is None:
+                                                    current[key] = "%s%s" % (randomInt(), BOUNDED_INJECTION_MARKER)
+                                                elif value == []:
+                                                    current[key] = ["%s%s" % (randomInt(), BOUNDED_INJECTION_MARKER)]
                                                 else:
                                                     current[key] = "%s%s" % (value, BOUNDED_INJECTION_MARKER)
                                                 candidates["%s (%s)" % (parameter, key)] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), r"\g<1>%s" % json.dumps(deserialized, separators=(',', ':') if ", " not in testableParameters[parameter] else None), parameters)
                                                 current[key] = original
+                                            elif isinstance(value, (list, tuple, set, dict)):
+                                                if value:
+                                                    walk(head, value)
 
                                 deserialized = json.loads(testableParameters[parameter])
                                 walk(deserialized)
@@ -943,28 +947,31 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
 
     retVal = message
 
-    if message and (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
+    if message:
-        if level is None:
+        if (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
-            levels = re.findall(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
+            if level is None:
+                levels = re.findall(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
-            if len(levels) == 1:
+                if len(levels) == 1:
-                level = levels[0]
+                    level = levels[0]
 
-        if bold or color:
+            if bold or color:
-            retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
+                retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
-        elif level:
+            elif level:
-            try:
+                try:
-                level = getattr(logging, level, None)
+                    level = getattr(logging, level, None)
-            except:
+                except:
-                level = None
+                    level = None
-            retVal = LOGGER_HANDLER.colorize(message, level)
+                retVal = LOGGER_HANDLER.colorize(message, level)
-        else:
+            else:
-            match = re.search(r"\(([^)]*)\s*fork\)", message)
+                match = re.search(r"\(([^)]*)\s*fork\)", message)
-            if match:
+                if match:
-                retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
+                    retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
 
-            for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
+                for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
-                retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+                    retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+
+        message = message.strip()
 
     return retVal
 
@@ -988,6 +995,12 @@ def dataToStdout(data, forceOutput=False, bold=False, contentType=None, status=C
     Writes text to the stdout (console) stream
     """
 
+    if not IS_TTY and isinstance(data, six.string_types) and data.startswith("\r"):
+        if re.search(r"\(\d+%\)", data):
+            data = ""
+        else:
+            data = "\n%s" % data.strip("\r")
+
     if not kb.get("threadException"):
         if forceOutput or not (getCurrentThreadData().disableStdOut or kb.get("wizardMode")):
             multiThreadMode = isMultiThreadMode()
@@ -1213,9 +1226,9 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None):
     """
 
     if seed is not None:
-        _ = getCurrentThreadData().random
+        _random = getCurrentThreadData().random
-        _.seed(seed)
+        _random.seed(seed)
-        choice = _.choice
+        choice = _random.choice
     else:
         choice = random.choice
 
@@ -1247,10 +1260,12 @@ def getHeader(headers, key):
     """
 
     retVal = None
-    for _ in (headers or {}):
+
-        if _.upper() == key.upper():
+    for header in (headers or {}):
-            retVal = headers[_]
+        if header.upper() == key.upper():
+            retVal = headers[header]
             break
+
     return retVal
 
 def checkPipedInput():
@@ -1259,7 +1274,7 @@ def checkPipedInput():
     # Reference: https://stackoverflow.com/a/33873570
     """
 
-    return not os.isatty(sys.stdin.fileno()) if hasattr(sys.stdin, "fileno") else False
+    return hasattr(sys.stdin, "fileno") and not os.isatty(sys.stdin.fileno())
 
 def isZipFile(filename):
     """
@@ -1349,9 +1364,9 @@ def parsePasswordHash(password):
     >>> kb.forcedDbms = popValue()
     """
 
-    blank = " " * 8
+    blank = ' ' * 8
 
-    if isNoneValue(password) or password == " ":
+    if isNoneValue(password) or password == ' ':
         retVal = NULL
     else:
         retVal = password
@@ -1431,6 +1446,7 @@ def setPaths(rootPath):
             checkFile(path)
 
     if IS_WIN:
+        # Reference: https://pureinfotech.com/list-environment-variables-windows-10/
         if os.getenv("LOCALAPPDATA"):
             paths.SQLMAP_HOME_PATH = os.path.expandvars("%LOCALAPPDATA%\\sqlmap")
         elif os.getenv("USERPROFILE"):
@@ -1440,11 +1456,17 @@ def setPaths(rootPath):
     else:
         paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".sqlmap")
 
+        if not os.path.isdir(paths.SQLMAP_HOME_PATH):
+            if "XDG_DATA_HOME" in os.environ:
+                paths.SQLMAP_HOME_PATH = os.path.join(os.environ["XDG_DATA_HOME"], "sqlmap")
+            else:
+                paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".local", "share", "sqlmap")
+
     paths.SQLMAP_OUTPUT_PATH = getUnicode(paths.get("SQLMAP_OUTPUT_PATH", os.path.join(paths.SQLMAP_HOME_PATH, "output")), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.SQLMAP_DUMP_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "dump")
     paths.SQLMAP_FILES_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "files")
 
-    # history files
+    # History files
     paths.SQLMAP_HISTORY_PATH = getUnicode(os.path.join(paths.SQLMAP_HOME_PATH, "history"), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.API_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "api.hst")
     paths.OS_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "os.hst")
@@ -1488,7 +1510,7 @@ def parseTargetDirect():
         if details:
             conf.dbms = details.group("dbms")
 
-            if details.group('credentials'):
+            if details.group("credentials"):
                 conf.dbmsUser = details.group("user")
                 conf.dbmsPass = details.group("pass")
             else:
@@ -1600,8 +1622,8 @@ def parseTargetUrl():
 
     originalUrl = conf.url
 
-    if re.search(r"\[.+\]", conf.url) and not socket.has_ipv6:
+    if re.search(r"://\[.+\]", conf.url) and not socket.has_ipv6:
-        errMsg = "IPv6 addressing is not supported "
+        errMsg = "IPv6 communication is not supported "
         errMsg += "on this platform"
         raise SqlmapGenericException(errMsg)
 
@@ -1658,7 +1680,7 @@ def parseTargetUrl():
         conf.port = 80
 
     if conf.port < 1 or conf.port > 65535:
-        errMsg = "invalid target URL's port (%d)" % conf.port
+        errMsg = "invalid target URL port (%d)" % conf.port
         raise SqlmapSyntaxException(errMsg)
 
     conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))
@@ -3253,7 +3275,7 @@ def parseSqliteTableSchema(value):
     Parses table column names and types from specified SQLite table schema
 
     >>> kb.data.cachedColumns = {}
-    >>> parseSqliteTableSchema("CREATE TABLE users\\n\\t\\tid INTEGER\\n\\t\\tname TEXT\\n);")
+    >>> parseSqliteTableSchema("CREATE TABLE users(\\n\\t\\tid INTEGER,\\n\\t\\tname TEXT\\n);")
     True
     >>> repr(kb.data.cachedColumns).count(',') == 1
     True
@@ -3265,9 +3287,9 @@ def parseSqliteTableSchema(value):
         table = {}
         columns = {}
 
-        for match in re.finditer(r"(\w+)[\"'`]?\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b", decodeStringEscape(value), re.I):
+        for match in re.finditer(r"[(,]\s*[\"'`]?(\w+)[\"'`]?(?:\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b)?", decodeStringEscape(value), re.I):
             retVal = True
-            columns[match.group(1)] = match.group(2)
+            columns[match.group(1)] = match.group(2) or "TEXT"
 
         table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns
         kb.data.cachedColumns[conf.db] = table

lib/core/convert.py

@@ -256,7 +256,10 @@ def getBytes(value, encoding=None, errors="strict", unsafe=True):
             if unsafe:
                 retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
         else:
-            retVal = value.encode(encoding, errors)
+            try:
+                retVal = value.encode(encoding, errors)
+            except UnicodeError:
+                retVal = value.encode(UNICODE_ENCODING, errors="replace")
 
             if unsafe:
                 retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)

lib/core/option.py

@@ -371,7 +371,7 @@ def _doSearch():
 
         for link in links:
             link = urldecode(link)
-            if re.search(r"(.*?)\?(.+)", link):
+            if re.search(r"(.*?)\?(.+)", link) or conf.forms:
                 kb.targets.add((link, conf.method, conf.data, conf.cookie, None))
             elif re.search(URI_INJECTABLE_REGEX, link, re.I):
                 if kb.data.onlyGETs is None and conf.data is None and not conf.googleDork:
@@ -387,14 +387,18 @@ def _doSearch():
 
         if kb.targets:
             infoMsg = "found %d results for your " % len(links)
-            infoMsg += "search dork expression, "
+            infoMsg += "search dork expression"
 
-            if len(links) == len(kb.targets):
+            if not conf.forms:
-                infoMsg += "all "
+                infoMsg += ", "
-            else:
+
-                infoMsg += "%d " % len(kb.targets)
+                if len(links) == len(kb.targets):
+                    infoMsg += "all "
+                else:
+                    infoMsg += "%d " % len(kb.targets)
+
+                infoMsg += "of them are testable targets"
 
-            infoMsg += "of them are testable targets"
             logger.info(infoMsg)
             break
 
@@ -1867,6 +1871,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.cache.content = {}
     kb.cache.encoding = {}
     kb.cache.alphaBoundaries = None
+    kb.cache.hashRegex = None
     kb.cache.intBoundaries = None
     kb.cache.parsedDbms = {}
     kb.cache.regex = {}

lib/core/optiondict.py

@@ -201,6 +201,7 @@ optDict = {
         "trafficFile": "string",
         "answers": "string",
         "batch": "boolean",
+        "base64Parameter": "string",
         "binaryFields": "string",
         "charset": "string",
         "checkInternet": "boolean",

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.3.12"
+VERSION = "1.4.6.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -249,7 +249,7 @@ PYVERSION = sys.version.split()[0]
 IS_WIN = PLATFORM == "nt"
 
 # Check if running in terminal
-IS_TTY = os.isatty(sys.stdout.fileno())
+IS_TTY = hasattr(sys.stdout, "fileno") and os.isatty(sys.stdout.fileno())
 
 # DBMS system databases
 MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
@@ -917,7 +917,18 @@ for key, value in os.environ.items():
     if key.upper().startswith("%s_" % SQLMAP_ENVIRONMENT_PREFIX):
         _ = key[len(SQLMAP_ENVIRONMENT_PREFIX) + 1:].upper()
         if _ in globals():
-            globals()[_] = value
+            original = globals()[_]
+            if isinstance(original, int):
+                try:
+                    globals()[_] = int(value)
+                except ValueError:
+                    pass
+            elif isinstance(original, bool):
+                globals()[_] = value.lower() in ('1', 'true')
+            elif isinstance(original, (list, tuple)):
+                globals()[_] = [__.strip() for __ in _.split(',')]
+            else:
+                globals()[_] = value
 
 # Installing "reversible" unicode (decoding) error handler
 def _reversible(ex):

lib/core/testing.py

@@ -39,8 +39,8 @@ def vulnTest():
 
     TESTS = (
         ("-h", ("to see full list of options run with '-hh'",)),
-        ("-u <url> --flush-session --wizard --check-internet", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.", "~no connection detected")),
         ("--dependencies", ("sqlmap requires", "third-party library")),
+        ("-u <url> --flush-session --wizard", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.")),
         (u"-c <config> --flush-session --roles --statements --hostname --privileges --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=U", (u": '\u0161u\u0107uraj'", "on SQLite it is not possible")),
         (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=B --no-escape --string=luther --unstable", (u": '\u0161u\u0107uraj'",)),
         ("--dummy", ("all tested parameters do not appear to be injectable", "does not seem to be injectable", "there is not at least one", "~might be injectable")),
@@ -48,6 +48,7 @@ def vulnTest():
         ("-r <request> --flush-session -v 5 --test-skip='heavy' --save=<tmp>", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")),
         ("-l <log> --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")),
         ("-l <log> --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")),
+        ("-u <url> --flush-session --banner --technique=B --not-string 'no results'", ("banner: '3.",)),
         ("-u <url> --flush-session --banner --technique=B --first=1 --last=2", ("banner: '3.'",)),
         ("-u <url> --flush-session --encoding=ascii --forms --crawl=2 --threads=2 --banner", ("total of 2 targets", "might be injectable", "Type: UNION query", "banner: '3.")),
         ("-u <url> --flush-session --data='{\"id\": 1}' --banner", ("might be injectable", "3 columns", "Payload: {\"id\"", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.")),
@@ -85,9 +86,13 @@ def vulnTest():
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         try:
             s.connect((address, port))
-            break
+            s.send(b"GET / HTTP/1.0\r\n\r\n")
+            if b"vulnserver" in s.recv(4096):
+                break
         except:
             time.sleep(1)
+        finally:
+            s.close()
 
     handle, config = tempfile.mkstemp(suffix=".conf")
     os.close(handle)
@@ -326,7 +331,7 @@ def smokeTest():
     count, length = 0, 0
 
     for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-        if any(_ in root for _ in ("thirdparty", "extra")):
+        if any(_ in root for _ in ("thirdparty", "extra", "interbase")):
             continue
 
         for filename in files:
@@ -334,7 +339,7 @@ def smokeTest():
                 length += 1
 
     for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-        if any(_ in root for _ in ("thirdparty", "extra")):
+        if any(_ in root for _ in ("thirdparty", "extra", "interbase")):
             continue
 
         for filename in files:

lib/parse/cmdline.py

@@ -616,6 +616,9 @@ def cmdLineParser(argv=None):
         general.add_argument("--answers", dest="answers",
             help="Set predefined answers (e.g. \"quit=N,follow=N\")")
 
+        general.add_argument("--base64", dest="base64Parameter",
+            help="Parameter(s) containing Base64 encoded data")
+
         general.add_argument("--batch", dest="batch", action="store_true",
             help="Never ask for user input, use the default behavior")
 
@@ -746,9 +749,6 @@ def cmdLineParser(argv=None):
             help="Simple wizard interface for beginner users")
 
         # Hidden and/or experimental options
-        parser.add_argument("--base64", dest="base64Parameter",
-            help=SUPPRESS)  # "Parameter(s) containing Base64 encoded values"
-
         parser.add_argument("--crack", dest="hashFile",
             help=SUPPRESS)  # "Load and crack hashes from a file (standalone)"
 
@@ -916,18 +916,25 @@ def cmdLineParser(argv=None):
             except ValueError as ex:
                 raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % getSafeExString(ex))
 
+        longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
+        longSwitches = set(re.findall(r"\-\-([^= ]+?)\s", parser.format_help()))
+
         for i in xrange(len(argv)):
-            longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
+            # Reference: https://en.wiktionary.org/wiki/-
-            longSwitches = set(re.findall(r"\-\-([^= ]+?)\s", parser.format_help()))
+            argv[i] = re.sub(u"\A(\u2010|\u2013|\u2212|\u2014|\u4e00|\u1680|\uFE63|\uFF0D)+", lambda match: '-' * len(match.group(0)), argv[i])
+
+            # Reference: https://unicode-table.com/en/sets/quotation-marks/
+            argv[i] = argv[i].strip(u"\u00AB\u2039\u00BB\u203A\u201E\u201C\u201F\u201D\u2019\u0022\u275D\u275E\u276E\u276F\u2E42\u301D\u301E\u301F\uFF02\u201A\u2018\u201B\u275B\u275C")
+
             if argv[i] == "-hh":
                 argv[i] = "-h"
             elif i == 1 and re.search(r"\A(http|www\.|\w[\w.-]+\.\w{2,})", argv[i]) is not None:
                 argv[i] = "--url=%s" % argv[i]
             elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
-                dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
+                dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is illegal (%s)\n" % argv[i])
                 raise SystemExit
             elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
-                dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n" % argv[i])
+                dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is illegal (%s)\n" % argv[i])
                 raise SystemExit
             elif re.search(r"\A-\w=.+", argv[i]):
                 dataToStdout("[!] potentially miswritten (illegal '=') short option detected ('%s')\n" % argv[i])

lib/request/comparison.py

@@ -63,13 +63,19 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
     if any((conf.string, conf.notString, conf.regexp)):
         rawResponse = "%s%s" % (listToStrValue(_ for _ in headers.headers if not _.startswith("%s:" % URI_HTTP_HEADER)) if headers else "", page)
 
-        # String to match in page when the query is True and/or valid
+        # String to match in page when the query is True
         if conf.string:
             return conf.string in rawResponse
 
-        # String to match in page when the query is False and/or invalid
+        # String to match in page when the query is False
         if conf.notString:
-            return conf.notString not in rawResponse
+            if conf.notString in rawResponse:
+                return False
+            else:
+                if kb.errorIsNone and (wasLastResponseDBMSError() or wasLastResponseHTTPError()):
+                    return None
+                else:
+                    return True
 
         # Regular expression to match in page when the query is True and/or valid
         if conf.regexp:

lib/request/connect.py

@@ -83,9 +83,9 @@ from lib.core.enums import WEB_PLATFORM
 from lib.core.exception import SqlmapCompressionException
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
+from lib.core.exception import SqlmapSkipTargetException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapTokenException
-from lib.core.exception import SqlmapUserQuitException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import ASTERISK_MARKER
 from lib.core.settings import BOUNDARY_BACKSLASH_MARKER
@@ -787,7 +787,7 @@ class Connect(object):
                     kb.connErrorChoice = readInput(message, default='N', boolean=True)
 
                 if kb.connErrorChoice is False:
-                    raise SqlmapUserQuitException
+                    raise SqlmapSkipTargetException
 
             if "forcibly closed" in tbMsg:
                 logger.critical(warnMsg)
@@ -1150,7 +1150,7 @@ class Connect(object):
 
         if conf.evalCode:
             delimiter = conf.paramDel or DEFAULT_GET_POST_DELIMITER
-            variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals()}
+            variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals(), "cookie": cookie}
             originals = {}
 
             if not get and PLACE.URI in conf.parameters:
@@ -1218,6 +1218,7 @@ class Connect(object):
                     variables[unsafeVariableNaming(variable)] = value
 
             uri = variables["uri"]
+            cookie = variables["cookie"]
 
             for name, value in variables.items():
                 if name != "__builtins__" and originals.get(name, "") != value:

lib/request/httpshandler.py

@@ -27,6 +27,7 @@ except ImportError:
 
 _protocols = filterNone(getattr(ssl, _, None) for _ in ("PROTOCOL_TLSv1_2", "PROTOCOL_TLSv1_1", "PROTOCOL_TLSv1", "PROTOCOL_SSLv3", "PROTOCOL_SSLv23", "PROTOCOL_SSLv2"))
 _lut = dict((getattr(ssl, _), _) for _ in dir(ssl) if _.startswith("PROTOCOL_"))
+_contexts = {}
 
 class HTTPSConnection(_http_client.HTTPSConnection):
     """
@@ -36,6 +37,12 @@ class HTTPSConnection(_http_client.HTTPSConnection):
     """
 
     def __init__(self, *args, **kwargs):
+        # NOTE: Dirty patch for https://bugs.python.org/issue38251 / https://github.com/sqlmapproject/sqlmap/issues/4158
+        if hasattr(ssl, "_create_default_https_context"):
+            if None not in _contexts:
+                _contexts[None] = ssl._create_default_https_context()
+            kwargs["context"] = _contexts[None]
+
         _http_client.HTTPSConnection.__init__(self, *args, **kwargs)
 
     def connect(self):
@@ -54,11 +61,12 @@ class HTTPSConnection(_http_client.HTTPSConnection):
             for protocol in [_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1]:
                 try:
                     sock = create_sock()
-                    context = ssl.SSLContext(protocol)
+                    if protocol not in _contexts:
-                    _ = context.wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
+                        _contexts[protocol] = ssl.SSLContext(protocol)
-                    if _:
+                    result = _contexts[protocol].wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
+                    if result:
                         success = True
-                        self.sock = _
+                        self.sock = result
                         _protocols.remove(protocol)
                         _protocols.insert(0, protocol)
                         break

lib/request/redirecthandler.py

@@ -154,6 +154,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                 except AttributeError:
                     def _(self):
                         return getattr(self, "hdrs") or {}
+
                     result.info = types.MethodType(_, result)
 
                 if not hasattr(result, "read"):
@@ -164,6 +165,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                             retVal = ""
                         finally:
                             return retVal
+
                     result.read = types.MethodType(_, result)
 
                 if not getattr(result, "url", None):

lib/techniques/blind/inference.py

@@ -137,7 +137,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
         if partialValue:
             firstChar = len(partialValue)
-        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN)\(", expression):
+        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
             firstChar = 0
         elif conf.firstChar is not None and (isinstance(conf.firstChar, int) or (hasattr(conf.firstChar, "isdigit") and conf.firstChar.isdigit())):
             firstChar = int(conf.firstChar) - 1
@@ -148,7 +148,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         else:
             firstChar = 0
 
-        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN)\(", expression):
+        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
             lastChar = 0
         elif conf.lastChar is not None and (isinstance(conf.lastChar, int) or (hasattr(conf.lastChar, "isdigit") and conf.lastChar.isdigit())):
             lastChar = int(conf.lastChar)

lib/techniques/union/use.py

@@ -167,6 +167,12 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 warnMsg += "(probably due to its length and/or content): "
                 warnMsg += safecharencode(trimmed)
                 logger.warn(warnMsg)
+            elif re.search(r"ORDER BY [^ ]+\Z", expression):
+                debugMsg = "retrying failed SQL query without the ORDER BY clause"
+                logger.debug(debugMsg)
+
+                expression = re.sub(r"\s*ORDER BY [^ ]+\Z", "", expression)
+                retVal = _oneShotUnionUse(expression, unpack, limited)
     else:
         vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
         kb.unionDuplicates = vector[7]

lib/utils/api.py

@@ -49,6 +49,7 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import RESTAPI_DEFAULT_ADAPTER
 from lib.core.settings import RESTAPI_DEFAULT_ADDRESS
 from lib.core.settings import RESTAPI_DEFAULT_PORT
+from lib.core.settings import VERSION_STRING
 from lib.core.shell import autoCompletion
 from lib.core.subprocessng import Popen
 from lib.parse.cmdline import cmdLineParser
@@ -657,6 +658,15 @@ def download(taskid, target, filename):
         logger.warning("[%s] File does not exist %s" % (taskid, target))
         return jsonize({"success": False, "message": "File does not exist"})
 
+@get("/version")
+def version(token=None):
+    """
+    Fetch server version
+    """
+
+    logger.debug("Fetched version (%s)" % ("admin" if is_admin(token) else request.remote_addr))
+    return jsonize({"success": True, "version": VERSION_STRING.split('/')[-1]})
+
 def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=RESTAPI_DEFAULT_ADAPTER, username=None, password=None):
     """
     REST-JSON API server
@@ -760,7 +770,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
             logger.critical(errMsg)
             return
 
-    commands = ("help", "new", "use", "data", "log", "status", "option", "stop", "kill", "list", "flush", "exit", "bye", "quit")
+    commands = ("help", "new", "use", "data", "log", "status", "option", "stop", "kill", "list", "flush", "version", "exit", "bye", "quit")
     autoCompletion(AUTOCOMPLETE_TYPE.API, commands=commands)
 
     taskid = None
@@ -849,6 +859,13 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
                 continue
             logger.info("Switching to task ID '%s' " % taskid)
 
+        elif command in ("version",):
+            raw = _client("%s/%s" % (addr, command))
+            res = dejsonize(raw)
+            if not res["success"]:
+                logger.error("Failed to execute command %s" % command)
+            dataToStdout("%s\n" % raw)
+
         elif command in ("list", "flush"):
             raw = _client("%s/admin/%s" % (addr, command))
             res = dejsonize(raw)
@@ -873,6 +890,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
             msg += "stop           Stop current task\n"
             msg += "kill           Kill current task\n"
             msg += "list           Display all tasks\n"
+            msg += "version        Fetch server version\n"
             msg += "flush          Flush tasks (delete all tasks)\n"
             msg += "exit           Exit this client\n"
 

lib/utils/hash.py

@@ -741,7 +741,9 @@ def hashRecognition(value):
     if value and len(value) >= 8 and ' ' not in value:   # Note: pre-filter condition (for optimization purposes)
         isOracle, isMySQL = Backend.isDbms(DBMS.ORACLE), Backend.isDbms(DBMS.MYSQL)
 
-        if isinstance(value, six.string_types):
+        if kb.cache.hashRegex is None:
+            parts = []
+
             for name, regex in getPublicTypeMembers(HASH):
                 # Hashes for Oracle and old MySQL look the same hence these checks
                 if isOracle and regex == HASH.MYSQL_OLD or isMySQL and regex == HASH.ORACLE_OLD:
@@ -749,9 +751,16 @@ def hashRecognition(value):
                 elif regex == HASH.CRYPT_GENERIC:
                     if any((value.lower() == value, value.upper() == value)):
                         continue
-                elif re.match(regex, value):
+                else:
-                    retVal = regex
+                    parts.append("(?P<%s>%s)" % (name, regex))
-                    break
+
+            kb.cache.hashRegex = ('|'.join(parts)).replace("(?i)", "")
+
+        if isinstance(value, six.string_types):
+            match = re.search(kb.cache.hashRegex, value, re.I)
+            if match:
+                algorithm, _ = [_ for _ in match.groupdict().items() if _[1] is not None][0]
+                retVal = getattr(HASH, algorithm)
 
     return retVal
 

lib/utils/hashdb.py

@@ -68,7 +68,7 @@ class HashDB(object):
 
     @staticmethod
     def hashKey(key):
-        key = getBytes(key if isinstance(key, six.text_type) else repr(key))
+        key = getBytes(key if isinstance(key, six.text_type) else repr(key), errors="xmlcharrefreplace")
         retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff  # Reference: http://stackoverflow.com/a/4448400
         return retVal
 

lib/utils/purge.py

@@ -80,8 +80,6 @@ def purge(directory):
             pass
 
     logger.debug("deleting the whole directory tree")
-    os.chdir(os.path.join(directory, ".."))
-
     try:
         shutil.rmtree(directory)
     except OSError as ex:

lib/utils/search.py

@@ -104,6 +104,8 @@ def _search(dork):
 
     page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
 
+    page = getUnicode(page)  # Note: if upper function call fails (Issue #4202)
+
     retVal = [_urllib.parse.unquote(match.group(1) or match.group(2)) for match in re.finditer(GOOGLE_REGEX, page, re.I)]
 
     if not retVal and "detected unusual traffic" in page:

plugins/dbms/postgresql/takeover.py

@@ -12,6 +12,7 @@ from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
 from lib.core.common import flattenValue
 from lib.core.common import isListLike
+from lib.core.common import isNoneValue
 from lib.core.common import isStackingAvailable
 from lib.core.common import randomStr
 from lib.core.data import kb
@@ -105,7 +106,10 @@ class Takeover(GenericTakeover):
             output = inject.getValue(query, resumeValue=False)
 
             if isListLike(output):
-                output = os.linesep.join(flattenValue(output))
+                output = flattenValue(output)
+
+            if not isNoneValue(output):
+                output = os.linesep.join(output)
 
             self._cleanupCmd = "DROP TABLE %s" % self.cmdTblName
             inject.goStacked(self._cleanupCmd)

plugins/generic/custom.py

@@ -19,6 +19,7 @@ from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.dicts import SQL_STATEMENTS
 from lib.core.enums import AUTOCOMPLETE_TYPE
+from lib.core.enums import DBMS
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.settings import NULL
 from lib.core.settings import PARAMETER_SPLITTING_REGEX
@@ -46,10 +47,15 @@ class Custom(object):
                         sqlType = sqlTitle
                         break
 
-            if not any(_ in query.upper() for _ in ("OPENROWSET", "INTO")) and (not sqlType or "SELECT" in sqlType):
+            if not re.search(r"\b(OPENROWSET|INTO)\b", query, re.I) and (not sqlType or "SELECT" in sqlType):
                 infoMsg = "fetching %s query output: '%s'" % (sqlType if sqlType is not None else "SQL", query)
                 logger.info(infoMsg)
 
+                if Backend.isDbms(DBMS.MSSQL):
+                    match = re.search(r"(\bFROM\s+)([^\s]+)", query, re.I)
+                    if match and match.group(2).count('.') == 1:
+                        query = query.replace(match.group(0), "%s%s" % (match.group(1), match.group(2).replace('.', ".dbo.")))
+
                 output = inject.getValue(query, fromUser=True)
 
                 return output

plugins/generic/entries.py

@@ -131,6 +131,8 @@ class Entries(object):
             try:
                 if Backend.isDbms(DBMS.INFORMIX):
                     kb.dumpTable = "%s:%s" % (conf.db, tbl)
+                elif Backend.isDbms(DBMS.SQLITE):
+                    kb.dumpTable = tbl
                 else:
                     kb.dumpTable = "%s.%s" % (conf.db, tbl)
 
@@ -156,7 +158,7 @@ class Entries(object):
                     logger.warn(warnMsg)
                     continue
 
-                kb.dumpColumns = colList
+                kb.dumpColumns = [unsafeSQLIdentificatorNaming(_) for _ in colList]
                 colNames = colString = ", ".join(column for column in colList)
                 rootQuery = queries[Backend.getIdentifiedDbms()].dump_table
 

sqlmap.conf

@@ -693,6 +693,9 @@ trafficFile =
 # Set predefined answers (e.g. "quit=N,follow=N").
 answers =
 
+# Parameter(s) containing Base64 encoded data
+base64Parameter =
+
 # Never ask for user input, use the default behaviour.
 # Valid: True or False
 batch = False

sqlmap.py

@@ -238,6 +238,8 @@ def main():
         errMsg = getSafeExString(ex)
         logger.critical(errMsg)
 
+        os._exitcode = 1
+
         raise SystemExit
 
     except KeyboardInterrupt:
@@ -249,8 +251,8 @@ def main():
         errMsg = "exit"
         logger.error(errMsg)
 
-    except SystemExit:
+    except SystemExit as ex:
-        pass
+        os._exitcode = ex.code or 0
 
     except:
         print()
@@ -258,6 +260,8 @@ def main():
         excMsg = traceback.format_exc()
         valid = checkIntegrity()
 
+        os._exitcode = 255
+
         if any(_ in excMsg for _ in ("MemoryError", "Cannot allocate memory")):
             errMsg = "memory exhaustion detected"
             logger.critical(errMsg)
@@ -530,4 +534,4 @@ if __name__ == "__main__":
             sys.exit(getattr(os, "_exitcode", 0))
 else:
     # cancelling postponed imports (because of Travis CI checks)
-    from lib.controller.controller import start
+    __import__("lib.controller.controller")

thirdparty/clientform/clientform.py

@@ -237,9 +237,9 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):
 
         repl = entities.get(ent)
         if repl is not None:
-            if type(repl) != type("") and encoding is not None:
+            if hasattr(repl, "decode") and encoding is not None:
                 try:
-                    repl = repl.encode(encoding)
+                    repl = repl.decode(encoding)
                 except UnicodeError:
                     repl = ent
         else:
@@ -255,15 +255,11 @@ def unescape_charref(data, encoding):
         name, base= name[1:], 16
     elif not name.isdigit():
         base = 16
-    uc = _unichr(int(name, base))
+
-    if encoding is None:
+    try:
-        return uc
+        return _unichr(int(name, base))
-    else:
+    except:
-        try:
+        return data
-            repl = uc.encode(encoding)
-        except UnicodeError:
-            repl = "&#%s;" % data
-        return repl
 
 def get_entitydefs():
     from codecs import latin_1_decode
@@ -713,7 +709,7 @@ class _AbstractFormParser:
                 data = data[1:]
             map[key] = data
         else:
-            map[key] = (map[key].decode("utf8") if isinstance(map[key], six.binary_type) else map[key]) + data
+            map[key] = (map[key].decode("utf8", "replace") if isinstance(map[key], six.binary_type) else map[key]) + data
 
     def do_button(self, attrs):
         debug("%s", attrs)

thirdparty/colorama/ansitowin32.py

@@ -47,7 +47,7 @@ class AnsiToWin32(object):
     win32 function calls.
     '''
     ANSI_CSI_RE = re.compile('\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?')     # Control Sequence Introducer
-    ANSI_OSC_RE = re.compile('\001?\033\\]((?:.|;)*?)(\x07)\002?')         # Operating System Command
+    ANSI_OSC_RE = re.compile('\001?\033\\]([^\a]*)(\a)\002?')               # Operating System Command (Note: https://github.com/tartley/colorama/issues/247)
 
     def __init__(self, wrapped, convert=None, strip=None, autoreset=False):
         # The wrapped stream (normally sys.stdout or sys.stderr)