Update regarding #6145

Every tests/test_*.py file provides an

    if __name__ == "__main__":
        unittest.main(...)

entrypoint so it can be run directly, except test_ssti.py and
test_xpath.py. Running either of those directly produced no output and
executed none of its tests; they only ran under `unittest discover`.

Add the missing entrypoint to both so they behave like the rest of the
suite. No test logic changed (62 and 48 tests respectively still pass).

data/xml/payloads/boolean_blind.xml

@@ -1365,6 +1365,27 @@ Tag: <test>
         </details>
     </test>
 
+    <!-- No parentheses or functions (e.g. value-wide character whitelists, #6145): TRUE keeps the original
+         sort column, FALSE raises a runtime regular expression error ('(' pattern) -->
+    <test>
+        <title>MySQL boolean-based blind - ORDER BY clause (RLIKE)</title>
+        <stype>1</stype>
+        <level>3</level>
+        <risk>1</risk>
+        <clause>3</clause>
+        <where>3</where>
+        <vector>CASE WHEN [INFERENCE] THEN [ORIGVALUE] ELSE 1 RLIKE 0x28 END</vector>
+        <request>
+            <payload>CASE WHEN [RANDNUM]=[RANDNUM] THEN [ORIGVALUE] ELSE 1 RLIKE 0x28 END</payload>
+        </request>
+        <response>
+            <comparison>CASE WHEN [RANDNUM]=[RANDNUM1] THEN [ORIGVALUE] ELSE 1 RLIKE 0x28 END</comparison>
+        </response>
+        <details>
+            <dbms>MySQL</dbms>
+        </details>
+    </test>
+
     <!-- Works in MySQL, Oracle, etc. -->
     <test>
         <title>HAVING boolean-based blind - WHERE, GROUP BY clause</title>

lib/controller/checks.py

@@ -403,7 +403,7 @@ def checkSqlInjection(place, parameter, value):
 
             # Parse test's <request>
             comment = agent.getComment(test.request) if len(conf.boundaries) > 1 else None
-            fstPayload = agent.cleanupPayload(test.request.payload, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None)
+            fstPayload = agent.cleanupPayload(test.request.payload, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None, clause=clause)
 
             for boundary in boundaries:
                 injectable = False
@@ -517,13 +517,13 @@ def checkSqlInjection(place, parameter, value):
                     # payload was successful
                     # Parse test's <response>
                     for method, check in test.response.items():
-                        check = agent.cleanupPayload(check, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None)
+                        check = agent.cleanupPayload(check, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None, clause=clause)
 
                         # In case of boolean-based blind SQL injection
                         if method == PAYLOAD.METHOD.COMPARISON:
                             # Generate payload used for comparison
                             def genCmpPayload():
-                                sndPayload = agent.cleanupPayload(test.response.comparison, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None)
+                                sndPayload = agent.cleanupPayload(test.response.comparison, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None, clause=clause)
 
                                 # Forge response payload by prepending with
                                 # boundary's prefix and appending the boundary's

lib/core/agent.py

@@ -61,6 +61,31 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.unescaper import unescaper
 from thirdparty import six
 
+def _isGroupOrderByClause(clause):
+    """
+    Returns True if (test's/injection's) clause is exclusively GROUP BY and/or ORDER BY
+
+    >>> _isGroupOrderByClause([PAYLOAD.CLAUSE.GROUP_BY, PAYLOAD.CLAUSE.ORDER_BY])
+    True
+    >>> _isGroupOrderByClause([PAYLOAD.CLAUSE.WHERE, PAYLOAD.CLAUSE.ORDER_BY])
+    False
+    """
+
+    return bool(clause) and set(clause) <= set((PAYLOAD.CLAUSE.GROUP_BY, PAYLOAD.CLAUSE.ORDER_BY))
+
+def _markerSafeRandom(generator):
+    """
+    Returns a random value (e.g. for [RANDNUM]) not containing the --string/--not-string page marker,
+    as a reflected one (e.g. echoed in a DBMS error message) would otherwise fake the comparison
+    """
+
+    markers = filterNone((conf.get("string"), conf.get("notString")))
+
+    while True:
+        retVal = getUnicode(generator())
+        if not any(_ in retVal for _ in markers):
+            return retVal
+
 class Agent(object):
     """
     This class defines the SQL agent methods.
@@ -184,7 +209,7 @@ class Agent(object):
 
             newValue = "%s%s" % (value, newValue)
 
-        newValue = self.cleanupPayload(newValue, origValue) or ""
+        newValue = self.cleanupPayload(newValue, origValue, clause=kb.injection.clause if kb.get("injection") else None) or ""
 
         if base64Encoding:
             _newValue = newValue
@@ -296,9 +321,9 @@ class Agent(object):
         # after the prefix or it is in GROUP BY / ORDER BY (<clause>)
         elif getTechnique() == PAYLOAD.TECHNIQUE.STACKED:
             query = kb.injection.prefix
-        elif kb.injection.clause == [2, 3] or kb.injection.clause == [2] or kb.injection.clause == [3]:
+        elif _isGroupOrderByClause(kb.injection.clause):
             query = kb.injection.prefix
-        elif clause == [2, 3] or clause == [2] or clause == [3]:
+        elif _isGroupOrderByClause(clause):
             query = prefix
 
         # In any other case prepend with the full prefix
@@ -356,7 +381,7 @@ class Agent(object):
 
         return re.sub(r";\W*;", ";", expression) if trimEmpty else expression
 
-    def cleanupPayload(self, payload, origValue=None):
+    def cleanupPayload(self, payload, origValue=None, clause=None):
         if not isinstance(payload, six.string_types):
             return
 
@@ -375,16 +400,18 @@ class Agent(object):
                 payload = payload.replace(value, replacements[value])
 
         for _ in set(re.findall(r"(?i)\[RANDNUM(?:\d+)?\]", payload)):
-            payload = payload.replace(_, str(randomInt()))
+            payload = payload.replace(_, _markerSafeRandom(randomInt))
 
         for _ in set(re.findall(r"(?i)\[RANDSTR(?:\d+)?\]", payload)):
-            payload = payload.replace(_, randomStr())
+            payload = payload.replace(_, _markerSafeRandom(randomStr))
 
         if origValue is not None:
             origValue = getUnicode(origValue)
 
             if "[ORIGVALUE]" in payload:
-                payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if origValue.isdigit() else unescaper.escape("'%s'" % origValue))
+                # Note: in a GROUP BY/ORDER BY payload a (plain identifier) original value is the column itself - quoting it would turn it into a constant
+                raw = origValue.isdigit() or (_isGroupOrderByClause(clause) and re.search(r"\A[\w.]+\Z", origValue) is not None)
+                payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if raw else unescaper.escape("'%s'" % origValue))
             if "[ORIGINAL]" in payload:
                 payload = getUnicode(payload).replace("[ORIGINAL]", origValue)
 
@@ -426,10 +453,10 @@ class Agent(object):
             payload = payload.replace(SINGLE_QUOTE_MARKER, "'")
 
             for _ in set(re.findall(r"\[RANDNUM(?:\d+)?\]", payload, re.I)):
-                payload = payload.replace(_, str(randomInt()))
+                payload = payload.replace(_, _markerSafeRandom(randomInt))
 
             for _ in set(re.findall(r"\[RANDSTR(?:\d+)?\]", payload, re.I)):
-                payload = payload.replace(_, randomStr())
+                payload = payload.replace(_, _markerSafeRandom(randomStr))
 
             if hashDBRetrieve(HASHDB_KEYS.DBMS_FORK) in (FORK.MEMSQL, FORK.TIDB, FORK.DRIZZLE):
                 payload = re.sub(r"(?i)\bORD\(", "ASCII(", payload)

lib/core/enums.py

@@ -352,18 +352,17 @@ class PAYLOAD(object):
         3: "High risk",
     }
 
-    CLAUSE = {
-        0: "Always",
-        1: "WHERE",
-        2: "GROUP BY",
-        3: "ORDER BY",
-        4: "LIMIT",
-        5: "OFFSET",
-        6: "TOP",
-        7: "Table name",
-        8: "Column name",
-        9: "Pre-WHERE (non-query)",
-    }
+    class CLAUSE(object):
+        ALWAYS = 0
+        WHERE = 1
+        GROUP_BY = 2
+        ORDER_BY = 3
+        LIMIT = 4
+        OFFSET = 5
+        TOP = 6
+        TABLE_NAME = 7
+        COLUMN_NAME = 8
+        PRE_WHERE = 9
 
     class METHOD(object):
         COMPARISON = "comparison"

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.10.0"
+VERSION = "1.10.10.3"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/request/connect.py

@@ -1638,7 +1638,7 @@ class Connect(object):
                                     match = re.search(r"['\"]", post)
                                     if match:
                                         quote = match.group(0)
-                                        post = re.sub(r"\}\Z", "%s%s}" % (',' if re.search(r"\w", post) else "", "%s%s%s:%s" % (quote, name, quote, entry if entry.isdigit() else "%s%s%s" % (quote, entry, quote))), post)
+                                        post = re.sub(r"\}\Z", "%s%s}" % (',' if re.search(r"\w", post) else "", "%s%s%s:%s" % (quote, name, quote, entry if entry.isdigit() else "%s%s%s" % (quote, entry.replace('\\', r'\\'), quote))), post)
                                 else:
                                     post += "%s%s=%s" % (delimiter, name, entry)
                             elif get is not None:

lib/utils/hash.py

@@ -79,6 +79,7 @@ from lib.core.enums import HASH
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.patch import dirtyPatches
 from lib.core.settings import COMMON_PASSWORD_SUFFIXES
 from lib.core.settings import COMMON_PASSWORDS
 from lib.core.settings import COMMON_USER_COLUMNS
@@ -101,6 +102,16 @@ from thirdparty import six
 from thirdparty.colorama.initialise import init as coloramainit
 from thirdparty.six.moves import queue as _queue
 
+# On Windows, multiprocessing's 'spawn' start method boots a fresh interpreter per cracking
+# worker (_bruteProcessVariantA/B/Salted below) to unpickle its target function, which re-imports
+# this module from scratch without ever running the main process's startup dirtyPatches() - so
+# process-wide monkeypatches it installs (e.g. the 'reversible' codec error handler used by
+# getUnicode()/openFile()) are missing in the worker, raising LookupError on every hashed entry
+# (#6143). Placing the (idempotent) call here means that very re-import applies it. POSIX fork()
+# workers inherit the parent's patches already, so this only needs to run on Windows.
+if IS_WIN:
+    dirtyPatches()
+
 def mysql_passwd(password, uppercase=True):
     """
     Reference(s):

tamper/between.py

@@ -38,6 +38,8 @@ def tamper(payload, **kwargs):
     '1 AND A BETWEEN B AND B--'
     >>> tamper('1 AND LAST_INSERT_ROWID()=LAST_INSERT_ROWID()')
     '1 AND LAST_INSERT_ROWID() BETWEEN LAST_INSERT_ROWID() AND LAST_INSERT_ROWID()'
+    >>> tamper('CASE WHEN 1=2 THEN id ELSE 0 END')
+    'CASE WHEN 1 BETWEEN 2 AND 2 THEN id ELSE 0 END'
     """
 
     retVal = payload
@@ -58,4 +60,7 @@ def tamper(payload, **kwargs):
                 _ = "%s %s BETWEEN %s AND %s" % (match.group(2), match.group(4), match.group(5), match.group(5))
                 retVal = retVal.replace(match.group(0), _)
 
+        # e.g. CASE WHEN 1=1 THEN ... (expression-position payloads with no AND/OR to anchor on)
+        retVal = re.sub(r"(?i)(\bWHEN\s+)([\w.]+)\s*(?<![<>!])=(?!=)\s*([\w.]+)(?=\s)", r"\g<1>\g<2> BETWEEN \g<3> AND \g<3>", retVal)
+
     return retVal

tests/test_agent.py

@@ -556,6 +556,21 @@ class TestCleanupPayloadOrigValue(DbmsStateMixin, unittest.TestCase):
         out = agent.cleanupPayload("p=[ORIGINAL]", origValue="raw")
         self.assertEqual(out, "p=raw")
 
+    def test_origvalue_identifier_raw_for_group_order_by(self):
+        self.assertEqual(agent.cleanupPayload("[ORIGVALUE]", origValue="p.name", clause=[2, 3]), "p.name")
+        self.assertIn("'p.name'", agent.cleanupPayload("[ORIGVALUE]", origValue="p.name", clause=[1, 2, 3]))
+
+    def test_random_values_avoid_string_marker(self):
+        # a reflected [RANDNUM] (e.g. quoted back in a DBMS error message) containing the --string
+        # marker would fake a True response (e.g. --string=98 vs. 8988)
+        saved = conf.string
+        conf.string = "9"
+        try:
+            for _ in range(200):
+                self.assertNotIn("9", agent.cleanupPayload("[RANDNUM] [RANDNUM1]"))
+        finally:
+            conf.string = saved
+
     def test_space_replace_marker(self):
         out = agent.cleanupPayload("a[SPACE_REPLACE]b")
         self.assertEqual(out, "a%sb" % kb.chars.space)

tests/test_ssti.py

@@ -827,3 +827,7 @@ class TestStruts2Header(unittest.TestCase):
         # so no start/end marker is found -> no fabricated 'output'
         ssti._s2045Send = lambda url, action: "reflected: %s" % action
         self.assertIsNone(ssti._executeStruts2Header("http://target", "id"))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/test_xpath.py

@@ -557,3 +557,7 @@ class TestRealXPathSyntax(unittest.TestCase):
                 self.fail("Boundary '%s' in '%s' with orig='%s' invalid: %s\n  payload: %s" % (bk, tkey, original, e, payload))
             self.assertIsInstance(count, int,
                 "Boundary '%s' in '%s' produced no count" % (bk, tkey))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)