Minor cleaning after #6146

A scan covering multiple targets inside a single API task (crawl,
'-m' bulk file, forms) ran every target through one StdDbOut keyed by
the single taskid, so _storeData()'s COMPLETE branch deleted each
earlier target's TARGET/TECHNIQUES rows - /scan/<id>/data returned
only the last target's results.

Mirror the --report-json fix for #6123: controller.start() now sets
kb.reportTaskId per target for API scans too, StdDbOut.write() keys
data rows '<taskid>-<n>' per target, and scan_data() folds the
per-target rows back into the task's flat result in target order.
Single-target scans keep the exact same response shape.

Regression tests in tests/test_api.py (TestStdDbOutMultiTarget,
TestScanDataMultiTarget).

data/xml/payloads/boolean_blind.xml

@@ -1365,6 +1365,27 @@ Tag: <test>
         </details>
     </test>
 
+    <!-- No parentheses or functions (e.g. value-wide character whitelists, #6145): TRUE keeps the original
+         sort column, FALSE raises a runtime regular expression error ('(' pattern) -->
+    <test>
+        <title>MySQL boolean-based blind - ORDER BY clause (RLIKE)</title>
+        <stype>1</stype>
+        <level>3</level>
+        <risk>1</risk>
+        <clause>3</clause>
+        <where>3</where>
+        <vector>CASE WHEN [INFERENCE] THEN [ORIGVALUE] ELSE 1 RLIKE 0x28 END</vector>
+        <request>
+            <payload>CASE WHEN [RANDNUM]=[RANDNUM] THEN [ORIGVALUE] ELSE 1 RLIKE 0x28 END</payload>
+        </request>
+        <response>
+            <comparison>CASE WHEN [RANDNUM]=[RANDNUM1] THEN [ORIGVALUE] ELSE 1 RLIKE 0x28 END</comparison>
+        </response>
+        <details>
+            <dbms>MySQL</dbms>
+        </details>
+    </test>
+
     <!-- Works in MySQL, Oracle, etc. -->
     <test>
         <title>HAVING boolean-based blind - WHERE, GROUP BY clause</title>

lib/controller/checks.py

@@ -403,7 +403,7 @@ def checkSqlInjection(place, parameter, value):
 
             # Parse test's <request>
             comment = agent.getComment(test.request) if len(conf.boundaries) > 1 else None
-            fstPayload = agent.cleanupPayload(test.request.payload, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None)
+            fstPayload = agent.cleanupPayload(test.request.payload, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None, clause=clause)
 
             for boundary in boundaries:
                 injectable = False
@@ -517,13 +517,13 @@ def checkSqlInjection(place, parameter, value):
                     # payload was successful
                     # Parse test's <response>
                     for method, check in test.response.items():
-                        check = agent.cleanupPayload(check, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None)
+                        check = agent.cleanupPayload(check, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None, clause=clause)
 
                         # In case of boolean-based blind SQL injection
                         if method == PAYLOAD.METHOD.COMPARISON:
                             # Generate payload used for comparison
                             def genCmpPayload():
-                                sndPayload = agent.cleanupPayload(test.response.comparison, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None)
+                                sndPayload = agent.cleanupPayload(test.response.comparison, origValue=value if place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER) and BOUNDED_INJECTION_MARKER not in (value or "") else None, clause=clause)
 
                                 # Forge response payload by prepending with
                                 # boundary's prefix and appending the boundary's

lib/controller/controller.py

@@ -376,9 +376,9 @@ def start():
     for targetUrl, targetMethod, targetData, targetCookie, targetHeaders in kb.targets:
         targetCount += 1
 
-        # --report-json: give each target its own taskid in the shared collector, so a multi-target
+        # --report-json / REST API: give each target its own taskid in the shared collector, so a multi-target
-        # run (e.g. '-m' bulk file) doesn't have later targets overwrite earlier ones (see _reportData)
+        # run (e.g. '-m' bulk file) doesn't have later targets overwrite earlier ones (see _reportData/StdDbOut.write)
-        if conf.reportJson:
+        if conf.reportJson or conf.api:
             kb.reportTaskId = targetCount
 
         try:

lib/core/agent.py

@@ -61,6 +61,31 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.unescaper import unescaper
 from thirdparty import six
 
+def _isGroupOrderByClause(clause):
+    """
+    Returns True if (test's/injection's) clause is exclusively GROUP BY and/or ORDER BY
+
+    >>> _isGroupOrderByClause([PAYLOAD.CLAUSE.GROUP_BY, PAYLOAD.CLAUSE.ORDER_BY])
+    True
+    >>> _isGroupOrderByClause([PAYLOAD.CLAUSE.WHERE, PAYLOAD.CLAUSE.ORDER_BY])
+    False
+    """
+
+    return bool(clause) and set(clause) <= set((PAYLOAD.CLAUSE.GROUP_BY, PAYLOAD.CLAUSE.ORDER_BY))
+
+def _markerSafeRandom(generator):
+    """
+    Returns a random value (e.g. for [RANDNUM]) not containing the --string/--not-string page marker,
+    as a reflected one (e.g. echoed in a DBMS error message) would otherwise fake the comparison
+    """
+
+    markers = filterNone((conf.get("string"), conf.get("notString")))
+
+    while True:
+        retVal = getUnicode(generator())
+        if not any(_ in retVal for _ in markers):
+            return retVal
+
 class Agent(object):
     """
     This class defines the SQL agent methods.
@@ -184,7 +209,7 @@ class Agent(object):
 
             newValue = "%s%s" % (value, newValue)
 
-        newValue = self.cleanupPayload(newValue, origValue) or ""
+        newValue = self.cleanupPayload(newValue, origValue, clause=kb.injection.clause if kb.get("injection") else None) or ""
 
         if base64Encoding:
             _newValue = newValue
@@ -296,9 +321,9 @@ class Agent(object):
         # after the prefix or it is in GROUP BY / ORDER BY (<clause>)
         elif getTechnique() == PAYLOAD.TECHNIQUE.STACKED:
             query = kb.injection.prefix
-        elif kb.injection.clause == [2, 3] or kb.injection.clause == [2] or kb.injection.clause == [3]:
+        elif _isGroupOrderByClause(kb.injection.clause):
             query = kb.injection.prefix
-        elif clause == [2, 3] or clause == [2] or clause == [3]:
+        elif _isGroupOrderByClause(clause):
             query = prefix
 
         # In any other case prepend with the full prefix
@@ -356,7 +381,7 @@ class Agent(object):
 
         return re.sub(r";\W*;", ";", expression) if trimEmpty else expression
 
-    def cleanupPayload(self, payload, origValue=None):
+    def cleanupPayload(self, payload, origValue=None, clause=None):
         if not isinstance(payload, six.string_types):
             return
 
@@ -375,16 +400,18 @@ class Agent(object):
                 payload = payload.replace(value, replacements[value])
 
         for _ in set(re.findall(r"(?i)\[RANDNUM(?:\d+)?\]", payload)):
-            payload = payload.replace(_, str(randomInt()))
+            payload = payload.replace(_, _markerSafeRandom(randomInt))
 
         for _ in set(re.findall(r"(?i)\[RANDSTR(?:\d+)?\]", payload)):
-            payload = payload.replace(_, randomStr())
+            payload = payload.replace(_, _markerSafeRandom(randomStr))
 
         if origValue is not None:
             origValue = getUnicode(origValue)
 
             if "[ORIGVALUE]" in payload:
-                payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if origValue.isdigit() else unescaper.escape("'%s'" % origValue))
+                # Note: in a GROUP BY/ORDER BY payload a (plain identifier) original value is the column itself - quoting it would turn it into a constant
+                raw = origValue.isdigit() or (_isGroupOrderByClause(clause) and re.search(r"\A[\w.]+\Z", origValue) is not None)
+                payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if raw else unescaper.escape("'%s'" % origValue))
             if "[ORIGINAL]" in payload:
                 payload = getUnicode(payload).replace("[ORIGINAL]", origValue)
 
@@ -426,10 +453,10 @@ class Agent(object):
             payload = payload.replace(SINGLE_QUOTE_MARKER, "'")
 
             for _ in set(re.findall(r"\[RANDNUM(?:\d+)?\]", payload, re.I)):
-                payload = payload.replace(_, str(randomInt()))
+                payload = payload.replace(_, _markerSafeRandom(randomInt))
 
             for _ in set(re.findall(r"\[RANDSTR(?:\d+)?\]", payload, re.I)):
-                payload = payload.replace(_, randomStr())
+                payload = payload.replace(_, _markerSafeRandom(randomStr))
 
             if hashDBRetrieve(HASHDB_KEYS.DBMS_FORK) in (FORK.MEMSQL, FORK.TIDB, FORK.DRIZZLE):
                 payload = re.sub(r"(?i)\bORD\(", "ASCII(", payload)

lib/core/enums.py

@@ -352,18 +352,17 @@ class PAYLOAD(object):
         3: "High risk",
     }
 
-    CLAUSE = {
+    class CLAUSE(object):
-        0: "Always",
+        ALWAYS = 0
-        1: "WHERE",
+        WHERE = 1
-        2: "GROUP BY",
+        GROUP_BY = 2
-        3: "ORDER BY",
+        ORDER_BY = 3
-        4: "LIMIT",
+        LIMIT = 4
-        5: "OFFSET",
+        OFFSET = 5
-        6: "TOP",
+        TOP = 6
-        7: "Table name",
+        TABLE_NAME = 7
-        8: "Column name",
+        COLUMN_NAME = 8
-        9: "Pre-WHERE (non-query)",
+        PRE_WHERE = 9
-    }
 
     class METHOD(object):
         COMPARISON = "comparison"

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.10.0"
+VERSION = "1.10.10.4"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/request/connect.py

@@ -1638,7 +1638,7 @@ class Connect(object):
                                     match = re.search(r"['\"]", post)
                                     if match:
                                         quote = match.group(0)
-                                        post = re.sub(r"\}\Z", "%s%s}" % (',' if re.search(r"\w", post) else "", "%s%s%s:%s" % (quote, name, quote, entry if entry.isdigit() else "%s%s%s" % (quote, entry, quote))), post)
+                                        post = re.sub(r"\}\Z", "%s%s}" % (',' if re.search(r"\w", post) else "", "%s%s%s:%s" % (quote, name, quote, entry if entry.isdigit() else "%s%s%s" % (quote, entry.replace('\\', r'\\'), quote))), post)
                                 else:
                                     post += "%s%s=%s" % (delimiter, name, entry)
                             elif get is not None:

lib/utils/api.py

@@ -444,7 +444,9 @@ class StdDbOut(object):
 
     def write(self, value, status=CONTENT_STATUS.IN_PROGRESS, content_type=None):
         if self.messagetype == "stdout":
-            _storeData(conf.databaseCursor, self.taskid, value, status, content_type)
+            # Note: per-target key in multi-target scans, so a later target's COMPLETE row doesn't delete an earlier one's (#2503)
+            taskid = "%s-%s" % (self.taskid, kb.reportTaskId) if kb.get("reportTaskId") else self.taskid
+            _storeData(conf.databaseCursor, taskid, value, status, content_type)
         else:
             conf.databaseCursor.execute("INSERT INTO errors VALUES(NULL, ?, ?)", (self.taskid, str(value) if value else ""))
 
@@ -818,6 +820,12 @@ def scan_data(taskid):
     # Read all data and error messages from the IPC database (shared assembler - same output as --report-json)
     result = _assembleData(DataStore.current_db, taskid)
 
+    # Note: fold in per-target rows ('<taskid>-<n>', see StdDbOut.write) of multi-target scans, in target order
+    prefix = "%s-" % taskid
+    subtaskids = [_[0] for _ in DataStore.current_db.execute("SELECT DISTINCT taskid FROM data WHERE taskid LIKE ?", (prefix + '%',)) if _[0][len(prefix):].isdigit()]
+    for subtaskid in sorted(subtaskids, key=lambda _: int(_[len(prefix):])):
+        result["data"].extend(_assembleData(DataStore.current_db, subtaskid)["data"])
+
     logger.debug("(%s) Retrieved scan data and error messages" % taskid)
     return jsonize(result)
 

lib/utils/hash.py

@@ -79,6 +79,7 @@ from lib.core.enums import HASH
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.patch import dirtyPatches
 from lib.core.settings import COMMON_PASSWORD_SUFFIXES
 from lib.core.settings import COMMON_PASSWORDS
 from lib.core.settings import COMMON_USER_COLUMNS
@@ -101,6 +102,16 @@ from thirdparty import six
 from thirdparty.colorama.initialise import init as coloramainit
 from thirdparty.six.moves import queue as _queue
 
+# On Windows, multiprocessing's 'spawn' start method boots a fresh interpreter per cracking
+# worker (_bruteProcessVariantA/B/Salted below) to unpickle its target function, which re-imports
+# this module from scratch without ever running the main process's startup dirtyPatches() - so
+# process-wide monkeypatches it installs (e.g. the 'reversible' codec error handler used by
+# getUnicode()/openFile()) are missing in the worker, raising LookupError on every hashed entry
+# (#6143). Placing the (idempotent) call here means that very re-import applies it. POSIX fork()
+# workers inherit the parent's patches already, so this only needs to run on Windows.
+if IS_WIN:
+    dirtyPatches()
+
 def mysql_passwd(password, uppercase=True):
     """
     Reference(s):

tamper/between.py

@@ -38,6 +38,8 @@ def tamper(payload, **kwargs):
     '1 AND A BETWEEN B AND B--'
     >>> tamper('1 AND LAST_INSERT_ROWID()=LAST_INSERT_ROWID()')
     '1 AND LAST_INSERT_ROWID() BETWEEN LAST_INSERT_ROWID() AND LAST_INSERT_ROWID()'
+    >>> tamper('CASE WHEN 1=2 THEN id ELSE 0 END')
+    'CASE WHEN 1 BETWEEN 2 AND 2 THEN id ELSE 0 END'
     """
 
     retVal = payload
@@ -58,4 +60,7 @@ def tamper(payload, **kwargs):
                 _ = "%s %s BETWEEN %s AND %s" % (match.group(2), match.group(4), match.group(5), match.group(5))
                 retVal = retVal.replace(match.group(0), _)
 
+        # e.g. CASE WHEN 1=1 THEN ... (expression-position payloads with no AND/OR to anchor on)
+        retVal = re.sub(r"(?i)(\bWHEN\s+)([\w.]+)\s*(?<![<>!])=(?!=)\s*([\w.]+)(?=\s)", r"\g<1>\g<2> BETWEEN \g<3> AND \g<3>", retVal)
+
     return retVal

tests/test_agent.py

@@ -556,6 +556,21 @@ class TestCleanupPayloadOrigValue(DbmsStateMixin, unittest.TestCase):
         out = agent.cleanupPayload("p=[ORIGINAL]", origValue="raw")
         self.assertEqual(out, "p=raw")
 
+    def test_origvalue_identifier_raw_for_group_order_by(self):
+        self.assertEqual(agent.cleanupPayload("[ORIGVALUE]", origValue="p.name", clause=[2, 3]), "p.name")
+        self.assertIn("'p.name'", agent.cleanupPayload("[ORIGVALUE]", origValue="p.name", clause=[1, 2, 3]))
+
+    def test_random_values_avoid_string_marker(self):
+        # a reflected [RANDNUM] (e.g. quoted back in a DBMS error message) containing the --string
+        # marker would fake a True response (e.g. --string=98 vs. 8988)
+        saved = conf.string
+        conf.string = "9"
+        try:
+            for _ in range(200):
+                self.assertNotIn("9", agent.cleanupPayload("[RANDNUM] [RANDNUM1]"))
+        finally:
+            conf.string = saved
+
     def test_space_replace_marker(self):
         out = agent.cleanupPayload("a[SPACE_REPLACE]b")
         self.assertEqual(out, "a%sb" % kb.chars.space)

tests/test_api.py

@@ -30,6 +30,7 @@ bootstrap()
 
 import lib.utils.api as api
 from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.convert import encodeBase64
 from lib.core.enums import CONTENT_STATUS, CONTENT_TYPE
 from thirdparty.bottle.bottle import default_app
@@ -308,6 +309,63 @@ class TestStdDbOutAndLogRecorder(unittest.TestCase):
         self.assertEqual(rows[0][2], "hello world")
 
 
+class TestStdDbOutMultiTarget(unittest.TestCase):
+    """
+    Regression coverage for issue #2503: a multi-target scan (crawl or '-m' bulk
+    file) inside a single REST API task runs every target through one StdDbOut
+    instance. StdDbOut.write() keys each target's data rows by kb.reportTaskId
+    ('<taskid>-<n>'), so a later target's COMPLETE row cannot delete an earlier
+    target's rows of the same content_type in _storeData() - mirroring the
+    --report-json fix for #6123 (see test_report.py's TestMultiTargetTaskId).
+    """
+
+    def setUp(self):
+        self.db = api.Database(":memory:")
+        self.db.connect("client")
+        self.db.init()
+        self._saved_stdout = sys.stdout
+        self._saved_cursor = conf.get("databaseCursor")
+        self._saved_partRun = kb.get("partRun")
+        self._had_reportTaskId = "reportTaskId" in kb
+        self._saved_reportTaskId = kb.get("reportTaskId")
+        conf.databaseCursor = self.db
+        kb.partRun = None
+
+    def tearDown(self):
+        sys.stdout = self._saved_stdout
+        conf.databaseCursor = self._saved_cursor
+        kb.partRun = self._saved_partRun
+        if self._had_reportTaskId:
+            kb.reportTaskId = self._saved_reportTaskId
+        elif "reportTaskId" in kb:
+            del kb.reportTaskId
+        self.db.disconnect()
+
+    def test_second_target_does_not_overwrite_first(self):
+        std = api.StdDbOut("t1", messagetype="stdout")
+        sys.stdout = self._saved_stdout    # StdDbOut hijacked it; call write() directly
+
+        kb.reportTaskId = 1
+        std.write({"url": "http://host1/?id=1"}, status=CONTENT_STATUS.COMPLETE, content_type=CONTENT_TYPE.TARGET)
+        kb.reportTaskId = 2
+        std.write({"url": "http://host2/?id=1"}, status=CONTENT_STATUS.COMPLETE, content_type=CONTENT_TYPE.TARGET)
+
+        rows = self.db.execute("SELECT taskid FROM data ORDER BY id")
+        self.assertEqual([row[0] for row in rows], ["t1-1", "t1-2"])    # neither target's row was deleted
+
+    def test_without_report_task_id_keeps_plain_taskid(self):
+        # e.g. output written before controller.start() reaches its target loop
+        if "reportTaskId" in kb:
+            del kb.reportTaskId
+        std = api.StdDbOut("t1", messagetype="stdout")
+        sys.stdout = self._saved_stdout
+
+        std.write({"url": "http://host1/?id=1"}, status=CONTENT_STATUS.COMPLETE, content_type=CONTENT_TYPE.TARGET)
+
+        rows = self.db.execute("SELECT taskid FROM data")
+        self.assertEqual([row[0] for row in rows], ["t1"])
+
+
 # ---------------------------------------------------------------------------
 # HTTP routes (WSGI test client)
 # ---------------------------------------------------------------------------
@@ -485,6 +543,26 @@ class TestScanQueryRoutes(_ApiServerCase):
         self.assertFalse(parsed["success"])
 
 
+class TestScanDataMultiTarget(_ApiServerCase):
+    """scan_data() folds the per-target '<taskid>-<n>' rows (see StdDbOut.write) back into the task's flat result, in target order."""
+
+    def test_data_merges_per_target_rows(self):
+        taskid = self._new_task()
+        api._storeData(self.db, "%s-2" % taskid, {"url": "http://host2/?id=1"}, CONTENT_STATUS.COMPLETE, CONTENT_TYPE.TARGET)
+        api._storeData(self.db, "%s-1" % taskid, {"url": "http://host1/?id=1"}, CONTENT_STATUS.COMPLETE, CONTENT_TYPE.TARGET)
+        code, parsed, _ = _wsgi_call("GET", "/scan/%s/data" % taskid)
+        self.assertEqual(code, 200)
+        self.assertTrue(parsed["success"])
+        self.assertEqual([d["value"]["url"] for d in parsed["data"]], ["http://host1/?id=1", "http://host2/?id=1"])
+
+    def test_data_ignores_other_tasks_rows(self):
+        taskid = self._new_task()
+        api._storeData(self.db, "othertask-1", {"url": "http://other/?id=1"}, CONTENT_STATUS.COMPLETE, CONTENT_TYPE.TARGET)
+        code, parsed, _ = _wsgi_call("GET", "/scan/%s/data" % taskid)
+        self.assertTrue(parsed["success"])
+        self.assertEqual(parsed["data"], [])
+
+
 class TestScanStart(_ApiServerCase):
     """scan_start, with the subprocess-spawning seam (engine_start) monkeypatched."""
 

tests/test_ssti.py

@@ -827,3 +827,7 @@ class TestStruts2Header(unittest.TestCase):
         # so no start/end marker is found -> no fabricated 'output'
         ssti._s2045Send = lambda url, action: "reflected: %s" % action
         self.assertIsNone(ssti._executeStruts2Header("http://target", "id"))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/test_xpath.py

@@ -557,3 +557,7 @@ class TestRealXPathSyntax(unittest.TestCase):
                 self.fail("Boundary '%s' in '%s' with orig='%s' invalid: %s\n  payload: %s" % (bk, tkey, original, e, payload))
             self.assertIsInstance(count, int,
                 "Boundary '%s' in '%s' produced no count" % (bk, tkey))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)