From 5b8a2faab1ca014467e73c80ead03deacc1878af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Wed, 21 Oct 2026 12:10:18 +0200 Subject: [PATCH] Doing some cleaning for web search requests --- lib/core/settings.py | 9 +-------- lib/parse/cmdline.py | 8 ++++---- lib/utils/search.py | 11 +++++------ sqlmap.conf | 6 +++--- tests/test_dork_search.py | 16 +++++++++++++++- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 431dff718..df3fea86e 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -12,7 +12,6 @@ import random import re import string import sys -import time from lib.core.enums import DBMS from lib.core.enums import DBMS_DIRECTORY_NAME @@ -20,7 +19,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.10.16" +VERSION = "1.10.10.17" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -204,12 +203,6 @@ JITTER_JUNK_RESPONSES = ( ("", 200), ) -# Regular expression used for extracting results from Google search -GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)" - -# Google Search consent cookie -GOOGLE_CONSENT_COOKIE = "CONSENT=YES+shp.gws-%s-0-RC1.%s+FX+740" % (time.strftime("%Y%m%d"), "".join(random.sample(string.ascii_lowercase, 2))) - # Regular expression used for extracting results from DuckDuckGo search DUCKDUCKGO_REGEX = r' 1 else 1 @@ -99,11 +96,13 @@ def _search(dork): encoded = urlencode(dork, convall=True) - # Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one) + # Note: DuckDuckGo's first result page holds 10 entries, while each subsequent one holds 15 (hence the non-linear offset) + ddgOffset = 0 if gpage == 1 else 10 + (gpage - 2) * 15 + + # Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (Google is not in here as it serves results only to JavaScript capable clients) engines = ( - ("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")), + ("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, ddgOffset), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")), ("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: decodeBase64(match.group(1), binary=False) if match.group(1) else match.group(2).replace("&", "&")), - ("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)), ) for name, url, data, regex, flags, extract in engines: diff --git a/sqlmap.conf b/sqlmap.conf index b844e8598..b6bbb7d5f 100644 --- a/sqlmap.conf +++ b/sqlmap.conf @@ -25,8 +25,8 @@ bulkFile = # Example (file content): POST /login.jsp HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/4.0\n\nuserid=joe&password=guessme requestFile = -# Rather than providing a target URL, let Google return target -# hosts as result of your Google dork expression. For a list of Google +# Rather than providing a target URL, let the search engine return target +# hosts as result of your dork expression. For a list of # dorks see Google Hacking Database at # https://www.exploit-db.com/google-hacking-database # Example: +ext:php +inurl:"&id=" +intext:"powered by " @@ -797,7 +797,7 @@ forms = False # Valid: True or False freshQueries = False -# Use Google dork results from specified page number. +# Use dork results from specified page number. # Valid: integer # Default: 1 googlePage = 1 diff --git a/tests/test_dork_search.py b/tests/test_dork_search.py index 00c046a60..09a3fe951 100644 --- a/tests/test_dork_search.py +++ b/tests/test_dork_search.py @@ -42,7 +42,13 @@ class TestDorkSearch(unittest.TestCase): """Patches _fetch to serve `pages` in engine order (None meaning no reply)""" served = list(pages) - smod._fetch = lambda *args, **kwargs: served.pop(0) if served else None + self.requested = [] + + def fetch(url, headers, data=None): + self.requested.append((url, data)) + return served.pop(0) if served else None + + smod._fetch = fetch def test_duckduckgo_results(self): self._pages(DUCKDUCKGO_PAGE) @@ -52,6 +58,14 @@ class TestDorkSearch(unittest.TestCase): self._pages(None, BING_PAGE) self.assertEqual(smod._search("dork"), ["https://www.example.com/a.php?id=1"]) + def test_duckduckgo_paging_offset(self): + # Note: first result page holds 10 entries, each subsequent one 15 (and not the uniform 30 as previously used) + for gpage, offset in ((1, 0), (2, 10), (3, 25), (4, 40)): + conf.googlePage = gpage + self._pages(DUCKDUCKGO_PAGE) + smod._search("dork") + self.assertEqual(self.requested[0][1], "q=dork&s=%d" % offset) + def test_abuse_page_switches_proxy(self): # Note: without '--proxy-list' there is nothing to switch to, hence no exception for marker in SEARCH_ABUSE_MARKERS: