@@ -191,7 +191,7 @@ EasySpider is a completely free and ad-free open-source software. The developmen
diff --git a/ElectronJS/src/taskGrid/logic.js b/ElectronJS/src/taskGrid/logic.js
index ddacfb2..32ca03b 100644
--- a/ElectronJS/src/taskGrid/logic.js
+++ b/ElectronJS/src/taskGrid/logic.js
@@ -491,7 +491,7 @@ if (mobile == "true") {
}
let serviceInfo = {
- "version": "0.6.3"
+ "version": "0.6.5"
};
function saveService(type) {
@@ -625,7 +625,7 @@ function saveService(type) {
"links": links,
"create_time": $("#create_time").val(),
"update_time": formatDateTime(new Date()),
- "version": "0.6.3",
+ "version": "0.6.5",
"saveThreshold": saveThreshold,
// "cloudflare": cloudflare,
"quitWaitTime": parseInt($("#quitWaitTime").val()),
diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py
index 321aa0f..614e061 100644
--- a/ExecuteStage/easyspider_executestage.py
+++ b/ExecuteStage/easyspider_executestage.py
@@ -2310,7 +2310,7 @@ if __name__ == '__main__':
"remote_control_key": "", # 如果开启远程控制，那么初始化时需要填写此 key，保证安全性
"keyboard": True, # 是否监听键盘输入
"pause_key": "p", # 暂停键
- "version": "0.6.3",
+ "version": "0.6.5",
"docker_driver": "",
"user_folder": "",
}
@@ -2326,8 +2326,6 @@ if __name__ == '__main__':
os.mkdir(os.getcwd() + "/Data")
if sys.platform == "darwin" and platform.architecture()[0] == "64bit":
options.binary_location = "EasySpider.app/Contents/Resources/app/chrome_mac64.app/Contents/MacOS/Google Chrome"
- options.add_extension(
- "EasySpider.app/Contents/Resources/app/XPathHelper.crx")
driver_path = "EasySpider.app/Contents/Resources/app/chromedriver_mac64"
print(driver_path)
if c.config_folder == "":
@@ -2341,7 +2339,6 @@ if __name__ == '__main__':
# 相对于下一条检查语句，这里去掉了 EasySpider 文件夹这一层
options.binary_location = "resources/app/chrome_linux64/chrome"
driver_path = "resources/app/chrome_linux64/chromedriver_linux64"
- options.add_extension("resources/app/XPathHelper.crx")
elif os.path.exists(os.getcwd() + "/EasySpider/resources"): # 打包后的路径
print("Finding chromedriver in EasySpider",
os.getcwd() + "/EasySpider")
@@ -2350,17 +2347,14 @@ if __name__ == '__main__':
os.getcwd(), "EasySpider/resources/app/chrome_win32/chrome.exe") # 指定chrome位置
driver_path = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win32/chromedriver_win32.exe")
- options.add_extension("EasySpider/resources/app/XPathHelper.crx")
elif sys.platform == "win32" and platform.architecture()[0] == "64bit":
options.binary_location = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win64/chrome.exe")
driver_path = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win64/chromedriver_win64.exe")
- options.add_extension("EasySpider/resources/app/XPathHelper.crx")
elif sys.platform == "linux" and platform.architecture()[0] == "64bit":
options.binary_location = "EasySpider/resources/app/chrome_linux64/chrome"
driver_path = "EasySpider/resources/app/chrome_linux64/chromedriver_linux64"
- options.add_extension("EasySpider/resources/app/XPathHelper.crx")
else:
print("Unsupported platform")
sys.exit()
@@ -2375,22 +2369,18 @@ if __name__ == '__main__':
os.getcwd(), "EasySpider/resources/app/chrome_win32/chrome.exe") # 指定chrome位置
driver_path = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win32/chromedriver_win32.exe")
- options.add_extension("EasySpider/resources/app/XPathHelper.crx")
elif sys.platform == "win32" and platform.architecture()[0] == "64bit":
options.binary_location = "../ElectronJS/chrome_win64/chrome.exe" # 指定chrome位置
driver_path = "../ElectronJS/chrome_win64/chromedriver_win64.exe"
- options.add_extension("../ElectronJS/XPathHelper.crx")
elif sys.platform == "linux" and platform.architecture()[0] == "64bit":
options.binary_location = "../ElectronJS/chrome_linux64/chrome"
driver_path = "../ElectronJS/chrome_linux64/chromedriver_linux64"
- options.add_extension("../ElectronJS/XPathHelper.crx")
else:
print("Unsupported platform for automatic detection. You need to specify chrome executable path, chromedriver path in code.")
sys.exit()
else:
options.binary_location = "./chrome.exe" # 指定chrome位置
driver_path = "./chromedriver.exe"
- options.add_extension("XPathHelper.crx")
options.add_experimental_option(
'excludeSwitches', ['enable-automation']) # 以开发者模式
diff --git a/ExecuteStage/generateEXE_win64.cmd b/ExecuteStage/generateEXE_win64.cmd
index 0f5c647..6222bf1 100644
--- a/ExecuteStage/generateEXE_win64.cmd
+++ b/ExecuteStage/generateEXE_win64.cmd
@@ -1,7 +1,27 @@
-rmdir /s /Q build
-rmdir /s /Q dist
-@REM --add-data "C:\Python311\Lib\site-packages\ddddocr\common.onnx;ddddocr"
-@REM pyinstaller -F --icon=favicon.ico easyspider_executestage.py
-pyinstaller -F --icon=favicon.ico --add-data "C:\Python311\Lib\site-packages\onnxruntime\capi\onnxruntime_providers_shared.dll;onnxruntime\capi" --add-data "C:\Python311\Lib\site-packages\ddddocr\common_old.onnx;ddddocr" easyspider_executestage.py
-del ..\ElectronJS\chrome_win64\easyspider_executestage.exe
-copy dist\easyspider_executestage.exe ..\ElectronJS\chrome_win64\easyspider_executestage.exe
\ No newline at end of file
+@echo off
+setlocal
+
+if exist build rmdir /s /Q build
+if exist dist rmdir /s /Q dist
+
+for /f "delims=" %%P in ('python -c "import onnxruntime, pathlib; print(pathlib.Path(onnxruntime.__file__).parent / 'capi' / 'onnxruntime_providers_shared.dll')"') do set "ONNX_DLL=%%P"
+for /f "delims=" %%P in ('python -c "import ddddocr, pathlib; print(pathlib.Path(ddddocr.__file__).parent / 'common_old.onnx')"') do set "DDDDOCR_MODEL=%%P"
+
+if not exist "%ONNX_DLL%" (
+ echo Could not find onnxruntime_providers_shared.dll in the active Python environment.
+ exit /b 1
+)
+if not exist "%DDDDOCR_MODEL%" (
+ echo Could not find ddddocr common_old.onnx in the active Python environment.
+ exit /b 1
+)
+
+pyinstaller -F --icon=favicon.ico --hidden-import selenium.webdriver.ie.webdriver --add-data "%ONNX_DLL%;onnxruntime\capi" --add-data "%DDDDOCR_MODEL%;ddddocr" easyspider_executestage.py
+if errorlevel 1 exit /b %errorlevel%
+if not exist dist\easyspider_executestage.exe exit /b 1
+
+if not exist ..\ElectronJS\chrome_win64 mkdir ..\ElectronJS\chrome_win64
+if exist ..\ElectronJS\chrome_win64\easyspider_executestage.exe del /f /q ..\ElectronJS\chrome_win64\easyspider_executestage.exe
+copy /Y dist\easyspider_executestage.exe ..\ElectronJS\chrome_win64\easyspider_executestage.exe
+if errorlevel 1 exit /b %errorlevel%
+echo Windows x64 execution stage packaged successfully.
diff --git a/ExecuteStage/generateExecutable_Linux64.sh b/ExecuteStage/generateExecutable_Linux64.sh
index 566426a..b9ef912 100755
--- a/ExecuteStage/generateExecutable_Linux64.sh
+++ b/ExecuteStage/generateExecutable_Linux64.sh
@@ -1,11 +1,47 @@
-#!/bin/bash
+#!/usr/bin/env bash
+set -euo pipefail
-rm -r build
-rm -r dist
-# 一定要先source一下，不然会出现找不到conda命令的错误！！！
-source ~/miniconda3/etc/profile.d/conda.sh
-conda activate easyspider
-# Python一定要是3.11版本，不然会出现浏览器弹出崩溃的错误！！！原来使用的3.8，崩溃原因未知。
-pyinstaller -F --add-data "/home/naibo/miniconda3/envs/easyspider/lib/python3.11/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-311-x86_64-linux-gnu.so:onnxruntime/capi" --add-data "/home/naibo/miniconda3/envs/easyspider/lib/python3.11/site-packages/ddddocr/common_old.onnx:ddddocr" easyspider_executestage.py
-rm ../ElectronJS/chrome_linux64/easyspider_executestage
-cp dist/easyspider_executestage ../ElectronJS/chrome_linux64/easyspider_executestage
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+
+if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then
+ echo "This script must run on a native Linux x86_64 build host." >&2
+ exit 1
+fi
+
+PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+PYTHON_ARCH="$($PYTHON_BIN -c 'import platform; print(platform.machine())')"
+if [[ "$PYTHON_VERSION" != "3.11" || "$PYTHON_ARCH" != "x86_64" ]]; then
+ echo "Python 3.11 x86_64 is required; found Python $PYTHON_VERSION $PYTHON_ARCH." >&2
+ exit 1
+fi
+
+DDDDOCR_MODEL="$($PYTHON_BIN -c 'import ddddocr, pathlib; print(pathlib.Path(ddddocr.__file__).resolve().parent / "common_old.onnx")')"
+ONNX_STATE="$($PYTHON_BIN -c 'import onnxruntime, pathlib; capi = pathlib.Path(onnxruntime.__file__).resolve().parent / "capi"; matches = sorted(capi.glob("onnxruntime_pybind11_state*.so")); print(matches[0] if matches else "")')"
+
+if [[ ! -f "$DDDDOCR_MODEL" ]]; then
+ echo "Could not find ddddocr/common_old.onnx in the active Python environment." >&2
+ exit 1
+fi
+if [[ ! -f "$ONNX_STATE" ]]; then
+ echo "Could not find onnxruntime_pybind11_state*.so in the active Python environment." >&2
+ exit 1
+fi
+
+cd "$SCRIPT_DIR"
+rm -rf build dist
+
+"$PYTHON_BIN" -m PyInstaller \
+ --noconfirm \
+ --clean \
+ --onefile \
+ --hidden-import selenium.webdriver.ie.webdriver \
+ --add-binary "$ONNX_STATE:onnxruntime/capi" \
+ --add-data "$DDDDOCR_MODEL:ddddocr" \
+ easyspider_executestage.py
+
+TARGET_DIR="$REPO_ROOT/ElectronJS/chrome_linux64"
+mkdir -p "$TARGET_DIR"
+install -m 755 dist/easyspider_executestage "$TARGET_DIR/easyspider_executestage"
+echo "Linux x64 execution stage packaged successfully."
diff --git a/ExecuteStage/generateExecutable_Macos.sh b/ExecuteStage/generateExecutable_Macos.sh
index a0a873d..c1fd440 100755
--- a/ExecuteStage/generateExecutable_Macos.sh
+++ b/ExecuteStage/generateExecutable_Macos.sh
@@ -1,22 +1,73 @@
-# 先打包一个不带ddddocr和pandas的版本，然后再打包一个带的版本，不带ddddocr和pandas的版本运行速度会快很多
-rm -r build
-rm -r dist
-pyinstaller -F --icon=favicon.ico easyspider_executestage.py --exclude-module ddddocr --exclude-module onnxruntime --exclude-module onnx --exclude-module onnxruntime_pybind11_state.so --exclude-module pillow --exclude-module pandas --exclude-module numpy --exclude-module scipy --exclude-module sklearn
+#!/usr/bin/env bash
+set -euo pipefail
-rm ../.temp_to_pub/EasySpider_MacOS/easyspider_executestage
-cp dist/easyspider_executestage ../.temp_to_pub/EasySpider_MacOS/easyspider_executestage
-# mv dist/easyspider_executestage ../ElectronJS/easyspider_executestage
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
-echo "With ddddocr and pandas"
+if [[ "$(uname -s)" != "Darwin" ]]; then
+ echo "This script must run on macOS." >&2
+ exit 1
+fi
-# # 打包带ddddocr和pandas的版本
-rm -r build
-rm -r dist
-# Get the site-packages path for ddddocr and onnxruntime
-# 如果当前终端激活了 conda 环境，下方脚本应当可以正确的从 conda 环境安装的包中获得数据文件位置
-ddddocr_path=$(python3 -c "import ddddocr; print(ddddocr.__path__[0])")
-onnxruntime_path=$(python3 -c "import onnxruntime; print(onnxruntime.__path__[0])")
+HOST_ARCH="$(uname -m)"
+PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+PYTHON_ARCH="$($PYTHON_BIN -c 'import platform; print(platform.machine())')"
+if [[ "$HOST_ARCH" != "arm64" && "$HOST_ARCH" != "x86_64" ]]; then
+ echo "Unsupported macOS architecture: $HOST_ARCH" >&2
+ exit 1
+fi
+if [[ "$PYTHON_VERSION" != "3.11" || "$PYTHON_ARCH" != "$HOST_ARCH" ]]; then
+ echo "Native Python 3.11 $HOST_ARCH is required; found Python $PYTHON_VERSION $PYTHON_ARCH." >&2
+ exit 1
+fi
-pyinstaller -F --icon=favicon.ico --add-data "$onnxruntime_path/capi/onnxruntime_pybind11_state.so:onnxruntime/capi" --add-data "$ddddocr_path/common_old.onnx:ddddocr" easyspider_executestage.py
-rm ../.temp_to_pub/EasySpider_MacOS/easyspider_executestage_full
-cp dist/easyspider_executestage ../.temp_to_pub/EasySpider_MacOS/easyspider_executestage_full
\ No newline at end of file
+STAGE_DIR="$REPO_ROOT/.temp_to_pub/EasySpider_MacOS"
+mkdir -p "$STAGE_DIR"
+cd "$SCRIPT_DIR"
+
+# The lightweight executor omits OCR and Pandas but retains Pillow, which is
+# imported by the execution-stage program independently of OCR.
+rm -rf build dist
+"$PYTHON_BIN" -m PyInstaller \
+ --noconfirm \
+ --clean \
+ --onefile \
+ --target-arch "$HOST_ARCH" \
+ --icon favicon.ico \
+ --hidden-import selenium.webdriver.ie.webdriver \
+ --exclude-module ddddocr \
+ --exclude-module onnxruntime \
+ --exclude-module onnx \
+ --exclude-module pandas \
+ --exclude-module numpy \
+ --exclude-module scipy \
+ --exclude-module sklearn \
+ easyspider_executestage.py
+install -m 755 dist/easyspider_executestage "$STAGE_DIR/easyspider_executestage"
+
+DDDDOCR_MODEL="$($PYTHON_BIN -c 'import ddddocr, pathlib; print(pathlib.Path(ddddocr.__file__).resolve().parent / "common_old.onnx")')"
+ONNX_STATE="$($PYTHON_BIN -c 'import onnxruntime, pathlib; capi = pathlib.Path(onnxruntime.__file__).resolve().parent / "capi"; matches = sorted(capi.glob("onnxruntime_pybind11_state*.so")); print(matches[0] if matches else "")')"
+
+if [[ ! -f "$DDDDOCR_MODEL" ]]; then
+ echo "Could not find ddddocr/common_old.onnx in the active Python environment." >&2
+ exit 1
+fi
+if [[ ! -f "$ONNX_STATE" ]]; then
+ echo "Could not find onnxruntime_pybind11_state*.so in the active Python environment." >&2
+ exit 1
+fi
+
+rm -rf build dist
+"$PYTHON_BIN" -m PyInstaller \
+ --noconfirm \
+ --clean \
+ --onefile \
+ --target-arch "$HOST_ARCH" \
+ --icon favicon.ico \
+ --hidden-import selenium.webdriver.ie.webdriver \
+ --add-binary "$ONNX_STATE:onnxruntime/capi" \
+ --add-data "$DDDDOCR_MODEL:ddddocr" \
+ easyspider_executestage.py
+install -m 755 dist/easyspider_executestage "$STAGE_DIR/easyspider_executestage_full"
+echo "macOS $HOST_ARCH execution stages packaged successfully."
diff --git a/ExecuteStage/requirements.txt b/ExecuteStage/requirements.txt
index 496b90f..8b1a1c5 100644
--- a/ExecuteStage/requirements.txt
+++ b/ExecuteStage/requirements.txt
@@ -1,14 +1,14 @@
commandline_config==2.2.3
-requests==2.32.3
-selenium==4.27.1
-pyinstaller==5.13.2
-Pillow==10.2.0
-xlsxwriter==3.2.0
-openpyxl==3.1.2
-pymysql==1.1.1
-lxml==5.3.0
-ddddocr==1.5.6
-pynput==1.7.6
-beautifulsoup4==4.12.2
-undetected-chromedriver==3.4.7
-pandas==2.2.3
+requests==2.34.2
+selenium==4.47.0
+pyinstaller==6.22.1
+Pillow==12.3.0
+xlsxwriter==3.2.9
+openpyxl==3.1.5
+pymysql==1.2.0
+lxml==6.1.1
+ddddocr==1.6.1
+pynput==1.8.2
+beautifulsoup4==4.15.0
+undetected-chromedriver==3.5.5
+pandas==3.0.5
diff --git a/Extension/manifest_v3/package-lock.json b/Extension/manifest_v3/package-lock.json
index 7c14b2e..627da8b 100644
--- a/Extension/manifest_v3/package-lock.json
+++ b/Extension/manifest_v3/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "EasySpider",
- "version": "0.6.3",
+ "version": "0.6.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "EasySpider",
- "version": "0.6.3",
+ "version": "0.6.5",
"license": "AGPL-3.0",
"dependencies": {
"crx": "^5.0.1",
diff --git a/Extension/manifest_v3/package.json b/Extension/manifest_v3/package.json
index 9c054ad..7e6e42a 100644
--- a/Extension/manifest_v3/package.json
+++ b/Extension/manifest_v3/package.json
@@ -1,6 +1,6 @@
{
"name": "EasySpider",
- "version": "0.6.3",
+ "version": "0.6.5",
"type": "module",
"scripts": {
"build": "rollup -c",
@@ -48,4 +48,4 @@
"rollup-plugin-zip": "^1.0.1",
"typescript": "^4.4.3"
}
-}
\ No newline at end of file
+}
diff --git a/Extension/manifest_v3/src/manifest.json b/Extension/manifest_v3/src/manifest.json
index 9f0cc8a..5c0efb4 100644
--- a/Extension/manifest_v3/src/manifest.json
+++ b/Extension/manifest_v3/src/manifest.json
@@ -1,6 +1,6 @@
{
"name": "EasySpider",
- "version": "0.6.3",
+ "version": "0.6.5",
"description": "EasySpider's chrome extension",
"author": "Naibo Wang",
"manifest_version": 3,
@@ -57,4 +57,4 @@
"activeTab",
"notifications"
]
-}
\ No newline at end of file
+}
diff --git a/scripts/smoke_python_dependencies.py b/scripts/smoke_python_dependencies.py
new file mode 100644
index 0000000..fd4b9a3
--- /dev/null
+++ b/scripts/smoke_python_dependencies.py
@@ -0,0 +1,122 @@
+"""Small, side-effect-free smoke tests for EasySpider's Python dependencies."""
+
+from __future__ import annotations
+
+import io
+import json
+from importlib import metadata
+
+
+def package_version(distribution: str) -> str:
+ return metadata.version(distribution)
+
+
+def main() -> None:
+ results: dict[str, object] = {}
+
+ # Image: exercise both Pillow and OpenCV using an in-memory PNG.
+ import cv2
+ import numpy as np
+ from PIL import Image, ImageDraw
+
+ image = Image.new("RGB", (96, 32), "white")
+ ImageDraw.Draw(image).text((8, 8), "1234", fill="black")
+ png = io.BytesIO()
+ image.save(png, format="PNG")
+ png_bytes = png.getvalue()
+ decoded = cv2.imdecode(np.frombuffer(png_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
+ assert image.rotate(90, expand=True).size == (32, 96)
+ assert decoded is not None and decoded.shape[:2] == (32, 96)
+ results["image"] = {
+ "Pillow": package_version("Pillow"),
+ "opencv-python": package_version("opencv-python"),
+ "decoded_shape": list(decoded.shape),
+ }
+
+ # OCR: initialize the bundled ONNX model and perform one inference.
+ import ddddocr
+
+ ocr = ddddocr.DdddOcr(show_ad=False)
+ ocr_text = ocr.classification(png_bytes)
+ assert isinstance(ocr_text, str)
+ results["ocr"] = {
+ "ddddocr": package_version("ddddocr"),
+ "onnxruntime": package_version("onnxruntime"),
+ "result_type": type(ocr_text).__name__,
+ }
+
+ # HTML/XML: parse HTML with BeautifulSoup and evaluate XML XPath with lxml.
+ from bs4 import BeautifulSoup
+ from lxml import etree
+
+ soup = BeautifulSoup("
EasySpider
", "html.parser")
+ assert soup.select_one("h1").get_text(strip=True) == "EasySpider"
+ xml_root = etree.fromstring(b"
- ok
")
+ assert xml_root.xpath("string(/root/item[@id='1'])") == "ok"
+ results["html_xml"] = {
+ "beautifulsoup4": package_version("beautifulsoup4"),
+ "lxml": package_version("lxml"),
+ }
+
+ # Excel: write with XlsxWriter and read the same workbook with openpyxl.
+ import openpyxl
+ import xlsxwriter
+
+ workbook_bytes = io.BytesIO()
+ workbook = xlsxwriter.Workbook(workbook_bytes, {"in_memory": True})
+ sheet = workbook.add_worksheet("Smoke")
+ sheet.write(0, 0, "EasySpider")
+ sheet.write_number(0, 1, 65)
+ workbook.close()
+ workbook_bytes.seek(0)
+ loaded = openpyxl.load_workbook(workbook_bytes, read_only=True, data_only=True)
+ assert loaded["Smoke"]["A1"].value == "EasySpider"
+ assert loaded["Smoke"]["B1"].value == 65
+ loaded.close()
+ results["excel"] = {
+ "xlsxwriter": package_version("xlsxwriter"),
+ "openpyxl": package_version("openpyxl"),
+ }
+
+ # Pandas: exercise DataFrame creation, grouping, and aggregation.
+ import pandas as pd
+
+ frame = pd.DataFrame({"group": ["a", "a", "b"], "value": [1, 2, 3]})
+ totals = frame.groupby("group", sort=True)["value"].sum().to_dict()
+ assert totals == {"a": 3, "b": 3}
+ results["pandas"] = {"pandas": package_version("pandas"), "totals": totals}
+
+ # Selenium: construct real Chrome options and an XPath locator without starting Chrome.
+ from selenium import webdriver
+ from selenium.webdriver.common.by import By
+
+ options = webdriver.ChromeOptions()
+ options.add_argument("--headless=new")
+ locator = (By.XPATH, "//h1")
+ assert "--headless=new" in options.arguments and locator == ("xpath", "//h1")
+ results["selenium"] = {
+ "selenium": package_version("selenium"),
+ "arguments": options.arguments,
+ "locator": list(locator),
+ }
+
+ # Keyboard: construct key representations without injecting input into the OS.
+ from pynput.keyboard import Key, KeyCode
+
+ pause_key = KeyCode.from_char("p")
+ assert pause_key.char == "p" and Key.space.value is not None
+ results["keyboard"] = {"pynput": package_version("pynput"), "pause_key": pause_key.char}
+
+ # MySQL client: exercise PyMySQL's query-string escaping without needing credentials.
+ import pymysql
+ from pymysql.converters import escape_string
+
+ escaped = escape_string("O'Reilly")
+ assert escaped == "O\\'Reilly"
+ results["mysql_client"] = {"PyMySQL": package_version("PyMySQL"), "escaped": escaped}
+
+ print(json.dumps(results, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/test_bundled_chrome.py b/scripts/test_bundled_chrome.py
new file mode 100644
index 0000000..b4218ea
--- /dev/null
+++ b/scripts/test_bundled_chrome.py
@@ -0,0 +1,66 @@
+"""Verify the Chrome/ChromeDriver bundled in a Windows EasySpider package."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--stage",
+ type=Path,
+ default=Path(".temp_to_pub/EasySpider_windows_x64"),
+ help="root of the staged Windows package",
+ )
+ parser.add_argument("--url", default="https://example.com/")
+ args = parser.parse_args()
+
+ app = args.stage.resolve() / "EasySpider" / "resources" / "app"
+ chrome_binary = app / "chrome_win64" / "chrome.exe"
+ chrome_driver = app / "chrome_win64" / "chromedriver_win64.exe"
+ if not chrome_binary.is_file() or not chrome_driver.is_file():
+ raise FileNotFoundError(f"bundled browser files are missing below {app}")
+
+ options = webdriver.ChromeOptions()
+ options.binary_location = str(chrome_binary)
+ options.add_argument("--headless=new")
+ options.add_argument("--disable-gpu")
+ options.add_argument("--no-first-run")
+ options.add_argument("--no-default-browser-check")
+ options.add_argument("--ignore-certificate-errors")
+
+ driver = webdriver.Chrome(service=Service(str(chrome_driver)), options=options)
+ try:
+ driver.set_page_load_timeout(30)
+ driver.get(args.url)
+ heading = driver.find_element(By.TAG_NAME, "h1").text.strip()
+ assert driver.title == "Example Domain", driver.title
+ assert heading == "Example Domain", heading
+ print(
+ json.dumps(
+ {
+ "url": driver.current_url,
+ "title": driver.title,
+ "heading": heading,
+ "browser_version": driver.capabilities.get("browserVersion"),
+ "platform": driver.capabilities.get("platformName"),
+ "chrome_binary": str(chrome_binary),
+ "chrome_driver": str(chrome_driver),
+ },
+ ensure_ascii=False,
+ indent=2,
+ )
+ )
+ finally:
+ driver.quit()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/test_xpath_helper_bidi.js b/scripts/test_xpath_helper_bidi.js
new file mode 100644
index 0000000..af0248a
--- /dev/null
+++ b/scripts/test_xpath_helper_bidi.js
@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+
+// Windows smoke test for the two unpacked extensions loaded by EasySpider.
+// It uses the same Chrome, ChromeDriver, WebDriver BiDi helper and Chrome
+// flags as ElectronJS/main.js, but serves a deterministic local test page.
+
+const http = require('http');
+const path = require('path');
+
+const repoRoot = path.resolve(__dirname, '..');
+const electronRoot = process.env.EASYSPIDER_ELECTRON_ROOT
+ ? path.resolve(process.env.EASYSPIDER_ELECTRON_ROOT)
+ : path.join(repoRoot, 'ElectronJS');
+const seleniumRoot = path.join(electronRoot, 'node_modules', 'selenium-webdriver');
+
+const {Builder, By, Key, until} = require(seleniumRoot);
+const chrome = require(path.join(seleniumRoot, 'chrome'));
+const getWebExtensionInstance = require(
+ path.join(electronRoot, 'selenium-webdriver-bidi-webExtension', 'webExtension')
+);
+const ExtensionData = require(
+ path.join(electronRoot, 'selenium-webdriver-bidi-webExtension', 'extensionData')
+);
+
+const chromeBinary = path.join(electronRoot, 'chrome_win64', 'chrome.exe');
+const chromeDriver = path.join(
+ electronRoot,
+ 'chrome_win64',
+ 'chromedriver_win64.exe'
+);
+const easySpiderExtension = path.join(electronRoot, 'EasySpider_zh');
+const xpathHelperExtension = path.join(electronRoot, 'XPathHelper_MV3');
+const legacyXpathHelper = path.join(electronRoot, 'XPathHelper.crx');
+
+function startTestServer() {
+ const html = `
+
+
XPath Helper BiDi Test
+
+
XPath Smoke Test
+
first
+
second
+
+`;
+
+ const server = http.createServer((request, response) => {
+ response.writeHead(200, {'Content-Type': 'text/html; charset=utf-8'});
+ response.end(html);
+ });
+
+ return new Promise((resolve, reject) => {
+ server.once('error', reject);
+ server.listen(0, '127.0.0.1', () => resolve(server));
+ });
+}
+
+function closeServer(server) {
+ return new Promise((resolve) => server.close(resolve));
+}
+
+async function main() {
+ const server = await startTestServer();
+ const address = server.address();
+ let driver;
+
+ try {
+ const options = new chrome.Options();
+ options.setChromeBinaryPath(chromeBinary);
+ options.set('webSocketUrl', true);
+ options.set('excludeSwitches', ['enable-automation']);
+ options.set('useAutomationExtension', false);
+ options.addArguments('--enable-unsafe-extension-debugging');
+ options.addArguments('--remote-debugging-pipe');
+ options.addArguments('--disable-blink-features=AutomationControlled');
+ options.addArguments('--disable-infobars');
+ options.addArguments('--disable-gpu');
+ options.addArguments('--no-first-run');
+ options.addArguments('--no-default-browser-check');
+ if (process.env.HEADED !== '1') {
+ options.addArguments('--headless=new');
+ }
+
+ const serviceBuilder = new chrome.ServiceBuilder(chromeDriver);
+ driver = await new Builder()
+ .forBrowser('chrome')
+ .setChromeOptions(options)
+ .setChromeService(serviceBuilder)
+ .build();
+
+ const webExtension = await getWebExtensionInstance(driver);
+ let legacyArchiveError = null;
+ let legacyPathError = null;
+ if (process.env.TEST_LEGACY === '1') {
+ try {
+ await webExtension.install(ExtensionData.setArchivePath(legacyXpathHelper));
+ } catch (error) {
+ legacyArchiveError = String(error && (error.stack || error.message || error));
+ }
+ }
+ if (process.env.LEGACY_UNPACKED_PATH) {
+ try {
+ await webExtension.install(
+ ExtensionData.setPath(path.resolve(process.env.LEGACY_UNPACKED_PATH))
+ );
+ } catch (error) {
+ legacyPathError = String(error && (error.stack || error.message || error));
+ }
+ }
+
+ const easySpiderId = await webExtension.install(
+ ExtensionData.setPath(easySpiderExtension)
+ );
+ const xpathHelperId = await webExtension.install(
+ ExtensionData.setPath(xpathHelperExtension)
+ );
+
+ await driver.get(`http://127.0.0.1:${address.port}/`);
+ await driver.wait(until.elementLocated(By.id('smoke-title')), 10000);
+
+ await driver.actions()
+ .keyDown(Key.CONTROL)
+ .keyDown(Key.SHIFT)
+ .sendKeys('x')
+ .keyUp(Key.SHIFT)
+ .keyUp(Key.CONTROL)
+ .perform();
+
+ const xpathFrame = await driver.wait(
+ until.elementLocated(By.id('xh-bar')),
+ 10000
+ );
+ await driver.wait(async () => {
+ return !(await xpathFrame.getAttribute('class')).split(/\s+/).includes('hidden');
+ }, 10000);
+
+ await driver.switchTo().frame(xpathFrame);
+ const query = await driver.findElement(By.id('query'));
+ await query.clear();
+ await query.sendKeys('//h1');
+
+ const nodeCount = await driver.findElement(By.id('node-count'));
+ await driver.wait(async () => (await nodeCount.getText()) === '1', 10000);
+ const nodeCountText = await nodeCount.getText();
+ const resultText = await driver.findElement(By.id('results')).getAttribute('value');
+
+ await driver.switchTo().defaultContent();
+ const highlighted = await driver.executeScript(
+ "return document.querySelectorAll('.xh-highlight').length"
+ );
+ const title = await driver.getTitle();
+
+ if (highlighted !== 1) {
+ throw new Error(`Expected one highlighted XPath match, got ${highlighted}`);
+ }
+
+ console.log(JSON.stringify({
+ chromeBinary,
+ chromeDriver,
+ easySpiderId,
+ xpathHelperId,
+ title,
+ nodeCount: nodeCountText,
+ resultText,
+ highlighted,
+ legacyArchiveTested: process.env.TEST_LEGACY === '1',
+ legacyArchiveRejected: legacyArchiveError !== null,
+ legacyArchiveError,
+ legacyPathTested: Boolean(process.env.LEGACY_UNPACKED_PATH),
+ legacyPathRejected: legacyPathError !== null,
+ legacyPathError,
+ }, null, 2));
+ } finally {
+ if (driver) {
+ await driver.quit();
+ }
+ await closeServer(server);
+ }
+}
+
+main().catch((error) => {
+ console.error(error && (error.stack || error));
+ process.exitCode = 1;
+});