Prepare EasySpider 0.6.5 cross-platform release

2026-10-16 21:13:18 +08:00 · 2026-08-18 18:06:08 +08:00
parent 191bd6d754
commit d6548ed75a
44 changed files with 1693 additions and 244 deletions
@@ -0,0 +1,122 @@
+"""Small, side-effect-free smoke tests for EasySpider's Python dependencies."""
+
+from __future__ import annotations
+
+import io
+import json
+from importlib import metadata
+
+
+def package_version(distribution: str) -> str:
+    return metadata.version(distribution)
+
+
+def main() -> None:
+    results: dict[str, object] = {}
+
+    # Image: exercise both Pillow and OpenCV using an in-memory PNG.
+    import cv2
+    import numpy as np
+    from PIL import Image, ImageDraw
+
+    image = Image.new("RGB", (96, 32), "white")
+    ImageDraw.Draw(image).text((8, 8), "1234", fill="black")
+    png = io.BytesIO()
+    image.save(png, format="PNG")
+    png_bytes = png.getvalue()
+    decoded = cv2.imdecode(np.frombuffer(png_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
+    assert image.rotate(90, expand=True).size == (32, 96)
+    assert decoded is not None and decoded.shape[:2] == (32, 96)
+    results["image"] = {
+        "Pillow": package_version("Pillow"),
+        "opencv-python": package_version("opencv-python"),
+        "decoded_shape": list(decoded.shape),
+    }
+
+    # OCR: initialize the bundled ONNX model and perform one inference.
+    import ddddocr
+
+    ocr = ddddocr.DdddOcr(show_ad=False)
+    ocr_text = ocr.classification(png_bytes)
+    assert isinstance(ocr_text, str)
+    results["ocr"] = {
+        "ddddocr": package_version("ddddocr"),
+        "onnxruntime": package_version("onnxruntime"),
+        "result_type": type(ocr_text).__name__,
+    }
+
+    # HTML/XML: parse HTML with BeautifulSoup and evaluate XML XPath with lxml.
+    from bs4 import BeautifulSoup
+    from lxml import etree
+
+    soup = BeautifulSoup("<main><h1>EasySpider</h1></main>", "html.parser")
+    assert soup.select_one("h1").get_text(strip=True) == "EasySpider"
+    xml_root = etree.fromstring(b"<root><item id='1'>ok</item></root>")
+    assert xml_root.xpath("string(/root/item[@id='1'])") == "ok"
+    results["html_xml"] = {
+        "beautifulsoup4": package_version("beautifulsoup4"),
+        "lxml": package_version("lxml"),
+    }
+
+    # Excel: write with XlsxWriter and read the same workbook with openpyxl.
+    import openpyxl
+    import xlsxwriter
+
+    workbook_bytes = io.BytesIO()
+    workbook = xlsxwriter.Workbook(workbook_bytes, {"in_memory": True})
+    sheet = workbook.add_worksheet("Smoke")
+    sheet.write(0, 0, "EasySpider")
+    sheet.write_number(0, 1, 65)
+    workbook.close()
+    workbook_bytes.seek(0)
+    loaded = openpyxl.load_workbook(workbook_bytes, read_only=True, data_only=True)
+    assert loaded["Smoke"]["A1"].value == "EasySpider"
+    assert loaded["Smoke"]["B1"].value == 65
+    loaded.close()
+    results["excel"] = {
+        "xlsxwriter": package_version("xlsxwriter"),
+        "openpyxl": package_version("openpyxl"),
+    }
+
+    # Pandas: exercise DataFrame creation, grouping, and aggregation.
+    import pandas as pd
+
+    frame = pd.DataFrame({"group": ["a", "a", "b"], "value": [1, 2, 3]})
+    totals = frame.groupby("group", sort=True)["value"].sum().to_dict()
+    assert totals == {"a": 3, "b": 3}
+    results["pandas"] = {"pandas": package_version("pandas"), "totals": totals}
+
+    # Selenium: construct real Chrome options and an XPath locator without starting Chrome.
+    from selenium import webdriver
+    from selenium.webdriver.common.by import By
+
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless=new")
+    locator = (By.XPATH, "//h1")
+    assert "--headless=new" in options.arguments and locator == ("xpath", "//h1")
+    results["selenium"] = {
+        "selenium": package_version("selenium"),
+        "arguments": options.arguments,
+        "locator": list(locator),
+    }
+
+    # Keyboard: construct key representations without injecting input into the OS.
+    from pynput.keyboard import Key, KeyCode
+
+    pause_key = KeyCode.from_char("p")
+    assert pause_key.char == "p" and Key.space.value is not None
+    results["keyboard"] = {"pynput": package_version("pynput"), "pause_key": pause_key.char}
+
+    # MySQL client: exercise PyMySQL's query-string escaping without needing credentials.
+    import pymysql
+    from pymysql.converters import escape_string
+
+    escaped = escape_string("O'Reilly")
+    assert escaped == "O\\'Reilly"
+    results["mysql_client"] = {"PyMySQL": package_version("PyMySQL"), "escaped": escaped}
+
+    print(json.dumps(results, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,66 @@
+"""Verify the Chrome/ChromeDriver bundled in a Windows EasySpider package."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--stage",
+        type=Path,
+        default=Path(".temp_to_pub/EasySpider_windows_x64"),
+        help="root of the staged Windows package",
+    )
+    parser.add_argument("--url", default="https://example.com/")
+    args = parser.parse_args()
+
+    app = args.stage.resolve() / "EasySpider" / "resources" / "app"
+    chrome_binary = app / "chrome_win64" / "chrome.exe"
+    chrome_driver = app / "chrome_win64" / "chromedriver_win64.exe"
+    if not chrome_binary.is_file() or not chrome_driver.is_file():
+        raise FileNotFoundError(f"bundled browser files are missing below {app}")
+
+    options = webdriver.ChromeOptions()
+    options.binary_location = str(chrome_binary)
+    options.add_argument("--headless=new")
+    options.add_argument("--disable-gpu")
+    options.add_argument("--no-first-run")
+    options.add_argument("--no-default-browser-check")
+    options.add_argument("--ignore-certificate-errors")
+
+    driver = webdriver.Chrome(service=Service(str(chrome_driver)), options=options)
+    try:
+        driver.set_page_load_timeout(30)
+        driver.get(args.url)
+        heading = driver.find_element(By.TAG_NAME, "h1").text.strip()
+        assert driver.title == "Example Domain", driver.title
+        assert heading == "Example Domain", heading
+        print(
+            json.dumps(
+                {
+                    "url": driver.current_url,
+                    "title": driver.title,
+                    "heading": heading,
+                    "browser_version": driver.capabilities.get("browserVersion"),
+                    "platform": driver.capabilities.get("platformName"),
+                    "chrome_binary": str(chrome_binary),
+                    "chrome_driver": str(chrome_driver),
+                },
+                ensure_ascii=False,
+                indent=2,
+            )
+        )
+    finally:
+        driver.quit()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+
+// Windows smoke test for the two unpacked extensions loaded by EasySpider.
+// It uses the same Chrome, ChromeDriver, WebDriver BiDi helper and Chrome
+// flags as ElectronJS/main.js, but serves a deterministic local test page.
+
+const http = require('http');
+const path = require('path');
+
+const repoRoot = path.resolve(__dirname, '..');
+const electronRoot = process.env.EASYSPIDER_ELECTRON_ROOT
+  ? path.resolve(process.env.EASYSPIDER_ELECTRON_ROOT)
+  : path.join(repoRoot, 'ElectronJS');
+const seleniumRoot = path.join(electronRoot, 'node_modules', 'selenium-webdriver');
+
+const {Builder, By, Key, until} = require(seleniumRoot);
+const chrome = require(path.join(seleniumRoot, 'chrome'));
+const getWebExtensionInstance = require(
+  path.join(electronRoot, 'selenium-webdriver-bidi-webExtension', 'webExtension')
+);
+const ExtensionData = require(
+  path.join(electronRoot, 'selenium-webdriver-bidi-webExtension', 'extensionData')
+);
+
+const chromeBinary = path.join(electronRoot, 'chrome_win64', 'chrome.exe');
+const chromeDriver = path.join(
+  electronRoot,
+  'chrome_win64',
+  'chromedriver_win64.exe'
+);
+const easySpiderExtension = path.join(electronRoot, 'EasySpider_zh');
+const xpathHelperExtension = path.join(electronRoot, 'XPathHelper_MV3');
+const legacyXpathHelper = path.join(electronRoot, 'XPathHelper.crx');
+
+function startTestServer() {
+  const html = `<!doctype html>
+<html>
+  <head><meta charset="utf-8"><title>XPath Helper BiDi Test</title></head>
+  <body>
+    <h1 id="smoke-title">XPath Smoke Test</h1>
+    <div class="item">first</div>
+    <div class="item">second</div>
+  </body>
+</html>`;
+
+  const server = http.createServer((request, response) => {
+    response.writeHead(200, {'Content-Type': 'text/html; charset=utf-8'});
+    response.end(html);
+  });
+
+  return new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => resolve(server));
+  });
+}
+
+function closeServer(server) {
+  return new Promise((resolve) => server.close(resolve));
+}
+
+async function main() {
+  const server = await startTestServer();
+  const address = server.address();
+  let driver;
+
+  try {
+    const options = new chrome.Options();
+    options.setChromeBinaryPath(chromeBinary);
+    options.set('webSocketUrl', true);
+    options.set('excludeSwitches', ['enable-automation']);
+    options.set('useAutomationExtension', false);
+    options.addArguments('--enable-unsafe-extension-debugging');
+    options.addArguments('--remote-debugging-pipe');
+    options.addArguments('--disable-blink-features=AutomationControlled');
+    options.addArguments('--disable-infobars');
+    options.addArguments('--disable-gpu');
+    options.addArguments('--no-first-run');
+    options.addArguments('--no-default-browser-check');
+    if (process.env.HEADED !== '1') {
+      options.addArguments('--headless=new');
+    }
+
+    const serviceBuilder = new chrome.ServiceBuilder(chromeDriver);
+    driver = await new Builder()
+      .forBrowser('chrome')
+      .setChromeOptions(options)
+      .setChromeService(serviceBuilder)
+      .build();
+
+    const webExtension = await getWebExtensionInstance(driver);
+    let legacyArchiveError = null;
+    let legacyPathError = null;
+    if (process.env.TEST_LEGACY === '1') {
+      try {
+        await webExtension.install(ExtensionData.setArchivePath(legacyXpathHelper));
+      } catch (error) {
+        legacyArchiveError = String(error && (error.stack || error.message || error));
+      }
+    }
+    if (process.env.LEGACY_UNPACKED_PATH) {
+      try {
+        await webExtension.install(
+          ExtensionData.setPath(path.resolve(process.env.LEGACY_UNPACKED_PATH))
+        );
+      } catch (error) {
+        legacyPathError = String(error && (error.stack || error.message || error));
+      }
+    }
+
+    const easySpiderId = await webExtension.install(
+      ExtensionData.setPath(easySpiderExtension)
+    );
+    const xpathHelperId = await webExtension.install(
+      ExtensionData.setPath(xpathHelperExtension)
+    );
+
+    await driver.get(`http://127.0.0.1:${address.port}/`);
+    await driver.wait(until.elementLocated(By.id('smoke-title')), 10000);
+
+    await driver.actions()
+      .keyDown(Key.CONTROL)
+      .keyDown(Key.SHIFT)
+      .sendKeys('x')
+      .keyUp(Key.SHIFT)
+      .keyUp(Key.CONTROL)
+      .perform();
+
+    const xpathFrame = await driver.wait(
+      until.elementLocated(By.id('xh-bar')),
+      10000
+    );
+    await driver.wait(async () => {
+      return !(await xpathFrame.getAttribute('class')).split(/\s+/).includes('hidden');
+    }, 10000);
+
+    await driver.switchTo().frame(xpathFrame);
+    const query = await driver.findElement(By.id('query'));
+    await query.clear();
+    await query.sendKeys('//h1');
+
+    const nodeCount = await driver.findElement(By.id('node-count'));
+    await driver.wait(async () => (await nodeCount.getText()) === '1', 10000);
+    const nodeCountText = await nodeCount.getText();
+    const resultText = await driver.findElement(By.id('results')).getAttribute('value');
+
+    await driver.switchTo().defaultContent();
+    const highlighted = await driver.executeScript(
+      "return document.querySelectorAll('.xh-highlight').length"
+    );
+    const title = await driver.getTitle();
+
+    if (highlighted !== 1) {
+      throw new Error(`Expected one highlighted XPath match, got ${highlighted}`);
+    }
+
+    console.log(JSON.stringify({
+      chromeBinary,
+      chromeDriver,
+      easySpiderId,
+      xpathHelperId,
+      title,
+      nodeCount: nodeCountText,
+      resultText,
+      highlighted,
+      legacyArchiveTested: process.env.TEST_LEGACY === '1',
+      legacyArchiveRejected: legacyArchiveError !== null,
+      legacyArchiveError,
+      legacyPathTested: Boolean(process.env.LEGACY_UNPACKED_PATH),
+      legacyPathRejected: legacyPathError !== null,
+      legacyPathError,
+    }, null, 2));
+  } finally {
+    if (driver) {
+      await driver.quit();
+    }
+    await closeServer(server);
+  }
+}
+
+main().catch((error) => {
+  console.error(error && (error.stack || error));
+  process.exitCode = 1;
+});