mirror of
https://github.com/getmaxun/maxun.git
synced 2026-10-16 20:21:03 +08:00
chore: drop jest test infra to keep PR scoped to persistence fix
Per maintainer review (RohitR311 on #1138): the repo has no existing test infra or CI to run tests, so introducing jest here is out of scope. Removes jest configs, test files, tsconfig.test.json, the test:server script, and jest/ts-jest/@types/jest/jest-environment-jsdom dependencies from both maxun-core and server. output-validation.ts stays - it's a real production module (not test code) extracted to fix the duplicated isPartial bug CodeRabbit flagged; only its test file is dropped.
This commit is contained in:
hmhngx
@@ -1,10 +0,0 @@
|
||||
/** @type {import('jest').Config} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'jsdom',
|
||||
testMatch: ['**/src/__tests__/**/*.test.ts'],
|
||||
testTimeout: 30000,
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.test.json' }],
|
||||
},
|
||||
};
|
||||
@@ -35,11 +35,5 @@
|
||||
"joi": "^17.6.0",
|
||||
"playwright-core": "1.57.0",
|
||||
"turndown": "^7.2.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.12",
|
||||
"jest": "^29.7.0",
|
||||
"jest-environment-jsdom": "^29.7.0",
|
||||
"ts-jest": "^29.1.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
/**
|
||||
* Regression tests for getmaxun/maxun#1104: crawl/search robot runs used to accumulate every
|
||||
* visited page in a local array and only call `serializableCallback` once, after the whole
|
||||
* page-by-page loop finished. If the run was killed/timed out/aborted mid-loop, everything
|
||||
* scraped so far was lost. These tests drive the real `Interpreter` (no mocking of the crawl/
|
||||
* search action logic itself) against a mocked Playwright `Page` backed by jsdom, and assert
|
||||
* that `serializableCallback` is invoked incrementally, once per page, with a strictly growing
|
||||
* snapshot — not once at the very end.
|
||||
*/
|
||||
import type { Page } from 'playwright-core';
|
||||
import Interpreter from '../interpret';
|
||||
import type { WorkflowFile } from '../types/workflow';
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
interface MockPageOptions {
|
||||
initialUrl: string;
|
||||
/** Exact-match or startsWith-prefix map from URL to the HTML to load into jsdom's document. */
|
||||
fixtures: Record<string, string>;
|
||||
/** URLs for which page.goto() should reject (simulating a hard navigation failure). */
|
||||
failGotoUrls?: string[];
|
||||
/** URLs for which page.evaluate() should reject while the page is "on" that URL. */
|
||||
failEvaluateUrls?: string[];
|
||||
}
|
||||
|
||||
function resolveFixture(fixtures: Record<string, string>, url: string): string {
|
||||
if (fixtures[url]) return fixtures[url];
|
||||
const prefixMatch = Object.keys(fixtures).find((k) => url.startsWith(k));
|
||||
return prefixMatch ? fixtures[prefixMatch] : '<html><body></body></html>';
|
||||
}
|
||||
|
||||
function createMockPage(opts: MockPageOptions) {
|
||||
let currentUrl = opts.initialUrl;
|
||||
|
||||
const navigateTo = (url: string) => {
|
||||
currentUrl = url;
|
||||
document.documentElement.innerHTML = resolveFixture(opts.fixtures, url);
|
||||
};
|
||||
|
||||
navigateTo(opts.initialUrl);
|
||||
|
||||
const page: any = {
|
||||
url: jest.fn(() => currentUrl),
|
||||
context: jest.fn(() => ({})),
|
||||
setDefaultNavigationTimeout: jest.fn(),
|
||||
isClosed: jest.fn(() => false),
|
||||
close: jest.fn().mockResolvedValue(undefined),
|
||||
on: jest.fn(),
|
||||
removeListener: jest.fn(),
|
||||
waitForLoadState: jest.fn().mockResolvedValue(undefined),
|
||||
waitForSelector: jest.fn().mockResolvedValue({}),
|
||||
waitForFunction: jest.fn().mockResolvedValue(undefined),
|
||||
addInitScript: jest.fn().mockResolvedValue(undefined),
|
||||
keyboard: { press: jest.fn().mockResolvedValue(undefined) },
|
||||
goto: jest.fn(async (url: string) => {
|
||||
if (opts.failGotoUrls?.includes(url)) {
|
||||
throw new Error(`Simulated navigation failure for ${url}`);
|
||||
}
|
||||
navigateTo(url);
|
||||
return null;
|
||||
}),
|
||||
evaluate: jest.fn((fn: any, ...args: any[]) => {
|
||||
if (opts.failEvaluateUrls?.includes(currentUrl)) {
|
||||
return Promise.reject(new Error(`Simulated evaluate failure at ${currentUrl}`));
|
||||
}
|
||||
return Promise.resolve(fn(...args));
|
||||
}),
|
||||
};
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
function crawlWorkflow(baseUrl: string, crawlConfig: Record<string, any>): WorkflowFile {
|
||||
return {
|
||||
workflow: [
|
||||
{
|
||||
where: { url: baseUrl },
|
||||
what: [{ action: 'crawl', args: [crawlConfig] }],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* serializableCallback is invoked with the SAME mutable array reference every time (the
|
||||
* production code intentionally reuses one array/object, relying on each call being awaited
|
||||
* before the next page mutates it further — see persistCrawlProgress/persistSearchProgress in
|
||||
* interpret.ts). jest's `mock.calls` only stores that reference, so inspecting it after the run
|
||||
* finishes would show every call as the final, fully-mutated state. Deep-clone at call time
|
||||
* instead, to actually capture what was "persisted" at each point in time.
|
||||
*/
|
||||
function createSnapshottingCallback() {
|
||||
const snapshots: any[] = [];
|
||||
const callback = jest.fn(async (payload: any) => {
|
||||
snapshots.push(JSON.parse(JSON.stringify(payload)));
|
||||
});
|
||||
return { callback, snapshots };
|
||||
}
|
||||
|
||||
function searchWorkflow(searchConfig: Record<string, any>): WorkflowFile {
|
||||
return {
|
||||
workflow: [
|
||||
{
|
||||
where: { url: 'https://duckduckgo.com/' },
|
||||
what: [{ action: 'search', args: [searchConfig] }],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('crawl action — incremental persistence', () => {
|
||||
const BASE = 'https://example.com/';
|
||||
const PAGE1 = 'https://example.com/page1';
|
||||
const PAGE2 = 'https://example.com/page2';
|
||||
|
||||
const fixtures = {
|
||||
[BASE]: `<html><head><title>Home</title></head><body><a href="${PAGE1}">Page1</a></body></html>`,
|
||||
[PAGE1]: `<html><head><title>Page1</title></head><body><a href="${PAGE2}">Page2</a></body></html>`,
|
||||
[PAGE2]: '<html><head><title>Page2</title></head><body>No more links</body></html>',
|
||||
};
|
||||
|
||||
const baseCrawlConfig = {
|
||||
mode: 'domain',
|
||||
limit: 3,
|
||||
maxDepth: 5,
|
||||
includePaths: [],
|
||||
excludePaths: [],
|
||||
useSitemap: false,
|
||||
followLinks: true,
|
||||
respectRobots: false,
|
||||
};
|
||||
|
||||
function crawlSnapshots(snapshots: any[]): any[][] {
|
||||
return snapshots
|
||||
.map((payload) => payload?.crawl?.['Crawl Results'])
|
||||
.filter((v: any) => Array.isArray(v));
|
||||
}
|
||||
|
||||
it('persists results after every page, not just once at the end', async () => {
|
||||
const { callback: serializableCallback, snapshots: rawSnapshots } = createSnapshottingCallback();
|
||||
const interpreter = new Interpreter(crawlWorkflow(BASE, baseCrawlConfig), {
|
||||
serializableCallback,
|
||||
binaryCallback: jest.fn(),
|
||||
});
|
||||
const page = createMockPage({ initialUrl: BASE, fixtures });
|
||||
|
||||
await interpreter.run(page as unknown as Page);
|
||||
|
||||
const snapshots = crawlSnapshots(rawSnapshots);
|
||||
|
||||
// The old, buggy behavior only ever produced ONE snapshot (the full array, at the very
|
||||
// end). The fix must produce a snapshot after every page, growing by exactly one each time.
|
||||
expect(snapshots.length).toBeGreaterThanOrEqual(3);
|
||||
expect(snapshots[0]).toHaveLength(1);
|
||||
expect(snapshots[1]).toHaveLength(2);
|
||||
expect(snapshots[snapshots.length - 1]).toHaveLength(3);
|
||||
|
||||
for (let i = 1; i < snapshots.length; i++) {
|
||||
expect(snapshots[i].length).toBeGreaterThanOrEqual(snapshots[i - 1].length);
|
||||
}
|
||||
|
||||
const finalUrls = snapshots[snapshots.length - 1].map((r: any) => r.metadata?.url);
|
||||
expect(finalUrls).toEqual(expect.arrayContaining([BASE, PAGE1, PAGE2]));
|
||||
});
|
||||
|
||||
it('keeps already-scraped pages recorded when a later page fails outright (simulated kill mid-run)', async () => {
|
||||
const { callback: serializableCallback, snapshots: rawSnapshots } = createSnapshottingCallback();
|
||||
const interpreter = new Interpreter(crawlWorkflow(BASE, baseCrawlConfig), {
|
||||
serializableCallback,
|
||||
binaryCallback: jest.fn(),
|
||||
});
|
||||
const page = createMockPage({ initialUrl: BASE, fixtures, failGotoUrls: [PAGE1] });
|
||||
|
||||
await expect(interpreter.run(page as unknown as Page)).resolves.toBeUndefined();
|
||||
|
||||
const snapshots = crawlSnapshots(rawSnapshots);
|
||||
|
||||
// A snapshot containing ONLY the first (successful) page must have been persisted before
|
||||
// the second page's failure was even known — proving the first page isn't held only in
|
||||
// memory waiting for the whole loop to finish.
|
||||
expect(snapshots.some((s) => s.length === 1 && !s[0].error && s[0].metadata?.url === BASE)).toBe(true);
|
||||
|
||||
const final = snapshots[snapshots.length - 1];
|
||||
expect(final).toHaveLength(2);
|
||||
const failedEntry = final.find((r: any) => r.metadata?.url === PAGE1);
|
||||
expect(failedEntry).toBeDefined();
|
||||
expect(failedEntry.error).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('search action (scrape mode) — incremental persistence', () => {
|
||||
const SITE_A = 'https://site-a.example/';
|
||||
const SITE_B = 'https://site-b.example/';
|
||||
|
||||
const ddgResultsHtml = `
|
||||
<html><body>
|
||||
<article data-testid="result">
|
||||
<h2><a href="${SITE_A}">Site A</a></h2>
|
||||
<div data-testid="result-snippet">Description A</div>
|
||||
</article>
|
||||
<article data-testid="result">
|
||||
<h2><a href="${SITE_B}">Site B</a></h2>
|
||||
<div data-testid="result-snippet">Description B</div>
|
||||
</article>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const fixtures = {
|
||||
'https://duckduckgo.com/': ddgResultsHtml,
|
||||
[SITE_A]: '<html><head><title>Site A Title</title></head><body>Site A body</body></html>',
|
||||
[SITE_B]: '<html><head><title>Site B Title</title></head><body>Site B body</body></html>',
|
||||
};
|
||||
|
||||
const baseSearchConfig = {
|
||||
query: 'test query',
|
||||
limit: 2,
|
||||
provider: 'duckduckgo',
|
||||
mode: 'scrape',
|
||||
};
|
||||
|
||||
function searchSnapshots(snapshots: any[]): any[][] {
|
||||
return snapshots
|
||||
.map((payload) => payload?.search?.['Search Results']?.results)
|
||||
.filter((v: any) => Array.isArray(v));
|
||||
}
|
||||
|
||||
it('persists scraped search results after every result page, not just once at the end', async () => {
|
||||
const { callback: serializableCallback, snapshots: rawSnapshots } = createSnapshottingCallback();
|
||||
const interpreter = new Interpreter(searchWorkflow(baseSearchConfig), {
|
||||
serializableCallback,
|
||||
binaryCallback: jest.fn(),
|
||||
});
|
||||
const page = createMockPage({ initialUrl: 'https://duckduckgo.com/', fixtures });
|
||||
|
||||
await interpreter.run(page as unknown as Page);
|
||||
|
||||
const snapshots = searchSnapshots(rawSnapshots);
|
||||
|
||||
expect(snapshots.length).toBeGreaterThanOrEqual(2);
|
||||
expect(snapshots[0]).toHaveLength(1);
|
||||
expect(snapshots[snapshots.length - 1]).toHaveLength(2);
|
||||
|
||||
const finalUrls = snapshots[snapshots.length - 1].map((r: any) => r.metadata?.url);
|
||||
expect(finalUrls).toEqual(expect.arrayContaining([SITE_A, SITE_B]));
|
||||
});
|
||||
|
||||
it('keeps the first scraped result recorded when the second result fails', async () => {
|
||||
const { callback: serializableCallback, snapshots: rawSnapshots } = createSnapshottingCallback();
|
||||
const interpreter = new Interpreter(searchWorkflow(baseSearchConfig), {
|
||||
serializableCallback,
|
||||
binaryCallback: jest.fn(),
|
||||
});
|
||||
const page = createMockPage({
|
||||
initialUrl: 'https://duckduckgo.com/',
|
||||
fixtures,
|
||||
failEvaluateUrls: [SITE_B],
|
||||
});
|
||||
|
||||
await expect(interpreter.run(page as unknown as Page)).resolves.toBeUndefined();
|
||||
|
||||
const snapshots = searchSnapshots(rawSnapshots);
|
||||
|
||||
expect(snapshots.some((s) => s.length === 1 && !s[0].error && s[0].metadata?.url === SITE_A)).toBe(true);
|
||||
|
||||
const final = snapshots[snapshots.length - 1];
|
||||
expect(final).toHaveLength(2);
|
||||
const failedEntry = final.find((r: any) => r.url === SITE_B || r.metadata?.url === SITE_B);
|
||||
expect(failedEntry).toBeDefined();
|
||||
expect(failedEntry.error).toBeDefined();
|
||||
});
|
||||
|
||||
it('stops scraping further results as soon as the interpreter is aborted', async () => {
|
||||
let interpreter: Interpreter;
|
||||
const rawSnapshots: any[] = [];
|
||||
const serializableCallback = jest.fn(async (payload: any) => {
|
||||
rawSnapshots.push(JSON.parse(JSON.stringify(payload)));
|
||||
const results = payload?.search?.['Search Results']?.results;
|
||||
if (Array.isArray(results) && results.length === 1) {
|
||||
interpreter.abort();
|
||||
}
|
||||
});
|
||||
interpreter = new Interpreter(searchWorkflow(baseSearchConfig), {
|
||||
serializableCallback,
|
||||
binaryCallback: jest.fn(),
|
||||
});
|
||||
const page = createMockPage({ initialUrl: 'https://duckduckgo.com/', fixtures });
|
||||
|
||||
await interpreter.run(page as unknown as Page);
|
||||
|
||||
const snapshots = searchSnapshots(rawSnapshots);
|
||||
const final = snapshots[snapshots.length - 1];
|
||||
|
||||
expect(final).toHaveLength(1);
|
||||
expect(final[0].metadata?.url).toBe(SITE_A);
|
||||
expect(page.goto).not.toHaveBeenCalledWith(SITE_B);
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,5 @@
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["src/__tests__"]
|
||||
"include": ["src"]
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"types": ["node", "jest"],
|
||||
"strict": false,
|
||||
"noImplicitAny": false
|
||||
},
|
||||
"exclude": []
|
||||
}
|
||||
+1
-5
@@ -105,8 +105,7 @@
|
||||
"seed": "sequelize-cli db:seed:all",
|
||||
"seed:undo:all": "sequelize-cli db:seed:undo:all",
|
||||
"migration:generate": "sequelize-cli migration:generate --name",
|
||||
"mcp:build": "tsc --project server/tsconfig.mcp.json",
|
||||
"test:server": "jest --config server/jest.config.js"
|
||||
"mcp:build": "tsc --project server/tsconfig.mcp.json"
|
||||
},
|
||||
"eslintConfig": {
|
||||
"extends": [
|
||||
@@ -122,7 +121,6 @@
|
||||
"@types/lodash": "^4.17.14",
|
||||
"@types/loglevel": "^1.6.3",
|
||||
"@types/multer": "^2.1.0",
|
||||
"@types/jest": "^29.5.12",
|
||||
"@types/node": "22.7.9",
|
||||
"@types/node-cron": "^3.0.11",
|
||||
"@types/node-fetch": "^2.6.12",
|
||||
@@ -135,11 +133,9 @@
|
||||
"concurrently": "^7.0.0",
|
||||
"cross-env": "^7.0.3",
|
||||
"esbuild": "^0.28.1",
|
||||
"jest": "^29.7.0",
|
||||
"js-cookie": "^3.0.5",
|
||||
"nodemon": "^2.0.15",
|
||||
"sequelize-cli": "^6.6.2",
|
||||
"ts-jest": "^29.1.2",
|
||||
"ts-node": "^10.4.0",
|
||||
"vite": "^6.4.2",
|
||||
"zod": "^3.25.62"
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
const path = require('path');
|
||||
|
||||
/** @type {import('jest').Config} */
|
||||
module.exports = {
|
||||
rootDir: __dirname,
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
testMatch: ['**/src/**/__tests__/**/*.test.ts'],
|
||||
testTimeout: 15000,
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: path.join(__dirname, 'tsconfig.test.json') }],
|
||||
},
|
||||
};
|
||||
@@ -1,137 +0,0 @@
|
||||
/**
|
||||
* Regression tests for two CodeRabbit findings on PR #1138:
|
||||
*
|
||||
* 1. (Major) hasData's scrapeSchema/scrapeList checks used `.length > 0`, but both are always
|
||||
* plain objects keyed by action name, never arrays - `.length` on a plain object is
|
||||
* `undefined`, so `undefined > 0` is always false. isPartial could never be true for a run
|
||||
* whose only partial output was scrapeSchema/scrapeList.
|
||||
* 2. (Major) abortRun reloaded the run before checking for partial data, but never forced the
|
||||
* interpreter's persistence buffer to flush first - scrapeList isn't on the immediate-flush
|
||||
* allowlist, so a buffered-but-not-yet-written scrapeList page could be invisible to the
|
||||
* reload.
|
||||
*
|
||||
* Both bugs are now fixed in one place (hasPartialOutput / flushReloadAndCheckPartialOutput)
|
||||
* instead of duplicated inline in task-runner.ts's two call sites, which is how the same bug
|
||||
* ended up in two places to begin with.
|
||||
*/
|
||||
import { hasPartialOutput, flushReloadAndCheckPartialOutput, RunLike, BrowserLike } from '../output-validation';
|
||||
|
||||
describe('hasPartialOutput', () => {
|
||||
it('returns true when scrapeSchema has keys (regression: .length > 0 never matched a plain object)', () => {
|
||||
expect(hasPartialOutput({ scrapeSchema: { field1: 'value' } }, {})).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when scrapeList has keys (same regression, scrapeList side)', () => {
|
||||
expect(hasPartialOutput({ scrapeList: { 'List 1': [{ a: 1 }] } }, {})).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when crawl has keys', () => {
|
||||
expect(hasPartialOutput({ crawl: { 'Crawl Results': [{ url: 'p1' }] } }, {})).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when search has keys', () => {
|
||||
expect(hasPartialOutput({ search: { 'Search Results': { results: [] } } }, {})).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when only binaryOutput has keys', () => {
|
||||
expect(hasPartialOutput({}, { 'screenshot-1': { data: 'abc' } })).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when every field is an empty object', () => {
|
||||
expect(hasPartialOutput(
|
||||
{ scrapeSchema: {}, scrapeList: {}, crawl: {}, search: {} },
|
||||
{}
|
||||
)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for null/undefined inputs without throwing', () => {
|
||||
expect(hasPartialOutput(null, null)).toBe(false);
|
||||
expect(hasPartialOutput(undefined, undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it('is not fooled by an actual array slipping in (defensive, matches Interpreter.ts normalization)', () => {
|
||||
// Interpreter.ts normalizes these to {} if ever caught as arrays, but hasPartialOutput
|
||||
// should behave sanely either way rather than assuming the object shape.
|
||||
expect(hasPartialOutput({ scrapeSchema: [] }, {})).toBe(false);
|
||||
expect(hasPartialOutput({ scrapeSchema: [{ a: 1 }] }, {})).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('flushReloadAndCheckPartialOutput', () => {
|
||||
function fakeRun(overrides: Partial<RunLike> = {}): RunLike & { reload: jest.Mock } {
|
||||
return {
|
||||
serializableOutput: {},
|
||||
binaryOutput: {},
|
||||
reload: jest.fn().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
} as RunLike & { reload: jest.Mock };
|
||||
}
|
||||
|
||||
function fakeBrowser(flushImpl?: () => Promise<void>): BrowserLike & { interpreter: { flushPersistenceBuffer: jest.Mock } } {
|
||||
return {
|
||||
interpreter: {
|
||||
flushPersistenceBuffer: jest.fn(flushImpl ?? (() => Promise.resolve())),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it('flushes the persistence buffer before reloading the run (ordering matters)', async () => {
|
||||
const callOrder: string[] = [];
|
||||
const run = fakeRun();
|
||||
(run.reload as jest.Mock).mockImplementation(async () => { callOrder.push('reload'); });
|
||||
const browser = fakeBrowser(async () => { callOrder.push('flush'); });
|
||||
|
||||
await flushReloadAndCheckPartialOutput(run, browser);
|
||||
|
||||
expect(callOrder).toEqual(['flush', 'reload']);
|
||||
});
|
||||
|
||||
it('reflects data that only appears after reload (proves it reads run state post-reload, not pre-reload)', async () => {
|
||||
const run = fakeRun({ serializableOutput: {} });
|
||||
(run.reload as jest.Mock).mockImplementation(async () => {
|
||||
// Simulate the reload picking up a scrapeList page that was buffered/flushed concurrently.
|
||||
run.serializableOutput = { scrapeList: { 'List 1': [{ a: 1 }] } };
|
||||
});
|
||||
const browser = fakeBrowser();
|
||||
|
||||
const result = await flushReloadAndCheckPartialOutput(run, browser);
|
||||
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it('still reloads and checks when there is no browser/interpreter at all', async () => {
|
||||
const run = fakeRun({ serializableOutput: { crawl: { 'Crawl Results': [{ url: 'p1' }] } } });
|
||||
|
||||
const result = await flushReloadAndCheckPartialOutput(run, null);
|
||||
|
||||
expect(run.reload).toHaveBeenCalledTimes(1);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it('does not call flushPersistenceBuffer when interpreter is missing', async () => {
|
||||
const run = fakeRun();
|
||||
const browserWithoutInterpreter: BrowserLike = {};
|
||||
|
||||
await flushReloadAndCheckPartialOutput(run, browserWithoutInterpreter);
|
||||
|
||||
expect(run.reload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('swallows a flushPersistenceBuffer rejection and still reloads and checks', async () => {
|
||||
const run = fakeRun({ serializableOutput: { crawl: { 'Crawl Results': [{ url: 'p1' }] } } });
|
||||
const browser = fakeBrowser(() => Promise.reject(new Error('flush boom')));
|
||||
|
||||
const result = await expect(flushReloadAndCheckPartialOutput(run, browser)).resolves.toBe(true);
|
||||
expect(run.reload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('swallows a run.reload rejection and still checks whatever is already on run', async () => {
|
||||
const run = fakeRun({ serializableOutput: { search: { 'Search Results': { results: [{ url: 's1' }] } } } });
|
||||
(run.reload as jest.Mock).mockRejectedValue(new Error('reload boom'));
|
||||
const browser = fakeBrowser();
|
||||
|
||||
const result = await flushReloadAndCheckPartialOutput(run, browser);
|
||||
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
});
|
||||
-248
@@ -1,248 +0,0 @@
|
||||
/**
|
||||
* Regression tests for the O(n^2) fix on top of getmaxun/maxun#1104's incremental persistence:
|
||||
* crawl/search deliver the FULL cumulative snapshot on every page (so the in-memory cache and
|
||||
* socket broadcast stay simple), but persisting that full snapshot on every single page write
|
||||
* would resend every prior page's data again and again. computeCrawlSearchPersistDelta slices
|
||||
* off just the new items before they reach the persistence buffer, and mergeCrawlDelta /
|
||||
* mergeSearchDelta append that delta onto the existing DB state instead of overwriting it.
|
||||
*
|
||||
* These are pure, stateful-but-isolated methods (no DB/socket involved), extracted specifically
|
||||
* so this logic can be verified directly without driving a full interpreter run.
|
||||
*/
|
||||
jest.mock('../../../models/Run', () => ({ __esModule: true, default: {} }));
|
||||
// Interpreter.ts imports BinaryOutputService for the per-page screenshot upload path (#1105) -
|
||||
// mock it so importing the module under test doesn't attempt a real MinIO connection.
|
||||
jest.mock('../../../storage/mino', () => ({ BinaryOutputService: jest.fn() }));
|
||||
// flushPersistenceBuffer lazily requires storage/db for its transaction - mock it so the
|
||||
// retry-exhaustion tests below can force every write to fail without a real Postgres connection.
|
||||
jest.mock('../../../storage/db', () => ({
|
||||
__esModule: true,
|
||||
default: { transaction: jest.fn().mockRejectedValue(new Error('db unavailable')) },
|
||||
}));
|
||||
|
||||
import { WorkflowInterpreter } from '../Interpreter';
|
||||
import type { Socket } from 'socket.io';
|
||||
|
||||
function createInterpreter(): any {
|
||||
const fakeSocket = { emit: jest.fn(), nsp: { emit: jest.fn() } } as unknown as Socket;
|
||||
return new WorkflowInterpreter(fakeSocket, 'test-run-id') as any;
|
||||
}
|
||||
|
||||
describe('computeCrawlSearchPersistDelta', () => {
|
||||
it('passes non-crawl/search types through unchanged', () => {
|
||||
const interp = createInterpreter();
|
||||
const result = interp.computeCrawlSearchPersistDelta('scrapeList', 'List 1', { a: 1 });
|
||||
expect(result).toEqual({ dataToPersist: { a: 1 }, skip: false });
|
||||
});
|
||||
|
||||
it('slices only the new crawl pages on each successive call', () => {
|
||||
const interp = createInterpreter();
|
||||
|
||||
const call1 = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }]);
|
||||
expect(call1.skip).toBe(false);
|
||||
expect(call1.dataToPersist).toEqual([{ url: 'p1' }]);
|
||||
|
||||
const call2 = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }, { url: 'p2' }]);
|
||||
expect(call2.skip).toBe(false);
|
||||
expect(call2.dataToPersist).toEqual([{ url: 'p2' }]);
|
||||
|
||||
const call3 = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }, { url: 'p2' }, { url: 'p3' }]);
|
||||
expect(call3.dataToPersist).toEqual([{ url: 'p3' }]);
|
||||
});
|
||||
|
||||
it('always persists the first call even with an empty crawl result set', () => {
|
||||
const interp = createInterpreter();
|
||||
const result = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', []);
|
||||
expect(result.skip).toBe(false);
|
||||
expect(result.dataToPersist).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips a redundant repeat call with nothing new after the first persist', () => {
|
||||
const interp = createInterpreter();
|
||||
interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }]);
|
||||
// e.g. the harmless final safety-net call after the loop, when nothing changed since the
|
||||
// last per-page call
|
||||
const redundant = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }]);
|
||||
expect(redundant.skip).toBe(true);
|
||||
expect(redundant.dataToPersist).toEqual([]);
|
||||
});
|
||||
|
||||
it('tracks crawl and search deltas independently even with the same underlying counts', () => {
|
||||
const interp = createInterpreter();
|
||||
interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'c1' }]);
|
||||
const searchFirst = interp.computeCrawlSearchPersistDelta('search', 'Search Results', {
|
||||
query: 'q', provider: 'duckduckgo', results: [{ url: 's1' }], resultsCount: 1,
|
||||
});
|
||||
expect(searchFirst.skip).toBe(false);
|
||||
expect(searchFirst.dataToPersist.results).toEqual([{ url: 's1' }]);
|
||||
|
||||
// A second crawl page must not be affected by the search call above, and vice versa.
|
||||
const crawlSecond = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'c1' }, { url: 'c2' }]);
|
||||
expect(crawlSecond.dataToPersist).toEqual([{ url: 'c2' }]);
|
||||
});
|
||||
|
||||
it('slices only the new search results and refreshes metadata on each call', () => {
|
||||
const interp = createInterpreter();
|
||||
|
||||
const call1 = interp.computeCrawlSearchPersistDelta('search', 'Search Results', {
|
||||
query: 'test', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1',
|
||||
});
|
||||
expect(call1.dataToPersist).toEqual({
|
||||
query: 'test', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1',
|
||||
});
|
||||
|
||||
const call2 = interp.computeCrawlSearchPersistDelta('search', 'Search Results', {
|
||||
query: 'test', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }, { url: 's2' }], resultsCount: 2, searchedAt: 't2',
|
||||
});
|
||||
// Only the new result should be in the delta, but metadata (searchedAt) is still the latest.
|
||||
expect(call2.dataToPersist.results).toEqual([{ url: 's2' }]);
|
||||
expect(call2.dataToPersist.searchedAt).toBe('t2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeCrawlDelta', () => {
|
||||
it('appends a delta onto an empty/missing existing state', () => {
|
||||
const interp = createInterpreter();
|
||||
const merged = interp.mergeCrawlDelta(undefined, { 'Crawl Results': [{ url: 'p1' }] });
|
||||
expect(merged).toEqual({ 'Crawl Results': [{ url: 'p1' }] });
|
||||
});
|
||||
|
||||
it('appends successive deltas without dropping or duplicating prior pages', () => {
|
||||
const interp = createInterpreter();
|
||||
let state: Record<string, any> = {};
|
||||
state = interp.mergeCrawlDelta(state, { 'Crawl Results': [{ url: 'p1' }] });
|
||||
state = interp.mergeCrawlDelta(state, { 'Crawl Results': [{ url: 'p2' }] });
|
||||
state = interp.mergeCrawlDelta(state, { 'Crawl Results': [{ url: 'p3' }] });
|
||||
|
||||
expect(state['Crawl Results']).toEqual([{ url: 'p1' }, { url: 'p2' }, { url: 'p3' }]);
|
||||
});
|
||||
|
||||
it('does not mutate the previous state object in place', () => {
|
||||
const interp = createInterpreter();
|
||||
const original = { 'Crawl Results': [{ url: 'p1' }] };
|
||||
const merged = interp.mergeCrawlDelta(original, { 'Crawl Results': [{ url: 'p2' }] });
|
||||
|
||||
expect(original['Crawl Results']).toEqual([{ url: 'p1' }]);
|
||||
expect(merged['Crawl Results']).toEqual([{ url: 'p1' }, { url: 'p2' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeSearchDelta', () => {
|
||||
it('appends results and refreshes metadata, recomputing resultsCount from the merged array', () => {
|
||||
const interp = createInterpreter();
|
||||
let state: Record<string, any> = {};
|
||||
|
||||
state = interp.mergeSearchDelta(state, {
|
||||
'Search Results': { query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1' },
|
||||
});
|
||||
state = interp.mergeSearchDelta(state, {
|
||||
'Search Results': { query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's2' }], resultsCount: 2, searchedAt: 't2' },
|
||||
});
|
||||
|
||||
expect(state['Search Results'].results).toEqual([{ url: 's1' }, { url: 's2' }]);
|
||||
expect(state['Search Results'].resultsCount).toBe(2);
|
||||
expect(state['Search Results'].searchedAt).toBe('t2');
|
||||
});
|
||||
|
||||
it('appends a delta onto an empty/missing existing state', () => {
|
||||
const interp = createInterpreter();
|
||||
const merged = interp.mergeSearchDelta(undefined, {
|
||||
'Search Results': { query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1' },
|
||||
});
|
||||
expect(merged).toEqual({
|
||||
'Search Results': { query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1' },
|
||||
});
|
||||
});
|
||||
|
||||
it('does not mutate the previous state or delta inputs in place', () => {
|
||||
const interp = createInterpreter();
|
||||
const original = {
|
||||
'Search Results': { query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1' },
|
||||
};
|
||||
const delta = {
|
||||
'Search Results': { query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's2' }], resultsCount: 1, searchedAt: 't2' },
|
||||
};
|
||||
const originalSnapshot = JSON.parse(JSON.stringify(original));
|
||||
const deltaSnapshot = JSON.parse(JSON.stringify(delta));
|
||||
|
||||
const merged = interp.mergeSearchDelta(original, delta);
|
||||
|
||||
expect(original).toEqual(originalSnapshot);
|
||||
expect(delta).toEqual(deltaSnapshot);
|
||||
expect(merged['Search Results'].results).toEqual([{ url: 's1' }, { url: 's2' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeCrawlSearchPersistDelta + mergeCrawlDelta end-to-end', () => {
|
||||
it('reconstructs the exact full page sequence with no duplicates or gaps across many pages', () => {
|
||||
const interp = createInterpreter();
|
||||
let dbState: Record<string, any> = {};
|
||||
const cumulative: any[] = [];
|
||||
|
||||
for (let i = 1; i <= 10; i++) {
|
||||
cumulative.push({ url: `page${i}` });
|
||||
const { dataToPersist, skip } = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [...cumulative]);
|
||||
if (!skip) {
|
||||
dbState.crawl = interp.mergeCrawlDelta(dbState.crawl, { 'Crawl Results': dataToPersist });
|
||||
}
|
||||
}
|
||||
|
||||
expect(dbState.crawl['Crawl Results']).toEqual(cumulative);
|
||||
expect(dbState.crawl['Crawl Results']).toHaveLength(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe('flushPersistenceBuffer retry-exhaustion rollback', () => {
|
||||
// A failed flush schedules a real backoff retry via setTimeout (Interpreter.ts's
|
||||
// persistenceRetryTimer/persistenceTimer). These tests drive the retry-exhaustion path by
|
||||
// calling flushPersistenceBuffer directly instead of waiting on those timers, so clearState()
|
||||
// cancels whatever's still pending afterwards rather than leaking real timers past the test.
|
||||
let interp: any;
|
||||
|
||||
afterEach(async () => {
|
||||
await interp?.clearState();
|
||||
});
|
||||
|
||||
it('rolls back the crawl delta cursor when a write is dropped after max retries, so the next page recovers the lost data', async () => {
|
||||
interp = createInterpreter();
|
||||
interp.setRunId('test-run-id');
|
||||
|
||||
const call1 = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }]);
|
||||
expect(call1.skip).toBe(false);
|
||||
|
||||
// sequelize.transaction always rejects (mocked above), so this drives every retry attempt
|
||||
// through the failure branch. MAX_PERSISTENCE_RETRIES is 3, so it takes 1 initial attempt +
|
||||
// 3 manual retries to exhaust the budget and hit the "dropping N items" branch.
|
||||
await interp.persistDataToDatabase('crawl', { 'Crawl Results': call1.dataToPersist });
|
||||
await interp.flushPersistenceBuffer();
|
||||
await interp.flushPersistenceBuffer();
|
||||
await interp.flushPersistenceBuffer();
|
||||
|
||||
// The dropped page was never actually written - the next page's delta must include it again
|
||||
// instead of the cursor treating it as already persisted.
|
||||
const call2 = interp.computeCrawlSearchPersistDelta('crawl', 'Crawl Results', [{ url: 'p1' }, { url: 'p2' }]);
|
||||
expect(call2.skip).toBe(false);
|
||||
expect(call2.dataToPersist).toEqual([{ url: 'p1' }, { url: 'p2' }]);
|
||||
});
|
||||
|
||||
it('rolls back the search delta cursor the same way', async () => {
|
||||
interp = createInterpreter();
|
||||
interp.setRunId('test-run-id');
|
||||
|
||||
const call1 = interp.computeCrawlSearchPersistDelta('search', 'Search Results', {
|
||||
query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }], resultsCount: 1, searchedAt: 't1',
|
||||
});
|
||||
expect(call1.skip).toBe(false);
|
||||
|
||||
await interp.persistDataToDatabase('search', { 'Search Results': call1.dataToPersist });
|
||||
await interp.flushPersistenceBuffer();
|
||||
await interp.flushPersistenceBuffer();
|
||||
await interp.flushPersistenceBuffer();
|
||||
|
||||
const call2 = interp.computeCrawlSearchPersistDelta('search', 'Search Results', {
|
||||
query: 'q', provider: 'duckduckgo', mode: 'scrape', results: [{ url: 's1' }, { url: 's2' }], resultsCount: 2, searchedAt: 't2',
|
||||
});
|
||||
expect(call2.skip).toBe(false);
|
||||
expect(call2.dataToPersist.results).toEqual([{ url: 's1' }, { url: 's2' }]);
|
||||
});
|
||||
});
|
||||
@@ -26,7 +26,6 @@
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"src/mcp-worker.ts",
|
||||
"src/**/__tests__/**",
|
||||
"../src/components/**/*", // Exclude frontend components
|
||||
"../src/pages/**/*", // Exclude frontend pages
|
||||
"../src/app/**/*", // Exclude other frontend-specific code
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2020",
|
||||
"module": "commonjs",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"moduleResolution": "node",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"types": ["node", "jest"]
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user