diff --git a/__tests__/ui-highlight-bounded.test.ts b/__tests__/ui-highlight-bounded.test.ts new file mode 100644 index 000000000..af0982242 --- /dev/null +++ b/__tests__/ui-highlight-bounded.test.ts @@ -0,0 +1,28 @@ +/** + * A grammar that never finishes a slice must not take the viewer's server + * down. The COBOL grammar's scanner loops forever on a line like ` .` — a + * period in the fixed-format sequence area — which cobolcraft's free-format + * paragraphs end with, so a Flow card's window froze the server and every + * request behind it. Slices are classified in a worker the server ends past + * a deadline; the slice is served plain, and the next one gets a new worker. + */ +import { describe, it, expect, afterAll } from 'vitest'; +import { highlightLines, clearHighlightCache } from '../src/ui-server/highlight'; +import { stopHighlightWorker } from '../src/ui-server/highlight/bounded-tokenize'; + +afterAll(() => stopHighlightWorker()); + +describe('highlighting a slice', () => { + it('serves a slice the grammar never finishes plain, and keeps answering', async () => { + clearHighlightCache(); + const started = Date.now(); + const hung = await highlightLines([' PERFORM AssertOk', ' .'], { language: 'cobol' }); + expect(hung.engine).toBe('plain'); + expect(hung.reason).toMatch(/took too long/); + expect(Date.now() - started).toBeLessThan(10_000); + + // The next slice gets a fresh worker and is classified as usual. + const fine = await highlightLines(['export const x: number = 1;'], { language: 'typescript' }); + expect(fine.engine).toBe('tree-sitter'); + }, 20_000); +}); diff --git a/docs/viewer-launch-changelog.md b/docs/viewer-launch-changelog.md index e7e864499..7376e5517 100644 --- a/docs/viewer-launch-changelog.md +++ b/docs/viewer-launch-changelog.md @@ -171,6 +171,8 @@ These describe `codegraph ui` and its screens. They were taken out of `## [Unrel ## Fixes — Symbols, tests and the viewer +- **One slow file can no longer freeze the viewer.** Code is highlighted with the same grammars the index uses. The COBOL grammar never finishes a line like ` .`, which free-format COBOL paragraphs end with, so opening a Flow through cobolcraft froze `codegraph ui`, and every page after it waited forever. Highlighting now runs where it can be stopped: a piece of code that takes too long is shown unhighlighted, and everything else keeps working. + - **The Map opens on a picture, not a single box.** A project whose code sits in one flat folder, like Express's `lib/`, an R package's `R/`, an Erlang app's `src/` or fmt's `include/fmt/`, opened as one box with nothing to say. It now opens on the whole repository when that draws more. Files at the repository's top level also count when the Map picks where the program lives: git's hundreds of top-level `.c` files had made `builtin/` look like most of the code. - **A Flow between two symbols finds names of any shape.** A Flow opened from a Steps link, an entry point or the search box names its two ends by the symbols' own names, and names that don't look like plain identifiers were dropped: an Objective-C selector like `initWithFrame:`, a Ruby method like `save!` or `valid?`, and any name shorter than three letters, like `ok`. The strip then said the other end "names nothing in this index", even though it was there. Both ends are now looked up exactly as written. diff --git a/src/extraction/parse-budget.ts b/src/extraction/parse-budget.ts new file mode 100644 index 000000000..276e388b7 --- /dev/null +++ b/src/extraction/parse-budget.ts @@ -0,0 +1,23 @@ +/** + * A parse that answers a request — the viewer's syntax highlighting, the + * branch guards a Steps or Flow card reads — runs on the server's one thread, + * so a parse that does not finish takes every other request down with it. + * Indexing parses in workers it can kill; these cannot be killed, so they are + * cancelled instead: tree-sitter asks its progress callback as it goes, and a + * `true` stops the parse. A free-format COBOL window handed to the + * fixed-format grammar kept cobolcraft's Flow strip — and the whole viewer + * after it — parsing for good. + */ + +import type { Parser, Tree } from 'web-tree-sitter'; + +/** Longest a request-time parse may run before it is given up — far past any file that parses at all. */ +export const REQUEST_PARSE_BUDGET_MS = 3000; + +/** `parser.parse(source)`, or null when it runs past the budget. */ +export function parseWithinBudget(parser: Parser, source: string, budgetMs = REQUEST_PARSE_BUDGET_MS): Tree | null { + const deadline = Date.now() + budgetMs; + // The runtime hands the callback's return value to the parser: `true` cancels. + const cancel = (() => Date.now() > deadline) as unknown as () => void; + return parser.parse(source, null, { progressCallback: cancel }) ?? null; +} diff --git a/src/extraction/syntax-tokens.ts b/src/extraction/syntax-tokens.ts index 56a19f085..67c0d1b71 100644 --- a/src/extraction/syntax-tokens.ts +++ b/src/extraction/syntax-tokens.ts @@ -42,6 +42,7 @@ import type { Node as SyntaxNode } from 'web-tree-sitter'; import { Language } from '../types'; import { EXTRACTORS } from './languages'; import { getParser, loadGrammarsForLanguages } from './grammars'; +import { parseWithinBudget } from './parse-budget'; import type { LanguageExtractor } from './tree-sitter-types'; /* ------------------------------------------------------------- the classes -- */ @@ -450,7 +451,7 @@ async function tokenizeRegion( await loadGrammarsForLanguages([language]); const parser = getParser(language); if (!parser) return null; - const tree = parser.parse(source); + const tree = parseWithinBudget(parser, source); if (!tree?.rootNode) return null; try { return classifyTree(tree.rootNode, source, language, offset); diff --git a/src/graph/branch-guards.ts b/src/graph/branch-guards.ts index 76097b634..bc45ef45e 100644 --- a/src/graph/branch-guards.ts +++ b/src/graph/branch-guards.ts @@ -33,6 +33,7 @@ import * as fs from 'fs'; import type { Node as SyntaxNode, Tree } from 'web-tree-sitter'; import type { Language } from '../types'; import { getParser, loadGrammarsForLanguages } from '../extraction/grammars'; +import { parseWithinBudget } from '../extraction/parse-budget'; // ============================================================================= // Public shape @@ -213,7 +214,7 @@ async function parse(source: string, language: Language): Promise { await loadGrammarsForLanguages([language]); const parser = getParser(language); if (!parser) return null; - return parser.parse(source) ?? null; + return parseWithinBudget(parser, source); } catch { return null; } @@ -293,7 +294,7 @@ export function guardsForFileSync( } catch { return out; } - const tree = parser.parse(source); + const tree = parseWithinBudget(parser, source); if (!tree) return out; cached = { key, tree, source }; remember(absPath, cached); diff --git a/src/ui-server/highlight/bounded-tokenize.ts b/src/ui-server/highlight/bounded-tokenize.ts new file mode 100644 index 000000000..9147538f3 --- /dev/null +++ b/src/ui-server/highlight/bounded-tokenize.ts @@ -0,0 +1,112 @@ +/** + * Highlighting that cannot take the server down. + * + * The viewer's server answers every request on one thread, and a slice is + * classified by handing it to a tree-sitter grammar — which can loop forever on + * a fragment it was never written for: cobolcraft's free-format COBOL, cut to a + * Flow card's window, kept the fixed-format grammar's scanner spinning inside + * WebAssembly, and every request after it — the page's own `/api/stats` + * included — waited behind it for good. A parse inside WebAssembly cannot be + * interrupted from JavaScript, so the slice is classified in a worker the + * server can terminate: past the deadline the worker is ended, the slice is + * served plain, and the next slice gets a fresh worker. + */ + +import { existsSync } from 'fs'; +import * as path from 'path'; +import { Worker } from 'worker_threads'; +import { tokenizeSource, type TokenizeResult } from '../../extraction/syntax-tokens'; +import type { Language } from '../../types'; + +/** Longest a slice may take to classify — far past any file the grammars can parse at all. */ +export const TOKENIZE_DEADLINE_MS = 2000; + +/** What a bounded classification came to: the result, or that it ran out of time. */ +export type BoundedTokenize = { result: TokenizeResult | null } | { timedOut: true }; + +interface Pending { + resolve: (outcome: BoundedTokenize) => void; + timer: NodeJS.Timeout; +} + +let worker: Worker | null = null; +let nextId = 1; +const pending = new Map(); + +/** + * The compiled worker beside us; Vitest loads `src/` but builds `dist/` before + * the tests, so it uses that copy. Null when neither exists. + */ +function workerFile(): string | null { + const sibling = path.join(__dirname, 'tokenize-worker.js'); + if (existsSync(sibling)) return sibling; + const built = path.resolve(__dirname, '../../../dist/ui-server/highlight/tokenize-worker.js'); + return existsSync(built) ? built : null; +} + +/** Settle everything waiting on the current worker and forget it. */ +function retire(outcome: BoundedTokenize): void { + const dead = worker; + worker = null; + for (const [id, p] of pending) { + clearTimeout(p.timer); + pending.delete(id); + p.resolve(outcome); + } + if (dead) void dead.terminate().catch(() => {}); +} + +function ensureWorker(file: string): Worker { + if (worker) return worker; + const w = new Worker(file); + w.unref(); + w.on('message', (msg: { id: number; result: TokenizeResult | null }) => { + const p = pending.get(msg.id); + if (!p) return; + clearTimeout(p.timer); + pending.delete(msg.id); + p.resolve({ result: msg.result }); + }); + w.on('error', () => { + if (worker === w) retire({ result: null }); + }); + w.on('exit', () => { + if (worker === w) retire({ result: null }); + }); + worker = w; + return w; +} + +/** Classify `text` in the highlight worker, giving up after {@link TOKENIZE_DEADLINE_MS}. */ +export async function tokenizeBounded( + text: string, + language: Language, + deadlineMs = TOKENIZE_DEADLINE_MS +): Promise { + const file = workerFile(); + // No compiled worker (a source checkout that was never built): classify + // in-process, as before. + if (!file) return { result: await tokenizeSource(text, language) }; + let w: Worker; + try { + w = ensureWorker(file); + } catch { + return { result: await tokenizeSource(text, language) }; + } + const id = nextId++; + return new Promise((resolve) => { + const timer = setTimeout(() => { + // Everything queued behind the slice that hung waits on the same + // worker: end it, serve them all plain, and start over next time. + if (worker === w) retire({ timedOut: true }); + }, deadlineMs); + timer.unref(); + pending.set(id, { resolve, timer }); + w.postMessage({ id, text, language }); + }); +} + +/** End the highlight worker — for tests, and for a server that is shutting down. */ +export function stopHighlightWorker(): void { + retire({ result: null }); +} diff --git a/src/ui-server/highlight/index.ts b/src/ui-server/highlight/index.ts index 11e69861e..3e3b5a9cc 100644 --- a/src/ui-server/highlight/index.ts +++ b/src/ui-server/highlight/index.ts @@ -42,7 +42,8 @@ * whole-file view pages the same file repeatedly. */ -import { SYNTAX_TOKEN_CLASSES, tokenizeSource, type SyntaxTokenClass } from '../../extraction/syntax-tokens'; +import { SYNTAX_TOKEN_CLASSES, type SyntaxTokenClass } from '../../extraction/syntax-tokens'; +import { tokenizeBounded } from './bounded-tokenize'; import type { Language } from '../../types'; import { grammarFor } from './languages'; @@ -208,7 +209,11 @@ async function highlightUncached( return plain(lines, grammar, 'Too much text on too few lines to highlight (minified?).'); } - const tokenized = await tokenizeSource(text, language as Language); + const bounded = await tokenizeBounded(text, language as Language); + if ('timedOut' in bounded) { + return plain(lines, grammar, `The ${grammar} grammar took too long on this slice, so it is shown unhighlighted.`); + } + const tokenized = bounded.result; if (!tokenized || tokenized.spans.length === 0) { return plain(lines, grammar, `The ${grammar} grammar is not available in this build.`); } diff --git a/src/ui-server/highlight/tokenize-worker.ts b/src/ui-server/highlight/tokenize-worker.ts new file mode 100644 index 000000000..6c25fe6e1 --- /dev/null +++ b/src/ui-server/highlight/tokenize-worker.ts @@ -0,0 +1,25 @@ +/** + * The worker side of bounded highlighting (`bounded-tokenize.ts`): classifies + * one slice per message with the engine's own grammars, off the server's + * thread, so a grammar that never finishes a slice can be terminated. + */ + +import { parentPort } from 'worker_threads'; +import { tokenizeSource } from '../../extraction/syntax-tokens'; +import type { Language } from '../../types'; + +interface TokenizeRequest { + id: number; + text: string; + language: Language; +} + +parentPort?.on('message', async (msg: TokenizeRequest) => { + let result: Awaited> = null; + try { + result = await tokenizeSource(msg.text, msg.language); + } catch { + result = null; + } + parentPort?.postMessage({ id: msg.id, result }); +});