From f416357b9394e202ad815425d6ade04e725d1f46 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Thu, 1 Oct 2026 21:47:52 +0000 Subject: [PATCH] fix(cobol): a file ending inside the sequence area no longer hangs the parser (#2272) * fix(cobol): a file ending inside the sequence area no longer hangs the parser The vendored COBOL grammar's external scanner skips the fixed-format sequence area (columns 1-6) with `while (get_column() <= 5) advance()`. At end of input advance() is a no-op, so a file whose last line stops in that area (` .`, `X\n .`, a sentence closed by a short ` .` line) never reached column 7 and the scanner spun forever inside wasm. Indexing hit the parse worker's hard timeout three times (~93 s) and stored the file with zero symbols; the viewer's highlighter froze on the same text. The loop now also stops at eof(). It is upstream code (present at e99dbdc3), so the change is a standalone upstreamable fix; inputs that parsed before are unaffected (412 NIST + ocesql test sources give byte-identical trees with the old and new wasm, and the upstream corpus keeps its single pre-existing failure). Regenerates docs/grammars/tree-sitter-cobol.patch, rebuilds the wasm per docs/grammars/tree-sitter-cobol.md, and adds an out-of-process regression test under a hard deadline (a regression is an uninterruptible wasm loop). Co-Authored-By: Claude Opus 5.5 * test(viewer): force the highlight deadline instead of relying on a grammar bug The bounded-highlight test used the COBOL ` .` slice as its slice that never finishes; the grammar fix makes it parse, so the deadline is now forced through tokenizeBounded's deadline parameter. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/cobol-sequence-area-period.test.ts | 119 +++++++++++++++++++ __tests__/ui-highlight-bounded.test.ts | 34 ++++-- docs/grammars/tree-sitter-cobol.md | 13 +- docs/grammars/tree-sitter-cobol.patch | 20 ++-- src/extraction/wasm/tree-sitter-cobol.wasm | Bin 16355286 -> 16355308 bytes 6 files changed, 165 insertions(+), 22 deletions(-) create mode 100644 __tests__/cobol-sequence-area-period.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e165649c7..c4db43bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- A COBOL file whose last line ends inside the sequence area (columns 1-6), such as a closing period indented only four spaces, no longer stalls indexing for over a minute and then ends up in the index with no symbols. - In Laravel apps, a route whose controller is named by string now links to its handler, without needing a `Controller` suffix and keeping the namespace it's written with: `'Common\Uploads@inline'`, and `Route::resource('companies', 'Common\Companies', [...])` with an options array. Two same-named controllers in different namespaces are told apart by that namespace. Before, akaunting linked 304 of its 313 routes to nothing. - In Nuxt and Astro projects, a file-routed page is now linked to the component its file defines. An Astro endpoint is linked to the HTTP-verb handlers it exports, like `export const GET`. Before, a page route like Nuxt's `pages/admin.vue` or Astro's `src/pages/about.astro` linked to nothing, so callers, impact and flows stopped at the route. - In Rails apps, a `resources` line's `only:` and `except:` are now read in every form Rails accepts: `%i[new create index]`, `%w(index show)`, a quoted name, and the older `:only => [...]`. Before, only a bracketed list was read, so maybe's `resources :family_exports, only: %i[new create index]` became all seven routes, four of them to actions the controller doesn't have. diff --git a/__tests__/cobol-sequence-area-period.test.ts b/__tests__/cobol-sequence-area-period.test.ts new file mode 100644 index 000000000..79985c0f4 --- /dev/null +++ b/__tests__/cobol-sequence-area-period.test.ts @@ -0,0 +1,119 @@ +/** + * COBOL: a file that ends inside the sequence area must parse, not hang. + * + * The vendored COBOL grammar's external scanner skips the fixed-format + * sequence area (columns 1-6) by advancing until the lexer reaches column 7. + * At end of input `advance()` is a no-op, so a file whose last line stops + * inside that area — ` .` (four spaces and a period: the period lands in + * column 5), `X\n .`, a paragraph closed by a short ` .` line — never + * reached column 7 and the scanner spun forever inside WebAssembly. Indexing + * such a file ran into the parse worker's timeout (about a minute and a half + * with the retries) and stored it with zero symbols; the viewer's highlighter + * froze on the same text. The scanner now stops at end of input. + * + * Both checks run against the built engine in a child process under a hard + * timeout: a regression is an infinite loop inside wasm that no in-process + * timer can interrupt, so parsing in this worker would hang the suite instead + * of failing the test. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { WASM_RUNTIME_FLAGS } from '../src/extraction/wasm-runtime-flags'; + +const DIST = path.resolve(__dirname, '../dist'); + +/** A fixed-format program whose last line is a lone period in column 5. */ +const HANGY = [ + ' IDENTIFICATION DIVISION.', + ' PROGRAM-ID. HANGY.', + ' PROCEDURE DIVISION.', + ' MAIN-PARA.', + ' DISPLAY "X"', + ' .', + '', +].join('\n'); + +/** Under the parse worker's 30 s hard per-file timeout a hang would run into. */ +const DEADLINE_MS = 20_000; + +function runChild(script: string, args: string[]) { + return spawnSync(process.execPath, [...WASM_RUNTIME_FLAGS, '-e', script, DIST, ...args], { + encoding: 'utf8', + timeout: DEADLINE_MS, + env: { ...process.env, CODEGRAPH_TELEMETRY: '0' }, + }); +} + +describe('COBOL sequence-area period at end of file', () => { + it('extracts every source that ends inside the sequence area', () => { + const sources: Record = { + 'hangy.cob': HANGY, + 'hangy-no-newline.cob': HANGY.trimEnd(), + 'period-only.cob': ' .', + 'after-statement.cob': ' PERFORM AssertOk\n .', + 'after-word.cob': 'X\n .', + }; + const script = ` +const path = require('path'); +const { initGrammars, loadGrammarsForLanguages } = require(path.join(process.argv[1], 'extraction/grammars.js')); +const { extractFromSource } = require(path.join(process.argv[1], 'extraction/index.js')); +(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['cobol']); + const out = {}; + for (const [file, source] of Object.entries(JSON.parse(process.argv[2]))) { + const result = extractFromSource(file, source); + out[file] = result.nodes.map((n) => n.kind + ':' + n.name); + } + process.stdout.write(JSON.stringify(out)); +})(); +`; + const child = runChild(script, [JSON.stringify(sources)]); + expect(child.signal, `COBOL parse did not finish within ${DEADLINE_MS}ms: ${child.stderr}`).toBeNull(); + expect(child.status, child.stderr).toBe(0); + + const nodes = JSON.parse(child.stdout) as Record; + expect(Object.keys(nodes).sort()).toEqual(Object.keys(sources).sort()); + for (const file of ['hangy.cob', 'hangy-no-newline.cob']) { + expect(nodes[file]).toContain('module:HANGY'); + expect(nodes[file]).toContain('function:MAIN-PARA'); + } + }, DEADLINE_MS + 10_000); + + describe('indexing a project', () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cobol-seq-period-')); + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('indexes the file with its symbols instead of timing it out', () => { + fs.writeFileSync(path.join(root, 'hangy.cob'), HANGY); + const script = ` +const path = require('path'); +const { default: CodeGraph } = require(path.join(process.argv[1], 'index.js')); +const cg = CodeGraph.initSync(process.argv[2]); +cg.indexAll().then((result) => { + const nodes = cg.getNodesInFile('hangy.cob').map((n) => n.kind + ':' + n.name); + process.stdout.write(JSON.stringify({ errored: result.filesErrored, indexed: result.filesIndexed, nodes })); +}).finally(() => cg.close()); +`; + const child = runChild(script, [root]); + expect(child.signal, `indexing did not finish within ${DEADLINE_MS}ms: ${child.stderr}`).toBeNull(); + expect(child.status, child.stderr).toBe(0); + + const result = JSON.parse(child.stdout) as { errored: number; indexed: number; nodes: string[] }; + expect(result.errored).toBe(0); + expect(result.indexed).toBe(1); + expect(result.nodes).toContain('module:HANGY'); + expect(result.nodes).toContain('function:MAIN-PARA'); + }, DEADLINE_MS + 10_000); + }); +}); diff --git a/__tests__/ui-highlight-bounded.test.ts b/__tests__/ui-highlight-bounded.test.ts index af0982242..c0b497a5f 100644 --- a/__tests__/ui-highlight-bounded.test.ts +++ b/__tests__/ui-highlight-bounded.test.ts @@ -1,28 +1,36 @@ /** * A grammar that never finishes a slice must not take the viewer's server - * down. The COBOL grammar's scanner loops forever on a line like ` .` — a - * period in the fixed-format sequence area — which cobolcraft's free-format - * paragraphs end with, so a Flow card's window froze the server and every - * request behind it. Slices are classified in a worker the server ends past - * a deadline; the slice is served plain, and the next one gets a new worker. + * down. The COBOL grammar's scanner once looped forever on a slice ending in a + * line like ` .` (fixed in the grammar since), which froze cobolcraft's Flow + * strip and every request behind it: a parse inside WebAssembly cannot be + * interrupted from JavaScript. Slices are classified in a worker the server + * ends past a deadline; the slice is served plain, and the next one gets a + * fresh worker. The deadline is forced here rather than found in a grammar. */ import { describe, it, expect, afterAll } from 'vitest'; import { highlightLines, clearHighlightCache } from '../src/ui-server/highlight'; -import { stopHighlightWorker } from '../src/ui-server/highlight/bounded-tokenize'; +import { stopHighlightWorker, tokenizeBounded } from '../src/ui-server/highlight/bounded-tokenize'; afterAll(() => stopHighlightWorker()); describe('highlighting a slice', () => { - it('serves a slice the grammar never finishes plain, and keeps answering', async () => { - clearHighlightCache(); + it('gives up on a slice past the deadline, and the next slice gets a fresh worker', async () => { + const big = 'export const value: number = 1;\n'.repeat(2000); const started = Date.now(); - const hung = await highlightLines([' PERFORM AssertOk', ' .'], { language: 'cobol' }); - expect(hung.engine).toBe('plain'); - expect(hung.reason).toMatch(/took too long/); - expect(Date.now() - started).toBeLessThan(10_000); + const timedOut = await tokenizeBounded(big, 'typescript', 1); + expect('timedOut' in timedOut).toBe(true); + expect(Date.now() - started).toBeLessThan(5_000); - // The next slice gets a fresh worker and is classified as usual. + const fine = await tokenizeBounded('const y = 2;', 'typescript'); + expect('result' in fine && (fine.result?.spans.length ?? 0) > 0).toBe(true); + }, 20_000); + + it('still highlights through the worker', async () => { + clearHighlightCache(); const fine = await highlightLines(['export const x: number = 1;'], { language: 'typescript' }); expect(fine.engine).toBe('tree-sitter'); + // The slice that used to hang the COBOL grammar is classified now. + const cobol = await highlightLines([' PERFORM AssertOk', ' .'], { language: 'cobol' }); + expect(cobol.reason ?? '').not.toMatch(/took too long/); }, 20_000); }); diff --git a/docs/grammars/tree-sitter-cobol.md b/docs/grammars/tree-sitter-cobol.md index 02df0ed87..b2cf6aa97 100644 --- a/docs/grammars/tree-sitter-cobol.md +++ b/docs/grammars/tree-sitter-cobol.md @@ -54,6 +54,15 @@ mainframe and GnuCOBOL code: `SECURITY.` headers. 10. **`CALL ... GIVING`** — upstream *intended* to support it but a misnested `field()` call swallowed the GIVING alternative entirely. +11. **No hang when the input ends inside the sequence area.** The scanner + skips columns 1-6 by advancing until column 7, but `advance()` is a no-op + at end of input, so a file whose last line stops in that area (` .`, + `X\n .`, a sentence closed by a short ` .` line) spun forever inside + the scanner — parse timeouts and a zero-symbol file when indexing. The + loop now also stops at `eof()`. The loop is upstream code (present at + `e99dbdc3`), so this is a standalone upstream fix; inputs that parsed + before are unaffected (the NIST + ocesql test sources, 412 files, give + byte-identical trees before and after). ## Measured parse health (at vendoring time) @@ -100,7 +109,9 @@ Sent as [yutaro-sakamoto/tree-sitter-cobol#41](https://github.com/yutaro-sakamot (branch `real-world-cobol-sources` on the colbymchenry fork). If upstream merges it, the vendored wasm can track upstream releases instead of this patch. Until then, `git apply tree-sitter-cobol.patch` on upstream commit -`e99dbdc3` reproduces the fork exactly. The PR body as sent: +`e99dbdc3` reproduces the vendored grammar exactly. Item 11 (the +end-of-input hang) came after the PR was sent and is not on that branch +yet. The PR body as sent: > **Parse real-world CICS/DB2 and GnuCOBOL sources** > diff --git a/docs/grammars/tree-sitter-cobol.patch b/docs/grammars/tree-sitter-cobol.patch index 72191bd14..15540a0c5 100644 --- a/docs/grammars/tree-sitter-cobol.patch +++ b/docs/grammars/tree-sitter-cobol.patch @@ -303,7 +303,7 @@ index 6d48b47..e700a18 100644 _NOT_GREATER: $ => /([nN][oO][tT][ \t]+(>|[gG][rR][eE][aA][tT][eE][rR]))/, diff --git a/src/scanner.c b/src/scanner.c -index 6c1ae90..b69cf53 100644 +index 6c1ae90..3382988 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -1,4 +1,5 @@ @@ -372,14 +372,18 @@ index 6c1ae90..b69cf53 100644 if(valid_symbols[WHITE_SPACES]) { if(is_white_space(lexer->lookahead)) { while(is_white_space(lexer->lookahead)) { -@@ -106,9 +121,20 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -106,9 +121,23 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[LINE_PREFIX_COMMENT] && lexer->get_column(lexer) <= 5) { +- while(lexer->get_column(lexer) <= 5) { + // The sequence area is ignored content — but the free-format + // preprocessor plants the CGWIDE sentinel here on the first line. ++ // Stop at end of input: advance() is a no-op there, so a file that ++ // ends inside the sequence area (e.g. a last line " .") would ++ // never reach column 6 and the loop would spin forever. + int match = 0; - while(lexer->get_column(lexer) <= 5) { ++ while(lexer->get_column(lexer) <= 5 && !lexer->eof(lexer)) { + if(match >= 0 && match < 6 && lexer->lookahead == CG_WIDE_SENTINEL[match]) { + match++; + } else { @@ -393,7 +397,7 @@ index 6c1ae90..b69cf53 100644 lexer->result_symbol = LINE_PREFIX_COMMENT; lexer->mark_end(lexer); return true; -@@ -132,7 +158,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -132,7 +161,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[LINE_SUFFIX_COMMENT]) { @@ -402,7 +406,7 @@ index 6c1ae90..b69cf53 100644 while(lexer->lookahead != '\n' && lexer->lookahead != 0) { lexer->advance(lexer, true); } -@@ -143,7 +169,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -143,7 +172,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[COMMENT_ENTRY]) { @@ -411,7 +415,7 @@ index 6c1ae90..b69cf53 100644 lexer->mark_end(lexer); lexer->result_symbol = COMMENT_ENTRY; return true; -@@ -152,18 +178,66 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -152,18 +181,66 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } } @@ -482,7 +486,7 @@ index 6c1ae90..b69cf53 100644 lexer->mark_end(lexer); return true; } -@@ -187,7 +261,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -187,7 +264,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } lexer->advance(lexer, true); @@ -491,7 +495,7 @@ index 6c1ae90..b69cf53 100644 lexer->advance(lexer, true); } } -@@ -197,11 +271,19 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -197,11 +274,19 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } unsigned tree_sitter_COBOL_external_scanner_serialize(void *payload, char *buffer) { diff --git a/src/extraction/wasm/tree-sitter-cobol.wasm b/src/extraction/wasm/tree-sitter-cobol.wasm index 12496a95a2306c47cec47ace840d87a641d78d3c..c96722eb93d8c7f034997764b7c54a8c47e76d31 100755 GIT binary patch delta 1060 zcmWmA=bz67003a$-<@2Ja&%_+UC3o-lj4LpBYRwTLbi;GhL&;ajHH~s&R*Fgqhy^? z_Rb!aU6hrNH$A<0{(`w|drdLbOj147(gK!WZ${!QT5}6PV z#YN-@qSGHIH;4+Nqr+M1ha*xGQ_>Ps0zyO(Nmia98&Nz-cB08aPM+dvauGvr^6(6? zz0Xv*s}qd6^TNh?~@hPJe$JxRPlGXH;f;7vNx ziO#%57rN4o?)0E1z39!`^r0{P=+8U6%Xs;0#1^))jqU6plb!5hH+%S*U)ak&_H%%P z9O5uXILa}ObAprn$|+8BhO?aGJQujgB`)(Dzw-xIxXLxIbAy}Q;!pnKHh1`&f4Iv% R?(=|${JTCe<lwc>lmlT9O!Yg)Q9!frPC+0%sxu%j3e@9*s*MSuZ3gRcKInXmE5?WK2|KfM7l# z86T3I5I!OWDM>|Y(vX&Pq$dLz$wX$dkdrl%y1;DMMMxQJzn!Kt;m%j7n7IbE;64YE-8NHK|2y>JUy{>QSEtG^7!YX+l%J zpc&2ik`}b26|MP-uZf@yZF&FQj`l>+fsS;dGhOIPH@=}eJ@}UI=t(bn^F4j&%MbLU zKLZ%ZAb#X01~Y`A3}ZMW7|AF`6U`XLGL9I=Gl7XrVluHzVJg#@&J5z1$t-3whq=sS zJ_}gLA{MiRpIOQ>ma~GDtYS55Sj#%rvw@9lVl!L#geU&IwL(iqo9oEay1SA6(!fm$=LouJR`dT;n=7_=~^!hkv=rEpBs% m|48I6_qfjk9+Jc(9`l5!JmWbpc**~~;x%vLqaxqNmVXD7pTdCv