diff --git a/CHANGELOG.md b/CHANGELOG.md index e165649c7..c4db43bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- A COBOL file whose last line ends inside the sequence area (columns 1-6), such as a closing period indented only four spaces, no longer stalls indexing for over a minute and then ends up in the index with no symbols. - In Laravel apps, a route whose controller is named by string now links to its handler, without needing a `Controller` suffix and keeping the namespace it's written with: `'Common\Uploads@inline'`, and `Route::resource('companies', 'Common\Companies', [...])` with an options array. Two same-named controllers in different namespaces are told apart by that namespace. Before, akaunting linked 304 of its 313 routes to nothing. - In Nuxt and Astro projects, a file-routed page is now linked to the component its file defines. An Astro endpoint is linked to the HTTP-verb handlers it exports, like `export const GET`. Before, a page route like Nuxt's `pages/admin.vue` or Astro's `src/pages/about.astro` linked to nothing, so callers, impact and flows stopped at the route. - In Rails apps, a `resources` line's `only:` and `except:` are now read in every form Rails accepts: `%i[new create index]`, `%w(index show)`, a quoted name, and the older `:only => [...]`. Before, only a bracketed list was read, so maybe's `resources :family_exports, only: %i[new create index]` became all seven routes, four of them to actions the controller doesn't have. diff --git a/__tests__/cobol-sequence-area-period.test.ts b/__tests__/cobol-sequence-area-period.test.ts new file mode 100644 index 000000000..79985c0f4 --- /dev/null +++ b/__tests__/cobol-sequence-area-period.test.ts @@ -0,0 +1,119 @@ +/** + * COBOL: a file that ends inside the sequence area must parse, not hang. + * + * The vendored COBOL grammar's external scanner skips the fixed-format + * sequence area (columns 1-6) by advancing until the lexer reaches column 7. + * At end of input `advance()` is a no-op, so a file whose last line stops + * inside that area — ` .` (four spaces and a period: the period lands in + * column 5), `X\n .`, a paragraph closed by a short ` .` line — never + * reached column 7 and the scanner spun forever inside WebAssembly. Indexing + * such a file ran into the parse worker's timeout (about a minute and a half + * with the retries) and stored it with zero symbols; the viewer's highlighter + * froze on the same text. The scanner now stops at end of input. + * + * Both checks run against the built engine in a child process under a hard + * timeout: a regression is an infinite loop inside wasm that no in-process + * timer can interrupt, so parsing in this worker would hang the suite instead + * of failing the test. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { WASM_RUNTIME_FLAGS } from '../src/extraction/wasm-runtime-flags'; + +const DIST = path.resolve(__dirname, '../dist'); + +/** A fixed-format program whose last line is a lone period in column 5. */ +const HANGY = [ + ' IDENTIFICATION DIVISION.', + ' PROGRAM-ID. HANGY.', + ' PROCEDURE DIVISION.', + ' MAIN-PARA.', + ' DISPLAY "X"', + ' .', + '', +].join('\n'); + +/** Under the parse worker's 30 s hard per-file timeout a hang would run into. */ +const DEADLINE_MS = 20_000; + +function runChild(script: string, args: string[]) { + return spawnSync(process.execPath, [...WASM_RUNTIME_FLAGS, '-e', script, DIST, ...args], { + encoding: 'utf8', + timeout: DEADLINE_MS, + env: { ...process.env, CODEGRAPH_TELEMETRY: '0' }, + }); +} + +describe('COBOL sequence-area period at end of file', () => { + it('extracts every source that ends inside the sequence area', () => { + const sources: Record = { + 'hangy.cob': HANGY, + 'hangy-no-newline.cob': HANGY.trimEnd(), + 'period-only.cob': ' .', + 'after-statement.cob': ' PERFORM AssertOk\n .', + 'after-word.cob': 'X\n .', + }; + const script = ` +const path = require('path'); +const { initGrammars, loadGrammarsForLanguages } = require(path.join(process.argv[1], 'extraction/grammars.js')); +const { extractFromSource } = require(path.join(process.argv[1], 'extraction/index.js')); +(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['cobol']); + const out = {}; + for (const [file, source] of Object.entries(JSON.parse(process.argv[2]))) { + const result = extractFromSource(file, source); + out[file] = result.nodes.map((n) => n.kind + ':' + n.name); + } + process.stdout.write(JSON.stringify(out)); +})(); +`; + const child = runChild(script, [JSON.stringify(sources)]); + expect(child.signal, `COBOL parse did not finish within ${DEADLINE_MS}ms: ${child.stderr}`).toBeNull(); + expect(child.status, child.stderr).toBe(0); + + const nodes = JSON.parse(child.stdout) as Record; + expect(Object.keys(nodes).sort()).toEqual(Object.keys(sources).sort()); + for (const file of ['hangy.cob', 'hangy-no-newline.cob']) { + expect(nodes[file]).toContain('module:HANGY'); + expect(nodes[file]).toContain('function:MAIN-PARA'); + } + }, DEADLINE_MS + 10_000); + + describe('indexing a project', () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cobol-seq-period-')); + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('indexes the file with its symbols instead of timing it out', () => { + fs.writeFileSync(path.join(root, 'hangy.cob'), HANGY); + const script = ` +const path = require('path'); +const { default: CodeGraph } = require(path.join(process.argv[1], 'index.js')); +const cg = CodeGraph.initSync(process.argv[2]); +cg.indexAll().then((result) => { + const nodes = cg.getNodesInFile('hangy.cob').map((n) => n.kind + ':' + n.name); + process.stdout.write(JSON.stringify({ errored: result.filesErrored, indexed: result.filesIndexed, nodes })); +}).finally(() => cg.close()); +`; + const child = runChild(script, [root]); + expect(child.signal, `indexing did not finish within ${DEADLINE_MS}ms: ${child.stderr}`).toBeNull(); + expect(child.status, child.stderr).toBe(0); + + const result = JSON.parse(child.stdout) as { errored: number; indexed: number; nodes: string[] }; + expect(result.errored).toBe(0); + expect(result.indexed).toBe(1); + expect(result.nodes).toContain('module:HANGY'); + expect(result.nodes).toContain('function:MAIN-PARA'); + }, DEADLINE_MS + 10_000); + }); +}); diff --git a/__tests__/ui-highlight-bounded.test.ts b/__tests__/ui-highlight-bounded.test.ts index af0982242..c0b497a5f 100644 --- a/__tests__/ui-highlight-bounded.test.ts +++ b/__tests__/ui-highlight-bounded.test.ts @@ -1,28 +1,36 @@ /** * A grammar that never finishes a slice must not take the viewer's server - * down. The COBOL grammar's scanner loops forever on a line like ` .` — a - * period in the fixed-format sequence area — which cobolcraft's free-format - * paragraphs end with, so a Flow card's window froze the server and every - * request behind it. Slices are classified in a worker the server ends past - * a deadline; the slice is served plain, and the next one gets a new worker. + * down. The COBOL grammar's scanner once looped forever on a slice ending in a + * line like ` .` (fixed in the grammar since), which froze cobolcraft's Flow + * strip and every request behind it: a parse inside WebAssembly cannot be + * interrupted from JavaScript. Slices are classified in a worker the server + * ends past a deadline; the slice is served plain, and the next one gets a + * fresh worker. The deadline is forced here rather than found in a grammar. */ import { describe, it, expect, afterAll } from 'vitest'; import { highlightLines, clearHighlightCache } from '../src/ui-server/highlight'; -import { stopHighlightWorker } from '../src/ui-server/highlight/bounded-tokenize'; +import { stopHighlightWorker, tokenizeBounded } from '../src/ui-server/highlight/bounded-tokenize'; afterAll(() => stopHighlightWorker()); describe('highlighting a slice', () => { - it('serves a slice the grammar never finishes plain, and keeps answering', async () => { - clearHighlightCache(); + it('gives up on a slice past the deadline, and the next slice gets a fresh worker', async () => { + const big = 'export const value: number = 1;\n'.repeat(2000); const started = Date.now(); - const hung = await highlightLines([' PERFORM AssertOk', ' .'], { language: 'cobol' }); - expect(hung.engine).toBe('plain'); - expect(hung.reason).toMatch(/took too long/); - expect(Date.now() - started).toBeLessThan(10_000); + const timedOut = await tokenizeBounded(big, 'typescript', 1); + expect('timedOut' in timedOut).toBe(true); + expect(Date.now() - started).toBeLessThan(5_000); - // The next slice gets a fresh worker and is classified as usual. + const fine = await tokenizeBounded('const y = 2;', 'typescript'); + expect('result' in fine && (fine.result?.spans.length ?? 0) > 0).toBe(true); + }, 20_000); + + it('still highlights through the worker', async () => { + clearHighlightCache(); const fine = await highlightLines(['export const x: number = 1;'], { language: 'typescript' }); expect(fine.engine).toBe('tree-sitter'); + // The slice that used to hang the COBOL grammar is classified now. + const cobol = await highlightLines([' PERFORM AssertOk', ' .'], { language: 'cobol' }); + expect(cobol.reason ?? '').not.toMatch(/took too long/); }, 20_000); }); diff --git a/docs/grammars/tree-sitter-cobol.md b/docs/grammars/tree-sitter-cobol.md index 02df0ed87..b2cf6aa97 100644 --- a/docs/grammars/tree-sitter-cobol.md +++ b/docs/grammars/tree-sitter-cobol.md @@ -54,6 +54,15 @@ mainframe and GnuCOBOL code: `SECURITY.` headers. 10. **`CALL ... GIVING`** — upstream *intended* to support it but a misnested `field()` call swallowed the GIVING alternative entirely. +11. **No hang when the input ends inside the sequence area.** The scanner + skips columns 1-6 by advancing until column 7, but `advance()` is a no-op + at end of input, so a file whose last line stops in that area (` .`, + `X\n .`, a sentence closed by a short ` .` line) spun forever inside + the scanner — parse timeouts and a zero-symbol file when indexing. The + loop now also stops at `eof()`. The loop is upstream code (present at + `e99dbdc3`), so this is a standalone upstream fix; inputs that parsed + before are unaffected (the NIST + ocesql test sources, 412 files, give + byte-identical trees before and after). ## Measured parse health (at vendoring time) @@ -100,7 +109,9 @@ Sent as [yutaro-sakamoto/tree-sitter-cobol#41](https://github.com/yutaro-sakamot (branch `real-world-cobol-sources` on the colbymchenry fork). If upstream merges it, the vendored wasm can track upstream releases instead of this patch. Until then, `git apply tree-sitter-cobol.patch` on upstream commit -`e99dbdc3` reproduces the fork exactly. The PR body as sent: +`e99dbdc3` reproduces the vendored grammar exactly. Item 11 (the +end-of-input hang) came after the PR was sent and is not on that branch +yet. The PR body as sent: > **Parse real-world CICS/DB2 and GnuCOBOL sources** > diff --git a/docs/grammars/tree-sitter-cobol.patch b/docs/grammars/tree-sitter-cobol.patch index 72191bd14..15540a0c5 100644 --- a/docs/grammars/tree-sitter-cobol.patch +++ b/docs/grammars/tree-sitter-cobol.patch @@ -303,7 +303,7 @@ index 6d48b47..e700a18 100644 _NOT_GREATER: $ => /([nN][oO][tT][ \t]+(>|[gG][rR][eE][aA][tT][eE][rR]))/, diff --git a/src/scanner.c b/src/scanner.c -index 6c1ae90..b69cf53 100644 +index 6c1ae90..3382988 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -1,4 +1,5 @@ @@ -372,14 +372,18 @@ index 6c1ae90..b69cf53 100644 if(valid_symbols[WHITE_SPACES]) { if(is_white_space(lexer->lookahead)) { while(is_white_space(lexer->lookahead)) { -@@ -106,9 +121,20 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -106,9 +121,23 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[LINE_PREFIX_COMMENT] && lexer->get_column(lexer) <= 5) { +- while(lexer->get_column(lexer) <= 5) { + // The sequence area is ignored content — but the free-format + // preprocessor plants the CGWIDE sentinel here on the first line. ++ // Stop at end of input: advance() is a no-op there, so a file that ++ // ends inside the sequence area (e.g. a last line " .") would ++ // never reach column 6 and the loop would spin forever. + int match = 0; - while(lexer->get_column(lexer) <= 5) { ++ while(lexer->get_column(lexer) <= 5 && !lexer->eof(lexer)) { + if(match >= 0 && match < 6 && lexer->lookahead == CG_WIDE_SENTINEL[match]) { + match++; + } else { @@ -393,7 +397,7 @@ index 6c1ae90..b69cf53 100644 lexer->result_symbol = LINE_PREFIX_COMMENT; lexer->mark_end(lexer); return true; -@@ -132,7 +158,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -132,7 +161,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[LINE_SUFFIX_COMMENT]) { @@ -402,7 +406,7 @@ index 6c1ae90..b69cf53 100644 while(lexer->lookahead != '\n' && lexer->lookahead != 0) { lexer->advance(lexer, true); } -@@ -143,7 +169,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -143,7 +172,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[COMMENT_ENTRY]) { @@ -411,7 +415,7 @@ index 6c1ae90..b69cf53 100644 lexer->mark_end(lexer); lexer->result_symbol = COMMENT_ENTRY; return true; -@@ -152,18 +178,66 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -152,18 +181,66 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } } @@ -482,7 +486,7 @@ index 6c1ae90..b69cf53 100644 lexer->mark_end(lexer); return true; } -@@ -187,7 +261,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -187,7 +264,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } lexer->advance(lexer, true); @@ -491,7 +495,7 @@ index 6c1ae90..b69cf53 100644 lexer->advance(lexer, true); } } -@@ -197,11 +271,19 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, +@@ -197,11 +274,19 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } unsigned tree_sitter_COBOL_external_scanner_serialize(void *payload, char *buffer) { diff --git a/src/extraction/wasm/tree-sitter-cobol.wasm b/src/extraction/wasm/tree-sitter-cobol.wasm index 12496a95a..c96722eb9 100755 Binary files a/src/extraction/wasm/tree-sitter-cobol.wasm and b/src/extraction/wasm/tree-sitter-cobol.wasm differ