fix(cobol): a file ending inside the sequence area no longer hangs the parser (#2272)

* fix(cobol): a file ending inside the sequence area no longer hangs the parser

The vendored COBOL grammar's external scanner skips the fixed-format
sequence area (columns 1-6) with `while (get_column() <= 5) advance()`.
At end of input advance() is a no-op, so a file whose last line stops in
that area (`    .`, `X\n    .`, a sentence closed by a short `    .` line)
never reached column 7 and the scanner spun forever inside wasm. Indexing
hit the parse worker's hard timeout three times (~93 s) and stored the
file with zero symbols; the viewer's highlighter froze on the same text.

The loop now also stops at eof(). It is upstream code (present at
e99dbdc3), so the change is a standalone upstreamable fix; inputs that
parsed before are unaffected (412 NIST + ocesql test sources give
byte-identical trees with the old and new wasm, and the upstream corpus
keeps its single pre-existing failure).

Regenerates docs/grammars/tree-sitter-cobol.patch, rebuilds the wasm per
docs/grammars/tree-sitter-cobol.md, and adds an out-of-process regression
test under a hard deadline (a regression is an uninterruptible wasm loop).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(viewer): force the highlight deadline instead of relying on a grammar bug

The bounded-highlight test used the COBOL `    .` slice as its slice that
never finishes; the grammar fix makes it parse, so the deadline is now
forced through tokenizeBounded's deadline parameter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- A COBOL file whose last line ends inside the sequence area (columns 1-6), such as a closing period indented only four spaces, no longer stalls indexing for over a minute and then ends up in the index with no symbols.
 - In Laravel apps, a route whose controller is named by string now links to its handler, without needing a `Controller` suffix and keeping the namespace it's written with: `'Common\Uploads@inline'`, and `Route::resource('companies', 'Common\Companies', [...])` with an options array. Two same-named controllers in different namespaces are told apart by that namespace. Before, akaunting linked 304 of its 313 routes to nothing.
 - In Nuxt and Astro projects, a file-routed page is now linked to the component its file defines. An Astro endpoint is linked to the HTTP-verb handlers it exports, like `export const GET`. Before, a page route like Nuxt's `pages/admin.vue` or Astro's `src/pages/about.astro` linked to nothing, so callers, impact and flows stopped at the route.
 - In Rails apps, a `resources` line's `only:` and `except:` are now read in every form Rails accepts: `%i[new create index]`, `%w(index show)`, a quoted name, and the older `:only => [...]`. Before, only a bracketed list was read, so maybe's `resources :family_exports, only: %i[new create index]` became all seven routes, four of them to actions the controller doesn't have.

__tests__/cobol-sequence-area-period.test.ts

@@ -0,0 +1,119 @@
+/**
+ * COBOL: a file that ends inside the sequence area must parse, not hang.
+ *
+ * The vendored COBOL grammar's external scanner skips the fixed-format
+ * sequence area (columns 1-6) by advancing until the lexer reaches column 7.
+ * At end of input `advance()` is a no-op, so a file whose last line stops
+ * inside that area — `    .` (four spaces and a period: the period lands in
+ * column 5), `X\n    .`, a paragraph closed by a short `    .` line — never
+ * reached column 7 and the scanner spun forever inside WebAssembly. Indexing
+ * such a file ran into the parse worker's timeout (about a minute and a half
+ * with the retries) and stored it with zero symbols; the viewer's highlighter
+ * froze on the same text. The scanner now stops at end of input.
+ *
+ * Both checks run against the built engine in a child process under a hard
+ * timeout: a regression is an infinite loop inside wasm that no in-process
+ * timer can interrupt, so parsing in this worker would hang the suite instead
+ * of failing the test.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { WASM_RUNTIME_FLAGS } from '../src/extraction/wasm-runtime-flags';
+
+const DIST = path.resolve(__dirname, '../dist');
+
+/** A fixed-format program whose last line is a lone period in column 5. */
+const HANGY = [
+  '       IDENTIFICATION DIVISION.',
+  '       PROGRAM-ID. HANGY.',
+  '       PROCEDURE DIVISION.',
+  '       MAIN-PARA.',
+  '           DISPLAY "X"',
+  '    .',
+  '',
+].join('\n');
+
+/** Under the parse worker's 30 s hard per-file timeout a hang would run into. */
+const DEADLINE_MS = 20_000;
+
+function runChild(script: string, args: string[]) {
+  return spawnSync(process.execPath, [...WASM_RUNTIME_FLAGS, '-e', script, DIST, ...args], {
+    encoding: 'utf8',
+    timeout: DEADLINE_MS,
+    env: { ...process.env, CODEGRAPH_TELEMETRY: '0' },
+  });
+}
+
+describe('COBOL sequence-area period at end of file', () => {
+  it('extracts every source that ends inside the sequence area', () => {
+    const sources: Record<string, string> = {
+      'hangy.cob': HANGY,
+      'hangy-no-newline.cob': HANGY.trimEnd(),
+      'period-only.cob': '    .',
+      'after-statement.cob': '    PERFORM AssertOk\n    .',
+      'after-word.cob': 'X\n    .',
+    };
+    const script = `
+const path = require('path');
+const { initGrammars, loadGrammarsForLanguages } = require(path.join(process.argv[1], 'extraction/grammars.js'));
+const { extractFromSource } = require(path.join(process.argv[1], 'extraction/index.js'));
+(async () => {
+  await initGrammars();
+  await loadGrammarsForLanguages(['cobol']);
+  const out = {};
+  for (const [file, source] of Object.entries(JSON.parse(process.argv[2]))) {
+    const result = extractFromSource(file, source);
+    out[file] = result.nodes.map((n) => n.kind + ':' + n.name);
+  }
+  process.stdout.write(JSON.stringify(out));
+})();
+`;
+    const child = runChild(script, [JSON.stringify(sources)]);
+    expect(child.signal, `COBOL parse did not finish within ${DEADLINE_MS}ms: ${child.stderr}`).toBeNull();
+    expect(child.status, child.stderr).toBe(0);
+
+    const nodes = JSON.parse(child.stdout) as Record<string, string[]>;
+    expect(Object.keys(nodes).sort()).toEqual(Object.keys(sources).sort());
+    for (const file of ['hangy.cob', 'hangy-no-newline.cob']) {
+      expect(nodes[file]).toContain('module:HANGY');
+      expect(nodes[file]).toContain('function:MAIN-PARA');
+    }
+  }, DEADLINE_MS + 10_000);
+
+  describe('indexing a project', () => {
+    let root: string;
+
+    beforeEach(() => {
+      root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cobol-seq-period-'));
+    });
+
+    afterEach(() => {
+      fs.rmSync(root, { recursive: true, force: true });
+    });
+
+    it('indexes the file with its symbols instead of timing it out', () => {
+      fs.writeFileSync(path.join(root, 'hangy.cob'), HANGY);
+      const script = `
+const path = require('path');
+const { default: CodeGraph } = require(path.join(process.argv[1], 'index.js'));
+const cg = CodeGraph.initSync(process.argv[2]);
+cg.indexAll().then((result) => {
+  const nodes = cg.getNodesInFile('hangy.cob').map((n) => n.kind + ':' + n.name);
+  process.stdout.write(JSON.stringify({ errored: result.filesErrored, indexed: result.filesIndexed, nodes }));
+}).finally(() => cg.close());
+`;
+      const child = runChild(script, [root]);
+      expect(child.signal, `indexing did not finish within ${DEADLINE_MS}ms: ${child.stderr}`).toBeNull();
+      expect(child.status, child.stderr).toBe(0);
+
+      const result = JSON.parse(child.stdout) as { errored: number; indexed: number; nodes: string[] };
+      expect(result.errored).toBe(0);
+      expect(result.indexed).toBe(1);
+      expect(result.nodes).toContain('module:HANGY');
+      expect(result.nodes).toContain('function:MAIN-PARA');
+    }, DEADLINE_MS + 10_000);
+  });
+});

__tests__/ui-highlight-bounded.test.ts

@@ -1,28 +1,36 @@
 /**
  * A grammar that never finishes a slice must not take the viewer's server
- * down. The COBOL grammar's scanner loops forever on a line like `    .` — a
- * period in the fixed-format sequence area — which cobolcraft's free-format
- * paragraphs end with, so a Flow card's window froze the server and every
- * request behind it. Slices are classified in a worker the server ends past
- * a deadline; the slice is served plain, and the next one gets a new worker.
+ * down. The COBOL grammar's scanner once looped forever on a slice ending in a
+ * line like `    .` (fixed in the grammar since), which froze cobolcraft's Flow
+ * strip and every request behind it: a parse inside WebAssembly cannot be
+ * interrupted from JavaScript. Slices are classified in a worker the server
+ * ends past a deadline; the slice is served plain, and the next one gets a
+ * fresh worker. The deadline is forced here rather than found in a grammar.
  */
 import { describe, it, expect, afterAll } from 'vitest';
 import { highlightLines, clearHighlightCache } from '../src/ui-server/highlight';
-import { stopHighlightWorker } from '../src/ui-server/highlight/bounded-tokenize';
+import { stopHighlightWorker, tokenizeBounded } from '../src/ui-server/highlight/bounded-tokenize';
 
 afterAll(() => stopHighlightWorker());
 
 describe('highlighting a slice', () => {
-  it('serves a slice the grammar never finishes plain, and keeps answering', async () => {
-    clearHighlightCache();
+  it('gives up on a slice past the deadline, and the next slice gets a fresh worker', async () => {
+    const big = 'export const value: number = 1;\n'.repeat(2000);
     const started = Date.now();
-    const hung = await highlightLines(['    PERFORM AssertOk', '    .'], { language: 'cobol' });
-    expect(hung.engine).toBe('plain');
-    expect(hung.reason).toMatch(/took too long/);
-    expect(Date.now() - started).toBeLessThan(10_000);
+    const timedOut = await tokenizeBounded(big, 'typescript', 1);
+    expect('timedOut' in timedOut).toBe(true);
+    expect(Date.now() - started).toBeLessThan(5_000);
 
-    // The next slice gets a fresh worker and is classified as usual.
+    const fine = await tokenizeBounded('const y = 2;', 'typescript');
+    expect('result' in fine && (fine.result?.spans.length ?? 0) > 0).toBe(true);
+  }, 20_000);
+
+  it('still highlights through the worker', async () => {
+    clearHighlightCache();
     const fine = await highlightLines(['export const x: number = 1;'], { language: 'typescript' });
     expect(fine.engine).toBe('tree-sitter');
+    // The slice that used to hang the COBOL grammar is classified now.
+    const cobol = await highlightLines(['    PERFORM AssertOk', '    .'], { language: 'cobol' });
+    expect(cobol.reason ?? '').not.toMatch(/took too long/);
   }, 20_000);
 });

docs/grammars/tree-sitter-cobol.md

@@ -54,6 +54,15 @@ mainframe and GnuCOBOL code:
    `SECURITY.` headers.
 10. **`CALL ... GIVING`** — upstream *intended* to support it but a misnested
     `field()` call swallowed the GIVING alternative entirely.
+11. **No hang when the input ends inside the sequence area.** The scanner
+    skips columns 1-6 by advancing until column 7, but `advance()` is a no-op
+    at end of input, so a file whose last line stops in that area (`    .`,
+    `X\n    .`, a sentence closed by a short `    .` line) spun forever inside
+    the scanner — parse timeouts and a zero-symbol file when indexing. The
+    loop now also stops at `eof()`. The loop is upstream code (present at
+    `e99dbdc3`), so this is a standalone upstream fix; inputs that parsed
+    before are unaffected (the NIST + ocesql test sources, 412 files, give
+    byte-identical trees before and after).
 
 ## Measured parse health (at vendoring time)
 
@@ -100,7 +109,9 @@ Sent as [yutaro-sakamoto/tree-sitter-cobol#41](https://github.com/yutaro-sakamot
 (branch `real-world-cobol-sources` on the colbymchenry fork). If upstream
 merges it, the vendored wasm can track upstream releases instead of this
 patch. Until then, `git apply tree-sitter-cobol.patch` on upstream commit
-`e99dbdc3` reproduces the fork exactly. The PR body as sent:
+`e99dbdc3` reproduces the vendored grammar exactly. Item 11 (the
+end-of-input hang) came after the PR was sent and is not on that branch
+yet. The PR body as sent:
 
 > **Parse real-world CICS/DB2 and GnuCOBOL sources**
 >

docs/grammars/tree-sitter-cobol.patch

@@ -303,7 +303,7 @@ index 6d48b47..e700a18 100644
      _NOT_GREATER: $ => /([nN][oO][tT][ \t]+(>|[gG][rR][eE][aA][tT][eE][rR]))/,
  
 diff --git a/src/scanner.c b/src/scanner.c
-index 6c1ae90..b69cf53 100644
+index 6c1ae90..3382988 100644
 --- a/src/scanner.c
 +++ b/src/scanner.c
 @@ -1,4 +1,5 @@
@@ -372,14 +372,18 @@ index 6c1ae90..b69cf53 100644
      if(valid_symbols[WHITE_SPACES]) {
          if(is_white_space(lexer->lookahead)) {
              while(is_white_space(lexer->lookahead)) {
-@@ -106,9 +121,20 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -106,9 +121,23 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
      }
  
      if(valid_symbols[LINE_PREFIX_COMMENT] && lexer->get_column(lexer) <= 5) {
+-        while(lexer->get_column(lexer) <= 5) {
 +        // The sequence area is ignored content — but the free-format
 +        // preprocessor plants the CGWIDE sentinel here on the first line.
++        // Stop at end of input: advance() is a no-op there, so a file that
++        // ends inside the sequence area (e.g. a last line "    .") would
++        // never reach column 6 and the loop would spin forever.
 +        int match = 0;
-         while(lexer->get_column(lexer) <= 5) {
++        while(lexer->get_column(lexer) <= 5 && !lexer->eof(lexer)) {
 +            if(match >= 0 && match < 6 && lexer->lookahead == CG_WIDE_SENTINEL[match]) {
 +                match++;
 +            } else {
@@ -393,7 +397,7 @@ index 6c1ae90..b69cf53 100644
          lexer->result_symbol = LINE_PREFIX_COMMENT;
          lexer->mark_end(lexer);
          return true;
-@@ -132,7 +158,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -132,7 +161,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
      }
  
      if(valid_symbols[LINE_SUFFIX_COMMENT]) {
@@ -402,7 +406,7 @@ index 6c1ae90..b69cf53 100644
              while(lexer->lookahead != '\n' && lexer->lookahead != 0) {
                  lexer->advance(lexer, true);
              }
-@@ -143,7 +169,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -143,7 +172,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
      }
  
      if(valid_symbols[COMMENT_ENTRY]) {
@@ -411,7 +415,7 @@ index 6c1ae90..b69cf53 100644
              lexer->mark_end(lexer);
              lexer->result_symbol = COMMENT_ENTRY;
              return true;
-@@ -152,18 +178,66 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -152,18 +181,66 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
          }
      }
  
@@ -482,7 +486,7 @@ index 6c1ae90..b69cf53 100644
                  lexer->mark_end(lexer);
                  return true;
              }
-@@ -187,7 +261,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -187,7 +264,7 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
              }
  
              lexer->advance(lexer, true);
@@ -491,7 +495,7 @@ index 6c1ae90..b69cf53 100644
                  lexer->advance(lexer, true);
              }
          }
-@@ -197,11 +271,19 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -197,11 +274,19 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer,
  }
  
  unsigned tree_sitter_COBOL_external_scanner_serialize(void *payload, char *buffer) {