From 1dbb3d90f1e3aa85f0078eb804485c81490bd03a Mon Sep 17 00:00:00 2001 From: danusha2345 Date: Tue, 22 Sep 2026 18:01:22 +0300 Subject: [PATCH] fix(extraction): an MPEG-TS video named .ts is not TypeScript (#1910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `.ts` mapped to TypeScript by extension alone, so MPEG transport-stream fixtures (`testdata/*.ts`) were parsed by tree-sitter: ~28 s of CPU for a 900 KB clip, minutes for a directory of them, for zero symbols. Recognise the stream from the head of the file — the 0x47 sync byte at offsets 0, 188, 376 and 564 plus a NUL byte, which every stream carries in its first packets and UTF-8 source never does — and drop it at discovery: not indexed, not parsed, not counted, not tallied as an unsupported language. The check reads under 1 KB and only for `.ts` files; the batch reader sniffs the bytes it already read, and the single-file path (sync, watcher) does the same head read, so a clip handed in by name is skipped too. The skip happens before parse dispatch, so no kernel mirror is needed. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 2 + __tests__/mpeg-ts-not-typescript.test.ts | 156 +++++++++++++++++++++++ src/extraction/grammars.ts | 47 +++++++ src/extraction/index.ts | 58 ++++++++- 4 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 __tests__/mpeg-ts-not-typescript.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 01aed81c9..7f2f2965f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -178,6 +178,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Interrupted extension installs and changes recover their last consistent configuration and graph, while conflicting user edits produce repair instructions instead of being overwritten. - An empty or table-less `codegraph.db` left behind in a parent directory (by an interrupted `codegraph init`, or a never-populated `~/.codegraph/`) no longer counts as an initialized project, so it can no longer hide the real index of every project beneath it. Only a database that actually carries the codegraph schema is treated as initialized, and `codegraph init` in the directory with the broken file now repairs it instead of refusing with "Already initialized". (#1895) +- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped as non-source instead of being fed to the TypeScript parser — a 900 KB clip used to cost about 28 seconds of CPU per file for no symbols, and a folder of them minutes. Real TypeScript is never affected. (#1910) + - Rust calls on `self` now stay with the enclosing type instead of linking to an unrelated type’s same-named method. Thanks @L4XB. (#1861) - Turning telemetry off now resets its identity and stops running processes from recording, sending, or restoring unsent data. (#1869) diff --git a/__tests__/mpeg-ts-not-typescript.test.ts b/__tests__/mpeg-ts-not-typescript.test.ts new file mode 100644 index 000000000..2a6e12870 --- /dev/null +++ b/__tests__/mpeg-ts-not-typescript.test.ts @@ -0,0 +1,156 @@ +/** + * An MPEG transport stream named `.ts` is not TypeScript (#1910). + * + * Golden video fixtures (`testdata/*.ts`) share TypeScript's extension; fed to + * the tree-sitter TypeScript parser a 900 KB clip costs ~28 s of CPU for zero + * symbols. The fix recognises the stream from the head of the file (0x47 sync + * byte at each 188-byte packet boundary, plus a NUL byte no UTF-8 source has) + * and drops it at discovery — not indexed, not parsed, not counted, not + * reported as an unsupported language. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { CodeGraph } from '../src'; +import { scanDirectoryAsync, type ScanSkipStats } from '../src/extraction'; +import { detectLanguage, isMpegTransportStream, MPEG_TS_SNIFF_BYTES } from '../src/extraction/grammars'; + +const PACKET = 188; + +/** A synthetic transport stream: `packets` × 188 bytes, 0x47 then pseudo-random payload. */ +function makeMpegTs(packets: number, seed = 1): Buffer { + const buf = Buffer.alloc(packets * PACKET); + let x = seed >>> 0; + for (let i = 0; i < buf.length; i++) { + x = (x * 1664525 + 1013904223) >>> 0; + buf[i] = i % PACKET === 0 ? 0x47 : x >>> 24; + } + // Every real stream opens with PSI tables whose pointer field is 0x00. + buf[4] = 0; + return buf; +} + +/** + * Real TypeScript engineered to put the letter `G` (0x47) at offsets 0, 188, + * 376 and 564 — the sync-byte pattern alone. It must stay TypeScript. + */ +function makeGammaSource(): string { + const lines: string[] = []; + let text = ''; + for (let i = 0; i < 4; i++) { + const line = `Gamma${i}();`; + const pad = PACKET - line.length - 1; + text += line + ' '.repeat(pad) + '\n'; + lines.push(line); + } + text += 'export function Gamma0() { return 0; }\n'; + text += 'export function Gamma1() { return 1; }\n'; + text += 'export function Gamma2() { return 2; }\n'; + text += 'export function Gamma3() { return 3; }\n'; + for (const off of [0, PACKET, 2 * PACKET, 3 * PACKET]) { + if (text.charCodeAt(off) !== 0x47) throw new Error(`fixture: expected G at ${off}`); + } + void lines; + return text; +} + +const tempDirs: string[] = []; +function createProject(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-')); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const d of tempDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); +}); + +describe('isMpegTransportStream', () => { + it('recognises a transport stream from its head', () => { + const ts = makeMpegTs(40); + expect(isMpegTransportStream(ts.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true); + expect(isMpegTransportStream(ts)).toBe(true); + }); + + it('needs four aligned sync bytes — a head too short, or one packet off, is not video', () => { + const ts = makeMpegTs(40); + expect(isMpegTransportStream(ts.subarray(0, 3 * PACKET))).toBe(false); + const broken = Buffer.from(ts); + broken[2 * PACKET] = 0x48; + expect(isMpegTransportStream(broken)).toBe(false); + expect(isMpegTransportStream(Buffer.alloc(0))).toBe(false); + }); + + it('does not take source text with G at every 188th byte for video', () => { + const bytes = Buffer.from(makeGammaSource(), 'utf-8'); + expect(bytes[0]).toBe(0x47); + expect(bytes[3 * PACKET]).toBe(0x47); + expect(isMpegTransportStream(bytes)).toBe(false); + expect(detectLanguage('gamma.ts', makeGammaSource())).toBe('typescript'); + }); +}); + +describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910)', () => { + it('drops the clip at discovery: no file record, no nodes, no unsupported-language report', async () => { + const dir = createProject(); + fs.mkdirSync(path.join(dir, 'testdata')); + fs.writeFileSync(path.join(dir, 'testdata', 'clip.ts'), makeMpegTs(40)); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n'); + fs.writeFileSync(path.join(dir, 'gamma.ts'), makeGammaSource()); + + const stats: ScanSkipStats = { unsupportedByExtension: new Map() }; + const scanned = await scanDirectoryAsync(dir, undefined, stats); + expect(scanned.sort()).toEqual(['app.ts', 'gamma.ts']); + expect(stats.unsupportedByExtension.size).toBe(0); + + const cg = await CodeGraph.init(dir, { index: true }); + try { + const files = cg.getFiles().map((f) => f.path).sort(); + expect(files).toEqual(['app.ts', 'gamma.ts']); + expect(cg.searchNodes('greet').some((r) => r.node.name === 'greet')).toBe(true); + expect(cg.searchNodes('Gamma2').some((r) => r.node.name === 'Gamma2')).toBe(true); + + // A named re-sync (the watcher / `sync` path hands files in by name) must + // not let the clip back in either. + await cg.sync({ paths: ['testdata/clip.ts', 'app.ts'] }); + expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma.ts']); + } finally { + await cg.close(); + } + }); + + it('reports nothing skipped for a project of only source and video', async () => { + const dir = createProject(); + fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40)); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n'); + const cg = await CodeGraph.init(dir); + try { + const result = await cg.indexAll(); + expect(result.filesSkippedUnsupported).toBeUndefined(); + expect(result.topUnsupportedExtensions).toBeUndefined(); + expect(result.errors.filter((e) => e.filePath === 'clip.ts')).toEqual([]); + expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']); + } finally { + await cg.close(); + } + }); + + it('indexes past a 900 KB clip in well under two seconds', async () => { + const dir = createProject(); + fs.mkdirSync(path.join(dir, 'testdata')); + fs.writeFileSync(path.join(dir, 'testdata', 'golden.ts'), makeMpegTs(Math.ceil((900 * 1024) / PACKET))); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n'); + const cg = await CodeGraph.init(dir); + try { + const t0 = Date.now(); + const result = await cg.indexAll(); + const elapsed = Date.now() - t0; + expect(result.filesIndexed).toBe(1); + expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']); + expect(elapsed).toBeLessThan(2000); + } finally { + await cg.close(); + } + }); +}); diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index c7710f200..d75180e4e 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -172,6 +172,53 @@ export const EXTENSION_MAP: Record = { '.tofu': 'terraform', }; +/** MPEG transport stream: fixed 188-byte packets, each opening with 0x47. */ +const MPEG_TS_PACKET_SIZE = 188; +const MPEG_TS_SYNC_BYTE = 0x47; +/** Consecutive packets whose sync byte must line up before a file counts as video. */ +const MPEG_TS_MIN_PACKETS = 4; +/** + * How many bytes of a file's head `isMpegTransportStream` needs — enough to + * see `MPEG_TS_MIN_PACKETS` sync bytes plus the packets between them. + */ +export const MPEG_TS_SNIFF_BYTES = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS; + +/** + * Whether these leading bytes are an MPEG transport stream — the OTHER thing a + * `.ts` file can be. Golden video fixtures (`testdata/*.ts`, e2e clips) share + * TypeScript's extension, and tree-sitter takes ~28 s to chew through a 900 KB + * clip for zero symbols (#1910), so the decision has to be made from the head + * of the file, before any parse. + * + * Two conditions, both required: + * 1. the sync byte 0x47 sits at offsets 0, 188, 376 and 564 — every packet + * of a transport stream opens with it, and nothing else pads to 188; + * 2. a NUL byte appears somewhere in the head — every stream carries one + * within its first packets (the PSI pointer field, table reserved bits, + * the `00 00 01` PES start codes), and UTF-8 source text never does. + * 0x47 is the letter `G`, so (1) alone could in principle match a source file + * whose lines happen to put a `G` at four 188-byte strides; (2) closes that + * door, because a text file with a NUL in it is not TypeScript either. + * + * `head` is the first `MPEG_TS_SNIFF_BYTES` (or fewer) bytes of the file. + */ +export function isMpegTransportStream(head: Uint8Array): boolean { + const lastSync = MPEG_TS_PACKET_SIZE * (MPEG_TS_MIN_PACKETS - 1); + if (head.length <= lastSync) return false; + for (let off = 0; off <= lastSync; off += MPEG_TS_PACKET_SIZE) { + if (head[off] !== MPEG_TS_SYNC_BYTE) return false; + } + for (let i = 0; i < head.length; i++) { + if (head[i] === 0) return true; + } + return false; +} + +/** Whether `filePath` carries the one extension MPEG-TS shares with a language. */ +export function hasMpegTsExtension(filePath: string): boolean { + return filePath.length > 3 && filePath.slice(-3).toLowerCase() === '.ts'; +} + /** * Whether a file is one CodeGraph can parse, based purely on its extension. * This is the single source of truth for "should we index this file" — derived diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 2c27c758e..403f426d4 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -27,7 +27,7 @@ import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './ import { StoreWriter, StoreBundle, finalizeStoreBundle } from './store-writer'; import { materializeKernelResult } from './kernel'; import { detectGeneratedFile } from './generated-detection'; -import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars'; +import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes, isMpegTransportStream, hasMpegTsExtension, MPEG_TS_SNIFF_BYTES } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns, PROJECT_CONFIG_FILENAME } from '../project-config'; import { isCodeGraphDataDir } from '../directory'; import { logDebug, logWarn } from '../errors'; @@ -560,6 +560,7 @@ function collectIncludedFiles( if (!include.ignores(rel)) return; if (exclude && exclude.ignores(rel)) return; if (!isSourceFile(rel, overrides)) return; + if (isMpegTsVideoFile(rootDir, rel)) return; out.add(rel); } }; @@ -1498,6 +1499,7 @@ export function scanDirectory( let count = 0; for (const filePath of gitFiles) { if (isSourceFile(filePath, overrides)) { + if (isMpegTsVideoFile(rootDir, filePath)) continue; files.push(filePath); count++; onProgress?.(count, filePath); @@ -1514,6 +1516,31 @@ export function scanDirectory( * Async variant of scanDirectory that yields to the event loop periodically, * allowing worker threads to receive and render progress messages. */ +/** + * Whether a `.ts` file on disk is an MPEG transport stream rather than + * TypeScript (#1910). Reads only the file's head — `MPEG_TS_SNIFF_BYTES`, under + * 1 KB — so the check costs one small read per `.ts` file at discovery, never a + * whole-file read; any file the extension does not make ambiguous is not + * touched at all. A file that cannot be read is left to the indexing path, + * which reports the read error itself. + */ +function isMpegTsVideoFile(rootDir: string, relativePath: string): boolean { + if (!hasMpegTsExtension(relativePath)) return false; + let fd: number | null = null; + try { + fd = fs.openSync(path.join(rootDir, relativePath), 'r'); + const head = Buffer.allocUnsafe(MPEG_TS_SNIFF_BYTES); + const n = fs.readSync(fd, head, 0, head.length, 0); + if (!isMpegTransportStream(head.subarray(0, n))) return false; + } catch { + return false; + } finally { + if (fd !== null) fs.closeSync(fd); + } + logDebug('Skipping MPEG transport stream named .ts — not TypeScript', { filePath: relativePath }); + return true; +} + /** * What a scan saw but could not index, tallied by extension. * @@ -1548,6 +1575,7 @@ export async function scanDirectoryAsync( let count = 0; for (const filePath of gitFiles) { if (isSourceFile(filePath, overrides)) { + if (isMpegTsVideoFile(rootDir, filePath)) continue; files.push(filePath); count++; onProgress?.(count, filePath); @@ -1656,6 +1684,7 @@ function scanDirectoryWalk( } else if (stat.isFile()) { if (!isIgnored(fullPath, false, active)) { if (isSourceFile(relativePath, overrides)) { + if (isMpegTsVideoFile(rootDir, relativePath)) continue; files.push(relativePath); count++; onProgress?.(count, relativePath); @@ -1677,6 +1706,7 @@ function scanDirectoryWalk( } else if (entry.isFile()) { if (!isIgnored(fullPath, false, active)) { if (isSourceFile(relativePath, overrides)) { + if (isMpegTsVideoFile(rootDir, relativePath)) continue; files.push(relativePath); count++; onProgress?.(count, relativePath); @@ -2270,7 +2300,16 @@ export class ExtractionOrchestrator { logWarn('Path traversal blocked in batch reader', { filePath: fp }); return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: new Error('Path traversal blocked') }; } - const content = await fsp.readFile(fullPath, 'utf-8'); + // Read bytes, not text: a `.ts` that is really an MPEG transport + // stream (#1910) is recognised from its head here, at no extra I/O, + // and never decoded or parsed. The scan already drops these; this + // guards the paths that hand files in by name (sync, watcher). + const bytes = await fsp.readFile(fullPath); + if (hasMpegTsExtension(fp) && isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))) { + logDebug('Skipping MPEG transport stream named .ts — not TypeScript', { filePath: fp }); + return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: null as Error | null, skipped: true }; + } + const content = bytes.toString('utf-8'); const stats = await fsp.stat(fullPath); return { filePath: fp, content, stats, error: null as Error | null }; } catch (err) { @@ -2281,9 +2320,16 @@ export class ExtractionOrchestrator { // Dispatch each readable file into the bounded parse window; the window // stores results on the main thread as they arrive. - for (const { filePath, content, stats, error } of fileContents) { + for (const { filePath, content, stats, error, skipped } of fileContents) { if (signal?.aborted) { aborted = true; break; } + if (skipped) { + // Not a source file after all — counted as done, stored as nothing. + processed++; + onProgress?.({ phase: 'parsing', current: processed, total }); + continue; + } + if (error || content === null || stats === null) { processed++; filesErrored++; @@ -2635,6 +2681,12 @@ export class ExtractionOrchestrator { }; } + // An MPEG transport stream named `.ts` is not TypeScript (#1910): one + // sub-KB head read decides it, before the language lookup and the parse. + if (isMpegTsVideoFile(this.rootDir, relativePath)) { + return { nodes: [], edges: [], unresolvedReferences: [], errors: [], durationMs: 0 }; + } + const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir)); // Check file size