fix(extraction): an MPEG-TS video named .ts is not TypeScript (#1910)

`.ts` mapped to TypeScript by extension alone, so MPEG transport-stream
fixtures (`testdata/*.ts`) were parsed by tree-sitter: ~28 s of CPU for a
900 KB clip, minutes for a directory of them, for zero symbols.

Recognise the stream from the head of the file — the 0x47 sync byte at
offsets 0, 188, 376 and 564 plus a NUL byte, which every stream carries in
its first packets and UTF-8 source never does — and drop it at discovery:
not indexed, not parsed, not counted, not tallied as an unsupported
language. The check reads under 1 KB and only for `.ts` files; the batch
reader sniffs the bytes it already read, and the single-file path (sync,
watcher) does the same head read, so a clip handed in by name is skipped
too. The skip happens before parse dispatch, so no kernel mirror is needed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

CHANGELOG.md

@@ -178,6 +178,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Interrupted extension installs and changes recover their last consistent configuration and graph, while conflicting user edits produce repair instructions instead of being overwritten.
 
 - An empty or table-less `codegraph.db` left behind in a parent directory (by an interrupted `codegraph init`, or a never-populated `~/.codegraph/`) no longer counts as an initialized project, so it can no longer hide the real index of every project beneath it. Only a database that actually carries the codegraph schema is treated as initialized, and `codegraph init` in the directory with the broken file now repairs it instead of refusing with "Already initialized". (#1895)
+- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped as non-source instead of being fed to the TypeScript parser — a 900 KB clip used to cost about 28 seconds of CPU per file for no symbols, and a folder of them minutes. Real TypeScript is never affected. (#1910)
+
 - Rust calls on `self` now stay with the enclosing type instead of linking to an unrelated type’s same-named method. Thanks @L4XB. (#1861)
 
 - Turning telemetry off now resets its identity and stops running processes from recording, sending, or restoring unsent data. (#1869)

__tests__/mpeg-ts-not-typescript.test.ts

@@ -0,0 +1,156 @@
+/**
+ * An MPEG transport stream named `.ts` is not TypeScript (#1910).
+ *
+ * Golden video fixtures (`testdata/*.ts`) share TypeScript's extension; fed to
+ * the tree-sitter TypeScript parser a 900 KB clip costs ~28 s of CPU for zero
+ * symbols. The fix recognises the stream from the head of the file (0x47 sync
+ * byte at each 188-byte packet boundary, plus a NUL byte no UTF-8 source has)
+ * and drops it at discovery — not indexed, not parsed, not counted, not
+ * reported as an unsupported language.
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { CodeGraph } from '../src';
+import { scanDirectoryAsync, type ScanSkipStats } from '../src/extraction';
+import { detectLanguage, isMpegTransportStream, MPEG_TS_SNIFF_BYTES } from '../src/extraction/grammars';
+
+const PACKET = 188;
+
+/** A synthetic transport stream: `packets` × 188 bytes, 0x47 then pseudo-random payload. */
+function makeMpegTs(packets: number, seed = 1): Buffer {
+  const buf = Buffer.alloc(packets * PACKET);
+  let x = seed >>> 0;
+  for (let i = 0; i < buf.length; i++) {
+    x = (x * 1664525 + 1013904223) >>> 0;
+    buf[i] = i % PACKET === 0 ? 0x47 : x >>> 24;
+  }
+  // Every real stream opens with PSI tables whose pointer field is 0x00.
+  buf[4] = 0;
+  return buf;
+}
+
+/**
+ * Real TypeScript engineered to put the letter `G` (0x47) at offsets 0, 188,
+ * 376 and 564 — the sync-byte pattern alone. It must stay TypeScript.
+ */
+function makeGammaSource(): string {
+  const lines: string[] = [];
+  let text = '';
+  for (let i = 0; i < 4; i++) {
+    const line = `Gamma${i}();`;
+    const pad = PACKET - line.length - 1;
+    text += line + ' '.repeat(pad) + '\n';
+    lines.push(line);
+  }
+  text += 'export function Gamma0() { return 0; }\n';
+  text += 'export function Gamma1() { return 1; }\n';
+  text += 'export function Gamma2() { return 2; }\n';
+  text += 'export function Gamma3() { return 3; }\n';
+  for (const off of [0, PACKET, 2 * PACKET, 3 * PACKET]) {
+    if (text.charCodeAt(off) !== 0x47) throw new Error(`fixture: expected G at ${off}`);
+  }
+  void lines;
+  return text;
+}
+
+const tempDirs: string[] = [];
+function createProject(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  for (const d of tempDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true });
+});
+
+describe('isMpegTransportStream', () => {
+  it('recognises a transport stream from its head', () => {
+    const ts = makeMpegTs(40);
+    expect(isMpegTransportStream(ts.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true);
+    expect(isMpegTransportStream(ts)).toBe(true);
+  });
+
+  it('needs four aligned sync bytes — a head too short, or one packet off, is not video', () => {
+    const ts = makeMpegTs(40);
+    expect(isMpegTransportStream(ts.subarray(0, 3 * PACKET))).toBe(false);
+    const broken = Buffer.from(ts);
+    broken[2 * PACKET] = 0x48;
+    expect(isMpegTransportStream(broken)).toBe(false);
+    expect(isMpegTransportStream(Buffer.alloc(0))).toBe(false);
+  });
+
+  it('does not take source text with G at every 188th byte for video', () => {
+    const bytes = Buffer.from(makeGammaSource(), 'utf-8');
+    expect(bytes[0]).toBe(0x47);
+    expect(bytes[3 * PACKET]).toBe(0x47);
+    expect(isMpegTransportStream(bytes)).toBe(false);
+    expect(detectLanguage('gamma.ts', makeGammaSource())).toBe('typescript');
+  });
+});
+
+describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910)', () => {
+  it('drops the clip at discovery: no file record, no nodes, no unsupported-language report', async () => {
+    const dir = createProject();
+    fs.mkdirSync(path.join(dir, 'testdata'));
+    fs.writeFileSync(path.join(dir, 'testdata', 'clip.ts'), makeMpegTs(40));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n');
+    fs.writeFileSync(path.join(dir, 'gamma.ts'), makeGammaSource());
+
+    const stats: ScanSkipStats = { unsupportedByExtension: new Map() };
+    const scanned = await scanDirectoryAsync(dir, undefined, stats);
+    expect(scanned.sort()).toEqual(['app.ts', 'gamma.ts']);
+    expect(stats.unsupportedByExtension.size).toBe(0);
+
+    const cg = await CodeGraph.init(dir, { index: true });
+    try {
+      const files = cg.getFiles().map((f) => f.path).sort();
+      expect(files).toEqual(['app.ts', 'gamma.ts']);
+      expect(cg.searchNodes('greet').some((r) => r.node.name === 'greet')).toBe(true);
+      expect(cg.searchNodes('Gamma2').some((r) => r.node.name === 'Gamma2')).toBe(true);
+
+      // A named re-sync (the watcher / `sync` path hands files in by name) must
+      // not let the clip back in either.
+      await cg.sync({ paths: ['testdata/clip.ts', 'app.ts'] });
+      expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma.ts']);
+    } finally {
+      await cg.close();
+    }
+  });
+
+  it('reports nothing skipped for a project of only source and video', async () => {
+    const dir = createProject();
+    fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n');
+    const cg = await CodeGraph.init(dir);
+    try {
+      const result = await cg.indexAll();
+      expect(result.filesSkippedUnsupported).toBeUndefined();
+      expect(result.topUnsupportedExtensions).toBeUndefined();
+      expect(result.errors.filter((e) => e.filePath === 'clip.ts')).toEqual([]);
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+    } finally {
+      await cg.close();
+    }
+  });
+
+  it('indexes past a 900 KB clip in well under two seconds', async () => {
+    const dir = createProject();
+    fs.mkdirSync(path.join(dir, 'testdata'));
+    fs.writeFileSync(path.join(dir, 'testdata', 'golden.ts'), makeMpegTs(Math.ceil((900 * 1024) / PACKET)));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n');
+    const cg = await CodeGraph.init(dir);
+    try {
+      const t0 = Date.now();
+      const result = await cg.indexAll();
+      const elapsed = Date.now() - t0;
+      expect(result.filesIndexed).toBe(1);
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+      expect(elapsed).toBeLessThan(2000);
+    } finally {
+      await cg.close();
+    }
+  });
+});

src/extraction/grammars.ts

@@ -172,6 +172,53 @@ export const EXTENSION_MAP: Record<string, Language> = {
   '.tofu': 'terraform',
 };
 
+/** MPEG transport stream: fixed 188-byte packets, each opening with 0x47. */
+const MPEG_TS_PACKET_SIZE = 188;
+const MPEG_TS_SYNC_BYTE = 0x47;
+/** Consecutive packets whose sync byte must line up before a file counts as video. */
+const MPEG_TS_MIN_PACKETS = 4;
+/**
+ * How many bytes of a file's head `isMpegTransportStream` needs — enough to
+ * see `MPEG_TS_MIN_PACKETS` sync bytes plus the packets between them.
+ */
+export const MPEG_TS_SNIFF_BYTES = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS;
+
+/**
+ * Whether these leading bytes are an MPEG transport stream — the OTHER thing a
+ * `.ts` file can be. Golden video fixtures (`testdata/*.ts`, e2e clips) share
+ * TypeScript's extension, and tree-sitter takes ~28 s to chew through a 900 KB
+ * clip for zero symbols (#1910), so the decision has to be made from the head
+ * of the file, before any parse.
+ *
+ * Two conditions, both required:
+ *   1. the sync byte 0x47 sits at offsets 0, 188, 376 and 564 — every packet
+ *      of a transport stream opens with it, and nothing else pads to 188;
+ *   2. a NUL byte appears somewhere in the head — every stream carries one
+ *      within its first packets (the PSI pointer field, table reserved bits,
+ *      the `00 00 01` PES start codes), and UTF-8 source text never does.
+ * 0x47 is the letter `G`, so (1) alone could in principle match a source file
+ * whose lines happen to put a `G` at four 188-byte strides; (2) closes that
+ * door, because a text file with a NUL in it is not TypeScript either.
+ *
+ * `head` is the first `MPEG_TS_SNIFF_BYTES` (or fewer) bytes of the file.
+ */
+export function isMpegTransportStream(head: Uint8Array): boolean {
+  const lastSync = MPEG_TS_PACKET_SIZE * (MPEG_TS_MIN_PACKETS - 1);
+  if (head.length <= lastSync) return false;
+  for (let off = 0; off <= lastSync; off += MPEG_TS_PACKET_SIZE) {
+    if (head[off] !== MPEG_TS_SYNC_BYTE) return false;
+  }
+  for (let i = 0; i < head.length; i++) {
+    if (head[i] === 0) return true;
+  }
+  return false;
+}
+
+/** Whether `filePath` carries the one extension MPEG-TS shares with a language. */
+export function hasMpegTsExtension(filePath: string): boolean {
+  return filePath.length > 3 && filePath.slice(-3).toLowerCase() === '.ts';
+}
+
 /**
  * Whether a file is one CodeGraph can parse, based purely on its extension.
  * This is the single source of truth for "should we index this file" — derived

src/extraction/index.ts

@@ -27,7 +27,7 @@ import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './
 import { StoreWriter, StoreBundle, finalizeStoreBundle } from './store-writer';
 import { materializeKernelResult } from './kernel';
 import { detectGeneratedFile } from './generated-detection';
-import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars';
+import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes, isMpegTransportStream, hasMpegTsExtension, MPEG_TS_SNIFF_BYTES } from './grammars';
 import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns, PROJECT_CONFIG_FILENAME } from '../project-config';
 import { isCodeGraphDataDir } from '../directory';
 import { logDebug, logWarn } from '../errors';
@@ -560,6 +560,7 @@ function collectIncludedFiles(
       if (!include.ignores(rel)) return;
       if (exclude && exclude.ignores(rel)) return;
       if (!isSourceFile(rel, overrides)) return;
+      if (isMpegTsVideoFile(rootDir, rel)) return;
       out.add(rel);
     }
   };
@@ -1498,6 +1499,7 @@ export function scanDirectory(
     let count = 0;
     for (const filePath of gitFiles) {
       if (isSourceFile(filePath, overrides)) {
+        if (isMpegTsVideoFile(rootDir, filePath)) continue;
         files.push(filePath);
         count++;
         onProgress?.(count, filePath);
@@ -1514,6 +1516,31 @@ export function scanDirectory(
  * Async variant of scanDirectory that yields to the event loop periodically,
  * allowing worker threads to receive and render progress messages.
  */
+/**
+ * Whether a `.ts` file on disk is an MPEG transport stream rather than
+ * TypeScript (#1910). Reads only the file's head — `MPEG_TS_SNIFF_BYTES`, under
+ * 1 KB — so the check costs one small read per `.ts` file at discovery, never a
+ * whole-file read; any file the extension does not make ambiguous is not
+ * touched at all. A file that cannot be read is left to the indexing path,
+ * which reports the read error itself.
+ */
+function isMpegTsVideoFile(rootDir: string, relativePath: string): boolean {
+  if (!hasMpegTsExtension(relativePath)) return false;
+  let fd: number | null = null;
+  try {
+    fd = fs.openSync(path.join(rootDir, relativePath), 'r');
+    const head = Buffer.allocUnsafe(MPEG_TS_SNIFF_BYTES);
+    const n = fs.readSync(fd, head, 0, head.length, 0);
+    if (!isMpegTransportStream(head.subarray(0, n))) return false;
+  } catch {
+    return false;
+  } finally {
+    if (fd !== null) fs.closeSync(fd);
+  }
+  logDebug('Skipping MPEG transport stream named .ts — not TypeScript', { filePath: relativePath });
+  return true;
+}
+
 /**
  * What a scan saw but could not index, tallied by extension.
  *
@@ -1548,6 +1575,7 @@ export async function scanDirectoryAsync(
     let count = 0;
     for (const filePath of gitFiles) {
       if (isSourceFile(filePath, overrides)) {
+        if (isMpegTsVideoFile(rootDir, filePath)) continue;
         files.push(filePath);
         count++;
         onProgress?.(count, filePath);
@@ -1656,6 +1684,7 @@ function scanDirectoryWalk(
           } else if (stat.isFile()) {
             if (!isIgnored(fullPath, false, active)) {
               if (isSourceFile(relativePath, overrides)) {
+                if (isMpegTsVideoFile(rootDir, relativePath)) continue;
                 files.push(relativePath);
                 count++;
                 onProgress?.(count, relativePath);
@@ -1677,6 +1706,7 @@ function scanDirectoryWalk(
       } else if (entry.isFile()) {
         if (!isIgnored(fullPath, false, active)) {
           if (isSourceFile(relativePath, overrides)) {
+            if (isMpegTsVideoFile(rootDir, relativePath)) continue;
             files.push(relativePath);
             count++;
             onProgress?.(count, relativePath);
@@ -2270,7 +2300,16 @@ export class ExtractionOrchestrator {
               logWarn('Path traversal blocked in batch reader', { filePath: fp });
               return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: new Error('Path traversal blocked') };
             }
-            const content = await fsp.readFile(fullPath, 'utf-8');
+            // Read bytes, not text: a `.ts` that is really an MPEG transport
+            // stream (#1910) is recognised from its head here, at no extra I/O,
+            // and never decoded or parsed. The scan already drops these; this
+            // guards the paths that hand files in by name (sync, watcher).
+            const bytes = await fsp.readFile(fullPath);
+            if (hasMpegTsExtension(fp) && isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))) {
+              logDebug('Skipping MPEG transport stream named .ts — not TypeScript', { filePath: fp });
+              return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: null as Error | null, skipped: true };
+            }
+            const content = bytes.toString('utf-8');
             const stats = await fsp.stat(fullPath);
             return { filePath: fp, content, stats, error: null as Error | null };
           } catch (err) {
@@ -2281,9 +2320,16 @@ export class ExtractionOrchestrator {
 
       // Dispatch each readable file into the bounded parse window; the window
       // stores results on the main thread as they arrive.
-      for (const { filePath, content, stats, error } of fileContents) {
+      for (const { filePath, content, stats, error, skipped } of fileContents) {
         if (signal?.aborted) { aborted = true; break; }
 
+        if (skipped) {
+          // Not a source file after all — counted as done, stored as nothing.
+          processed++;
+          onProgress?.({ phase: 'parsing', current: processed, total });
+          continue;
+        }
+
         if (error || content === null || stats === null) {
           processed++;
           filesErrored++;
@@ -2635,6 +2681,12 @@ export class ExtractionOrchestrator {
       };
     }
 
+    // An MPEG transport stream named `.ts` is not TypeScript (#1910): one
+    // sub-KB head read decides it, before the language lookup and the parse.
+    if (isMpegTsVideoFile(this.rootDir, relativePath)) {
+      return { nodes: [], edges: [], unresolvedReferences: [], errors: [], durationMs: 0 };
+    }
+
     const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir));
 
     // Check file size