diff --git a/__tests__/generated-flag-index.test.ts b/__tests__/generated-flag-index.test.ts index 35898e3a1..2edc9e766 100644 --- a/__tests__/generated-flag-index.test.ts +++ b/__tests__/generated-flag-index.test.ts @@ -124,27 +124,25 @@ describe('generated flag — schema migration to v9', () => { if (dir) fs.rmSync(dir, { recursive: true, force: true }); }); - /** A pre-v9 `files` table: no `generated` column, no partial index. */ + /** A pre-v9 database: no `generated` column or partial index. */ function makeLegacyDb(): SqliteDatabase { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-genmigrate-')); const conn = createDatabase(path.join(dir, 'legacy.db')).db; + db = conn; + // Later migrations also run, so retain the rest of the real graph schema. + conn.exec(fs.readFileSync(path.join(__dirname, '../src/db/schema.sql'), 'utf8')); conn.exec(` - CREATE TABLE schema_versions (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL, description TEXT); + DELETE FROM schema_versions; INSERT INTO schema_versions VALUES (8, 0, 'legacy'); - CREATE TABLE files ( - path TEXT PRIMARY KEY, - content_hash TEXT NOT NULL, - language TEXT NOT NULL, - size INTEGER NOT NULL, - modified_at INTEGER NOT NULL, - indexed_at INTEGER NOT NULL, - node_count INTEGER DEFAULT 0, - errors TEXT - ); + DROP INDEX idx_files_generated; + ALTER TABLE files DROP COLUMN generated; + DROP TABLE synthesis_inputs; + DROP INDEX idx_edges_synthesis_site; + DROP INDEX idx_nodes_kind; + CREATE INDEX idx_nodes_kind ON nodes(kind); INSERT INTO files VALUES ('x/bank/types/tx.pb.go', 'h1', 'go', 10, 0, 0, 1, NULL); INSERT INTO files VALUES ('internal/payroll/payroll.go', 'h2', 'go', 10, 0, 0, 1, NULL); `); - db = conn; return conn; } diff --git a/__tests__/sync-rebuild-convergence.test.ts b/__tests__/sync-rebuild-convergence.test.ts index 688e5e37c..36967d6b8 100644 --- a/__tests__/sync-rebuild-convergence.test.ts +++ b/__tests__/sync-rebuild-convergence.test.ts @@ -41,6 +41,8 @@ import * as path from 'path'; import * as os from 'os'; import CodeGraph from '../src/index'; import { createDatabase } from '../src/db/sqlite-adapter'; +import { QueryBuilder } from '../src/db/queries'; +import { ToolHandler } from '../src/mcp/tools'; describe('Incremental sync converges to a full rebuild (CG-33)', () => { let testDir: string; @@ -518,3 +520,291 @@ describe('Same-name candidate order is content-derived, not insertion-derived (C expect(keys[0]).toContain('src/alpha.ts'); }); }); + +/** Synthesis owns a whole-graph result, including registrations in third files. */ +describe('Synthesized edges converge after sync (#1988)', () => { + let dir: string; + let cg: CodeGraph; + const write = (file: string, content: string) => fs.writeFileSync(path.join(dir, file), content); + const bus = `import { EventEmitter } from 'events'; +export const bus = new EventEmitter(); +export function fire(): void { bus.emit('ping'); } +export function onPing(): void {} +`; + const wiring = "import { bus, onPing } from './bus';\nbus.on('ping', onPing);\n"; + const readEdges = (synthesized = true) => { + const { db } = createDatabase(path.join(dir, '.codegraph', 'codegraph.db'), { readOnly: true }); + try { + return db.prepare(`SELECT source, target, kind, metadata, line, col, provenance FROM edges + ${synthesized ? "WHERE json_extract(metadata, '$.synthesizedBy') IS NOT NULL" : ''} + ORDER BY source, target, kind, line, col, metadata`).all(); + } finally { db.close(); } + }; + const load = async () => { + cg = CodeGraph.initSync(dir, { config: { exclude: [] } }); + await cg.indexAll(); + }; + const converges = async () => { + const synced = readEdges(); + cg.close(); + cg = await CodeGraph.recreate(dir, { silent: true }); + await cg.indexAll(); + expect(synced).toEqual(readEdges()); + }; + beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-synthesis-')); }); + afterEach(() => { + cg?.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it.each([false, true])('adds and removes registration without changing endpoints (scoped=%s)', async (scoped) => { + write('bus.ts', bus); + write('wiring.ts', "import { bus, onPing } from './bus';\n"); + await load(); + expect(readEdges()).toHaveLength(0); + write('wiring.ts', wiring); + const phases: string[] = []; + await cg.sync({ paths: scoped ? ['wiring.ts'] : undefined, onProgress: p => phases.push(p.phase) }); + expect(readEdges()).toHaveLength(1); + expect(phases.indexOf('linking')).toBeGreaterThan(phases.lastIndexOf('resolving')); + await converges(); + write('wiring.ts', "import { bus, onPing } from './bus';\n"); + await cg.sync({ paths: scoped ? ['wiring.ts'] : undefined }); + expect(readEdges()).toHaveLength(0); + await converges(); + }); + + it('adds, renames and deletes a separate registration file', async () => { + write('bus.ts', bus); + await load(); + write('wiring.ts', wiring); + await cg.sync({ paths: ['wiring.ts'] }); + expect(readEdges()).toHaveLength(1); + await converges(); + fs.renameSync(path.join(dir, 'wiring.ts'), path.join(dir, 'renamed.ts')); + await cg.sync({ paths: ['wiring.ts', 'renamed.ts'] }); + expect(JSON.parse(readEdges()[0].metadata).registeredAt).toBe('renamed.ts:2'); + await converges(); + fs.unlinkSync(path.join(dir, 'renamed.ts')); + await cg.sync({ paths: ['renamed.ts'] }); + expect(readEdges()).toHaveLength(0); + await converges(); + }); + + it('refreshes when a dispatcher loses its emit pattern before its edges cascade', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + await load(); + write('bus.ts', bus.replace("bus.emit('ping');", '')); + await cg.sync({ paths: ['bus.ts'] }); + expect(readEdges()).toHaveLength(0); + await converges(); + }); + + it('skips synthesis for no-op syncs and unrelated ordinary edits', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + write('math.ts', 'export function square(n: number) { return n * n; }\n'); + await load(); + const before = readEdges(); + const phases: string[] = []; + await cg.sync({ onProgress: p => phases.push(p.phase) }); + write('math.ts', 'export function square(n: number) { return n * n + 1; }\n'); + await cg.sync({ paths: ['math.ts'], onProgress: p => phases.push(p.phase) }); + expect(phases).not.toContain('linking'); + expect(readEdges()).toEqual(before); + }); + + it('refreshes all event channels when a registration crosses the global fan-out cap', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + for (let i = 0; i < 5; i++) { + write(`handler${i}.ts`, `import { bus } from './bus';\nfunction handler${i}() {}\nbus.on('ping', handler${i});\n`); + } + await load(); + expect(readEdges()).toHaveLength(6); + write('extra.ts', "import { bus } from './bus';\nfunction extra() {}\nbus.on('ping', extra);\n"); + await cg.sync({ paths: ['extra.ts'] }); + expect(readEdges()).toHaveLength(0); + await converges(); + fs.unlinkSync(path.join(dir, 'extra.ts')); + await cg.sync({ paths: ['extra.ts'] }); + expect(readEdges()).toHaveLength(6); + await converges(); + }); + + it('converges after C table, header and header-deletion edits', async () => { + const header = 'struct ops { int (*fn)(void); };\n'; + write('ops.h', header); + write('handlers.c', 'int first(void) { return 1; }\nint second(void) { return 2; }\n'); + write('table.c', '#include "ops.h"\nextern int first(void);\nextern int second(void);\nstruct ops table = { .fn = first };\n'); + write('dispatch.c', '#include "ops.h"\nint dispatch(struct ops *p) { return p->fn(); }\n'); + await load(); + expect(readEdges().some(e => JSON.parse(e.metadata).synthesizedBy === 'fn-pointer-dispatch')).toBe(true); + write('table.c', '#include "ops.h"\nextern int first(void);\nextern int second(void);\nstruct ops table = { .fn = second };\n'); + await cg.sync({ paths: ['table.c'] }); + await converges(); + write('ops.h', 'struct ops { int fn; };\n'); + await cg.sync({ paths: ['ops.h'] }); + expect(readEdges()).toHaveLength(0); + await converges(); + write('ops.h', header); + await cg.sync({ paths: ['ops.h'] }); + expect(readEdges().length).toBeGreaterThan(0); + await converges(); + fs.unlinkSync(path.join(dir, 'ops.h')); + await cg.sync({ paths: ['ops.h'] }); + expect(readEdges()).toHaveLength(0); + await converges(); + }); + + it('keeps C layout precedence independent of file insertion order', async () => { + write('alpha.c', 'struct ops { int (*first)(void); };\nint a(void) { return 1; }\nstruct ops one = { .first = a };\nint runA(struct ops *p) { return p->first(); }\n'); + write('zeta.c', 'struct ops { int (*second)(void); };\nint z(void) { return 2; }\nstruct ops two = { .second = z };\nint runZ(struct ops *p) { return p->second(); }\n'); + await load(); + fs.appendFileSync(path.join(dir, 'alpha.c'), '// edit earlier file\n'); + await cg.sync({ paths: ['alpha.c'] }); + await converges(); + }); + + it('recovers synthesis after extraction was interrupted before resolution', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + await load(); + fs.appendFileSync(path.join(dir, 'bus.ts'), '// interrupted index\n'); + await cg.indexFiles(['bus.ts']); + expect(cg.getPendingReferenceCount()).toBeGreaterThan(0); + const result = await cg.sync(); + expect(result.filesAdded + result.filesModified + result.filesRemoved).toBe(0); + expect(readEdges()).toHaveLength(1); + await converges(); + }); + + it('keeps old synthesis on pass failure and retries on a no-op sync', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + await load(); + const before = readEdges(); + write('wiring.ts', wiring.replace("'ping'", "'pong'")); + await expect(cg.sync({ paths: ['wiring.ts'], onProgress: p => { + if (p.phase === 'linking' && p.current > 0) throw new Error('interrupted synthesis'); + } })).rejects.toThrow('interrupted synthesis'); + expect(readEdges()).toEqual(before); + await cg.sync({ paths: ['wiring.ts'] }); + expect(readEdges()).toHaveLength(0); + await converges(); + }); + + it('rolls back a failed replacement without removing ordinary or old synthesized edges', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + await load(); + const before = readEdges(); + const ordinary = readEdges(false).filter(e => e.provenance !== 'heuristic'); + const { db } = createDatabase(path.join(dir, '.codegraph', 'codegraph.db')); + try { + db.exec(`CREATE TRIGGER fail_synthesis BEFORE INSERT ON edges + WHEN NEW.provenance = 'heuristic' BEGIN SELECT RAISE(FAIL, 'publish failed'); END`); + write('wiring.ts', wiring + '// changed registration file\n'); + await expect(cg.sync({ paths: ['wiring.ts'] })).rejects.toThrow('publish failed'); + expect(readEdges()).toEqual(before); + expect(readEdges(false).filter(e => e.provenance !== 'heuristic')).toEqual(ordinary); + db.exec('DROP TRIGGER fail_synthesis'); + } finally { db.close(); } + await cg.sync({ paths: ['wiring.ts'] }); + expect(readEdges()).toEqual(before); + await converges(); + }); + + it('migrates an existing index and repairs synthesis without requiring a file edit', async () => { + write('bus.ts', bus); + write('wiring.ts', wiring); + await load(); + cg.close(); + const { db } = createDatabase(path.join(dir, '.codegraph', 'codegraph.db')); + try { + db.exec(`DELETE FROM schema_versions WHERE version >= 10; + INSERT OR IGNORE INTO schema_versions(version, applied_at, description) VALUES (9, 0, 'legacy fixture'); + DROP TABLE synthesis_inputs; + DROP INDEX idx_edges_synthesis_site; + DROP INDEX idx_nodes_kind; + CREATE INDEX idx_nodes_kind ON nodes(kind); + DELETE FROM edges WHERE provenance = 'heuristic'`); + } finally { db.close(); } + cg = CodeGraph.openSync(dir); + await cg.sync(); + expect(readEdges()).toHaveLength(1); + await converges(); + }); + + it('refreshes Go prerequisites before interface dispatch', async () => { + write('api.go', 'package demo\ntype Runner interface { Run() }\ntype Worker struct {}\n'); + await load(); + write('worker.go', 'package demo\nfunc (w Worker) Run() {}\n'); + await cg.sync({ paths: ['worker.go'] }); + expect(readEdges().some(e => JSON.parse(e.metadata).synthesizedBy === 'go-implements')).toBe(true); + await converges(); + write('worker.go', 'package demo\nfunc (w Worker) Other() {}\n'); + await cg.sync({ paths: ['worker.go'] }); + expect(readEdges().some(e => JSON.parse(e.metadata).synthesizedBy === 'go-implements')).toBe(false); + await converges(); + }); + + it('keeps cross-file Go method containment structural after indexing and incremental refresh', async () => { + write('types.go', 'package demo\ntype Worker struct {}\n'); + write('worker.go', 'package demo\nfunc (w Worker) Run() {}\n'); + await load(); + const assertStructural = async () => { + const contains = readEdges().filter(e => JSON.parse(e.metadata).synthesizedBy === 'go-method-contains'); + expect(contains).toHaveLength(1); + expect(contains[0].kind).toBe('contains'); + expect.soft(contains[0].provenance).toBeNull(); + const result = await new ToolHandler(cg).execute('codegraph_explore', { query: 'Worker Run' }); + expect(result.isError).not.toBe(true); + const text = result.content.map(c => c.text ?? '').join('\n'); + expect(text).toContain('func (w Worker) Run()'); + expect(text).not.toMatch(/\[dynamic\b/i); + const { db } = createDatabase(path.join(dir, '.codegraph', 'codegraph.db'), { readOnly: true }); + try { + const queries = new QueryBuilder(db); + expect(queries.hasSynthesizedEdgesTouchingFile('types.go')).toBe(true); + expect(queries.hasSynthesizedEdgesTouchingFile('worker.go')).toBe(true); + } finally { db.close(); } + }; + await assertStructural(); + // Refresh with both endpoints untouched, then with the method re-extracted. + write('unrelated.go', 'package demo\ntype Extra struct {}\n'); + await cg.sync({ paths: ['unrelated.go'] }); + await assertStructural(); + await converges(); + write('worker.go', 'package demo\n\nfunc (w Worker) Run() {}\n'); + await cg.sync({ paths: ['worker.go'] }); + await assertStructural(); + await converges(); + }); + + it('migrates legacy Go containment ownership without changing provenance', async () => { + write('types.go', 'package demo\ntype Worker struct {}\n'); + write('worker.go', 'package demo\nfunc (w Worker) Run() {}\n'); + await load(); + cg.close(); + const { db } = createDatabase(path.join(dir, '.codegraph', 'codegraph.db')); + try { + db.exec(`DELETE FROM schema_versions WHERE version >= 10; + INSERT OR IGNORE INTO schema_versions(version, applied_at, description) VALUES (9, 0, 'legacy fixture'); + DROP TABLE synthesis_inputs; + DROP INDEX idx_edges_synthesis_site; + UPDATE edges SET provenance = NULL, metadata = NULL + WHERE json_extract(metadata, '$.synthesizedBy') = 'go-method-contains'`); + } finally { db.close(); } + cg = CodeGraph.openSync(dir); + // Inspect the migration itself before sync can replace its output. + const migrated = readEdges(); + expect(migrated).toHaveLength(1); + expect(migrated[0].provenance).toBeNull(); + expect(JSON.parse(migrated[0].metadata).synthesizedBy).toBe('go-method-contains'); + await cg.sync(); + expect(readEdges()).toEqual(migrated); + await converges(); + }); +}); diff --git a/src/db/index.ts b/src/db/index.ts index d38310e06..3b53ba3a1 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -361,6 +361,7 @@ export class DatabaseConnection { 'idx_edges_source_kind', 'idx_edges_target_kind', 'idx_edges_provenance', + 'idx_edges_synthesis_site', ] as const; /** diff --git a/src/db/migrations.ts b/src/db/migrations.ts index b6c82a5e5..b3b0a1805 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 9; +export const CURRENT_SCHEMA_VERSION = 10; /** * Migration definition @@ -177,6 +177,30 @@ const migrations: Migration[] = [ ); }, }, + { + version: 10, + description: 'Track synthesis inputs and stabilize synthesis traversal for incremental refresh (#1988)', + up: (db) => { + db.exec(` + DROP INDEX IF EXISTS idx_nodes_kind; + CREATE INDEX idx_nodes_kind ON nodes(kind, file_path, start_line, id); + CREATE TABLE IF NOT EXISTS synthesis_inputs ( + file_path TEXT PRIMARY KEY REFERENCES files(path) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_edges_synthesis_site ON edges(json_extract(metadata, '$.registeredAt')) + WHERE json_extract(metadata, '$.synthesizedBy') IS NOT NULL; + UPDATE edges SET metadata = json_set(COALESCE(metadata, '{}'), '$.synthesizedBy', 'go-method-contains') + WHERE kind = 'contains' AND provenance IS NULL AND EXISTS ( + SELECT 1 FROM nodes s JOIN nodes t ON t.id = edges.target + WHERE s.id = edges.source AND s.language = 'go' AND t.language = 'go' + AND s.kind IN ('struct', 'class', 'interface', 'enum', 'type_alias') AND t.kind = 'method' + AND s.file_path != t.file_path + ); + INSERT OR REPLACE INTO project_metadata(key, value, updated_at) + VALUES ('synthesis_pending', '1', 0); + `); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index 05a4152c3..5acfbf971 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -953,7 +953,7 @@ export class QueryBuilder { getNodesByFile(filePath: string): Node[] { if (!this.stmts.getNodesByFile) { this.stmts.getNodesByFile = this.db.prepare( - 'SELECT * FROM nodes WHERE file_path = ? ORDER BY start_line' + 'SELECT * FROM nodes WHERE file_path = ? ORDER BY start_line, id' ); } const rows = this.stmts.getNodesByFile.all(filePath) as NodeRow[]; @@ -1155,7 +1155,7 @@ export class QueryBuilder { */ getNodesByKind(kind: NodeKind): Node[] { if (!this.stmts.getNodesByKind) { - this.stmts.getNodesByKind = this.db.prepare('SELECT * FROM nodes WHERE kind = ?'); + this.stmts.getNodesByKind = this.db.prepare('SELECT * FROM nodes WHERE kind = ? ORDER BY file_path, start_line, id'); } const rows = this.stmts.getNodesByKind.all(kind) as NodeRow[]; return rows.map(rowToNode); @@ -1171,7 +1171,10 @@ export class QueryBuilder { *iterateNodesByKind(kind: NodeKind): IterableIterator { // Fresh statement per call (not a cached one): an iterator holds an open // cursor, so a shared statement would conflict across overlapping scans. - const stmt = this.db.prepare('SELECT * FROM nodes WHERE kind = ?'); + // Synthesis uses first/last-match precedence and caps: insertion order + // changes on sync. idx_nodes_kind streams this canonical order without + // materializing/sorting all of a large project's methods in memory. + const stmt = this.db.prepare('SELECT * FROM nodes WHERE kind = ? ORDER BY file_path, start_line, id'); for (const row of stmt.iterate(kind)) { yield rowToNode(row as NodeRow); } @@ -1198,7 +1201,7 @@ export class QueryBuilder { // Fresh statement per call — an iterator holds an open cursor (see // iterateNodesByKind). const stmt = this.db.prepare( - "SELECT * FROM nodes WHERE language = ? AND decorators LIKE '%' || ? || '%'" + "SELECT * FROM nodes WHERE language = ? AND decorators LIKE '%' || ? || '%' ORDER BY file_path, start_line, id" ); for (const row of stmt.iterate(language, `"${decorator}"`)) { yield rowToNode(row as NodeRow); @@ -1237,7 +1240,7 @@ export class QueryBuilder { getNodesByName(name: string): Node[] { if (!this.stmts.getNodesByName) { this.stmts.getNodesByName = this.db.prepare( - 'SELECT * FROM nodes WHERE name = ? ORDER BY file_path, start_line' + 'SELECT * FROM nodes WHERE name = ? ORDER BY file_path, start_line, id' ); } const rows = this.stmts.getNodesByName.all(name) as NodeRow[]; @@ -1278,7 +1281,7 @@ export class QueryBuilder { getNodesByQualifiedNameExact(qualifiedName: string): Node[] { if (!this.stmts.getNodesByQualifiedNameExact) { this.stmts.getNodesByQualifiedNameExact = this.db.prepare( - 'SELECT * FROM nodes WHERE qualified_name = ?' + 'SELECT * FROM nodes WHERE qualified_name = ? ORDER BY file_path, start_line, id' ); } const rows = this.stmts.getNodesByQualifiedNameExact.all(qualifiedName) as NodeRow[]; @@ -1873,6 +1876,32 @@ export class QueryBuilder { // Edge Operations // =========================================================================== + /** Must run before file replacement/deletion cascades the endpoint edges. */ + hasSynthesizedEdgesTouchingFile(filePath: string): boolean { + const owned = "json_extract(e.metadata, '$.synthesizedBy') IS NOT NULL"; + for (const endpoint of ['source', 'target']) { + if (this.db.prepare(`SELECT 1 FROM nodes n JOIN edges e ON e.${endpoint} = n.id + WHERE n.file_path = ? AND ${owned} LIMIT 1`).get(filePath)) return true; + } + // Wiring often lives in a third file, with neither endpoint in it. + return !!this.db.prepare(`SELECT 1 FROM edges e WHERE ${owned} + AND json_extract(e.metadata, '$.registeredAt') >= ? + AND json_extract(e.metadata, '$.registeredAt') < ? LIMIT 1` + ).get(`${filePath}:`, `${filePath};`); + } + + wasSynthesisInput(filePath: string): boolean { + return !!this.db.prepare('SELECT 1 FROM synthesis_inputs WHERE file_path = ?').get(filePath); + } + + replaceSynthesisInputs(files: string[]): void { + this.db.transaction(() => { + this.db.exec('DELETE FROM synthesis_inputs'); + const insert = this.db.prepare('INSERT INTO synthesis_inputs(file_path) VALUES (?)'); + for (const file of files) insert.run(file); + })(); + } + /** * Insert a new edge */ @@ -1961,12 +1990,13 @@ export class QueryBuilder { params.push(provenance); } + sql += ' ORDER BY target, kind, line, col'; const rows = this.db.prepare(sql).all(...params) as EdgeRow[]; return rows.map(rowToEdge); } if (!this.stmts.getEdgesBySource) { - this.stmts.getEdgesBySource = this.db.prepare('SELECT * FROM edges WHERE source = ?'); + this.stmts.getEdgesBySource = this.db.prepare('SELECT * FROM edges WHERE source = ? ORDER BY target, kind, line, col'); } const rows = this.stmts.getEdgesBySource.all(sourceId) as EdgeRow[]; return rows.map(rowToEdge); @@ -1977,13 +2007,13 @@ export class QueryBuilder { */ getIncomingEdges(targetId: string, kinds?: EdgeKind[]): Edge[] { if (kinds && kinds.length > 0) { - const sql = `SELECT * FROM edges WHERE target = ? AND kind IN (${kinds.map(() => '?').join(',')})`; + const sql = `SELECT * FROM edges WHERE target = ? AND kind IN (${kinds.map(() => '?').join(',')}) ORDER BY source, kind, line, col`; const rows = this.db.prepare(sql).all(targetId, ...kinds) as EdgeRow[]; return rows.map(rowToEdge); } if (!this.stmts.getEdgesByTarget) { - this.stmts.getEdgesByTarget = this.db.prepare('SELECT * FROM edges WHERE target = ?'); + this.stmts.getEdgesByTarget = this.db.prepare('SELECT * FROM edges WHERE target = ? ORDER BY source, kind, line, col'); } const rows = this.stmts.getEdgesByTarget.all(targetId) as EdgeRow[]; return rows.map(rowToEdge); diff --git a/src/db/schema.sql b/src/db/schema.sql index 6237d2d64..e49bf6937 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -105,7 +105,7 @@ CREATE TABLE IF NOT EXISTS unresolved_refs ( -- ============================================================================= -- Node indexes -CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind); +CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind, file_path, start_line, id); CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name); CREATE INDEX IF NOT EXISTS idx_nodes_qualified_name ON nodes(qualified_name); CREATE INDEX IF NOT EXISTS idx_nodes_file_path ON nodes(file_path); @@ -198,6 +198,14 @@ CREATE INDEX IF NOT EXISTS idx_unresolved_from_name ON unresolved_refs(from_node CREATE INDEX IF NOT EXISTS idx_unresolved_status ON unresolved_refs(status); CREATE INDEX IF NOT EXISTS idx_unresolved_failed_tail ON unresolved_refs(name_tail) WHERE status = 'failed'; CREATE INDEX IF NOT EXISTS idx_edges_provenance ON edges(provenance); +-- Sync's third-file wiring lookup must not scan every synthesized edge. +CREATE INDEX IF NOT EXISTS idx_edges_synthesis_site ON edges(json_extract(metadata, '$.registeredAt')) + WHERE json_extract(metadata, '$.synthesizedBy') IS NOT NULL; + +-- Retain the cheap source-gate verdict when a later sync deletes the source. +CREATE TABLE IF NOT EXISTS synthesis_inputs ( + file_path TEXT PRIMARY KEY REFERENCES files(path) ON DELETE CASCADE +); -- Project metadata for version/provenance tracking CREATE TABLE IF NOT EXISTS project_metadata ( diff --git a/src/db/synthesis-stage.ts b/src/db/synthesis-stage.ts new file mode 100644 index 000000000..7764a1c06 --- /dev/null +++ b/src/db/synthesis-stage.ts @@ -0,0 +1,86 @@ +import { createDatabase, type SqliteDatabase } from './sqlite-adapter'; +import { QueryBuilder } from './queries'; +import { createYielder } from '../resolution/cooperative-yield'; + +// Ownership is independent of provenance: Go method containment is structural. +export const SYNTHESIZED_EDGE = "json_extract(metadata, '$.synthesizedBy') IS NOT NULL"; + +/** A private edge overlay: passes see base edges plus their new Go prerequisites. */ +export class SynthesisStage { + readonly db: SqliteDatabase; + readonly queries: QueryBuilder; + + constructor(dbPath: string) { + this.db = createDatabase(dbPath).db; + try { + this.db.pragma('busy_timeout = 5000'); + this.db.pragma('foreign_keys = ON'); + this.db.pragma('synchronous = NORMAL'); + this.db.pragma('wal_autocheckpoint = 0'); + this.db.exec(` + CREATE TEMP TABLE synthesis_inputs (file_path TEXT PRIMARY KEY); + CREATE TEMP TABLE synthesis_edges ( + id INTEGER PRIMARY KEY, source TEXT, target TEXT, kind TEXT, + metadata TEXT, line INTEGER, col INTEGER, provenance TEXT + ); + CREATE UNIQUE INDEX temp.synthesis_identity ON synthesis_edges + (source, target, kind, IFNULL(line, -1), IFNULL(col, -1)); + CREATE INDEX temp.synthesis_source ON synthesis_edges(source, kind); + CREATE INDEX temp.synthesis_target ON synthesis_edges(target, kind); + CREATE TEMP VIEW edges AS + SELECT * FROM main.edges WHERE NOT COALESCE((${SYNTHESIZED_EDGE}), 0) + UNION ALL SELECT * FROM synthesis_edges; + CREATE TEMP TRIGGER synthesis_insert INSTEAD OF INSERT ON edges BEGIN + INSERT OR IGNORE INTO synthesis_edges + (source, target, kind, metadata, line, col, provenance) + SELECT NEW.source, NEW.target, NEW.kind, NEW.metadata, NEW.line, NEW.col, NEW.provenance + WHERE NOT EXISTS ( + SELECT 1 FROM main.edges WHERE source = NEW.source AND target = NEW.target + AND kind = NEW.kind AND IFNULL(line, -1) = IFNULL(NEW.line, -1) + AND IFNULL(col, -1) = IFNULL(NEW.col, -1) + AND NOT COALESCE((${SYNTHESIZED_EDGE}), 0) + ); + END; + `); + this.queries = new QueryBuilder(this.db); + } catch (error) { + this.db.close(); + throw error; + } + } + + async publish(backpressure?: () => Promise | null): Promise { + await backpressure?.(); + const yieldToLoop = createYielder(); + // Keep the replacement atomic for other connections. Yield between bounded + // writes, but checkpoint only OUTSIDE this transaction (uncommitted frames + // cannot be folded). Pass execution/staging above never writes the main WAL. + this.db.exec('BEGIN IMMEDIATE'); + try { + const remove = this.db.prepare(`DELETE FROM main.edges WHERE id IN ( + SELECT id FROM main.edges WHERE ${SYNTHESIZED_EDGE} LIMIT 2000 + )`); + while (remove.run().changes > 0) await yieldToLoop(); + const insert = this.db.prepare(`INSERT OR IGNORE INTO main.edges + (source, target, kind, metadata, line, col, provenance) + SELECT source, target, kind, metadata, line, col, provenance + FROM synthesis_edges WHERE id > ? AND id <= ?`); + const max = this.db.prepare('SELECT MAX(id) AS id FROM synthesis_edges').get()?.id ?? 0; + for (let i = 0; i < max; i += 2000) { + insert.run(i, i + 2000); + await yieldToLoop(); + } + this.db.exec(`DELETE FROM main.synthesis_inputs; + INSERT INTO main.synthesis_inputs SELECT * FROM temp.synthesis_inputs`); + this.db.exec('COMMIT'); + } catch (error) { + this.db.exec('ROLLBACK'); + throw error; + } + await backpressure?.(); + } + + close(): void { + this.db.close(); + } +} diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 612eab58c..9dbc7ef03 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -3037,7 +3037,9 @@ export class ExtractionOrchestrator { * is stored, when no extraction transaction is open, so a checkpoint can * safely catch up before the next file grows the WAL further. */ - backpressure?: () => Promise | null + backpressure?: () => Promise | null, + /** Inspect changed inputs before deletion/re-extraction cascades their edges. */ + onFileChange?: (filePath: string, content?: string) => void ): Promise { await initGrammars(); // Initialize WASM runtime (grammars loaded lazily below) const startTime = Date.now(); @@ -3153,6 +3155,7 @@ export class ExtractionOrchestrator { this.queries.insertUnresolvedRefsBatch(resurrected); } } + onFileChange?.(tracked.path); this.queries.deleteFile(tracked.path); filesRemoved++; } @@ -3202,10 +3205,12 @@ export class ExtractionOrchestrator { const contentHash = hashContent(content); if (!tracked) { + onFileChange?.(filePath, content); filesToIndex.push(filePath); changedFilePaths.push(filePath); filesAdded++; } else if (tracked.contentHash !== contentHash) { + onFileChange?.(filePath, content); filesToIndex.push(filePath); changedFilePaths.push(filePath); filesModified++; diff --git a/src/index.ts b/src/index.ts index 75f7fa081..1a7c5814c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,6 +47,7 @@ import { createResolver, ResolutionResult, } from './resolution'; +import { hasSynthesisPattern } from './resolution/callback-synthesizer'; import { GraphTraverser, GraphQueryManager } from './graph'; import { ContextBuilder, createContextBuilder } from './context'; import { Mutex, FileLock } from './utils'; @@ -839,7 +840,20 @@ export class CodeGraph { const fullReconcile = !options.paths || options.paths.length === 0; const gitState = this.orchestrator.beginGitIndexState(fullReconcile); - const result = await this.orchestrator.sync(options.onProgress, options.paths, backpressure); + // An interrupted index may have absorbed its changed files before + // resolution/synthesis. Detect those orphans BEFORE this sync adds refs. + let refreshSynthesis = this.queries.getMetadata('synthesis_pending') === '1' || + this.queries.getUnresolvedReferencesCount() > 0; + if (refreshSynthesis) this.queries.setMetadata('synthesis_pending', '1'); + const result = await this.orchestrator.sync(options.onProgress, options.paths, backpressure, + (filePath, content) => { + if (!refreshSynthesis && (this.queries.hasSynthesizedEdgesTouchingFile(filePath) || + this.queries.wasSynthesisInput(filePath) || + (content !== undefined && hasSynthesisPattern(filePath, content)))) { + refreshSynthesis = true; + this.queries.setMetadata('synthesis_pending', '1'); + } + }); // Fold the store phase's WAL BEFORE the post-store reads below // (resolution reads on the main thread) — same rationale as @@ -938,7 +952,8 @@ export class CodeGraph { total: totalPasses, }); }, - backpressure + backpressure, + false ); } } @@ -1008,7 +1023,8 @@ export class CodeGraph { total: totalPasses, }); }, - backpressure + backpressure, + false ); result.pendingRefsProcessed = recovery.stats.total; result.pendingRefsResolved = recovery.stats.resolved; @@ -1025,9 +1041,16 @@ export class CodeGraph { await this.resolver.resolveDeferredThisMemberRefs(); } + if (refreshSynthesis) { + await this.resolver.refreshSynthesis(this.db.getPath(), (done, total) => { + options.onProgress?.({ phase: 'linking', current: done, total }); + }, backpressure); + this.queries.setMetadata('synthesis_pending', '0'); + } + // Refresh planner stats + checkpoint the WAL after bulk writes. // Off-thread — see indexAll's call site. - if (filesChanged || result.filesRemoved > 0 || orphanCount > 0) { + if (filesChanged || result.filesRemoved > 0 || orphanCount > 0 || refreshSynthesis) { await this.db.runMaintenance(); } @@ -1295,7 +1318,8 @@ export class CodeGraph { // resolution is timer-driven passive checkpoints, which the pool's // continuous reads keep perpetually partial — the WAL then accretes the // whole phase's write volume (22GB on a 4.6GB DB at kernel scale). - backpressure?: () => Promise | null + backpressure?: () => Promise | null, + synthesize: boolean = true ): Promise { return this.resolver.resolveAndPersistBatched(onProgress, undefined, onSynthesisProgress, { dbPath: this.db.getPath(), @@ -1313,7 +1337,7 @@ export class CodeGraph { end: () => this.db.endBulkRefLoad(), }, backpressure, - }); + }, synthesize); } /** diff --git a/src/resolution/callback-synthesizer.ts b/src/resolution/callback-synthesizer.ts index 3aff95c2b..23ede9921 100644 --- a/src/resolution/callback-synthesizer.ts +++ b/src/resolution/callback-synthesizer.ts @@ -35,7 +35,7 @@ import { tanstackLinkEdges } from './tanstack-router-synthesizer'; import { vueRouterLinkEdges } from './vue-router-synthesizer'; import { svelteKitLinkEdges, svelteKitPageComponentEdges } from './sveltekit-synthesizer'; import { createYielder, type MaybeYield } from './cooperative-yield'; -import { crossTierEdges } from './tier-synthesizer'; +import { crossTierEdges, hasCrossTierPattern } from './tier-synthesizer'; import { enclosingFn, makeLineAt } from './synth-utils'; import { resolveImportPath } from './import-resolver'; import { crossesCodeBoundary } from './name-matcher'; @@ -889,9 +889,10 @@ async function goImplementsEdges(queries: QueryBuilder, onYield: MaybeYield): Pr * * Go guarantees a method's receiver type is declared in the SAME PACKAGE as the * method, and a Go package is a single directory — so this is a deterministic - * structural link, not a heuristic: find the same-named type in the method's own - * directory and add the missing `contains` edge (no `provenance: 'heuristic'`, - * matching the same-file edges extraction already emits). Skips methods that + * structural link, not a heuristic: find the same-named type in the method's + * own directory and add the missing `contains` edge with no provenance, matching + * same-file extraction. Tag synthesis ownership so an incremental + * refresh can replace it alongside implicit `implements`. Skips methods that * already have a type parent (the same-file case). (#583, cross-file half) */ async function goCrossFileMethodContainsEdges(queries: QueryBuilder, onYield: MaybeYield): Promise { @@ -940,7 +941,8 @@ async function goCrossFileMethodContainsEdges(queries: QueryBuilder, onYield: Ma const key = `${owner.id}>${method.id}`; if (seen.has(key)) continue; seen.add(key); - edges.push({ source: owner.id, target: method.id, kind: 'contains', line: method.startLine }); + edges.push({ source: owner.id, target: method.id, kind: 'contains', line: method.startLine, + metadata: { synthesizedBy: 'go-method-contains' } }); } return edges; } @@ -3608,6 +3610,36 @@ export interface SynthPassDef { const ALWAYS = (): boolean => true; +/** Conservative input gates for SYNTH_PASSES; keep these in sync when adding a pass. */ +export function hasSynthesisPattern(filePath: string, content: string): boolean { + // These passes consume declarations/layouts as well as dispatch sites. A + // header or markup edit can change a channel whose endpoints live elsewhere. + if (/\.(?:vue|svelte|dfm|fmx|nix|xml)$/.test(filePath)) return true; + if (/\.(?:c|h|cc|cpp|cxx|hpp|hh|hxx|cppm|ipp|inl|tcc|def|inc|tbl)$/i.test(filePath) && + /\b(?:struct|union|typedef|virtual|override)\b|#\s*(?:include|define|if)|=|->|\[/.test(content)) return true; + if (/\b(?:class|interface|protocol|trait|impl|extends|implements|expect|actual)\b/.test(content)) return true; + if (/\.go$/.test(filePath) && /\b(?:struct|interface)\b|\bfunc\s*\(/.test(content)) return true; + if (hasCrossTierPattern(content)) return true; + if (/\b(?:render|build|setState|defineStore|createStore|createApi|Store|href|sendEvent|sendEventWithName)\b|<\/|\/>/.test(content)) return true; + if (/\.(?:forEach|append|add|push|insert|fire|dispatchEvent|addListener|Use|GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD|Any|Handle)\s*\(/.test(content)) return true; + if (/[\w$]\s*\[\s*[A-Za-z_$]/.test(content) || /\b(?:dispatch|commit)\s*\(/.test(content)) return true; + const patterns = [THUNK_DECL_RE, CELERY_TASK_DECORATOR_RE, CELERY_DISPATCH_RE, + SPRING_LISTENER_ANNO_RE, SPRING_APP_LISTENER_RE, SPRING_PUBLISH_RE, + MEDIATR_HANDLER_BASE_RE, MEDIATR_DISPATCH_RE, SIDEKIQ_WORKER_RE, SIDEKIQ_DISPATCH_RE, + ERLANG_CALLBACK_DECL_RE, ERLANG_DISPATCH_RE, LARAVEL_DISPATCH_RE, ARKUI_EMITTER_CALL_RE, + ARKUI_ROUTER_RE]; + for (const re of patterns) { + re.lastIndex = 0; + const matches = re.test(content); + re.lastIndex = 0; + if (matches) return true; + } + // Field-backed observers use method-name gates rather than fixed call names. + return (content.match(/[A-Za-z_$][\w$]*/g) ?? []).some( + (name) => REGISTRAR_NAME.test(name) || DISPATCHER_NAME.test(name) + ); +} + /** * The independent passes, in MERGE ORDER — the first-seen dedup in * synthesizeCallbackEdges follows this array, so reordering entries changes @@ -3883,6 +3915,15 @@ export async function synthesizeCallbackEdges( await yieldToLoop(); await foldIfOver(); } + // Remember source gates, including inputs that currently produce NO edges + // (e.g. an over-cap channel). Deleting one may make the full pass viable. + const inputs: string[] = []; + for (const file of ctx.getAllFiles()) { + const content = ctx.readFile(file); + if (content !== null && hasSynthesisPattern(file, content)) inputs.push(file); + await yieldToLoop(); + } + queries.replaceSynthesisInputs(inputs); __mark('insertMergedEdges'); return merged.length + goImpl.length + goMethodContains.length; } diff --git a/src/resolution/index.ts b/src/resolution/index.ts index a98ebd217..db40e50c4 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -8,6 +8,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Language, Node, UnresolvedReference, Edge } from '../types'; import { QueryBuilder } from '../db/queries'; +import { SynthesisStage } from '../db/synthesis-stage'; import { UnresolvedRef, ResolvedRef, @@ -1683,7 +1684,8 @@ export class ReferenceResolver { * each per-batch DELETE's B-tree work (DatabaseConnection.beginBulkRefLoad). */ refIndexLoad?: { begin: () => void; end: () => void | Promise }; backpressure?: () => Promise | null; - } + }, + synthesize: boolean = true ): Promise { // Resolution runs on the indexer's MAIN thread, and the #850 liveness // watchdog SIGKILLs a process whose event loop stalls past its window (60s @@ -2105,7 +2107,7 @@ export class ReferenceResolver { // loop. See docs/design/callback-edge-synthesis.md. const tSynth = Date.now(); try { - aggregateStats.byMethod['callback-synthesis'] = await synthesizeCallbackEdges( + if (synthesize) aggregateStats.byMethod['callback-synthesis'] = await synthesizeCallbackEdges( this.queries, this.context, onSynthesisProgress, @@ -2133,6 +2135,25 @@ export class ReferenceResolver { }; } + /** Replace synthesis only after every base-resolution pass has finished. */ + async refreshSynthesis( + dbPath: string, + onProgress?: (done: number, total: number) => void, + backpressure?: () => Promise | null + ): Promise { + this.clearCaches(); + const stage = new SynthesisStage(dbPath); + try { + const fresh = new ReferenceResolver(this.projectRoot, stage.queries); + const count = await synthesizeCallbackEdges(stage.queries, fresh.context, onProgress, null, backpressure); + await stage.publish(backpressure); + return count; + } finally { + stage.close(); + this.clearCaches(); + } + } + /** * Get detected frameworks */ diff --git a/src/resolution/tier-synthesizer.ts b/src/resolution/tier-synthesizer.ts index 2aee717ec..b146fdce0 100644 --- a/src/resolution/tier-synthesizer.ts +++ b/src/resolution/tier-synthesizer.ts @@ -880,6 +880,10 @@ const HTTP_GATE = /\b(?:fetch|\$fetch|ofetch|axios|ky|got|useFetch|useSWR)\b|\.\ const QUEUE_GATE = /\.\s*add\s*\(|@Processor\s*\(|\bnew\s+Worker\s*[<(]|\.\s*process\s*\(/; const EVENT_GATE = /\.\s*(?:emit|emitAsync|on|once)\s*\(|@OnEvent\s*\(|@SubscribeMessage\s*\(/; +export function hasCrossTierPattern(content: string): boolean { + return HTTP_GATE.test(content) || QUEUE_GATE.test(content) || EVENT_GATE.test(content); +} + export async function crossTierEdges(ctx: ResolutionContext, onYield: MaybeYield): Promise { const routes = httpRoutes(ctx); const httpSites: HttpSite[] = [];