diff --git a/CHANGELOG.md b/CHANGELOG.md index 33286c47c..ce2600b20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Extension preview -- Reconcile global semantic-extension links after sync or explicit file indexing with an atomic candidate rebuild. Failed passes preserve the working graph. Reuse bounded core parse results on warm project instances while rerunning all framework hooks, global resolution and semantic passes; cold or disabled reuse retains full parsing. Reject observed source changes before commit and reclaim interrupted graph-only candidates. +- Reconcile global semantic-extension links after sync or explicit file indexing with an atomic candidate rebuild. Failed passes preserve the working graph. Reuse bounded core parse results on warm project instances while rerunning all framework hooks, global resolution and semantic passes; cold or disabled reuse retains full parsing. Reject observed source changes before commit and reclaim interrupted graph-only candidates. Rebuild text search in bulk inside the atomic replacement transaction. - Prepare immutable Drupal extension 0.1.1: ignore unrelated static receivers and commented/quoted declarations, and resolve both explicit Symfony event argument orders. Preserve the reviewed 0.1.0 artifact and policy. - Keep installed and selected marketplace versions distinct while compatibility metadata refreshes; pending requests no longer display an incompatible catalog version as installed. diff --git a/docs/design/semantic-update-reuse.md b/docs/design/semantic-update-reuse.md index e09d06bc7..7f1658dfe 100644 --- a/docs/design/semantic-update-reuse.md +++ b/docs/design/semantic-update-reuse.md @@ -38,7 +38,11 @@ The project SQLite coordinator is authoritative, independent of PID reuse. Fresh opens reclaim the exact orphaned stage after process death; an invalid record blocks recovery without deleting files. During a live graph-only update, readers can use the old committed graph. The existing SQLite graph transaction -still determines old/new visibility. A commit marker invalidates long-lived +still determines old/new visibility. FTS triggers are suspended and rebuilt +inside that same transaction using the existing bulk-load API, avoiding per-row +FTS work during replacement. A kill rolls back the trigger schema and FTS data +with the graph; readers never see a committed missing-trigger window. +`CODEGRAPH_NO_SEMANTIC_BULK_FTS=1` retains the previous trigger path. A commit marker invalidates long-lived reader caches. Managed config/trust transactions retain their exclusive guard. Measurements and correctness/recovery coverage will be linked after validation. diff --git a/scripts/validation/semantic-update-poll.cjs b/scripts/validation/semantic-update-poll.cjs index 7d0e6dd93..aa14bf46e 100644 --- a/scripts/validation/semantic-update-poll.cjs +++ b/scripts/validation/semantic-update-poll.cjs @@ -1,4 +1,4 @@ -const diagnosticEvents=[];require('node:diagnostics_channel').channel('codegraph.semantic.update').subscribe(e=>diagnosticEvents.push(e)); +const diagnosticEvents=[];require('node:diagnostics_channel').channel('codegraph.semantic.update').subscribe(e=>diagnosticEvents.push({...e,atMs:performance.now()})); const fs=require('node:fs'),path=require('node:path'),crypto=require('node:crypto'),assert=require('node:assert/strict');const {CodeGraph}=require('../../dist');const {DatabaseSync}=require('node:sqlite'); Object.assign(process.env,{CODEGRAPH_TELEMETRY:'0',CODEGRAPH_PARSE_WORKERS:'2',CODEGRAPH_RESOLVE_WORKERS:'0',CODEGRAPH_SYNTH_TIMINGS:'all'}); const [root,file,out]=process.argv.slice(2),original=fs.readFileSync(path.join(root,file)),rows=[];fs.mkdirSync(path.dirname(out),{recursive:true}); diff --git a/scripts/validation/semantic-update-recovery.cjs b/scripts/validation/semantic-update-recovery.cjs index bb7766a7c..311a55392 100644 --- a/scripts/validation/semantic-update-recovery.cjs +++ b/scripts/validation/semantic-update-recovery.cjs @@ -4,7 +4,8 @@ const {spawn}=require('node:child_process'),{channel}=require('node:diagnostics_ const {CodeGraph,createExtensionProject,packExtension}=require('../../dist');const {ExtensionManager}=require('../../dist/plugins/manager'); const {DatabaseSync}=require('node:sqlite'); Object.assign(process.env,{CODEGRAPH_TELEMETRY:'0',CODEGRAPH_PARSE_WORKERS:'2',CODEGRAPH_RESOLVE_WORKERS:'2',CODEGRAPH_PARALLEL_RESOLVE_MIN:'0'}); -const phases=['candidate_ready','before_commit','in_commit','committed']; +const phases=['candidate_ready','before_commit','copy_started','in_commit','committed']; +function ftsAdded(root){const db=new DatabaseSync(path.join(root,'.codegraph/codegraph.db'),{readOnly:true});try{return db.prepare("SELECT count(*) n FROM nodes_fts WHERE nodes_fts MATCH 'added_function'").get().n}finally{db.close()}} function snapshot(root){const db=new DatabaseSync(path.join(root,'.codegraph/codegraph.db'),{readOnly:true});try{return db.prepare("select a.name source,b.name target from edges e join nodes a on a.id=e.source join nodes b on b.id=e.target where e.metadata like '%\"synthesizedBy\":\"kill-fixture\"%' order by a.name,b.name").all().map(r=>[r.source,r.target]);}finally{db.close()}} if(process.argv[2]==='--child'){ const [root,action,phase]=process.argv.slice(3); @@ -12,7 +13,7 @@ if(process.argv[2]==='--child'){ if(action==='update'){ await g.indexFiles(['binding.py']); // warm this process's cache on old source channel('codegraph.semantic.update').subscribe(e=>{if(e.phase===phase){fs.writeSync(1,JSON.stringify({phase,pid:process.pid})+'\n');Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);}}); - fs.writeFileSync(path.join(root,'binding.py'),'cancel');await g.sync(); + fs.writeFileSync(path.join(root,'binding.py'),'cancel');fs.writeFileSync(path.join(root,'added.py'),'def added_function():\n pass\n');await g.sync(); }else if(action==='retry')await g.sync(); console.log(JSON.stringify({edges:snapshot(root),stats:g.getStats()})); }finally{g.close()}})().catch(e=>{console.error(e.stack);process.exitCode=1}); @@ -27,12 +28,12 @@ if(process.argv[2]==='--child'){ for(const phase of phases){const root=path.join(lab,phase);fs.mkdirSync(root);fs.writeFileSync(path.join(root,'handlers.py'),'def dispatch():\n missing()\ndef receipt():\n pass\ndef cancel():\n pass\n');fs.writeFileSync(path.join(root,'binding.py'),'receipt');await new ExtensionManager(root).install({bytes});const config=fs.readFileSync(path.join(root,'codegraph.json'),'utf8'); const reader=await CodeGraph.open(root);assert.deepEqual(snapshot(root),[['dispatch','receipt']]);reader.getNodesByKind('function'); const c=child(root,'update',phase);let timer;try{await Promise.race([c.reached,c.done.then(r=>{throw Error('child ended '+r.stderr)}),new Promise((_,r)=>timer=setTimeout(()=>r(Error('timeout '+phase)),90000))])}finally{clearTimeout(timer)} - const expected=phase==='committed'?'cancel':'receipt';assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.ok(reader.getStats().nodeCount>0); + const expected=phase==='committed'?'cancel':'receipt';assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.ok(reader.getStats().nodeCount>0);assert.equal(ftsAdded(root),phase==='committed'?1:0); const concurrent=await child(root,'retry').done;assert.notEqual(concurrent.exit,0);assert.match(concurrent.stderr,/operation is active/); assert.equal(c.p.kill('SIGKILL'),true);const killed=await c.done;assert.ok(killed.signal==='SIGKILL'||(process.platform==='win32'&&killed.exit!==0)); - fs.writeFileSync(path.join(root,'keep.txt'),'unrelated user file');const inspect=await child(root,'inspect').done;assert.equal(inspect.exit,0,inspect.stderr);assert.deepEqual(snapshot(root),[['dispatch',expected]]); + fs.writeFileSync(path.join(root,'keep.txt'),'unrelated user file');const inspect=await child(root,'inspect').done;assert.equal(inspect.exit,0,inspect.stderr);assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.equal(ftsAdded(root),phase==='committed'?1:0); assert.equal(fs.existsSync(path.join(root,'.codegraph/plugins/candidate.json')),false);assert.deepEqual(fs.readdirSync(path.join(root,'.codegraph')).filter(f=>f.startsWith('extension-stage-')),[]); - assert.equal((await child(root,'inspect').done).exit,0);assert.equal((await child(root,'retry').done).exit,0);assert.deepEqual(snapshot(root),[['dispatch','cancel']]); + assert.equal((await child(root,'inspect').done).exit,0);assert.equal((await child(root,'retry').done).exit,0);assert.deepEqual(snapshot(root),[['dispatch','cancel']]);assert.equal(ftsAdded(root),1); const funcs=reader.getNodesByKind('function');const names=new Map(funcs.map(n=>[n.id,n.name]));assert.deepEqual(reader.getOutgoingEdgesFrom(funcs.map(n=>n.id)).filter(e=>e.metadata?.synthesizedBy==='kill-fixture').map(e=>[names.get(e.source),names.get(e.target)]),[['dispatch','cancel']]);reader.close(); assert.equal(fs.readFileSync(path.join(root,'codegraph.json'),'utf8'),config);assert.equal(fs.readFileSync(path.join(root,'keep.txt'),'utf8'),'unrelated user file');rows.push({phase,passed:true,edgesAfterKill:expected,concurrentExcluded:true,repeatedRecovery:true,readerFresh:true});save();console.log('PASS',phase); } diff --git a/scripts/validation/semantic-update-sample.cjs b/scripts/validation/semantic-update-sample.cjs index 94db85870..3d6bbd50b 100644 --- a/scripts/validation/semantic-update-sample.cjs +++ b/scripts/validation/semantic-update-sample.cjs @@ -1,4 +1,4 @@ -const diagnosticEvents=[];require('node:diagnostics_channel').channel('codegraph.semantic.update').subscribe(e=>diagnosticEvents.push(e)); +const diagnosticEvents=[];require('node:diagnostics_channel').channel('codegraph.semantic.update').subscribe(e=>diagnosticEvents.push({...e,atMs:performance.now()})); // Disposable existing corpus only. Keeps the graph open across an edit sequence. const fs=require('node:fs'),path=require('node:path'),crypto=require('node:crypto'),cp=require('node:child_process'),assert=require('node:assert/strict'); const {DatabaseSync}=require('node:sqlite');const {CodeGraph}=require('../../dist'); diff --git a/src/extraction/parse-worker.ts b/src/extraction/parse-worker.ts index b207381af..e61feceda 100644 --- a/src/extraction/parse-worker.ts +++ b/src/extraction/parse-worker.ts @@ -99,7 +99,9 @@ parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: st let result: ReusableExtraction | undefined; if (msg.reuseCore) { const core = msg.coreExtraction ?? extractFromSource(filePath!, content!, language); - result = applyFrameworkExtraction(structuredClone(core), filePath!, content!, language, frameworkNames); + // A received base was already cloned by postMessage; only a new base + // needs a second copy so the returned cache entry stays unmodified. + result = applyFrameworkExtraction(msg.coreExtraction ? core : structuredClone(core), filePath!, content!, language, frameworkNames); if (!msg.coreExtraction) result.coreExtraction = core; } const frameworksNeedDecode = diff --git a/src/index.ts b/src/index.ts index 7a7e24830..65600a2cd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -594,10 +594,17 @@ export class CodeGraph { throw new Error('Project sources or configuration changed during semantic indexing; previous graph retained. Retry the update.'); channel('codegraph.semantic.update').publish({ projectRoot: this.projectRoot, phase: 'before_commit' }); connection.transaction(() => { + // Candidate replacement is one bulk load. Keep the FTS trigger window + // INSIDE this SQLite transaction so a kill rolls back graph and FTS + // schema together; readers keep the prior committed index. + const bulkFts = process.env.CODEGRAPH_NO_SEMANTIC_BULK_FTS !== '1'; + if (bulkFts) this.db.beginBulkNodeLoad(); + channel('codegraph.semantic.update').publish({ projectRoot: this.projectRoot, phase: 'copy_started' }); this.queries.clear(); for (const table of ['files', 'nodes', 'edges', 'unresolved_refs']) { connection.exec(`INSERT INTO main.${table} SELECT * FROM extension_candidate.${table}`); } + if (bulkFts) this.db.endBulkNodeLoad(); this.queries.setMetadata('plugins_last_run', JSON.stringify(plugins.diagnostics)); this.queries.setMetadata('indexed_with_plugins', this.extensionStamp(plugins)); this.queries.setMetadata('indexed_with_version', CodeGraphPackageVersion);