GithubStar/codegraph
1
0
Fork 0
mirror of https://github.com/colbymchenry/codegraph.git synced 2026-10-18 22:02:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(extraction): preserve symbols with colliding same-line IDs (#1349) (#2030)

Browse Source 
Same-kind, same-name declarations on one line shared an ID and silently overwrote each other in SQLite.
Use per-extraction collision tracking to append a UTF-16 column only to later distinct declarations.
Apply the same rule across wasm, Rust language walkers, Liquid, and CFML while preserving file IDs and edge endpoints.
Add persistence, call-edge, identity-vector, and Unicode parity regressions.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-09-27 13:35:19 +00:00
committed by GitHub
parent e27d6da6b2
commit 86708bf519
25 changed files with 329 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__tests__/extraction.test.ts
+71
View File
@@ -14,6 +14,7 @@ import { extractFromSource, scanDirectory, scanDirectoryAsync, buildDefaultIgnor
import { detectLanguage, isLanguageSupported, getSupportedLanguages, initGrammars, loadAllGrammars, isSourceFile } from '../src/extraction/grammars';
import { stripCppTemplateArgs, blankCppExportMacros, blankCppInlineMacros, blankMetalAttributes, blankCudaConstructs, blankCppAnnotationMacroCalls, blankCppApiPrefixMacros, blankCppInlineAnnotationMacros, blankCLeadingAttrMacros, recoverMangledCppName } from '../src/extraction/languages/c-cpp';
import { normalizePath } from '../src/utils';
import { generateNodeId } from '../src/extraction/tree-sitter-helpers';
beforeAll(async () => {
await initGrammars();
@@ -32,6 +33,76 @@ function cleanupTempDir(dir: string): void {
}
}
describe('same-line node identity (#1349)', () => {
let dir: string;
let cg: CodeGraph | undefined;
let kernel: string | undefined;
beforeEach(() => {
dir = createTempDir();
kernel = process.env.CODEGRAPH_KERNEL;
});
afterEach(() => {
cg?.destroy();
cg = undefined;
cleanupTempDir(dir);
if (kernel === undefined) delete process.env.CODEGRAPH_KERNEL;
else process.env.CODEGRAPH_KERNEL = kernel;
});
it.each(['default', 'wasm'])('persists both accessors and their separate call edges (%s)', async (backend) => {
if (backend === 'wasm') process.env.CODEGRAPH_KERNEL = '0';
else delete process.env.CODEGRAPH_KERNEL;
fs.writeFileSync(path.join(dir, 'point.ts'), [
'function read() { return 1; } function write(v: number) {}',
'export class Point { /* é😀 */ get x() { return read(); } set x(v: number) { write(v); }',
' get y() { return read(); }',
' set y(v: number) { write(v); }',
'}',
].join('\n'));
cg = CodeGraph.initSync(dir, { config: { include: ['**/*.ts'], exclude: [] } });
await cg.indexAll();
cg.resolveReferences();
const nodes = cg.getNodesInFile('point.ts');
const x = nodes.filter((n) => n.name === 'x').sort((a, b) => a.startColumn - b.startColumn);
expect(x).toHaveLength(2);
expect(new Set(nodes.map((n) => n.id)).size).toBe(nodes.length);
expect(nodes.filter((n) => n.name === 'y')).toHaveLength(2);
expect(x[0]!.id).toBe(generateNodeId('point.ts', 'method', 'x', 2));
expect(x[1]!.id).toBe(`${x[0]!.id}:${x[1]!.startColumn}`);
const cls = nodes.find((n) => n.name === 'Point')!;
for (const accessor of x) {
expect(cg.getIncomingEdges(accessor.id)).toContainEqual(expect.objectContaining({ source: cls.id, kind: 'contains' }));
}
expect(cg.getCallees(x[0]!.id).map((c) => c.node.name)).toEqual(['read']);
expect(cg.getCallees(x[1]!.id).map((c) => c.node.name)).toEqual(['write']);
for (const node of nodes.filter((n) => n.name !== 'x' && n.kind !== 'file')) {
expect(node.id).toBe(generateNodeId(node.filePath, node.kind, node.name, node.startLine));
}
});
it.each([
['template.liquid', 'é😀 {% render "x" %}{% render "x" %}{% assign v = 1 %}{% assign v = 2 %}', 'component'],
['Service.cfc', '<cfcomponent><!--- é😀 ---><cffunction name="x"></cffunction><cffunction name="x"></cffunction></cfcomponent>', 'method'],
] as const)('persists repeated same-line declarations in %s', async (file, source, kind) => {
fs.writeFileSync(path.join(dir, file), source);
cg = CodeGraph.initSync(dir, { config: { include: [file], exclude: [] } });
await cg.indexAll();
const nodes = cg.getNodesInFile(file);
const xs = nodes.filter((n) => n.name === 'x' && n.kind === kind);
expect(xs).toHaveLength(2);
expect(new Set(xs.map((n) => n.id)).size).toBe(2);
for (const node of xs) {
expect(cg.getIncomingEdges(node.id).some((e) => e.kind === 'contains')).toBe(true);
}
if (file.endsWith('.liquid')) {
expect(nodes.filter((n) => n.kind === 'import' && n.name === 'x')).toHaveLength(2);
expect(nodes.filter((n) => n.name === 'v')).toHaveLength(2);
}
});
});
describe('Language Detection', () => {
it('should detect TypeScript files', () => {
expect(detectLanguage('src/index.ts')).toBe('typescript');
__tests__/kernel-lua-parity.test.ts
+3 -3
View File
@@ -13,7 +13,7 @@
* `calls "require"`, table fn-ref registries with dedupe and the
* `M.cb = cb` param-storage skip, the raw-text callee zoo with colon/
* bracket/call-result callees and the `(handler)` conversion, LuaDoc
* `- `-keeping docstrings, `<const>` attributes, one-line duplicate-id
* `- `-keeping docstrings, `<const>` attributes, one-line disambiguated
* declarations; torture.luau: `--!strict` docstring joining, `export type`
* isExported, verbatim `Generic<T>` alias names, the typeof(require(...))
* alias+import pair, typed signatures with return suffixes, interpolation/
@@ -194,12 +194,12 @@ describe.skipIf(!kernelBuilt)('kernel Lua/Luau extraction parity', () => {
expect(names).toContain('obj:foo');
});
it('one-line duplicate declarations emit duplicate-id rows verbatim', () => {
it('one-line duplicate declarations retain distinct IDs (#1349)', () => {
const src = 'local x = 1; local x = 2\n';
const result = assertParity('fixtures/dup.lua', src, 'lua', 2);
const xs = result.nodes.filter((n) => n.kind === 'variable' && n.name === 'x');
expect(xs).toHaveLength(2);
expect(xs[0]!.id).toBe(xs[1]!.id);
expect(xs[1]!.id).toBe(`${xs[0]!.id}:${xs[1]!.startColumn}`);
});
it('cross-dialect syntax defers to the wasm extractor', () => {
__tests__/kernel-node-id-parity.test.ts
+81
View File
@@ -0,0 +1,81 @@
import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
import * as fs from 'fs';
import * as path from 'path';
import { extractFromSource } from '../src/extraction';
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel';
import { generateNodeId } from '../src/extraction/tree-sitter-helpers';
import type { ExtractionResult, Language } from '../src/types';
// Exercise every sibling walker with a same-line collision after BMP + astral Unicode.
const fixtures: Array<[Language, string]> = [
['java', '/* é😀 */ class A { void x() {} void x(int n) {} }'],
['python', 'label = "é😀"; x = 1; x = 2'],
['go', 'package p\n/* é😀 */ type A struct{}; type B struct{}; func (a A) x(){}; func (b B) x(){}'],
['c', '/* é😀 */ enum A { x }; void f() { enum B { x }; }'],
['cpp', '/* é😀 */ int x() { return 1; } int x(int n) { return n; }'],
['rust', '/* é😀 */ struct A; struct B; impl A { fn x() {} } impl B { fn x() {} }'],
['csharp', '/* é😀 */ class A { void x() {} void x(int n) {} }'],
['ruby', 'label = "é😀"; class A; def x; end; end; class B; def x; end; end'],
['php', '<?php /* é😀 */ class A { function x() {} } class B { function x() {} }'],
['swift', '/* é😀 */ func x() {}; func x(_ n: Int) {}'],
['kotlin', '/* é😀 */ fun x() {}; fun x(n: Int) {}'],
['scala', '/* é😀 */ object A { def x(): Int = 1; def x(n: Int): Int = n }'],
['dart', '/* é😀 */ class A { void x() {} } class B { void x() {} }'],
['lua', 'local label = "é😀"; local x = 1; local x = 2'],
['luau', 'local label = "é😀"; local x = 1; local x = 2'],
['r', 'label <- "é😀"; x <- function() {}; x <- function() {}'],
];
function canon(result: ExtractionResult) {
return {
nodes: result.nodes.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort())).sort(),
edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(),
refs: result.unresolvedReferences.map((r) => JSON.stringify(r, Object.keys(r).sort())).sort(),
};
}
const kernelBuilt = fs.existsSync(path.join(__dirname, '..', 'codegraph-kernel', 'prebuilds',
`${process.platform}-${process.arch}`, 'codegraph-kernel.node'));
const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const;
let savedEnv: Record<string, string | undefined>;
describe.skipIf(!kernelBuilt)('collision-only node identity parity (#1349)', () => {
beforeAll(async () => {
await initGrammars();
await loadGrammarsForLanguages(fixtures.map(([language]) => language));
});
beforeEach(() => {
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
resetKernelForTests();
});
afterEach(() => {
for (const k of ENV_KEYS) {
if (savedEnv[k] === undefined) delete process.env[k];
else process.env[k] = savedEnv[k];
}
resetKernelForTests();
});
it.each(fixtures)('%s keeps distinct same-line symbols and UTF-16 identity suffixes', (language, source) => {
const file = `fixture.${language}`;
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
delete process.env.CODEGRAPH_KERNEL;
const native = tryKernelExtract(file, source, language);
expect(native, 'must exercise the native walker, not fallback').not.toBeNull();
process.env.CODEGRAPH_KERNEL = '0';
const wasm = extractFromSource(file, source, language);
expect(canon(native!)).toEqual(canon(wasm));
const xs = wasm.nodes.filter((n) => n.name === 'x');
expect(xs).toHaveLength(2);
expect(xs[0]!.startLine).toBe(xs[1]!.startLine);
expect(xs[0]!.id).toBe(generateNodeId(file, xs[0]!.kind, 'x', xs[0]!.startLine));
expect(xs[1]!.id).toBe(`${xs[0]!.id}:${xs[1]!.startColumn}`);
expect(new Set(wasm.nodes.map((n) => n.id)).size).toBe(wasm.nodes.length);
const ids = new Set(wasm.nodes.map((n) => n.id));
for (const edge of wasm.edges) {
expect(ids.has(edge.source)).toBe(true);
expect(ids.has(edge.target)).toBe(true);
}
});
});
__tests__/kernel-scaffold.test.ts
+13 -1
View File
@@ -15,7 +15,7 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
import * as fs from 'fs';
import * as path from 'path';
import { NODE_KINDS, EDGE_KINDS } from '../src/types';
import { generateNodeId } from '../src/extraction/tree-sitter-helpers';
import { generateNodeId, NodeIdAllocator } from '../src/extraction/tree-sitter-helpers';
import { getKernel, tryKernelExtract, kernelRoutes, resetKernelForTests } from '../src/extraction/kernel';
import { extractFromSource } from '../src/extraction';
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
@@ -29,6 +29,18 @@ const KERNEL_PATH = path.join(
'codegraph-kernel.node'
);
const kernelBuilt = fs.existsSync(KERNEL_PATH);
it('collision-only identity vectors preserve legacy and repeated-position IDs (#1349)', () => {
const ids = new NodeIdAllocator();
const base = 'function:bfb15544fed707794274a5c61006ea7b';
expect(ids.generate('src/a.ts', 'function', 'foo', 3, 2)).toBe(base);
expect(ids.generate('src/a.ts', 'function', 'foo', 3, 24)).toBe(`${base}:24`);
expect(ids.generate('src/a.ts', 'function', 'foo', 3, 48)).toBe(`${base}:48`);
expect(ids.generate('src/a.ts', 'function', 'foo', 3, 24)).toBe(`${base}:24`);
expect(ids.generate('src/a.ts', 'function', 'foo', 3, 2)).toBe(base);
expect(ids.generate('src/b.ts', 'function', 'foo', 3, 24)).toBe(generateNodeId('src/b.ts', 'function', 'foo', 3));
expect(new NodeIdAllocator().generate('src/a.ts', 'function', 'foo', 3, 24)).toBe(base);
});
const expectKernel = process.env.CODEGRAPH_KERNEL_EXPECT === '1';
const FIXTURE = [
__tests__/kernel-tsjs-parity.test.ts
+14
View File
@@ -96,6 +96,20 @@ describe.skipIf(!kernelBuilt)('kernel TS/JS extraction parity', () => {
return viaWasm;
}
it.each([
['ts', 'typescript'], ['tsx', 'tsx'], ['js', 'javascript'], ['jsx', 'jsx'],
] as const)('same-line accessors retain distinct identities after Unicode: %s (#1349)', (ext, language) => {
const source = 'class Point { /* é😀 */ get x() { return read(); } set x(v) { write(v); } }';
const result = assertParity(`point.${ext}`, source, language);
const x = result.nodes.filter((n) => n.name === 'x');
expect(x).toHaveLength(2);
expect(x[1]!.id).toBe(`${x[0]!.id}:${source.indexOf('set x')}`);
expect(result.unresolvedReferences.filter((r) => r.referenceKind === 'calls').map((r) => [r.fromNodeId, r.referenceName]))
.toEqual([[x[0]!.id, 'read'], [x[1]!.id, 'write']]);
// No state leaks between files or repeated extractions.
expect(canon(assertParity(`point.${ext}`, source, language))).toEqual(canon(result));
});
it.each([
['ts', 'typescript'], ['tsx', 'tsx'], ['js', 'javascript'], ['jsx', 'jsx'],
] as const)('leaves nested identifier receivers unresolved and keeps argument calls: %s (#1566)', (ext, language) => {
codegraph-kernel/src/ccpp/mod.rs
+4 -1
View File
@@ -343,6 +343,7 @@ pub struct Walker<'t> {
variant: Variant,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
nodes_meta: Vec<NodeMeta>,
@@ -388,6 +389,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
variant,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
nodes_meta: Vec::new(),
@@ -495,7 +497,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
// (c/cpp define no resolveBody hook, so createNode's endLine extension
// for sibling-body grammars never fires — endLine is the node's own.)
let end_line = node.end_position().row as u32 + 1;
codegraph-kernel/src/csharp.rs
+4 -1
View File
@@ -130,6 +130,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
/// Node id string per row — ids COLLIDE for same-(kind, name, line) nodes
@@ -162,6 +163,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -317,7 +319,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1; // no resolveBody for csharp
let qualified = {
codegraph-kernel/src/dart.rs
+4 -1
View File
@@ -126,6 +126,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -156,6 +157,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -265,7 +267,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let qualified = {
let mut parts: Vec<&str> = Vec::new();
codegraph-kernel/src/go.rs
+4 -1
View File
@@ -88,6 +88,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
nodes_meta: Vec<NodeMeta>,
@@ -119,6 +120,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
nodes_meta: Vec::new(),
@@ -224,7 +226,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1;
let qualified = extra.qualified_name.unwrap_or_else(|| {
codegraph-kernel/src/ids.rs
+55
View File
@@ -2,6 +2,8 @@
//! `generateNodeId` in `src/extraction/tree-sitter-helpers.ts`:
//!
//! `${kind}:${sha256(`${filePath}:${kind}:${name}:${line}`).hex[0..32]}`
//! NodeIdAllocator appends `:<UTF-16 column>` only for later, distinct source
//! positions sharing that legacy ID within an extraction.
//!
//! and the file-node special case in `TreeSitterExtractor.extract()`:
//!
@@ -12,6 +14,33 @@
//! in `__tests__/kernel-scaffold.test.ts`.
use sha2::{Digest, Sha256};
use std::collections::HashMap;
/// Per-extraction collision handling, matching NodeIdAllocator in the wasm path.
#[derive(Default)]
pub struct NodeIdAllocator {
first_columns: HashMap<String, u32>,
}
impl NodeIdAllocator {
pub fn generate(
&mut self,
file_path: &str,
kind: &str,
name: &str,
line: u32,
column: u32,
) -> String {
let id = node_id(file_path, kind, name, line);
// Zero-based UTF-16 column (util::col16), never tree-sitter's byte column.
let first_column = self.first_columns.entry(id.clone()).or_insert(column);
if *first_column == column {
id
} else {
format!("{id}:{column}")
}
}
}
pub fn node_id(file_path: &str, kind: &str, name: &str, line: u32) -> String {
let mut hasher = Sha256::new();
@@ -41,6 +70,32 @@ pub fn file_node_id(file_path: &str) -> String {
mod tests {
use super::*;
#[test]
fn collision_only_identity_vectors() {
let mut ids = NodeIdAllocator::default();
let base = "function:bfb15544fed707794274a5c61006ea7b";
for (column, expected) in [
(2, base.to_string()),
(24, format!("{base}:24")),
(48, format!("{base}:48")),
(24, format!("{base}:24")),
(2, base.to_string()),
] {
assert_eq!(
ids.generate("src/a.ts", "function", "foo", 3, column),
expected
);
}
assert_eq!(
ids.generate("src/b.ts", "function", "foo", 3, 24),
node_id("src/b.ts", "function", "foo", 3)
);
assert_eq!(
NodeIdAllocator::default().generate("src/a.ts", "function", "foo", 3, 24),
base
);
}
#[test]
fn matches_known_ts_output() {
// Pinned vector: node -e "crypto.createHash('sha256')
codegraph-kernel/src/java.rs
+4 -1
View File
@@ -124,6 +124,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
nodes_meta: Vec<NodeMeta>,
@@ -157,6 +158,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
nodes_meta: Vec::new(),
@@ -300,7 +302,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1; // no resolveBody for java
let qualified = {
codegraph-kernel/src/kotlin.rs
+4 -1
View File
@@ -191,6 +191,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -224,6 +225,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -383,7 +385,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
// endLine extension via resolveBody — LIVE for kotlin function/method
// kinds (in-range for this grammar, so practically a no-op — but the
// hook is part of the contract).
codegraph-kernel/src/lua.rs
+4 -1
View File
@@ -75,6 +75,7 @@ pub struct Walker<'t> {
is_luau: bool,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -103,6 +104,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
is_luau: language == "luau",
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -203,7 +205,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
// buildQualifiedName (1447-1460) — non-file stack NAMES joined `::`;
// the receiver override (extractMethod:1790-1792) replaces it whole.
codegraph-kernel/src/php.rs
+4 -1
View File
@@ -142,6 +142,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -172,6 +173,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -316,7 +318,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1; // no resolveBody for php
let qualified = {
codegraph-kernel/src/python.rs
+4 -1
View File
@@ -141,6 +141,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -171,6 +172,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -276,7 +278,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1;
let qualified = {
codegraph-kernel/src/rlang.rs
+4 -1
View File
@@ -69,6 +69,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
}
@@ -92,6 +93,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
};
@@ -180,7 +182,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
// buildQualifiedName (tree-sitter.ts:1447-1460): non-file stack names
// joined `::` (namespacePrefix is always empty for R).
codegraph-kernel/src/ruby.rs
+4 -1
View File
@@ -118,6 +118,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -148,6 +149,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -257,7 +259,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1; // no resolveBody for ruby
let qualified = {
codegraph-kernel/src/rustlang.rs
+4 -1
View File
@@ -112,6 +112,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
nodes_meta: Vec<NodeMeta>,
@@ -143,6 +144,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
nodes_meta: Vec::new(),
@@ -252,7 +254,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1;
let qualified = extra.qualified_name.unwrap_or_else(|| {
codegraph-kernel/src/scala.rs
+4 -1
View File
@@ -155,6 +155,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -187,6 +188,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -297,7 +299,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
// buildQualifiedName (:1447-1460) — non-file stack names, `::`-joined;
// namespacePrefix always empty (no C++ namespaces, no scala namespace).
codegraph-kernel/src/swift.rs
+4 -1
View File
@@ -152,6 +152,7 @@ pub struct Walker<'t> {
file_path: &'t str,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
node_ids: Vec<String>,
@@ -182,6 +183,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
file_path,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -332,7 +334,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
let end_line = node.end_position().row as u32 + 1; // no resolveBody for swift
let qualified = {
codegraph-kernel/src/tsjs/mod.rs
+4 -1
View File
@@ -167,6 +167,7 @@ pub struct Walker<'t> {
variant: Variant,
line_starts: Vec<usize>,
arena: Arena,
node_id_allocator: ids::NodeIdAllocator,
tables: Tables,
stack: Vec<Scope>,
/// Node id string per row. Rows are unique but IDS COLLIDE for same
@@ -222,6 +223,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
variant,
line_starts: util::line_starts(source),
arena: Arena::default(),
node_id_allocator: ids::NodeIdAllocator::default(),
tables: Tables::default(),
stack: Vec::new(),
node_ids: Vec::new(),
@@ -348,7 +350,8 @@ impl<'t> Walker<'t> {
return None;
}
let start_line = self.line_of(node);
let id = ids::node_id(self.file_path, kind, name, start_line);
let column = self.col_of(node);
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
// endLine body extension: resolveBody only (TS/JS: function-valued
// class fields whose body nests in the arrow / HOF-wrapped arrow).
src/extraction/cfml-extractor.ts
+4 -3
View File
@@ -1,6 +1,6 @@
import type { Node as SyntaxNode } from 'web-tree-sitter';
import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, Language } from '../types';
import { generateNodeId } from './tree-sitter-helpers';
import { generateNodeId, NodeIdAllocator } from './tree-sitter-helpers';
import { TreeSitterExtractor } from './tree-sitter';
import { getParser } from './grammars';
@@ -22,6 +22,7 @@ export class CfmlExtractor {
private source: string;
private language: Language;
private nodes: Node[] = [];
private nodeIds = new NodeIdAllocator();
private edges: Edge[] = [];
private unresolvedReferences: UnresolvedReference[] = [];
private errors: ExtractionError[] = [];
@@ -174,7 +175,7 @@ export class CfmlExtractor {
*/
private extractComponent(openTag: SyntaxNode, containerId: string | undefined): SyntaxNode {
const name = this.tagAttr(openTag, 'name') ?? this.componentNameFromPath();
const id = generateNodeId(this.filePath, 'class', name, openTag.startPosition.row + 1);
const id = this.nodeIds.generate(this.filePath, 'class', name, openTag.startPosition.row + 1, openTag.startPosition.column);
const classNode: Node = {
id,
@@ -261,7 +262,7 @@ export class CfmlExtractor {
if (!name) return;
const kind = parentClassId ? 'method' : 'function';
const id = generateNodeId(this.filePath, kind, name, tag.startPosition.row + 1);
const id = this.nodeIds.generate(this.filePath, kind, name, tag.startPosition.row + 1, tag.startPosition.column);
const access = this.tagAttr(tag, 'access');
const visibility = access === 'private' ? 'private'
: access === 'package' ? 'internal'
src/extraction/liquid-extractor.ts
+8 -7
View File
@@ -1,5 +1,5 @@
import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference } from '../types';
import { generateNodeId } from './tree-sitter-helpers';
import { generateNodeId, NodeIdAllocator } from './tree-sitter-helpers';
/**
* LiquidExtractor - Extracts relationships from Liquid template files
@@ -14,6 +14,7 @@ export class LiquidExtractor {
private filePath: string;
private source: string;
private nodes: Node[] = [];
private nodeIds = new NodeIdAllocator();
private edges: Edge[] = [];
private unresolvedReferences: UnresolvedReference[] = [];
private errors: ExtractionError[] = [];
@@ -217,7 +218,7 @@ export class LiquidExtractor {
const line = this.getLineNumber(match.index);
// Create an import node for searchability
const importNodeId = generateNodeId(this.filePath, 'import', snippetName!, line);
const importNodeId = this.nodeIds.generate(this.filePath, 'import', snippetName!, line, match.index - this.getLineStart(line));
const importNode: Node = {
id: importNodeId,
kind: 'import',
@@ -242,7 +243,7 @@ export class LiquidExtractor {
});
// Create a component node for the snippet reference
const nodeId = generateNodeId(this.filePath, 'component', `${tagType}:${snippetName}`, line);
const nodeId = this.nodeIds.generate(this.filePath, 'component', `${tagType}:${snippetName}`, line, match.index - this.getLineStart(line));
const node: Node = {
id: nodeId,
@@ -289,7 +290,7 @@ export class LiquidExtractor {
const line = this.getLineNumber(match.index);
// Create an import node for searchability
const importNodeId = generateNodeId(this.filePath, 'import', sectionName!, line);
const importNodeId = this.nodeIds.generate(this.filePath, 'import', sectionName!, line, match.index - this.getLineStart(line));
const importNode: Node = {
id: importNodeId,
kind: 'import',
@@ -314,7 +315,7 @@ export class LiquidExtractor {
});
// Create a component node for the section reference
const nodeId = generateNodeId(this.filePath, 'component', `section:${sectionName}`, line);
const nodeId = this.nodeIds.generate(this.filePath, 'component', `section:${sectionName}`, line, match.index - this.getLineStart(line));
const node: Node = {
id: nodeId,
@@ -378,7 +379,7 @@ export class LiquidExtractor {
}
// Create a node for the schema
const nodeId = generateNodeId(this.filePath, 'constant', `schema:${schemaName}`, startLine);
const nodeId = this.nodeIds.generate(this.filePath, 'constant', `schema:${schemaName}`, startLine, match.index - this.getLineStart(startLine));
const node: Node = {
id: nodeId,
@@ -419,7 +420,7 @@ export class LiquidExtractor {
const line = this.getLineNumber(match.index);
// Create a variable node
const nodeId = generateNodeId(this.filePath, 'variable', variableName!, line);
const nodeId = this.nodeIds.generate(this.filePath, 'variable', variableName!, line, match.index - this.getLineStart(line));
const node: Node = {
id: nodeId,
src/extraction/tree-sitter-helpers.ts
+17
View File
@@ -29,6 +29,23 @@ export function generateNodeId(
return `${kind}:${hash}`;
}
/** Per-extraction identities: preserve legacy IDs unless distinct source positions collide. */
export class NodeIdAllocator {
private firstColumns = new Map<string, number>();
generate(filePath: string, kind: NodeKind, name: string, line: number, column: number): string {
const id = generateNodeId(filePath, kind, name, line);
const firstColumn = this.firstColumns.get(id);
if (firstColumn === undefined) {
this.firstColumns.set(id, column);
return id;
}
// Columns are zero-based UTF-16 code units in both wasm and the kernel.
// Revisiting the same declaration must still produce the same identity.
return firstColumn === column ? id : `${id}:${column}`;
}
}
/**
* Extract text from a syntax node
*/
src/extraction/tree-sitter.ts
+3 -2
View File
@@ -16,7 +16,7 @@ import {
UnresolvedReference,
} from '../types';
import { getParser, detectLanguage, isLanguageSupported, isFileLevelOnlyLanguage } from './grammars';
import { generateNodeId, getNodeText, getChildByField, getPrecedingDocstring } from './tree-sitter-helpers';
import { NodeIdAllocator, getNodeText, getChildByField, getPrecedingDocstring } from './tree-sitter-helpers';
import { FN_REF_SPECS, captureFnRefCandidates, type FnRefSpec, type FnRefCandidate } from './function-ref';
import { isGeneratedFile } from './generated-detection';
import type { LanguageExtractor, ExtractorContext } from './tree-sitter-types';
@@ -454,6 +454,7 @@ export class TreeSitterExtractor {
private source: string;
private tree: Tree | null = null;
private nodes: Node[] = [];
private nodeIds = new NodeIdAllocator();
private edges: Edge[] = [];
private unresolvedReferences: UnresolvedReference[] = [];
// Value-reference edges (default ON; set CODEGRAPH_VALUE_REFS=0 to disable; see flushValueRefs).
@@ -1412,7 +1413,7 @@ export class TreeSitterExtractor {
return null;
}
const id = generateNodeId(this.filePath, kind, name, node.startPosition.row + 1);
const id = this.nodeIds.generate(this.filePath, kind, name, node.startPosition.row + 1, node.startPosition.column);
// Some grammars (e.g. Dart) model a function/method body as a *sibling* of
// the signature node, so the declaration node's own range is just the