mirror of
https://github.com/colbymchenry/codegraph.git
synced 2026-10-18 05:41:18 +08:00
Same-kind, same-name declarations on one line shared an ID and silently overwrote each other in SQLite. Use per-extraction collision tracking to append a UTF-16 column only to later distinct declarations. Apply the same rule across wasm, Rust language walkers, Liquid, and CFML while preserving file IDs and edge endpoints. Add persistence, call-edge, identity-vector, and Unicode parity regressions. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -343,6 +343,7 @@ pub struct Walker<'t> {
|
||||
variant: Variant,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
nodes_meta: Vec<NodeMeta>,
|
||||
@@ -388,6 +389,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
|
||||
variant,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
nodes_meta: Vec::new(),
|
||||
@@ -495,7 +497,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
// (c/cpp define no resolveBody hook, so createNode's endLine extension
|
||||
// for sibling-body grammars never fires — endLine is the node's own.)
|
||||
let end_line = node.end_position().row as u32 + 1;
|
||||
|
||||
@@ -130,6 +130,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
/// Node id string per row — ids COLLIDE for same-(kind, name, line) nodes
|
||||
@@ -162,6 +163,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -317,7 +319,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1; // no resolveBody for csharp
|
||||
|
||||
let qualified = {
|
||||
|
||||
@@ -126,6 +126,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -156,6 +157,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -265,7 +267,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
|
||||
let qualified = {
|
||||
let mut parts: Vec<&str> = Vec::new();
|
||||
|
||||
@@ -88,6 +88,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
nodes_meta: Vec<NodeMeta>,
|
||||
@@ -119,6 +120,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
nodes_meta: Vec::new(),
|
||||
@@ -224,7 +226,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1;
|
||||
|
||||
let qualified = extra.qualified_name.unwrap_or_else(|| {
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
//! `generateNodeId` in `src/extraction/tree-sitter-helpers.ts`:
|
||||
//!
|
||||
//! `${kind}:${sha256(`${filePath}:${kind}:${name}:${line}`).hex[0..32]}`
|
||||
//! NodeIdAllocator appends `:<UTF-16 column>` only for later, distinct source
|
||||
//! positions sharing that legacy ID within an extraction.
|
||||
//!
|
||||
//! and the file-node special case in `TreeSitterExtractor.extract()`:
|
||||
//!
|
||||
@@ -12,6 +14,33 @@
|
||||
//! in `__tests__/kernel-scaffold.test.ts`.
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Per-extraction collision handling, matching NodeIdAllocator in the wasm path.
|
||||
#[derive(Default)]
|
||||
pub struct NodeIdAllocator {
|
||||
first_columns: HashMap<String, u32>,
|
||||
}
|
||||
|
||||
impl NodeIdAllocator {
|
||||
pub fn generate(
|
||||
&mut self,
|
||||
file_path: &str,
|
||||
kind: &str,
|
||||
name: &str,
|
||||
line: u32,
|
||||
column: u32,
|
||||
) -> String {
|
||||
let id = node_id(file_path, kind, name, line);
|
||||
// Zero-based UTF-16 column (util::col16), never tree-sitter's byte column.
|
||||
let first_column = self.first_columns.entry(id.clone()).or_insert(column);
|
||||
if *first_column == column {
|
||||
id
|
||||
} else {
|
||||
format!("{id}:{column}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn node_id(file_path: &str, kind: &str, name: &str, line: u32) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -41,6 +70,32 @@ pub fn file_node_id(file_path: &str) -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn collision_only_identity_vectors() {
|
||||
let mut ids = NodeIdAllocator::default();
|
||||
let base = "function:bfb15544fed707794274a5c61006ea7b";
|
||||
for (column, expected) in [
|
||||
(2, base.to_string()),
|
||||
(24, format!("{base}:24")),
|
||||
(48, format!("{base}:48")),
|
||||
(24, format!("{base}:24")),
|
||||
(2, base.to_string()),
|
||||
] {
|
||||
assert_eq!(
|
||||
ids.generate("src/a.ts", "function", "foo", 3, column),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
ids.generate("src/b.ts", "function", "foo", 3, 24),
|
||||
node_id("src/b.ts", "function", "foo", 3)
|
||||
);
|
||||
assert_eq!(
|
||||
NodeIdAllocator::default().generate("src/a.ts", "function", "foo", 3, 24),
|
||||
base
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_known_ts_output() {
|
||||
// Pinned vector: node -e "crypto.createHash('sha256')
|
||||
|
||||
@@ -124,6 +124,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
nodes_meta: Vec<NodeMeta>,
|
||||
@@ -157,6 +158,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
nodes_meta: Vec::new(),
|
||||
@@ -300,7 +302,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1; // no resolveBody for java
|
||||
|
||||
let qualified = {
|
||||
|
||||
@@ -191,6 +191,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -224,6 +225,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -383,7 +385,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
// endLine extension via resolveBody — LIVE for kotlin function/method
|
||||
// kinds (in-range for this grammar, so practically a no-op — but the
|
||||
// hook is part of the contract).
|
||||
|
||||
@@ -75,6 +75,7 @@ pub struct Walker<'t> {
|
||||
is_luau: bool,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -103,6 +104,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
|
||||
is_luau: language == "luau",
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -203,7 +205,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
|
||||
// buildQualifiedName (1447-1460) — non-file stack NAMES joined `::`;
|
||||
// the receiver override (extractMethod:1790-1792) replaces it whole.
|
||||
|
||||
@@ -142,6 +142,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -172,6 +173,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -316,7 +318,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1; // no resolveBody for php
|
||||
|
||||
let qualified = {
|
||||
|
||||
@@ -141,6 +141,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -171,6 +172,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -276,7 +278,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1;
|
||||
|
||||
let qualified = {
|
||||
|
||||
@@ -69,6 +69,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
}
|
||||
@@ -92,6 +93,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
};
|
||||
@@ -180,7 +182,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
|
||||
// buildQualifiedName (tree-sitter.ts:1447-1460): non-file stack names
|
||||
// joined `::` (namespacePrefix is always empty for R).
|
||||
|
||||
@@ -118,6 +118,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -148,6 +149,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -257,7 +259,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1; // no resolveBody for ruby
|
||||
|
||||
let qualified = {
|
||||
|
||||
@@ -112,6 +112,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
nodes_meta: Vec<NodeMeta>,
|
||||
@@ -143,6 +144,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
nodes_meta: Vec::new(),
|
||||
@@ -252,7 +254,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1;
|
||||
|
||||
let qualified = extra.qualified_name.unwrap_or_else(|| {
|
||||
|
||||
@@ -155,6 +155,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -187,6 +188,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -297,7 +299,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
|
||||
// buildQualifiedName (:1447-1460) — non-file stack names, `::`-joined;
|
||||
// namespacePrefix always empty (no C++ namespaces, no scala namespace).
|
||||
|
||||
@@ -152,6 +152,7 @@ pub struct Walker<'t> {
|
||||
file_path: &'t str,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
node_ids: Vec<String>,
|
||||
@@ -182,6 +183,7 @@ pub fn extract(file_path: &str, source: &str) -> Result<EmitOut, String> {
|
||||
file_path,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -332,7 +334,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
let end_line = node.end_position().row as u32 + 1; // no resolveBody for swift
|
||||
|
||||
let qualified = {
|
||||
|
||||
@@ -167,6 +167,7 @@ pub struct Walker<'t> {
|
||||
variant: Variant,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
node_id_allocator: ids::NodeIdAllocator,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
/// Node id string per row. Rows are unique but IDS COLLIDE for same
|
||||
@@ -222,6 +223,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
|
||||
variant,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
node_id_allocator: ids::NodeIdAllocator::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
@@ -348,7 +350,8 @@ impl<'t> Walker<'t> {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
let column = self.col_of(node);
|
||||
let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column);
|
||||
|
||||
// endLine body extension: resolveBody only (TS/JS: function-valued
|
||||
// class fields whose body nests in the arrow / HOF-wrapped arrow).
|
||||
|
||||
Reference in New Issue
Block a user