From 47840a6ed47ada8b10bf28d67b574ccc290cb349 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Mon, 28 Sep 2026 06:20:54 +0000 Subject: [PATCH] fix(explore): return a qualified or line-anchored method's body instead of its signature (#2063) `SQLCompiler.as_sql pre_sql_setup get_select` on django returned get_select and pre_sql_setup in full and SQLCompiler.as_sql -- qualified, step 1 of the Flow, 226 lines -- as one signature line. compiler.py renders as a per-symbol focused view, which picked bodies in source order within a tier, so the unnamed bridge get_qualify_sql and get_select (both above as_sql in the file) took the cap. The follow-ups agents issue (`compiler.py:776`, `compiler.py lines 900-1003`) pinned the file and dropped the line numbers. Every agent run ended in a Read of compiler.py. - query-paths: keep `:776` / `:12-40` / `#L88-L120` suffixes as line anchors and bind prose ranges (`lines 900-1003`, `L900-L1003`, `lines 900 to 1003`) to the nearest single-file path. - EXACT targets: a qualified name resolving to <=3 callables, or the callable enclosing a single-line anchor. Multi-line anchors render as that span. - Focused view: tiers exact > named spine step > unnamed bridge > unique named > family co-named, spine in call order. A tier 0-1 body that does not fit is windowed (head + calls into the question's other symbols) with the explore range that returns each hole, never reduced to a signature. Priced in rendered chars net of the signature lines, so the section keeps its promise. - Cluster path: exact clusters rank first with the spine's bounded overshoot, shrink by exact rendered size, and hold back what lower-ranked exact clusters still owe; files holding one are funded like spine files. - Blast radius leads with exact targets. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 2 + __tests__/explore-exact-target.test.ts | 306 +++++++++++ __tests__/query-paths.test.ts | 60 ++- docs/design/adaptive-explore-sizing.md | 58 ++ src/mcp/tools.ts | 699 +++++++++++++++++-------- src/search/query-paths.ts | 128 ++++- 6 files changed, 1030 insertions(+), 223 deletions(-) create mode 100644 __tests__/explore-exact-target.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cef29ae95..4bcd635d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - When `codegraph_explore` trims a large file, the gaps in its source now name the symbols that were left out, using only room the file's budget has left, so the names never push out the code itself. (#1711) - `codegraph_explore` no longer reports "runtime dispatch" through a base class that doesn't declare the method, so a common name shared by many unrelated subclasses no longer opens an answer with a list of thousands of implementations. - When you name several functions in one file and unrelated code sits between them, `codegraph_explore` now returns the functions you named before the code around them, instead of spending the file's room on that code and leaving one of the functions you asked for with no source at all. +- `codegraph_explore` now returns the body of a method you name together with its class, such as `SQLCompiler.as_sql`, even in a file full of same-named overrides. It used to shrink the method you asked for to its signature line while smaller methods around it came back in full. A method too long for the answer now comes back as its opening lines plus the places it calls the other symbols you named, with the exact follow-up query that returns the rest. +- `codegraph_explore` now honours line numbers in a query: `compiler.py:776` returns the method containing that line, and `compiler.py lines 900-1003` (or `:900-1003`, `#L900-L1003`, `L900-L1003`) returns exactly those lines. It used to pin the file and ignore the numbers. #### MCP / indexing diff --git a/__tests__/explore-exact-target.test.ts b/__tests__/explore-exact-target.test.ts new file mode 100644 index 000000000..d54d36798 --- /dev/null +++ b/__tests__/explore-exact-target.test.ts @@ -0,0 +1,306 @@ +/** + * EXACT targets in `codegraph_explore`: a qualified name (`SQLCompiler.as_sql`) + * or a line anchor (`compiler.py:776`, `compiler.py lines 900-1003`). + * + * The originating gap (django, measured 2026-09-27 on main and on every branch + * since): `SQLCompiler.as_sql pre_sql_setup get_select` returned `get_select` + * and `pre_sql_setup` in full and `SQLCompiler.as_sql` — the qualified name, + * step 1 of the rendered Flow, 226 lines — as ONE signature line. compiler.py + * is a base+subclasses "family" file, so it renders as a per-symbol focused + * view, and that view chose bodies in SOURCE order within a tier: the unnamed + * bridge `get_qualify_sql` and `get_select` sit above `as_sql` in the file, took + * the cap, and the method the agent asked for collapsed. Its follow-ups + * (`SQLCompiler.as_sql full body compiler.py:776`, `compiler.py lines 900-1003`) + * pinned the file but dropped the line numbers, so they returned other methods' + * clusters — and every agent run ended in a Read of compiler.py. + * + * The fixture is that file in miniature (Python, like django): `SQLCompiler` + * with `get_select` and `get_qualify_sql` above a larger `as_sql`, three + * subclasses overriding `as_sql`, and a second family file with a base method + * too big for any budget. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { ToolHandler } from '../src/mcp/tools'; +import CodeGraph from '../src/index'; + +/** Line numbers rendered for `file` in an explore response. */ +function renderedLines(text: string, file: string): Set { + const out = new Set(); + let current: string | null = null; + let inFence = false; + for (const line of text.split('\n')) { + const header = /^\*\*`([^`]+)`\*\*/.exec(line); + if (header && !inFence) { current = header[1]!; continue; } + if (line.startsWith('```')) { inFence = !inFence; continue; } + if (inFence && current === file) { + const m = /^(\d+)\t/.exec(line); + if (m) out.add(Number(m[1])); + } + } + return out; +} + +/** The `**``**` section, header through the line before the next file header. */ +function sectionFor(text: string, file: string): string { + const lines = text.split('\n'); + const start = lines.findIndex((l) => l.startsWith(`**\`${file}\`**`)); + if (start < 0) return ''; + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + if (lines[i]!.startsWith('**`')) { end = i; break; } + } + return lines.slice(start, end).join('\n'); +} + +/** `n` filler statements, each a distinct line so nothing dedups or folds. */ +function filler(tag: string, n: number, indent = ' '): string { + return Array.from({ length: n }, (_, i) => + `${indent}${tag}_${i} = self.query.alias_refcount.get("${tag}_${i}", 0) + len(self.query.select)`, + ).join('\n'); +} + +/** + * Unrelated methods, so compiler.py is a real family file's size (django's is + * 2,291 lines): small enough to ship WHOLE, the file would answer every + * question by accident and hide the gap. + */ +function helpers(n: number): string { + return Array.from({ length: n }, (_, i) => ` + def helper_${i}(self, value): + """Unrelated helper ${i}.""" + first = self.query.alias_map.get(value) + second = self.query.alias_refcount.get(value, 0) + third = self.query.external_aliases.get(value, False) + return first, second, third`).join('\n'); +} + +const COMPILER = 'compiler.py'; +const PLANNER = 'planner.py'; + +describe('codegraph_explore — exact targets (qualified names, line anchors)', () => { + let testDir: string; + let cg: CodeGraph; + let handler: ToolHandler; + const lineOf: Record = {}; + + beforeAll(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-explore-exact-')); + + // compiler.py — get_select and get_qualify_sql ABOVE as_sql, as in django. + const compiler = ` +class SQLCompiler: + def pre_sql_setup(self, with_col_aliases=False): + # PRE_SQL_SETUP_BODY + self.setup_query(with_col_aliases=with_col_aliases) + order_by = self.get_order_by() + return order_by + + def setup_query(self, with_col_aliases=False): + self.select = self.get_select(with_col_aliases=with_col_aliases) + return self.select + + def get_order_by(self): + return [] + + def get_select(self, with_col_aliases=False): + # GET_SELECT_BODY +${filler('sel', 34)} + return [] + + def get_qualify_sql(self): + # GET_QUALIFY_BODY +${filler('qual', 34)} + inner = self.get_select() + return inner + + def as_sql(self, with_limits=True, with_col_aliases=False): + # AS_SQL_HEAD + order_by = self.pre_sql_setup(with_col_aliases=with_col_aliases) +${filler('head', 30)} + result = self.get_qualify_sql() +${filler('tail', 30)} + return result # AS_SQL_TAIL_MARKER + + def execute_sql(self): + sql = self.as_sql() + return sql + + +${helpers(40)} + +def render_sql(compiler): + return SQLCompiler.as_sql(compiler) + + +class SQLInsertCompiler(SQLCompiler): + def as_sql(self): + # INSERT_AS_SQL_BODY + return super().as_sql() + + def execute_sql(self): + sql = self.as_sql() + return sql + + +class SQLUpdateCompiler(SQLCompiler): + def as_sql(self): + # UPDATE_AS_SQL_BODY + return super().as_sql() + + def pre_sql_setup(self): + # UPDATE_PRE_SQL_SETUP_BODY + return super().pre_sql_setup() + + +class SQLDeleteCompiler(SQLCompiler): + def as_sql(self): + # DELETE_AS_SQL_BODY + return super().as_sql() +`.trimStart(); + fs.writeFileSync(path.join(testDir, COMPILER), compiler); + const compilerLines = compiler.split('\n'); + const find = (lines: string[], needle: string) => lines.findIndex((l) => l.includes(needle)) + 1; + lineOf.asSqlDef = find(compilerLines, 'def as_sql(self, with_limits'); + lineOf.asSqlTail = find(compilerLines, 'AS_SQL_TAIL_MARKER'); + lineOf.asSqlTailStart = find(compilerLines, 'result = self.get_qualify_sql()') + 1; + + // planner.py — a family whose base method is too big for ANY budget. It + // calls `stage()` (the spine's next hop) near its END, so a head-only cut + // would lose the very line the flow runs through. + const planner = ` +class Planner: + def plan(self, query): + # PLAN_HEAD +${filler('plan', 420)} + staged = self.stage(query) # PLAN_CALLS_STAGE +${filler('post', 6)} + return staged + + def stage(self, query): + return self.finalize(query) + + def finalize(self, query): + # FINALIZE_BODY + return query + + def explain_plan(self, query): + # EXPLAIN_PLAN_BODY + return str(query) + + +class HashPlanner(Planner): + def plan(self, query): + return super().plan(query) + + +class MergePlanner(Planner): + def plan(self, query): + return super().plan(query) + + +class NestedPlanner(Planner): + def plan(self, query): + return super().plan(query) +`.trimStart(); + fs.writeFileSync(path.join(testDir, PLANNER), planner); + lineOf.planCallsStage = find(planner.split('\n'), 'PLAN_CALLS_STAGE'); + + // Other `as_sql`s, so the bare name is a family (as django's 110 are). + fs.writeFileSync(path.join(testDir, 'lookups.py'), ` +class Exact: + def as_sql(self, compiler, connection): + return "%s = %s" + + +class IExact: + def as_sql(self, compiler, connection): + return "UPPER(%s) = UPPER(%s)" +`.trimStart()); + + cg = CodeGraph.initSync(testDir, { config: { include: ['**/*.py'], exclude: [] } }); + await cg.indexAll(); + handler = new ToolHandler(cg); + }); + + afterAll(() => { + if (cg) cg.destroy(); + if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true }); + }); + + const explore = async (query: string): Promise => { + const result = await handler.execute('codegraph_explore', { query }); + return result.content?.[0]?.text ?? ''; + }; + + it('fixture sanity: the flow runs as_sql → get_qualify_sql → get_select and compiler.py goes focused', async () => { + const text = await explore('SQLCompiler.as_sql pre_sql_setup get_select'); + expect(text).toContain('**Flow (call path among the symbols you queried)'); + expect(text).toMatch(/1\. as_sql \(compiler\.py:\d+\)[\s\S]*2\. get_qualify_sql[\s\S]*3\. get_select/); + expect(sectionFor(text, COMPILER), 'the family file renders as a per-symbol view').toContain('· focused'); + }); + + it('returns the qualified method\'s WHOLE body, not its signature line', async () => { + const text = await explore('SQLCompiler.as_sql pre_sql_setup get_select'); + const section = sectionFor(text, COMPILER); + const lines = renderedLines(text, COMPILER); + for (let ln = lineOf.asSqlDef!; ln <= lineOf.asSqlTail!; ln++) { + expect(lines.has(ln), `as_sql line ${ln} rendered`).toBe(true); + } + expect(section).toContain('AS_SQL_TAIL_MARKER'); + // The other named step still gets its body too. + expect(section).toContain('GET_SELECT_BODY'); + }); + + it('keeps the family skeleton for the subclasses: their as_sql overrides stay signatures', async () => { + const text = await explore('SQLCompiler.as_sql pre_sql_setup get_select'); + const section = sectionFor(text, COMPILER); + expect(section).not.toContain('INSERT_AS_SQL_BODY'); + expect(section).not.toContain('DELETE_AS_SQL_BODY'); + }); + + it('leads the blast radius with the qualified method, not a same-named override', async () => { + const text = await explore('SQLCompiler.as_sql pre_sql_setup get_select'); + const blast = text.slice(text.indexOf('**Blast radius'), text.indexOf('**Relationships**')); + const first = blast.split('\n').find((l) => l.startsWith('- `')); + expect(first).toContain(`\`as_sql\` (${COMPILER}:${lineOf.asSqlDef})`); + }); + + it('a `file:line` anchor returns the method enclosing that line, whole', async () => { + // No symbol named — the line alone has to say which method. + const text = await explore(`${COMPILER}:${lineOf.asSqlDef! + 40} full body`); + const lines = renderedLines(text, COMPILER); + for (let ln = lineOf.asSqlDef!; ln <= lineOf.asSqlTail!; ln++) { + expect(lines.has(ln), `as_sql line ${ln} rendered`).toBe(true); + } + }); + + it('a `lines A-B` range returns exactly that span of the pinned file', async () => { + // The tail of `as_sql` — the span a windowed render elides and the agent + // then asks for by number (django's `lines 900-1003 as_sql tail`). + const a = lineOf.asSqlTailStart!; + const b = lineOf.asSqlTail!; + const text = await explore(`${COMPILER} lines ${a}-${b} tail`); + const lines = renderedLines(text, COMPILER); + for (let ln = a; ln <= b; ln++) expect(lines.has(ln), `line ${ln} rendered`).toBe(true); + }); + + it('windows an exact body too big for the budget — head AND the call the flow runs through — and names the rest by range', async () => { + const text = await explore('Planner.plan finalize explain_plan'); + expect(text).toContain('**Flow (call path among the symbols you queried)'); + const section = sectionFor(text, PLANNER); + const lines = renderedLines(text, PLANNER); + expect(section).toContain('PLAN_HEAD'); + // The spine's next-hop call sits ~420 lines into the body: a head-only cut + // would drop it. + expect(lines.has(lineOf.planCallsStage!), 'the call into stage() is rendered').toBe(true); + // Windowed, not dumped: most of the 430-line body is elided... + expect(lines.size).toBeLessThan(300); + // ...and each hole is named with the explore query that returns it. + expect(section).toMatch(/lines \d+-\d+ of `plan` elided — codegraph_explore `planner\.py:\d+-\d+` returns them/); + // Steered to explore, never to Read (the tag's own "do NOT Read" aside). + expect(section.replace(/do NOT Read/g, '')).not.toMatch(/\bRead\b/); + }); +}); diff --git a/__tests__/query-paths.test.ts b/__tests__/query-paths.test.ts index 06ea29a5e..956c33c85 100644 --- a/__tests__/query-paths.test.ts +++ b/__tests__/query-paths.test.ts @@ -151,7 +151,7 @@ describe('extractQueryPaths — resolution and stripping', () => { it('passes through untouched when nothing resolves', () => { const q = 'plain prose question about scrolling'; const out = extractQueryPaths(q, INDEX); - expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [], lineAnchors: [] }); }); }); @@ -191,13 +191,13 @@ describe('extractQueryPaths — extension-less kebab basenames', () => { it('leaves kebab prose that names no indexed file untouched — and unreported', () => { const q = 'how does cross-call dedup make explore non-blocking'; const out = extractQueryPaths(q, INDEX); - expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [], lineAnchors: [] }); }); it('leaves a stem shared by too many files alone — one hot name must not pin half the repo', () => { const q = 'refactor the user-profile rendering'; const out = extractQueryPaths(q, INDEX); - expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [], lineAnchors: [] }); }); it('pins all files sharing a stem when within the ambiguity budget', () => { @@ -317,3 +317,57 @@ describe('extractQueryPaths — dotless slashed spans, decided on disk', () => { expect(p.asked).not.toContain('scripts/pre-commit'); }); }); + +describe('extractQueryPaths — line anchors', () => { + // The django follow-ups that went unanswered: an agent handed a signature for + // a 226-line method asks for the body by line, and explore used to pin the + // file but drop the lines — answering "the file's most relevant clusters" + // instead of the span it named. + const CHAT = 'src/lib/chat-manager.ts'; + + it('keeps a :line / :start-end / #L suffix as an anchor on the pinned file', () => { + expect(extractQueryPaths(`body of ${CHAT}:776`, INDEX).lineAnchors) + .toEqual([{ file: CHAT, start: 776, end: 776 }]); + expect(extractQueryPaths(`see ${CHAT}:12-40`, INDEX).lineAnchors) + .toEqual([{ file: CHAT, start: 12, end: 40 }]); + expect(extractQueryPaths('regression at src/lib/task-runner-manager.ts#L88-L120', INDEX).lineAnchors) + .toEqual([{ file: 'src/lib/task-runner-manager.ts', start: 88, end: 120 }]); + }); + + it('binds a prose line range to the path it sits next to and removes it from the query', () => { + const out = extractQueryPaths(`${CHAT} lines 900-1003 flushQueue tail`, INDEX); + expect(out.lineAnchors).toEqual([{ file: CHAT, start: 900, end: 1003 }]); + // `lines` would feed FTS a word every file holds; the numbers match nothing. + expect(out.strippedQuery).toBe('flushQueue tail'); + }); + + it('accepts the other spellings agents write', () => { + const anchor = (q: string) => extractQueryPaths(q, INDEX).lineAnchors; + expect(anchor(`lines 900 to 1003 of ${CHAT}`)).toEqual([{ file: CHAT, start: 900, end: 1003 }]); + expect(anchor(`L900-L1003 in ${CHAT}`)).toEqual([{ file: CHAT, start: 900, end: 1003 }]); + expect(anchor(`${CHAT} line 42`)).toEqual([{ file: CHAT, start: 42, end: 42 }]); + expect(anchor(`${CHAT} 900-1003`)).toEqual([{ file: CHAT, start: 900, end: 1003 }]); + expect(anchor(`${CHAT} (lines 1003-900)`)).toEqual([{ file: CHAT, start: 900, end: 1003 }]); + }); + + it('binds each range to its NEAREST path when the query names two files', () => { + const out = extractQueryPaths( + `${CHAT} lines 10-20 and src/lib/task-runner-manager.ts lines 30-40`, INDEX, + ); + expect(out.lineAnchors).toEqual([ + { file: CHAT, start: 10, end: 20 }, + { file: 'src/lib/task-runner-manager.ts', start: 30, end: 40 }, + ]); + }); + + it('leaves line numbers alone when there is no path, or the path is ambiguous', () => { + const noPath = extractQueryPaths('flushQueue lines 900-1003', INDEX); + expect(noPath.lineAnchors).toEqual([]); + // A bare number with no line context is not a line number either. + expect(extractQueryPaths(`${CHAT} retries 3 times`, INDEX).lineAnchors).toEqual([]); + // `generic-modal` pins two files; a line number means nothing across both. + const twoFiles = extractQueryPaths('generic-modal.tsx:40', INDEX); + expect(twoFiles.pinnedFiles).toHaveLength(2); + expect(twoFiles.lineAnchors).toEqual([]); + }); +}); diff --git a/docs/design/adaptive-explore-sizing.md b/docs/design/adaptive-explore-sizing.md index 5c72ef199..9e8135da2 100644 --- a/docs/design/adaptive-explore-sizing.md +++ b/docs/design/adaptive-explore-sizing.md @@ -56,6 +56,57 @@ source. > first, so the budget cap never binds it). OkHttp's interceptors stay a pure > signature skeleton (no named callable in them, don't define a supertype). +> **Refinement (2026-09-28) — exact targets.** The per-symbol view above still +> lost the one method the agent asked for by its qualified name. On a fresh django +> index `SQLCompiler.as_sql pre_sql_setup get_select` returned `get_select` and +> `pre_sql_setup` in full and `SQLCompiler.as_sql` — step 1 of the rendered Flow, +> 226 lines — as its signature line, on main and on every branch since. Bodies +> were chosen in SOURCE order within a tier, so the unnamed bridge +> `get_qualify_sql` and `get_select` (both above `as_sql` in the file) took the +> cap. The agents' follow-ups (`SQLCompiler.as_sql full body compiler.py:776`, +> `compiler.py lines 900-1003 as_sql tail`) pinned the file but dropped the line +> numbers, returning `SQLInsertCompiler.as_sql` and friends instead; every run +> Read compiler.py. The fix: +> 1. **Exact targets** — a qualified name that resolves to ≤3 callables +> (`SQLCompiler.as_sql`: 1 of 110), or the callable enclosing a line anchor +> (`compiler.py:776`). A bare `as_sql` is NOT exact, so the family rules above +> still keep the 110 overrides as signatures. +> 2. **Tiers in the focused view:** exact → spine step the agent named → unnamed +> spine bridge → uniquely-named off-spine → family co-named; the spine in CALL +> order within its tier. +> 3. **Window, never signature, for tiers 0–1.** A body too big for what is left +> is cut to its head plus the lines where it calls the question's other +> symbols, and each hole names the explore query that returns it +> (`codegraph_explore \`compiler.py:871-1001\``). +> 4. **Line anchors** (`query-paths.ts`): `file:line`, `file:a-b`, `#La-Lb`, and +> prose `lines a-b` / `L a-L b` next to a path. A range renders as exactly that +> span (cluster path, ranked first). +> 5. **Priced in rendered chars.** The view counted raw source while it rendered +> line numbers and ~20 signature lines; once windows fill the cap the +> undercount became a 19% overshoot that zeroed four excalidraw files below it. +> +> Deterministic replay (54 queries × 8 repos): 43 byte-identical; `as_sql` +> complete in all 5 distinct django queries that single it out (was 1, 0, 175, +> 0 and 226 of 226 lines), the `lines 900-1003` span 104/104 (was 1); named-body +> lines 8,706 → 9,443. The trade: where the file's budget cannot hold everything, a large named +> step now wins over several small off-spine bodies (`pb-compiler`: `as_sql` whole, +> `execute_sql`/`results_iter` to signatures). +> +> Agent A/B (`ab-new-vs-baseline.sh`, django, README question pair, Sonnet / +> `--effort high`, RUNS=3, baseline `e63fe2ec`): **Read 0 in 3/3 new runs vs a +> Read of compiler.py in 2/3 baseline runs** — each right after the agent's own +> `SQLCompiler.as_sql … compiler.py:776` explore, the exact follow-up the report +> described. Tool calls 3 → 4 median, retrieval residual 19.5K vs 22.9K tokens, +> median cost $0.195 vs $0.229 and 30s vs 35s (n=3, the last two within noise). +> +> Rebased onto #2062 (protected members paid before incidental ones across +> clusters), which leaves this gap open — `as_sql` 1/226 on that main. The two +> compose: #2062's protected/incidental rule is the cluster path's base, and +> exact targets sit on top of it (ranked first, every member priced in rendered +> chars, held back for lower exact clusters). Re-measured against that main, 59 +> queries: 43 byte-identical, named-body lines 9,804 → 10,731, every target span +> above complete. + --- ## TL;DR @@ -260,8 +311,15 @@ the agent's real query — that mismatch is what this refinement corrects.) `definesPolymorphicSupertype()` (a node HAS ≥3 incoming `implements`/`extends` — i.e. the file is the family base). The skeleton branch: `off-spine && isPolymorphicSibling && !(namedInFile && !definesSupertype)`. + `exactNodeIds` / `anchorSpans` (the exact targets) and `bodyFocusLines()` + (what a window of an oversize body must reach) are shared by the focused + view and the cluster path. +- `src/search/query-paths.ts` — `lineAnchors`: `file:line` / range parsing. - `__tests__/adaptive-explore-sizing.test.ts` — 7 cases incl. the named-callable spare (RealCall) and the supertype-family override (compiler.py). +- `__tests__/explore-exact-target.test.ts` — the django shape in miniature: + qualified body whole, subclass overrides still signatures, blast-radius lead, + `file:line` and `lines a-b` anchors, and a windowed body keeping its tail call. ## Frontier / future work diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 850e1644b..21a4bbdf0 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -34,7 +34,7 @@ import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; import { isTestFile, normalizeNameToken } from '../search/query-utils'; import { groupDefinitions, lastQualifierPart, matchesSymbol } from '../graph/symbol-lookup'; -import { extractQueryPaths, queryMightContainPaths } from '../search/query-paths'; +import { extractQueryPaths, queryMightContainPaths, type QueryLineAnchor } from '../search/query-paths'; import { existsSync, readFileSync, @@ -679,7 +679,10 @@ export interface ExploreAllocationCandidate { * self-referential generated file scores well on both. */ worth: number; - /** Carries a symbol on the rendered flow spine. */ + /** + * Carries a symbol on the rendered flow spine — or an exact target (a + * qualified name, a line anchor), which is the answer by the same argument. + */ spine: boolean; /** * The query named this file by PATH (see query-paths.ts). Pinned files are @@ -902,6 +905,23 @@ const POINTER_MAX_FILES = 10; */ const ELIDED_SYMBOL_CAP = 6; +/** + * Kinds a single-line anchor (`compiler.py:776`) resolves to: the innermost + * one containing the line is the symbol the agent is pointing at. A class + * enclosing the line is deliberately NOT a target — it spans most of its file, + * and "the whole class" is not what a line number asks for. + */ +const ANCHOR_CALLABLE_KINDS = new Set(['method', 'function', 'constructor', 'component']); +/** Lines either side of a single-line anchor that no callable encloses. */ +const ANCHOR_LINE_CONTEXT = 15; + +/** + * Most windows one oversize exact / named-spine body is cut into in the + * focused view: the head plus the call sites into the other symbols the + * question named. More turns a method into confetti. + */ +const MAX_BODY_FOCUS_LINES = 6; + type ElidedSymbolRef = { name: string; kind: string; startLine: number }; /** @@ -3441,7 +3461,17 @@ export class ToolHandler { * that have no dependents (nothing to warn about), and returns '' when none * qualify so a leaf-only exploration stays clean. */ - private buildBlastRadiusSection(cg: CodeGraph, subgraph: Subgraph): string { + private buildBlastRadiusSection( + cg: CodeGraph, + subgraph: Subgraph, + /** + * Exact targets (a qualified name, a line anchor) lead the list. The search + * roots are whatever FTS ranked first for the bare name, so without this a + * query for `SQLCompiler.as_sql` headlined `SQLInsertCompiler.as_sql` — and + * the agent took that as the tool having found the wrong method. + */ + leadingIds: Iterable = [], + ): string { const ROOT_CAP = 5; // only the symbols the query actually targeted const FILE_CAP = 4; // caller files listed per symbol before "+N more" const MEANINGFUL = new Set([ @@ -3450,7 +3480,7 @@ export class ToolHandler { ]); const rel = (p: string) => p.replace(/\\/g, '/'); - const roots = subgraph.roots + const roots = [...new Set([...leadingIds, ...subgraph.roots])] .map((id) => subgraph.nodes.get(id)) .filter((n): n is Node => !!n && MEANINGFUL.has(n.kind)) .slice(0, ROOT_CAP); @@ -3656,6 +3686,7 @@ export class ToolHandler { // `+page.svelte`), starving the very files the agent asked for. let pinnedFiles: string[] = []; let unresolvedPathSpans: string[] = []; + let lineAnchors: QueryLineAnchor[] = []; let matchQuery = query; if (queryMightContainPaths(rawQuery)) { try { @@ -3667,6 +3698,7 @@ export class ToolHandler { if (extraction.pinnedFiles.length > 0 || extraction.unresolvedPathSpans.length > 0) { pinnedFiles = extraction.pinnedFiles; unresolvedPathSpans = extraction.unresolvedPathSpans; + lineAnchors = extraction.lineAnchors; matchQuery = normalizeQuerySpelling(extraction.strippedQuery); } } catch { /* path pinning must never fail an explore call */ } @@ -3752,6 +3784,42 @@ export class ToolHandler { .forEach((n) => { if (!subgraph.nodes.has(n.id)) subgraph.nodes.set(n.id, n); }); } + // EXACT targets: callables the agent pointed at with no ambiguity left — a + // qualified name that resolves to at most a handful of defs + // (`SQLCompiler.as_sql`, filled in by the seeding loop below) or the + // callable enclosing a line anchor (`compiler.py:776`). Every render path + // puts these first and never reduces one to a signature line: the agent + // has already said which of the 110 `as_sql`s it means, and an answer that + // spends the file on the named method's neighbours instead is the one it + // Reads the file to get past. + const exactNodeIds = new Set(); + // A multi-line anchor (`lines 900-1003`, `foo.ts:12-40`) names a SPAN, not + // a symbol — often the tail of a body a previous response windowed — and + // is rendered as exactly that span. A single-line anchor with no enclosing + // callable (a module-level line) becomes a small span around the line. + const anchorSpans = new Map(); + for (const anchor of lineAnchors) { + let fileNodes: Node[] = []; + try { fileNodes = cg.getNodesInFile(anchor.file); } catch { continue; } + if (anchor.start === anchor.end) { + const enclosing = fileNodes + .filter((n) => ANCHOR_CALLABLE_KINDS.has(n.kind) + && n.startLine <= anchor.start && n.endLine >= anchor.start) + .sort((a, b) => (a.endLine - a.startLine) - (b.endLine - b.startLine))[0]; + if (enclosing) { + exactNodeIds.add(enclosing.id); + if (!subgraph.nodes.has(enclosing.id)) subgraph.nodes.set(enclosing.id, enclosing); + continue; + } + } + const span = anchor.start === anchor.end + ? { start: Math.max(1, anchor.start - ANCHOR_LINE_CONTEXT), end: anchor.start + ANCHOR_LINE_CONTEXT } + : { start: anchor.start, end: anchor.end }; + const spans = anchorSpans.get(anchor.file) ?? []; + spans.push(span); + anchorSpans.set(anchor.file, spans); + } + if (subgraph.nodes.size === 0) { diag?.finishEmpty('no relevant code found — empty subgraph'); const missNote = unresolvedPathSpans.length > 0 @@ -4040,6 +4108,11 @@ export class ToolHandler { // named-file sort below. (Previously only NEW injections were marked, // so a named symbol FTS already gathered never sorted to the top.) namedSeedIds.add(n.id); + // A QUALIFIED name that resolved to a handful of defs is the agent + // choosing one overload out of a family — `SQLCompiler.as_sql`, not + // the 109 other `as_sql`s. The bare name keeps the family rules; the + // qualified one is an exact target (see `exactNodeIds`). + if (isQual && cands.length <= 3 && CALLABLE.has(n.kind)) exactNodeIds.add(n.id); } // An interface's `method_signature` seeds (so RWR and the flow ranking // still see it, and a query that names it still reaches its file) but @@ -4578,7 +4651,7 @@ export class ToolHandler { // Blast radius (always-on, compact): for the entry symbols, who depends on // them + which tests cover them — locations only, no source — so the agent // knows what to update/verify before editing without a separate call. - const blastRadius = this.buildBlastRadiusSection(cg, subgraph); + const blastRadius = this.buildBlastRadiusSection(cg, subgraph, exactNodeIds); if (blastRadius) lines.push(blastRadius); // Relationship map — show how symbols connect @@ -4625,6 +4698,33 @@ export class ToolHandler { await warmBranchGuardGrammars(); const flow = this.buildFlowFromNamedSymbols(cg, matchQuery); + // The symbols the question is about: exact targets, the named ones, the spine. + const questionIds = new Set([...exactNodeIds, ...flow.pathNodeIds, ...flow.namedNodeIds]); + /** + * The lines a WINDOW of an oversize body must reach: an anchored line + * inside it, the spine's next-hop call, and every call it makes into + * another symbol the question is about — "the parts the question is + * about", in the graph's own terms rather than by matching words. Most + * important first, capped so a window stays a window. + */ + const bodyFocusLines = (n: Node): number[] => { + const focus: number[] = []; + for (const a of lineAnchors) { + if (a.file === n.filePath && a.start > n.startLine && a.start <= n.endLine) focus.push(a.start); + } + const call = flow.spineCallSites.get(n.id); + if (call) focus.push(call); + const calls: number[] = []; + try { + for (const e of cg.getOutgoingEdges(n.id)) { + if (e.target === n.id || !questionIds.has(e.target) || !e.line) continue; + if (e.line > n.startLine && e.line <= n.endLine) calls.push(e.line); + } + } catch { /* focus is a nicety; the head window still renders */ } + focus.push(...calls.sort((a, b) => a - b)); + return [...new Set(focus)].slice(0, MAX_BODY_FOCUS_LINES); + }; + // Snapshot every ranked candidate's scoring inputs, in final sort order, so // the diagnostic can show what each file's share of the envelope was BOUGHT // with (score, graph mass, term hits, flags) — not just what it cost. @@ -4668,7 +4768,11 @@ export class ToolHandler { // A pinned file's bytes are worth full price by definition — the agent // asked for the file itself, generated/test or not. worth: pinnedSet.has(fp) ? 1 : rankPenalty(fp), - spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), + // A file holding an EXACT target is funded like a spine file: it is the + // answer to the question as asked, and clipping it is what sends the + // agent to Read — the exact reason the spine gets this treatment. + spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id) || exactNodeIds.has(n.id)) + || anchorSpans.has(fp), pinned: pinnedSet.has(fp), })), budget, @@ -5146,6 +5250,171 @@ export class ToolHandler { } }; + // Window helpers — shared by the focused per-symbol view and the cluster + // path below, so an oversize body is cut by ONE set of rules wherever it + // renders. Parts are SPAN-KEYED: every part carries the exact line range + // its text was sliced from (see `buildSection`). + const SPINE_WINDOW = 28; // lines each side of a focus line (the next-hop call site) + type SectionPart = { range: ExploreLineRange; text: string }; + /** + * Bounded overshoot for one cluster's render (CG-30). + * + * `shrinkCluster` keeps the highest-importance member whole even when that + * member alone is oversize — an empty file section sends the agent to Read, + * which is exactly what explore exists to prevent. But "never empty" is not + * "any size": with nothing bounding it, one 22K member rendered against a + * 9K reservation (2.4x), which collapses the headroom every file ranked + * below it draws from. Past the ceiling the member is WINDOWED rather than + * dropped — a leading window (signature + head of the body), plus a window + * on the spine's call site when the head misses it, since on a flow cluster + * the call path IS the answer. + */ + const MIN_WINDOW_LINES = 12; + /** Rendered cost of one source line, line numbering included. */ + const lineCost = (ln: number): number => + (fileLines[ln - 1] ?? '').length + 1 + (withLineNumbers ? String(ln).length + 1 : 0); + /** `lineCostPrefix[n]` = rendered chars of lines 1..n, newlines included. */ + let lineCostPrefix: number[] | null = null; + /** Rendered chars of lines `start..end` (line numbers included), in O(1). */ + const renderedSpanCost = (start: number, end: number): number => { + if (!lineCostPrefix) { + lineCostPrefix = [0]; + for (let ln = 1; ln <= fileLines.length; ln++) lineCostPrefix.push(lineCostPrefix[ln - 1]! + lineCost(ln)); + } + const s = Math.max(1, start); + const e = Math.min(fileLines.length, end); + return e < s ? 0 : Math.max(0, lineCostPrefix[e]! - lineCostPrefix[s - 1]! - 1); + }; + /** + * Longest prefix of `r` that fits `room`. `minLines` is the never-empty + * floor — it may overrun `room`, so it is only ever asked for when nothing + * else has been emitted and the alternative is an empty section. + */ + const headWindowOf = ( + r: ExploreLineRange, room: number, minLines = 0, + ): ExploreLineRange | null => { + let end = r.start - 1; + let chars = 0; + for (let ln = r.start; ln <= r.end; ln++) { + const cost = lineCost(ln); + if (chars + cost > room && end - r.start + 1 >= minLines) break; + chars += cost; + end = ln; + } + return end >= r.start ? { start: r.start, end } : null; + }; + /** Widest window around `line` inside [lo, hi] that fits `room`. */ + const centeredWindowOf = ( + line: number, lo: number, hi: number, room: number, + ): ExploreLineRange | null => { + if (line < lo || line > hi) return null; + let start = line, end = line, chars = lineCost(line); + for (let grown = true; grown;) { + grown = false; + if (end + 1 <= hi && chars + lineCost(end + 1) <= room) { end += 1; chars += lineCost(end); grown = true; } + if (start - 1 >= lo && chars + lineCost(start - 1) <= room) { start -= 1; chars += lineCost(start); grown = true; } + } + return { start, end }; + }; + /** + * Reduce rendered parts to fit `ceiling`, never to nothing. Whole parts are + * kept while they fit; the first part that overruns is cut to a leading + * window on whole lines (a body is never cut mid-line), and everything past + * it is dropped. The GAP_MARKER between surviving parts — and the line-number + * jump — is what tells the agent the cut happened. + * + * A partial window shorter than MIN_WINDOW_LINES is not worth emitting, and + * emitting one is actively harmful: the session record then claims a 4-line + * sliver, and the NEXT call's dedup has to either shred a whole block around + * it or re-send it. Below that floor the part is simply dropped — unless + * nothing has been emitted at all, where the floor wins over the ceiling + * because an empty section is the one outcome worse than an oversize one. + * + * `focusLines` are the lines this trim must not lose: the spine's next-hop + * call site (CG-30) and every definition the agent NAMED inside the cluster + * (CG-38). The head fill is source-ordered, so a named def in the TAIL of a + * large file is otherwise always the first thing an over-ceiling render + * drops — the one span the agent asked for by name, cut in favour of + * head-of-file filler it did not ask for. The full-ceiling fill is tried + * FIRST and the 60% hold-back applies only when a focus line is actually + * left uncovered, so a cluster whose head already reaches its focus keeps + * the whole ceiling for source. + */ + const windowToCeiling = ( + parts: ReadonlyArray, + ceiling: number, + focusLines: ReadonlyArray = [], + ): SectionPart[] => { + const inParts = (line: number) => + parts.some((p) => line >= p.range.start && line <= p.range.end); + const focus = [...new Set(focusLines)] + .filter((l) => typeof l === 'number' && l > 0 && inParts(l)) + .sort((a, b) => a - b); + /** Source-ordered fill of whole parts, the overrunning one cut to a head window. */ + const fill = (room: number): { emit: ExploreLineRange[]; used: number } => { + const emit: ExploreLineRange[] = []; + let used = 0; + for (const p of parts) { + const join = emit.length > 0 ? GAP_MARKER.length : 0; + if (used + join + p.text.length <= room) { + emit.push(p.range); + used += join + p.text.length; + continue; + } + const first = emit.length === 0; + const win = headWindowOf( + p.range, Math.max(0, room - used - join), first ? MIN_WINDOW_LINES : 0); + if (win && (first || win.end - win.start + 1 >= MIN_WINDOW_LINES)) { + emit.push(win); + used += join + renderSpan(win).length; + } + break; + } + return { emit, used }; + }; + let { emit, used } = fill(ceiling); + const reached = () => (emit.length ? emit[emit.length - 1]!.end : 0); + if (focus.some((l) => l > reached())) { + // Hold room back for the focus windows so the head can't eat all of it. + ({ emit, used } = fill(Math.floor(ceiling * 0.6))); + } + // What is left is SPLIT between the uncovered focus lines rather than + // handed to them in order. Greedy-in-source-order reproduces the very bug + // this guards: on a prose query resolving four focus lines, the two + // earliest took the whole reserve and `flushQueuedMessages` at L1102 — + // named in the question — was dropped again. A skipped or undersized + // window returns its share to the pool for the ones after it. + let covered = reached(); + let room = Math.max(0, ceiling - used); + const pending = focus.filter((l) => l > covered); + for (let i = 0; i < pending.length; i++) { + const line = pending[i]!; + if (line <= covered) continue; // an earlier window already reached it + const share = Math.floor(room / (pending.length - i)) - GAP_MARKER.length; + if (share <= 0) continue; + const host = parts.find((p) => line >= p.range.start && line <= p.range.end)!; + const lo = Math.max(host.range.start, line - SPINE_WINDOW, covered + 1); + const hi = Math.min(host.range.end, line + SPINE_WINDOW); + const win = centeredWindowOf(line, lo, hi, share); + // Same sliver floor as the head window — a two-line peek at the call + // site teaches the next call's dedup to shred the block around it. + if (!win || win.end - win.start + 1 < MIN_WINDOW_LINES) continue; + emit.push(win); + const cost = GAP_MARKER.length + renderSpan(win).length; + used += cost; + room -= cost; + covered = win.end; + } + // Never empty: a section with no source sends the agent to Read. + if (emit.length === 0 && parts.length > 0) { + const first = headWindowOf(parts[0]!.range, ceiling, MIN_WINDOW_LINES); + if (first) emit.push(first); + } + return emit + .sort((a, b) => a.start - b.start) + .map((r) => ({ range: r, text: renderSpan(r) })); + }; + // Disk-drift gate (#1474): every render branch below except whole-file // slices fileContent (CURRENT bytes) at INDEXED line ranges. Content is // already in hand, so the check costs one stat (hash only on mismatch). @@ -5189,7 +5458,11 @@ export class ToolHandler { const onSpineGodFile = hasSpineNode && namedBodyChars > allowance && group.nodes.some(n => CALLABLE_BODY.has(n.kind) && flow.uniqueNamedNodeIds.has(n.id) && !flow.pathNodeIds.has(n.id)); + // A line RANGE the agent asked for (`lines 900-1003`) is not a symbol, so + // the per-symbol view has no way to show it — such a file takes the + // cluster path, which renders anchor spans first. if (!fileStale && adaptiveExploreEnabled() && flow.pathNodeIds.size > 0 + && !anchorSpans.has(filePath) && (onSpineGodFile || (!hasSpineNode && isPolymorphicSibling(group.nodes) && !spared))) { const syms = group.nodes .filter(n => n.kind !== 'import' && n.kind !== 'export' && n.startLine > 0) @@ -5197,33 +5470,92 @@ export class ToolHandler { // Pass 1: choose which symbols get a FULL body, by priority, greedily within // a per-file body cap — so one huge family file can't body every named method // and crowd out the other flow files (Django's query.py). A symbol earns a - // body if it's on-spine, or UNIQUELY named (`SQLCompiler.execute_sql`), or a - // co-named method WHEN this file DEFINES the family supertype (so the base - // `SQLCompiler.as_sql` body shows, but the 110 leaf `as_sql` overrides — and - // OkHttp's 5 `intercept`s if the agent names `intercept` — stay signatures). + // body if it's EXACT, on-spine, UNIQUELY named (`SQLCompiler.execute_sql`), + // or a co-named method WHEN this file DEFINES the family supertype (so the + // base `SQLCompiler.as_sql` body shows, but the 110 leaf `as_sql` overrides — + // and OkHttp's 5 `intercept`s if the agent names `intercept` — stay + // signatures). Tiers, most-wanted first: + // 0 EXACT — a qualified name or a line anchor (see `exactNodeIds`), + // 1 a spine step the agent NAMED, + // 2 a spine step it did not (the one unnamed bridge between two named), + // 3 a UNIQUELY named off-spine method, + // 4 a co-named method in the family's base file. + // Within a tier the spine keeps its CALL order, then source order. Source + // order alone let the smaller steps that happen to sit higher in the file + // take the cap: Django's `SQLCompiler.as_sql` — named, qualified, step 1 of + // the flow, 226 lines — lost to the unnamed bridge `get_qualify_sql` and to + // `get_select` and came back as its signature line, every time. + const chainPos = new Map([...flow.pathNodeIds].map((id, i) => [id, i])); const prio = (n: Node) => !CALLABLE_BODY.has(n.kind) ? 99 - : flow.pathNodeIds.has(n.id) ? 0 - : flow.uniqueNamedNodeIds.has(n.id) ? 1 - : (fileDefinesSuper && flow.namedNodeIds.has(n.id)) ? 2 : 99; + : exactNodeIds.has(n.id) ? 0 + : flow.pathNodeIds.has(n.id) ? (flow.namedNodeIds.has(n.id) ? 1 : 2) + : flow.uniqueNamedNodeIds.has(n.id) ? 3 + : (fileDefinesSuper && flow.namedNodeIds.has(n.id)) ? 4 : 99; + // Tiers 0-1 are never reduced to a signature: a body too big for what is + // left is WINDOWED instead — its head plus the lines where it calls the + // other symbols the question named. A signature is the one render of a + // method the agent explicitly asked for that is guaranteed to send it to + // Read. + const WINDOWED_TIER = 1; // One WINDOW per file, sized by this file's RESERVATION. syms are taken by - // priority (spine first, then uniquely-named, then family-base), and the cap - // applies to ALL of them — including the spine — so a big-spine god-file - // (tokio's worker.rs: run→run_task→next_task→steal_work) can't eat the whole - // response and starve the co-flow file (harness.rs's poll). The native agent - // windows such a file too (~190 lines at a time), so this mimics, not - // truncates. Always emit ≥1 (never an empty section). + // priority, and the cap applies to ALL of them — including the spine — so a + // big-spine god-file (tokio's worker.rs: run→run_task→next_task→steal_work) + // can't eat the whole response and starve the co-flow file (harness.rs's + // poll). The native agent windows such a file too (~190 lines at a time), + // so this mimics, not truncates. Always emit ≥1 (never an empty section). // // Held to `fundedHeadroom` as well (CG-31) so this path cannot spend a // reservation still owed below it either. It never exceeds `allowance` // today, so the bound only bites once the ceiling is genuinely tight — // but "every render path" has to mean every one, or the guard is just a // detour the next god-file takes. - const bodyCap = Math.min(allowance, fundedHeadroom); + // + // Priced in RENDERED chars — line numbers included — and net of the + // signature lines the view also prints. Raw source length undercounts + // both, and once an oversize body is windowed to the room that is left + // the undercount stops being slack: the section ran 19% past what it + // was funded (excalidraw's App.tsx) and the four files below it + // rendered nothing. + const SIG_MAX = Math.max(12, budget.maxSymbolsInFileHeader * 2); + const sigReserve = syms.slice(0, SIG_MAX).reduce((sum, n) => sum + lineCost(n.startLine), 0); + const bodyCap = Math.max(0, Math.min(allowance, fundedHeadroom) - sigReserve); const bodyIds = new Set(); + /** Tier 0-1 bodies too big to fit whole, as the windows they render as. */ + const bodyWindows = new Map(); let bodyChars = 0; - for (const n of syms.filter(n => prio(n) < 99 && n.endLine >= n.startLine).sort((a, b) => prio(a) - prio(b))) { - const sz = fileLines.slice(n.startLine - 1, n.endLine).join('\n').length; - if (bodyChars + sz > bodyCap && bodyIds.size > 0) continue; + const ranked = syms + .filter(n => prio(n) < 99 && n.endLine >= n.startLine) + .sort((a, b) => prio(a) - prio(b) + || (chainPos.get(a.id) ?? Number.MAX_SAFE_INTEGER) - (chainPos.get(b.id) ?? Number.MAX_SAFE_INTEGER) + || a.startLine - b.startLine); + const deferred: Node[] = []; + for (const n of ranked) { + if (prio(n) > WINDOWED_TIER) continue; + const sz = renderedSpanCost(n.startLine, n.endLine); + if (bodyChars + sz <= bodyCap) { + bodyIds.add(n.id); + bodyChars += sz; + } else { + deferred.push(n); + } + } + for (const n of deferred) { + const room = bodyCap - bodyChars; + // Below MIN_CHARS a window cannot hold a readable block; with nothing + // emitted yet the never-empty floor inside `windowToCeiling` wins. + if (room < EXPLORE_ALLOCATION.MIN_CHARS && bodyIds.size + bodyWindows.size > 0) continue; + const range = { start: n.startLine, end: n.endLine }; + const windows = windowToCeiling( + [{ range, text: renderSpan(range) }], Math.max(0, room), bodyFocusLines(n), + ).map((p) => p.range); + if (windows.length === 0) continue; + bodyWindows.set(n.id, windows); + bodyChars += windows.reduce((sum, w) => sum + renderedSpanCost(w.start, w.end), 0); + } + for (const n of ranked) { + if (prio(n) <= WINDOWED_TIER) continue; + const sz = renderedSpanCost(n.startLine, n.endLine); + if (bodyChars + sz > bodyCap && bodyIds.size + bodyWindows.size > 0) continue; bodyIds.add(n.id); bodyChars += sz; } @@ -5233,7 +5565,6 @@ export class ToolHandler { const skel: Array<{ range: ExploreLineRange; text: string }> = []; let coveredUntil = 0; // skip symbols already inside an emitted body let sigCount = 0, sigDropped = 0; - const SIG_MAX = Math.max(12, budget.maxSymbolsInFileHeader * 2); for (const n of syms) { if (n.startLine <= coveredUntil) continue; if (bodyIds.has(n.id)) { @@ -5244,6 +5575,19 @@ export class ToolHandler { text: withLineNumbers ? numberSourceLines(body, n.startLine) : body, }); coveredUntil = end; + } else if (bodyWindows.has(n.id)) { + // Each hole in a windowed body is named with the explore query that + // returns it — a line range, which explore now honours — so the + // follow-up is one call, not a Read of the file. + const windows = bodyWindows.get(n.id)!; + windows.forEach((w, k) => { + const holeEnd = (windows[k + 1]?.start ?? n.endLine + 1) - 1; + const hole = holeEnd > w.end + ? `\n… lines ${w.end + 1}-${holeEnd} of \`${n.name}\` elided — codegraph_explore \`${filePath}:${w.end + 1}-${holeEnd}\` returns them` + : ''; + skel.push({ range: w, text: renderSpan(w) + hole }); + }); + coveredUntil = n.endLine; } else { // Elide the body, emit the signature. node.startLine can point at a // decorator/annotation, so scan forward for the line that names the symbol. @@ -5273,7 +5617,7 @@ export class ToolHandler { // (Session.swift, DataRequest.swift) that fired an over-investigation // spiral (the agent Read the skeletonized file, then kept digging). // CLAUDE.md: explore output must never tell the agent to Read. - const tag = bodyIds.size > 0 + const tag = bodyIds.size + bodyWindows.size > 0 ? 'focused (the methods you named in full, the rest as signatures — codegraph_explore a signature by name for its body; do NOT Read)' : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; // Dedup runs on the per-symbol parts, so a body the agent already has @@ -5289,7 +5633,7 @@ export class ToolHandler { ranges: dd.parts.map((p) => p.range), covered: dd.covered, overhead: 120, - mode: bodyIds.size > 0 ? 'focused' : 'skeleton', + mode: bodyIds.size + bodyWindows.size > 0 ? 'focused' : 'skeleton', // Always "clipped": the per-symbol view elides bodies by construction. clipped: true, fullBody: withTail(skel), @@ -5485,17 +5829,24 @@ export class ToolHandler { // budget — the agent explicitly asked for these symbols. const rangeNodes = new Map(); for (const n of group.nodes) if (n.startLine > 0 && n.endLine > 0) rangeNodes.set(n.id, n); - for (const id of flow.namedNodeIds) { + for (const id of [...flow.namedNodeIds, ...exactNodeIds]) { if (rangeNodes.has(id)) continue; const n = cg.getNode(id); if (n && n.filePath === filePath && n.startLine > 0 && n.endLine > 0) rangeNodes.set(id, n); } + // EXACT targets outrank even the query's entry points: an entry is what the + // search matched, an exact target is the def the agent singled out. Their + // clusters rank first and get the spine's bounded overshoot (below), so a + // qualified `SQLCompiler.as_sql` is no longer outranked by a denser cluster + // around `SQLInsertCompiler.as_sql` that merely matched the bare name. + const EXACT_IMPORTANCE = 11; const ranges: Array<{ start: number; end: number; name: string; kind: string; importance: number; spine: boolean; spineCallLine?: number }> = [...rangeNodes.values()] // Drop whole-file envelope nodes (containers covering >50% of the file). .filter(n => !(ENVELOPE_KINDS.has(n.kind) && (n.endLine - n.startLine + 1) > fileLines.length * 0.5)) .map(n => { let importance = 1; - if (entryNodeIds.has(n.id)) importance = 10; + if (exactNodeIds.has(n.id)) importance = EXACT_IMPORTANCE; + else if (entryNodeIds.has(n.id)) importance = 10; else if (flow.namedNodeIds.has(n.id)) importance = 9; // agent named it → keep its cluster else if (glueNodeIds.has(n.id)) importance = 6; // bridging caller/callee of an entry else if (connectedToEntry.has(n.id)) importance = 3; @@ -5526,6 +5877,15 @@ export class ToolHandler { } } + // Line spans the agent anchored in this file (`lines 900-1003`) are exact + // targets too — rendered as exactly the span asked for, not as the + // symbols around it. + for (const span of anchorSpans.get(filePath) ?? []) { + if (span.start > fileLines.length) continue; + const end = Math.min(span.end, fileLines.length); + ranges.push({ start: span.start, end, name: `lines ${span.start}-${end}`, kind: 'range', importance: EXACT_IMPORTANCE, spine: false }); + } + ranges.sort((a, b) => a.start - b.start); if (ranges.length === 0) { @@ -5595,7 +5955,6 @@ export class ToolHandler { // the exact gap this closes. Bounded, so a god-method can't blow the budget yet // the spine's call still appears in context. const OVERSIZE_SPINE_LINES = 200; - const SPINE_WINDOW = 28; // lines each side of the next-hop call site // Returns the rendered text as SPAN-KEYED PARTS. Every part carries the // exact line range its text was sliced from, which two things depend on: // the session record (CG-17) — a record claiming lines it never sent would @@ -5604,7 +5963,6 @@ export class ToolHandler { // agent already holds the rest. Both read the spans from the function that // does the slicing; a second function mirroring these window/padding rules // would drift. - type SectionPart = { range: ExploreLineRange; text: string }; // Every fit decision below measures parts joined by BARE gap markers. // Naming what a gap skipped (#1711) is added once, at assembly, from what // the file's budget has left: measured with the names, a shrunk cluster @@ -5614,9 +5972,13 @@ export class ToolHandler { const sectionText = (parts: ReadonlyArray): string => joinPartsWithNamedGaps(filePath, parts, fileIndexNodes, 0); const buildSection = ( - c: { start: number; end: number; hasSpine?: boolean; spineCallLine?: number }, + c: { start: number; end: number; hasSpine?: boolean; spineCallLine?: number; maxImportance?: number }, ): SectionPart[] => { - if (c.hasSpine && c.spineCallLine && (c.end - c.start + 1) > OVERSIZE_SPINE_LINES) { + // Not for a cluster holding an exact target: its body is what was asked + // for, so it is kept whole up to the ceiling and only then windowed — + // by `windowToCeiling`, which keeps the call site AND the named defs. + if (c.hasSpine && c.spineCallLine && (c.end - c.start + 1) > OVERSIZE_SPINE_LINES + && (c.maxImportance ?? 0) < EXACT_IMPORTANCE) { const call = c.spineCallLine; const winStart = Math.max(c.start, call - SPINE_WINDOW); const winEnd = Math.min(c.end, call + SPINE_WINDOW); @@ -5648,9 +6010,6 @@ export class ToolHandler { }]; }; - /** Rendered cost of one source line, line numbering included. */ - const lineCost = (ln: number): number => - (fileLines[ln - 1] ?? '').length + 1 + (withLineNumbers ? String(ln).length + 1 : 0); /** Rendered cost of one member on its own, context padding included. */ const paddedCost = (r: ExploreRange): number => { let chars = 0; @@ -5665,19 +6024,23 @@ export class ToolHandler { * is incidental context that happens to sit within `gapThreshold` of it. */ const isProtectedMember = (r: ExploreRange): boolean => r.importance >= 9 || r.spine; - /** Whole member ranges, re-merged in source order and rendered. */ - const partsOfMembers = (members: ReadonlyArray): SectionPart[] => { - // Re-merge in source order so adjacent survivors read as one block - // rather than a stutter of one-symbol fragments. - const sorted = [...members].sort((a, b) => a.start - b.start); + /** + * Kept ranges re-merged in source order, the way a cluster merges — so + * adjacent survivors read as one block rather than a stutter of + * one-symbol fragments. + */ + const mergeKept = (keep: ReadonlyArray): Array<{ start: number; end: number }> => { const merged: Array<{ start: number; end: number }> = []; - for (const r of sorted) { + for (const r of [...keep].sort((a, b) => a.start - b.start)) { const last = merged[merged.length - 1]; if (last && r.start <= last.end + gapThreshold) last.end = Math.max(last.end, r.end); else merged.push({ start: r.start, end: r.end }); } - return merged.flatMap((m) => buildSection(m)); + return merged; }; + /** Whole member ranges, re-merged in source order and rendered. */ + const partsOfMembers = (members: ReadonlyArray): SectionPart[] => + mergeKept(members).flatMap((m) => buildSection(m)); /** * What this cluster's protected members cost to render on their own, after * the session history — the room a cluster ranked ABOVE it must leave for @@ -5726,6 +6089,14 @@ export class ToolHandler { * numbers) because it is a bound — the ceiling past which the render is * windowed, or bytes owed to a cluster below — not an estimate. */ + /** + * Exact rendered size of `keep` once merged, padded and joined — what + * `sectionText` will measure — via a prefix sum, so pricing a candidate + * member costs O(members), not a re-render. + */ + const renderedSizeOfKept = (keep: readonly ExploreRange[]): number => + mergeKept(keep).reduce((sum, m, i) => + sum + renderedSpanCost(m.start - contextPadding, m.end + contextPadding) + (i > 0 ? GAP_MARKER.length : 0), 0); const shrinkCluster = ( c: ExploreCluster, cap: number, incidentalCap?: number, ): SectionPart[] | null => { @@ -5734,178 +6105,52 @@ export class ToolHandler { const byImportance = [...c.members].sort((a, b) => (guarded ? Number(isProtectedMember(b)) - Number(isProtectedMember(a)) : 0) || b.importance - a.importance || (a.end - a.start) - (b.end - b.start) || a.start - b.start); - const sizeOf = (r: ExploreRange) => fileLines.slice(r.start - 1, r.end).join('\n').length; const keep: ExploreRange[] = []; - let kept = 0; - // Rendered size of what is kept so far, once the first incidental member - // is weighed. Every protected member sorts ahead of it, so this is the - // exact cost of the protected set; each incidental member then adds its - // own padded span, an over-estimate that errs toward the cluster below. - let rendered: number | undefined; - for (const r of byImportance) { - const sz = sizeOf(r) + GAP_MARKER.length; - // Always keep the most important range, even if it alone is oversize — - // an empty section sends the agent to Read, which costs far more. How - // far it may overshoot is bounded by the caller's ceiling (CG-30), which - // windows a runaway member instead of dropping it. - if (keep.length > 0 && kept + sz > cap) continue; - if (guarded && keep.length > 0 && !isProtectedMember(r)) { - rendered ??= sectionText(dedupeSpans(partsOfMembers(keep)).parts).length; - const cost = paddedCost(r) + GAP_MARKER.length; - if (rendered + cost > incidentalCap) continue; - rendered += cost; + if (c.maxImportance >= EXACT_IMPORTANCE) { + // An EXACT target prices EVERY member in rendered chars, protected + // ones included. The raw estimate below over-keeps on purpose and + // leaves the ceiling trim to cut — but that trim fills in SOURCE + // order, so any member above the exact body in the file (an entry + // point, a named neighbour) was paid for first and the body the agent + // asked for lost its tail (Django's `SQLCompiler.as_sql` stopped at + // L944 of 1001 behind a 15-line neighbour). Priced exactly, the kept + // set fits and the trim has nothing left to cut unless the exact body + // ALONE overruns the ceiling — where it is windowed on its focus + // lines. Incidental members also fit `incidentalCap`, as below. + for (const r of byImportance) { + const limit = guarded && !isProtectedMember(r) ? Math.min(cap, incidentalCap!) : cap; + if (keep.length > 0 && renderedSizeOfKept([...keep, r]) > limit) continue; + keep.push(r); + } + } else { + const sizeOf = (r: ExploreRange) => fileLines.slice(r.start - 1, r.end).join('\n').length; + let kept = 0; + // Rendered size of what is kept so far, once the first incidental member + // is weighed. Every protected member sorts ahead of it, so this is the + // exact cost of the protected set; each incidental member then adds its + // own padded span, an over-estimate that errs toward the cluster below. + let rendered: number | undefined; + for (const r of byImportance) { + const sz = sizeOf(r) + GAP_MARKER.length; + // Always keep the most important range, even if it alone is oversize — + // an empty section sends the agent to Read, which costs far more. How + // far it may overshoot is bounded by the caller's ceiling (CG-30), which + // windows a runaway member instead of dropping it. + if (keep.length > 0 && kept + sz > cap) continue; + if (guarded && keep.length > 0 && !isProtectedMember(r)) { + rendered ??= sectionText(dedupeSpans(partsOfMembers(keep)).parts).length; + const cost = paddedCost(r) + GAP_MARKER.length; + if (rendered + cost > incidentalCap!) continue; + rendered += cost; + } + keep.push(r); + kept += sz; } - keep.push(r); - kept += sz; } if (keep.length === c.members.length) return null; return partsOfMembers(keep); }; - /** - * Bounded overshoot for one cluster's render (CG-30). - * - * `shrinkCluster` keeps the highest-importance member whole even when that - * member alone is oversize — an empty file section sends the agent to Read, - * which is exactly what explore exists to prevent. But "never empty" is not - * "any size": with nothing bounding it, one 22K member rendered against a - * 9K reservation (2.4x), which collapses the headroom every file ranked - * below it draws from. Past the ceiling the member is WINDOWED rather than - * dropped — a leading window (signature + head of the body), plus a window - * on the spine's call site when the head misses it, since on a flow cluster - * the call path IS the answer. - */ - const MIN_WINDOW_LINES = 12; - /** - * Longest prefix of `r` that fits `room`. `minLines` is the never-empty - * floor — it may overrun `room`, so it is only ever asked for when nothing - * else has been emitted and the alternative is an empty section. - */ - const headWindowOf = ( - r: ExploreLineRange, room: number, minLines = 0, - ): ExploreLineRange | null => { - let end = r.start - 1; - let chars = 0; - for (let ln = r.start; ln <= r.end; ln++) { - const cost = lineCost(ln); - if (chars + cost > room && end - r.start + 1 >= minLines) break; - chars += cost; - end = ln; - } - return end >= r.start ? { start: r.start, end } : null; - }; - /** Widest window around `line` inside [lo, hi] that fits `room`. */ - const centeredWindowOf = ( - line: number, lo: number, hi: number, room: number, - ): ExploreLineRange | null => { - if (line < lo || line > hi) return null; - let start = line, end = line, chars = lineCost(line); - for (let grown = true; grown;) { - grown = false; - if (end + 1 <= hi && chars + lineCost(end + 1) <= room) { end += 1; chars += lineCost(end); grown = true; } - if (start - 1 >= lo && chars + lineCost(start - 1) <= room) { start -= 1; chars += lineCost(start); grown = true; } - } - return { start, end }; - }; - /** - * Reduce rendered parts to fit `ceiling`, never to nothing. Whole parts are - * kept while they fit; the first part that overruns is cut to a leading - * window on whole lines (a body is never cut mid-line), and everything past - * it is dropped. The GAP_MARKER between surviving parts — and the line-number - * jump — is what tells the agent the cut happened. - * - * A partial window shorter than MIN_WINDOW_LINES is not worth emitting, and - * emitting one is actively harmful: the session record then claims a 4-line - * sliver, and the NEXT call's dedup has to either shred a whole block around - * it or re-send it. Below that floor the part is simply dropped — unless - * nothing has been emitted at all, where the floor wins over the ceiling - * because an empty section is the one outcome worse than an oversize one. - * - * `focusLines` are the lines this trim must not lose: the spine's next-hop - * call site (CG-30) and every definition the agent NAMED inside the cluster - * (CG-38). The head fill is source-ordered, so a named def in the TAIL of a - * large file is otherwise always the first thing an over-ceiling render - * drops — the one span the agent asked for by name, cut in favour of - * head-of-file filler it did not ask for. The full-ceiling fill is tried - * FIRST and the 60% hold-back applies only when a focus line is actually - * left uncovered, so a cluster whose head already reaches its focus keeps - * the whole ceiling for source. - */ - const windowToCeiling = ( - parts: ReadonlyArray, - ceiling: number, - focusLines: ReadonlyArray = [], - ): SectionPart[] => { - const inParts = (line: number) => - parts.some((p) => line >= p.range.start && line <= p.range.end); - const focus = [...new Set(focusLines)] - .filter((l) => typeof l === 'number' && l > 0 && inParts(l)) - .sort((a, b) => a - b); - /** Source-ordered fill of whole parts, the overrunning one cut to a head window. */ - const fill = (room: number): { emit: ExploreLineRange[]; used: number } => { - const emit: ExploreLineRange[] = []; - let used = 0; - for (const p of parts) { - const join = emit.length > 0 ? GAP_MARKER.length : 0; - if (used + join + p.text.length <= room) { - emit.push(p.range); - used += join + p.text.length; - continue; - } - const first = emit.length === 0; - const win = headWindowOf( - p.range, Math.max(0, room - used - join), first ? MIN_WINDOW_LINES : 0); - if (win && (first || win.end - win.start + 1 >= MIN_WINDOW_LINES)) { - emit.push(win); - used += join + renderSpan(win).length; - } - break; - } - return { emit, used }; - }; - let { emit, used } = fill(ceiling); - const reached = () => (emit.length ? emit[emit.length - 1]!.end : 0); - if (focus.some((l) => l > reached())) { - // Hold room back for the focus windows so the head can't eat all of it. - ({ emit, used } = fill(Math.floor(ceiling * 0.6))); - } - // What is left is SPLIT between the uncovered focus lines rather than - // handed to them in order. Greedy-in-source-order reproduces the very bug - // this guards: on a prose query resolving four focus lines, the two - // earliest took the whole reserve and `flushQueuedMessages` at L1102 — - // named in the question — was dropped again. A skipped or undersized - // window returns its share to the pool for the ones after it. - let covered = reached(); - let room = Math.max(0, ceiling - used); - const pending = focus.filter((l) => l > covered); - for (let i = 0; i < pending.length; i++) { - const line = pending[i]!; - if (line <= covered) continue; // an earlier window already reached it - const share = Math.floor(room / (pending.length - i)) - GAP_MARKER.length; - if (share <= 0) continue; - const host = parts.find((p) => line >= p.range.start && line <= p.range.end)!; - const lo = Math.max(host.range.start, line - SPINE_WINDOW, covered + 1); - const hi = Math.min(host.range.end, line + SPINE_WINDOW); - const win = centeredWindowOf(line, lo, hi, share); - // Same sliver floor as the head window — a two-line peek at the call - // site teaches the next call's dedup to shred the block around it. - if (!win || win.end - win.start + 1 < MIN_WINDOW_LINES) continue; - emit.push(win); - const cost = GAP_MARKER.length + renderSpan(win).length; - used += cost; - room -= cost; - covered = win.end; - } - // Never empty: a section with no source sends the agent to Read. - if (emit.length === 0 && parts.length > 0) { - const first = headWindowOf(parts[0]!.range, ceiling, MIN_WINDOW_LINES); - if (first) emit.push(first); - } - return emit - .sort((a, b) => a.start - b.start) - .map((r) => ({ range: r, text: renderSpan(r) })); - }; - /** * The lines a ceiling trim of this cluster must not lose: the spine's * next-hop call site, and the definition line of every member the agent @@ -5921,7 +6166,14 @@ export class ToolHandler { .sort((a, b) => b.importance - a.importance || a.start - b.start) .slice(0, MAX_FOCUS_LINES) .map((m) => m.start); - return c.spineCallLine ? [c.spineCallLine, ...named] : named; + // An exact body cut by the ceiling keeps its calls into the question's + // other symbols too, not only its def line — the same focus the + // per-symbol view windows by. + const exactFocus = c.maxImportance < EXACT_IMPORTANCE ? [] : [...exactNodeIds] + .map((id) => subgraph.nodes.get(id)) + .filter((n): n is Node => !!n && n.filePath === filePath && n.startLine >= c.start && n.endLine <= c.end) + .flatMap((n) => bodyFocusLines(n)); + return [...exactFocus, ...(c.spineCallLine ? [c.spineCallLine] : []), ...named]; }; /** @@ -5995,10 +6247,14 @@ export class ToolHandler { const rankedClusters = clusters .map((c, i) => ({ idx: i, span: c.end - c.start + 1, c })) .sort((a, b) => { - // Spine clusters first — the rendered call path IS the flow answer, so it + // Exact-target clusters first — the def (or span) the agent singled out. + // Then spine clusters — the rendered call path IS the flow answer, so it // outranks any denser block of peripheral declarations (a low-density entry // method must not lose the budget to them). Within spine / within non-spine, // the existing importance → density → score → span order holds. + const aExact = a.c.maxImportance >= EXACT_IMPORTANCE; + const bExact = b.c.maxImportance >= EXACT_IMPORTANCE; + if (aExact !== bExact) return (bExact ? 1 : 0) - (aExact ? 1 : 0); if (a.c.hasSpine !== b.c.hasSpine) return (b.c.hasSpine ? 1 : 0) - (a.c.hasSpine ? 1 : 0); if (b.c.maxImportance !== a.c.maxImportance) return b.c.maxImportance - a.c.maxImportance; const densityA = a.c.score / a.span; @@ -6062,6 +6318,9 @@ export class ToolHandler { const spent = projectedChars + (chosenIndices.size > 0 ? GAP_MARKER.length : 0); return Math.min(room, fileBudget - spent) - owedBelow; }; + /** Rendered cost of a cluster's EXACT members on their own. */ + const exactCostOf = (c: ExploreCluster): number => + renderedSizeOfKept(c.members.filter((m) => m.importance >= EXACT_IMPORTANCE)); for (let p = 0; p < rankedClusters.length; p++) { const rc = rankedClusters[p]!; // The top-ranked cluster is always taken — an empty file section sends the @@ -6073,15 +6332,31 @@ export class ToolHandler { // A spine cluster (the rendered call path) is the flow answer — it may run // past the per-file budget up to the spine ceiling; non-spine clusters obey // the normal per-file budget. - const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget; + const cap = rc.c.hasSpine || rc.c.maxImportance >= EXACT_IMPORTANCE ? SPINE_CEILING : fileBudget; // CG-30: shrinking keeps the top member whole however big it is, so bound // how far that member may overshoot — the same 1.5x-of-reservation bound // SPINE_CEILING already draws, never below `cap` (a cluster that fits its // cap is never windowed). A spine cluster's cap already IS that bound, so // this holds it to it rather than letting the member rule walk past it. const ceiling = Math.max(cap, SPINE_CEILING); + // Several exact targets can land in different clusters of one big file — + // a pinned 13K-line App.tsx merges into a handful of giant clusters, and + // `App.triggerRender` and `App.handleCanvasPointerDown` sat in two. The + // one ranked first spent the whole cap on its own neighbours and the + // other target rendered nothing. So an exact cluster holds back what the + // exact clusters ranked BELOW it still owe — never cutting into its own + // targets — and each target renders, windowed if it must. + const owedExact = rc.c.maxImportance < EXACT_IMPORTANCE ? 0 + : rankedClusters.slice(p + 1).reduce((n, other) => (other.c.maxImportance < EXACT_IMPORTANCE + ? n : n + exactCostOf(other.c) + GAP_MARKER.length), 0); + const holdBack = (room: number): number => (owedExact > 0 + ? Math.min(room, Math.max(exactCostOf(rc.c), room - owedExact)) + : room); if (first) { - const section = renderCluster(rc.c, cap, ceiling, incidentalCapFor(p, cap)); + const firstCap = holdBack(cap); + const section = renderCluster( + rc.c, firstCap, owedExact > 0 ? firstCap : ceiling, incidentalCapFor(p, firstCap), + ); renderedClusters.set(rc.idx, section); anyClusterShrunk = anyClusterShrunk || section.shrunk; chosenIndices.add(rc.idx); @@ -6104,7 +6379,7 @@ export class ToolHandler { // nothing") applied between CLUSTERS. Below `MIN_CHARS` the remainder can't // hold one readable block, so it stays a drop rather than a stutter of // fragments the next call's dedup then has to shred around. - const room = cap - projectedChars - GAP_MARKER.length; + const room = holdBack(cap - projectedChars - GAP_MARKER.length); if (room < EXPLORE_ALLOCATION.MIN_CHARS) continue; const section = renderCluster(rc.c, room, room, incidentalCapFor(p, room)); const text = sectionText(section.parts); diff --git a/src/search/query-paths.ts b/src/search/query-paths.ts index 9855c8b7d..302c442a3 100644 --- a/src/search/query-paths.ts +++ b/src/search/query-paths.ts @@ -43,6 +43,23 @@ export interface QueryPathExtraction { * surfaced to the agent so the miss is visible instead of silent. */ unresolvedPathSpans: string[]; + /** + * Line spans the query anchored to a pinned file, 1-based and inclusive: + * `compiler.py:776`, `foo.ts:12-40`, `foo.ts#L88-L120`, or prose next to a + * path (`compiler.py lines 900-1003`, `L900-L1003 of compiler.py`). An agent + * writes these when it wants THOSE lines — typically the body a previous + * response elided — so a pin that drops them answers a different question + * (the file's most relevant clusters) than the one asked. Only a span whose + * path resolved to exactly one file is kept: a line number means nothing + * across two candidate files. + */ + lineAnchors: QueryLineAnchor[]; +} + +export interface QueryLineAnchor { + file: string; + start: number; + end: number; } /** @@ -106,15 +123,48 @@ function buildBasenameStems(indexedPaths: readonly string[]): Map MAX_LINE_NUMBER || end > MAX_LINE_NUMBER) return null; + return start <= end ? { start, end } : { start: end, end: start }; +} + +/** Prose punctuation a line-number token can carry: "lines 900-1003," / "(L88)". */ +function stripLineTokenPunctuation(token: string): string { + return token.replace(/^[('"`[]+/, '').replace(/[)'"`\].,;:!?]+$/, ''); +} + /** * Strip prose punctuation wrapped around a token without eating punctuation * that is PART of the path: quotes/backticks always strip; a trailing `)`/`]` * strips only when the token has no matching opener (so `(protected)` and * `[id]` segments survive, while "…(see src/foo.ts)" loses its parenthesis); * a leading `(`/`[` mirrors that. Trailing sentence punctuation strips last, - * so "src/foo.ts." resolves. + * so "src/foo.ts." resolves. A trailing line reference is split off and + * returned separately — the file is what resolves, the lines are what the + * agent wants from it. */ -function stripWrapping(token: string): string { +function stripWrapping(token: string): { path: string; lines: { start: number; end: number } | null } { let s = token; for (;;) { const first = s[0]; @@ -134,10 +184,12 @@ function stripWrapping(token: string): string { if (last === '}' && !s.includes('{')) { s = s.slice(0, -1); continue; } break; } - // Line references ride along in agent-written paths: `foo.ts:123`, - // `foo.ts:12-40`, `foo.ts#L88`. The file is what gets pinned. - s = s.replace(/(?::\d+(?:-\d+)?|#L\d+(?:-L?\d+)?)$/, ''); - return s; + const ref = LINE_REF_SUFFIX.exec(s); + if (!ref) return { path: s, lines: null }; + return { + path: s.slice(0, ref.index), + lines: ref[1] !== undefined ? lineSpan(ref[1], ref[2]) : lineSpan(ref[3], ref[4]), + }; } /** Normalize a span into the repo-relative shape the files table stores. */ @@ -213,6 +265,7 @@ export function extractQueryPaths( strippedQuery: query, pinnedFiles: [], unresolvedPathSpans: [], + lineAnchors: [], }; if (!query.trim() || indexedPaths.length === 0) return passthrough; @@ -226,12 +279,15 @@ export function extractQueryPaths( const pinned: string[] = []; const pinnedSeen = new Set(); const unresolved: string[] = []; + const anchors: QueryLineAnchor[] = []; + /** Token index → the ONE file it pinned, for binding prose line ranges. */ + const singleFileAt = new Map(); let candidatesExamined = 0; for (let i = 0; i < tokens.length; i++) { if (pinned.length >= maxPins) break; if (candidatesExamined >= MAX_CANDIDATE_SPANS) break; - const stripped = stripWrapping(tokens[i]!); + const { path: stripped, lines } = stripWrapping(tokens[i]!); if (stripped.length < 4) continue; const hasSlash = /[/\\]/.test(stripped); if (!hasSlash && !DOTTED_BASENAME.test(stripped)) continue; @@ -250,6 +306,10 @@ export function extractQueryPaths( pinnedSeen.add(m); pinned.push(m); } + if (matches.length === 1 && pinnedSeen.has(matches[0]!)) { + singleFileAt.set(i, matches[0]!); + if (lines) anchors.push({ file: matches[0]!, ...lines }); + } } else if ( ambiguous || isClearlyPathShaped(normalized) @@ -289,7 +349,7 @@ export function extractQueryPaths( let basenameStems: Map | null = null; for (let i = 0; i < tokens.length && pinned.length < maxPins; i++) { if (consumed.has(i)) continue; - const stripped = stripWrapping(tokens[i]!); + const { path: stripped, lines } = stripWrapping(tokens[i]!); if (stripped.length < 4 || !KEBAB_BASENAME.test(stripped)) continue; basenameStems ??= buildBasenameStems(indexedPaths); const matches = basenameStems.get(stripped.toLowerCase()); @@ -300,12 +360,64 @@ export function extractQueryPaths( pinnedSeen.add(m); pinned.push(m); } + if (matches.length === 1 && pinnedSeen.has(matches[0]!)) { + singleFileAt.set(i, matches[0]!); + if (lines) anchors.push({ file: matches[0]!, ...lines }); + } + } + + // Third pass — prose line ranges (`lines 900-1003`, `line 42`, `L88-L120`, + // `lines 900 to 1003`) bound to the NEAREST single-file path token. These + // are what an agent writes when a previous response elided a body and it + // asks for the rest; left in the query, the numbers match nothing and + // `lines` feeds FTS a word every file contains. A range with no path to + // bind to says nothing about which file, so it is left alone. + if (singleFileAt.size > 0) { + const pathIdx = [...singleFileAt.keys()]; + const nearestFile = (i: number): string => { + let best = pathIdx[0]!; + for (const p of pathIdx) if (Math.abs(p - i) < Math.abs(best - i)) best = p; + return singleFileAt.get(best)!; + }; + for (let i = 0; i < tokens.length; i++) { + if (consumed.has(i)) continue; + const tok = stripLineTokenPunctuation(tokens[i]!); + const afterWord = i > 0 && !consumed.has(i - 1) && LINE_WORD.test(stripLineTokenPunctuation(tokens[i - 1]!)); + const afterPath = singleFileAt.has(i - 1); + const m = LINE_NUMBER_TOKEN.exec(tok); + if (!m) continue; + // A bare number is a line number only in a line context; `L900` is one + // on its own. + if (!m[1] && !afterWord && !afterPath) continue; + let span = lineSpan(m[2], m[3]); + const used = [i]; + // `lines 900 to 1003` + if (span && m[3] === undefined && i + 2 < tokens.length + && RANGE_CONNECTIVE.test(stripLineTokenPunctuation(tokens[i + 1]!))) { + const tail = LINE_NUMBER_TOKEN.exec(stripLineTokenPunctuation(tokens[i + 2]!)); + if (tail && tail[3] === undefined) { + span = lineSpan(m[2], tail[2]); + used.push(i + 1, i + 2); + } + } + if (!span) continue; + anchors.push({ file: nearestFile(i), ...span }); + for (const u of used) consumed.add(u); + if (afterWord) consumed.add(i - 1); + } } if (consumed.size === 0) return passthrough; + const seenAnchor = new Set(); return { strippedQuery: tokens.filter((_, i) => !consumed.has(i)).join(' '), pinnedFiles: pinned, unresolvedPathSpans: unresolved, + lineAnchors: anchors.filter((a) => { + const key = `${a.file}:${a.start}-${a.end}`; + if (seenAnchor.has(key)) return false; + seenAnchor.add(key); + return true; + }), }; }