From b37f191f5a6b2346adf364462b33ad094f7136d9 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Mon, 3 Aug 2026 22:51:18 -0500 Subject: [PATCH 01/15] feat(mcp): per-file allocation diagnostic for explore (CG-4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit How codegraph_explore divides its byte envelope among files was unobservable — you could read a response and guess, but not say "this file took 16% and that one took 20%." Nothing else in the budget- allocation epic is measurable without that. CODEGRAPH_EXPLORE_DEBUG now emits one report per explore call (stderr table, stderr JSON, or a JSONL sidecar path). Per file: relevance score, graph mass, term hits, ranking flags, render mode, bytes allocated vs delivered, both shares, and whether it was clipped — plus why a ranked candidate never rendered. Totals cover envelope vs maxOutputChars vs the hard ceiling, the source/meta split, the selection funnel, and the score floor and relevance-gate thresholds applied. Allocated and delivered are reported separately on purpose: they diverge exactly when the 25K ceiling truncates, and conflating them is how a dropped trailing file goes unnoticed. Off by default and byte-identical when off — it ships in the product binary, and a diagnostic that perturbs the response by one byte would invalidate every A/B taken with it on. ExploreDiagnostics.start() returns null unless the env var is set, so every call site is a `diag?.` no-op. Baseline recorded in docs/design/explore-budget-allocation.md: on this repo, src/mcp/tools.ts gets 15.8% of the envelope while three weakly- relevant agent-eval scripts take 61% between them — despite tools.ts carrying 5.4x the score and 2.6x the graph mass of any of them. Small files ship whole; the large answer file is clipped at maxCharsPerFile. Rank ordering is correct and buys nothing. The loop also allocated 23,193 chars against an 18,000 budget. Co-Authored-By: Claude Opus 5 --- __tests__/explore-diagnostics.test.ts | 294 +++++++++++++ docs/design/explore-budget-allocation.md | 102 +++++ src/mcp/explore-diagnostics.ts | 528 +++++++++++++++++++++++ src/mcp/tools.ts | 72 +++- 4 files changed, 990 insertions(+), 6 deletions(-) create mode 100644 __tests__/explore-diagnostics.test.ts create mode 100644 docs/design/explore-budget-allocation.md create mode 100644 src/mcp/explore-diagnostics.ts diff --git a/__tests__/explore-diagnostics.test.ts b/__tests__/explore-diagnostics.test.ts new file mode 100644 index 000000000..e8b925ccc --- /dev/null +++ b/__tests__/explore-diagnostics.test.ts @@ -0,0 +1,294 @@ +/** + * Per-file allocation diagnostic for codegraph_explore (CG-4). + * + * The instrument ships in the product binary, so the load-bearing property is + * NOT what it reports — it's that it reports NOTHING unless asked. An explore + * response is the agent's context; a diagnostic that perturbs it by one byte + * invalidates every A/B measurement taken with it on, which is the exact thing + * the rest of the budget-allocation work depends on. + * + * So the first block pins byte-identical output across on/off, and only then + * do we assert the report's shape and internal consistency. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { ToolHandler } from '../src/mcp/tools'; +import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; +import CodeGraph from '../src/index'; + +const DEBUG_ENV = 'CODEGRAPH_EXPLORE_DEBUG'; + +/** Restore the env var to "unset" — `delete` matters; '' is a distinct case. */ +function clearDebugEnv(): void { + delete process.env[DEBUG_ENV]; +} + +describe('attributeSourceBytes', () => { + it('attributes a fenced block to the file section header above it', () => { + const text = [ + '**Exploration: x**', + '', + '**`src/a.ts`** — foo(function)', + '', + '```typescript', + '1\tconst a = 1;', + '2\tconst b = 2;', + '```', + '', + '**`src/b.ts`** — bar(function)', + '', + '```typescript', + '1\tconst c = 3;', + '```', + '', + ].join('\n'); + const bytes = attributeSourceBytes(text); + expect(bytes.get('src/a.ts')).toBe('1\tconst a = 1;\n2\tconst b = 2;'.length); + expect(bytes.get('src/b.ts')).toBe('1\tconst c = 3;'.length); + }); + + it('sums multiple fenced blocks under one file header', () => { + const text = [ + '**`src/a.ts`** — foo(function)', + '', + '```ts', + 'aa', + '```', + '', + '```ts', + 'bbb', + '```', + ].join('\n'); + expect(attributeSourceBytes(text).get('src/a.ts')).toBe('aa'.length + 'bbb'.length); + }); + + it('counts an unterminated block — the ceiling can cut mid-fence', () => { + const text = ['**`src/a.ts`** — foo(function)', '', '```ts', 'x'.repeat(40)].join('\n'); + expect(attributeSourceBytes(text).get('src/a.ts')).toBe(40); + }); + + it('returns nothing for text with no file sections', () => { + expect(attributeSourceBytes('No relevant code found for "zzz"').size).toBe(0); + expect(attributeSourceBytes('').size).toBe(0); + }); +}); + +describe('codegraph_explore allocation diagnostic', () => { + let testDir: string; + let sidecarDir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + const QUERY = 'Session method helper callSession'; + + beforeAll(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-explore-diag-')); + sidecarDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-explore-diag-out-')); + const srcDir = path.join(testDir, 'src'); + fs.mkdirSync(srcDir); + + // One fat file plus several small callers, so the render loop exercises + // more than one allocation branch (clusters for the fat file, whole-file + // for the small ones) and there is a real per-file split to report. + const fatLines: string[] = ['export class Session {']; + for (let i = 0; i < 30; i++) { + fatLines.push(` method${i}(arg: string): string {`); + fatLines.push(` return this.helper${i}(arg) + "${i}";`); + fatLines.push(` }`); + fatLines.push(` private helper${i}(arg: string): string {`); + fatLines.push(` return arg.repeat(${i + 1});`); + fatLines.push(` }`); + } + fatLines.push('}'); + fs.writeFileSync(path.join(srcDir, 'session.ts'), fatLines.join('\n')); + + for (let i = 0; i < 6; i++) { + fs.writeFileSync( + path.join(srcDir, `support${i}.ts`), + `import { Session } from './session';\n` + + `export function callSession${i}(s: Session) {\n` + + ` return s.method${i}('hi');\n` + + `}\n`, + ); + } + + clearDebugEnv(); + cg = CodeGraph.initSync(testDir, { config: { include: ['**/*.ts'], exclude: [] } }); + await cg.indexAll(); + handler = new ToolHandler(cg); + }); + + afterEach(() => { + clearDebugEnv(); + vi.restoreAllMocks(); + }); + + afterAll(() => { + clearDebugEnv(); + if (cg) cg.destroy(); + for (const dir of [testDir, sidecarDir]) { + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + const explore = async (): Promise => { + const result = await handler.execute('codegraph_explore', { query: QUERY }); + return result.content?.[0]?.text ?? ''; + }; + + it('produces byte-identical output whether the diagnostic is on or off', async () => { + clearDebugEnv(); + const off = await explore(); + expect(off.length).toBeGreaterThan(0); + + // Sanity: the tool itself is deterministic, so a difference below is + // attributable to the diagnostic and not to explore's own variance. + expect(await explore()).toBe(off); + + vi.spyOn(process.stderr, 'write').mockImplementation((() => true) as typeof process.stderr.write); + const sidecar = path.join(sidecarDir, 'identical.jsonl'); + for (const value of ['1', 'json', sidecar]) { + process.env[DEBUG_ENV] = value; + const on = await explore(); + clearDebugEnv(); + expect(on).toBe(off); + } + }); + + it('writes nothing to stderr when the env var is unset', async () => { + clearDebugEnv(); + const writes: string[] = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stderr.write); + await explore(); + expect(writes.join('')).toBe(''); + }); + + it('stays off for every falsy env value', async () => { + const writes: string[] = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stderr.write); + for (const value of ['', '0', 'false', 'off', 'no', 'OFF', ' 0 ']) { + process.env[DEBUG_ENV] = value; + await explore(); + } + expect(writes.join('')).toBe(''); + }); + + it('prints a per-file table to stderr when enabled', async () => { + const writes: string[] = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stderr.write); + + process.env[DEBUG_ENV] = '1'; + await explore(); + const out = writes.join(''); + + expect(out).toContain('codegraph explore diagnostic'); + // Totals: envelope vs budget, and the file-selection funnel with its floor. + expect(out).toMatch(/envelope [\d,]+ chars delivered · [\d,]+ allocated of [\d,]+ budget/); + expect(out).toMatch(/hard ceiling [\d,]+/); + expect(out).toMatch(/files [\d,]+ grouped .*past score floor \(>=\d+\).*in output \(maxFiles \d+\)/); + // Per-file columns. + expect(out).toMatch(/#\s+alloc%\s+deliv%\s+bytes\s+score\s+graph\s+hits\s+flags\s+render\s+file/); + expect(out).toContain('src/session.ts'); + expect(out).toMatch(/\d+\.\d%/); + }); + + it('appends one JSON report per call to a sidecar path', async () => { + const sidecar = path.join(sidecarDir, 'reports.jsonl'); + process.env[DEBUG_ENV] = sidecar; + await explore(); + await explore(); + clearDebugEnv(); + + const rows = fs.readFileSync(sidecar, 'utf-8').trim().split('\n'); + expect(rows).toHaveLength(2); + + const report = JSON.parse(rows[0]!); + expect(report.tool).toBe('codegraph_explore'); + expect(report.query).toBe(QUERY); + + // Totals the task asks for: envelope vs maxOutputChars, files considered + // vs included, and the score floor that was applied. + expect(report.budget.maxOutputChars).toBeGreaterThan(0); + expect(report.envelope.chars).toBeGreaterThan(0); + expect(report.selection.scoreFloor).toBeGreaterThan(0); + expect(report.selection.filesGrouped).toBeGreaterThanOrEqual(report.selection.filesPastScoreFloor); + expect(report.selection.filesPastScoreFloor).toBeGreaterThanOrEqual(report.selection.filesRanked); + expect(report.selection.filesRanked).toBeGreaterThanOrEqual(report.selection.filesInFinalOutput); + expect(report.selection.filesInFinalOutput).toBeGreaterThan(0); + expect(report.selection.filesInFinalOutput).toBeLessThanOrEqual(report.budget.maxFiles); + + // Per-file: score, bytes, share, clipped, spine. + const shown = report.files.filter((f: { finalChars: number }) => f.finalChars > 0); + expect(shown.length).toBeGreaterThan(0); + for (const f of shown) { + expect(typeof f.path).toBe('string'); + expect(typeof f.score).toBe('number'); + expect(typeof f.graphScore).toBe('number'); + expect(typeof f.clipped).toBe('boolean'); + expect(typeof f.spine).toBe('boolean'); + expect(f.finalChars).toBeGreaterThan(0); + expect(f.share).toBeGreaterThan(0); + expect(f.share).toBeLessThanOrEqual(1); + expect(f.render).toBeTruthy(); + } + expect(shown.some((f: { path: string }) => f.path === 'src/session.ts')).toBe(true); + }); + + it('attributes the envelope consistently — per-file bytes sum to the reported source total', async () => { + const sidecar = path.join(sidecarDir, 'consistency.jsonl'); + process.env[DEBUG_ENV] = sidecar; + const text = await explore(); + clearDebugEnv(); + + const report = JSON.parse(fs.readFileSync(sidecar, 'utf-8').trim()); + expect(report.envelope.chars).toBe(text.length); + + const summed = report.files.reduce( + (s: number, f: { finalChars: number }) => s + f.finalChars, 0, + ); + expect(summed).toBe(report.envelope.sourceChars); + expect(report.envelope.sourceChars + report.envelope.metaChars).toBe(report.envelope.chars); + // Shares are fractions of the delivered envelope, so they can't exceed it. + const shareSum = report.files.reduce((s: number, f: { share: number }) => s + f.share, 0); + expect(shareSum).toBeLessThanOrEqual(1.0001); + expect(shareSum).toBeCloseTo(report.envelope.sourceShare, 3); + }); + + it('survives an unwritable sink without failing the explore call', async () => { + clearDebugEnv(); + const expected = await explore(); + + // A directory is never a valid append target. + process.env[DEBUG_ENV] = sidecarDir; + const result = await handler.execute('codegraph_explore', { query: QUERY }); + clearDebugEnv(); + + expect(result.isError).toBeFalsy(); + expect(result.content?.[0]?.text).toBe(expected); + }); + + it('records a report even when explore finds nothing', async () => { + const sidecar = path.join(sidecarDir, 'empty.jsonl'); + process.env[DEBUG_ENV] = sidecar; + const result = await handler.execute('codegraph_explore', { + query: 'zzzznonexistentsymbolzzzz', + }); + clearDebugEnv(); + + expect(result.content?.[0]?.text).toContain('No relevant code found'); + const report = JSON.parse(fs.readFileSync(sidecar, 'utf-8').trim()); + expect(report.note).toContain('no relevant code found'); + expect(report.files).toEqual([]); + }); +}); diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md new file mode 100644 index 000000000..4371362e0 --- /dev/null +++ b/docs/design/explore-budget-allocation.md @@ -0,0 +1,102 @@ +# Explore budget allocation — the instrument and the baseline + +`codegraph_explore` has a fixed byte envelope (`getExploreOutputBudget().maxOutputChars`, +hard-capped at 25K so the host never externalizes the result). **How that envelope gets +divided among files** is decided by a long chain of gates, tiers and caps spread across +`handleExplore` — and until CG-4 that chain was unobservable. You could read an explore +response and guess; you could not say "this file took 16% and that one took 20%." + +This document covers the diagnostic that makes it measurable, and the baseline it recorded. + +## The diagnostic + +Set `CODEGRAPH_EXPLORE_DEBUG` and every `codegraph_explore` call (MCP tool or +`codegraph explore` CLI) emits one report: + +| value | sink | +|---|---| +| `1` / `true` / `on` / `yes` / `stderr` | human-readable table on stderr | +| `json` | one pretty-printed JSON report on stderr | +| anything else | treated as a path — one JSON report per line, appended (JSONL) | +| unset / `0` / `false` / `off` / `no` / empty | **off** | + +Per file it reports: relevance score, graph (RWR) mass, distinct query-term hits, ranking +flags (named / entry / central / spine / low-value / generated), render mode, bytes of +source allocated, bytes actually delivered, both shares, and whether it was clipped. For +files that never rendered it reports why (`max-files`, `budget-90pct`, `budget-whole-file`, +`budget-clusters`, `unreadable`, `no-ranges`). Totals cover the envelope (delivered vs +allocated vs `maxOutputChars` vs hard ceiling), the source/meta split, the file-selection +funnel at each stage, and the thresholds applied (score floor, graph-relevance gate). + +**It is off by default and produces byte-identical output when off** — it ships in the +product binary, and a diagnostic that perturbs the response by one byte would invalidate +every A/B measurement taken with it on. `ExploreDiagnostics.start()` returns `null` unless +the env var is set, so every call site is a `diag?.` no-op. Pinned by +`__tests__/explore-diagnostics.test.ts`. + +Two envelope numbers, deliberately kept separate: + +- **allocated** — what the render loop chose to emit, before the final hard-ceiling cut. + This is the allocator's own decision, and the number budget work is about. +- **delivered** — what the agent actually received. + +They diverge exactly when the ceiling truncates. Conflating them is how a dropped trailing +file goes unnoticed. + +## Baseline (2026-08-03, this repo at `main`) + +``` +codegraph explore "how does explore allocate its output budget across files" --path . +``` + +469 files indexed → small tier (`maxOutputChars` 18,000, `maxCharsPerFile` 3,800, +`defaultMaxFiles` 5). Envelope: **23,196 delivered / 23,193 allocated against an 18,000 +budget — 29% over**, absorbed only because the 25K hard ceiling sits above it. + +| # | share | bytes | score | graph | hits | flags | render | file | +|---|---|---|---|---|---|---|---|---| +| 4 | 21.2% | 4,928 | 10 | 0.125 | 1 | entry | whole | `scripts/agent-eval/offload-eval-hook.mjs` | +| 5 | 20.1% | 4,665 | 10 | 0.125 | 1 | entry | whole | `scripts/agent-eval/offload-eval-metrics.mjs` | +| 3 | 19.8% | 4,585 | 22 | 0.125 | 1 | entry central | whole | `scripts/agent-eval/parse-session.mjs` | +| 1 | **15.8%** | 3,659 | **54** | **0.322** | **4** | entry central | clusters* | `src/mcp/tools.ts` | +| 2 | 15.0% | 3,479 | 34 | 0.082 | 2 | entry | clusters* | `src/index.ts` | + +\* clipped. Ranked but never rendered: `scripts/agent-eval/offload-eval-cost.mjs` (#6) and +`src/resolution/lru-cache.ts` (#7), both cut by `maxFiles`. + +Files: 17 grouped → 10 past the score floor (≥3) → 10 past the low-value filter → 7 past +the relevance gate (graph ≥ 0.0193, 6% of max 0.3215) → 5 in the output. + +### What the baseline shows + +**Score does not drive allocation.** `src/mcp/tools.ts` — the file that actually answers +the query — carries 5.4× the relevance score, 2.6× the graph mass and 4× the term hits of +any `.mjs` script, and gets a *smaller* share than each of them. The three agent-eval +scripts take **61%** of the envelope between them; the answer file takes 16%. + +The mechanism is that the two allocation paths are decided by **file size, not relevance**: +a small file clears `WHOLE_FILE_MAX_LINES`/`WHOLE_FILE_MAX_CHARS` and ships entirely, while +a large file falls through to cluster selection and is clipped at `maxCharsPerFile`. So a +weakly-relevant 130-line script gets 100% of itself; the strongly-relevant 5,000-line file +gets 3,800 chars. Rank ordering is correct (tools.ts sorts #1) and buys nothing, because +rank has no effect on how many bytes a file receives. + +**The envelope is over-subscribed.** 23,193 allocated against an 18,000 budget means the +per-file caps do not compose into the total cap; the total is enforced only by the 25K +ceiling silently dropping whole trailing sections. Under a slightly different index state +(one more candidate file) the same query allocated 27,518 chars and the ceiling dropped a +7,678-char section — the single largest allocation in the response — with the only trace +being the truncation notice at the end. + +This is the gap the rest of the epic closes: relevance-proportional allocation with a +relative cliff (CG-12), on top of scoring that stops rewarding incidental name collisions +(CG-10). + +### Reproducing + +The query explores this repo, so **uncommitted edits to `src/mcp/tools.ts` change the +result** — the index picks them up and scores shift (the same query on the CG-4 working +tree reported tools.ts at 13–19% depending on the sync state). Measure against a clean +tree: restore `src/mcp/tools.ts` from `main`, remove `src/mcp/explore-diagnostics.ts`, +`codegraph sync`, then run the built `dist/` binary (which still carries the instrument). +Restore afterwards. diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts new file mode 100644 index 000000000..210a5cb06 --- /dev/null +++ b/src/mcp/explore-diagnostics.ts @@ -0,0 +1,528 @@ +/** + * Per-file allocation diagnostic for `codegraph_explore` (CG-4). + * + * The explore response is a fixed byte envelope (`budget.maxOutputChars`, hard- + * capped at 25K so the host never externalizes the result). WHICH files fill it, + * and in what proportion, is decided by a long chain of gates, tiers and caps + * spread across `handleExplore`. That chain is currently unobservable: you can + * read the output and guess, but you cannot say "this file took 16% of the + * envelope and that one took 21%" without hand-counting. + * + * This module is the instrument. Enabled by `CODEGRAPH_EXPLORE_DEBUG`, it + * records, for one explore call: + * - per candidate file: relevance score, graph (RWR) mass, distinct query-term + * hits, ranking flags, render mode, bytes of source actually emitted, that + * file's share of the final envelope, whether it was clipped, whether it + * carries a flow-spine symbol — and for the ones that didn't render, why; + * - totals: envelope vs `maxOutputChars` vs the hard ceiling, source bytes vs + * meta-text overhead, files considered at each filter stage, and the score + * floor / relevance-gate thresholds that were applied. + * + * HARD CONSTRAINT — this ships in the product binary: when the env var is unset + * the diagnostic must not exist. `start()` returns `null`, every call site is a + * `diag?.` no-op, and the agent-facing response is byte-identical. The + * diagnostic never mutates render state, and every method is wrapped so a bug in + * here can never fail an explore call. + * + * Sinks (value of `CODEGRAPH_EXPLORE_DEBUG`): + * `1` / `true` / `on` / `yes` / `stderr` → human-readable table on stderr + * `json` → one JSON object on stderr + * anything else → treated as a path; one JSON object + * per line appended (JSONL sidecar) + */ + +import { appendFileSync } from 'fs'; + +/** How a file's source was rendered into the response. */ +export type ExploreRenderMode = + | 'whole' // whole-file window + | 'clusters' // ranked contiguous clusters + | 'focused' // per-symbol view, named/spine bodies full + | 'skeleton' // per-symbol view, signatures only + | 'stale-omitted' // drifted on disk; source deliberately withheld + | 'dropped'; // rendered into `lines` but cut by the final hard ceiling + +/** Why a ranked candidate never reached the output. */ +export type ExploreSkipReason = + | 'max-files' // maxFiles reached before this file + | 'budget-90pct' // incidental file past the 90%-of-budget soft stop + | 'budget-whole-file' // incidental whole-file render wouldn't fit + | 'budget-clusters' // incidental cluster render wouldn't fit + | 'unreadable' // outside root, missing, or read error + | 'no-ranges'; // no renderable line ranges in this file + +/** Ranking inputs for one candidate file, captured before the render loop. */ +export interface ExploreCandidateMeta { + rank: number; + score: number; + graphScore: number; + termHits: number; + nodes: number; + named: boolean; + central: boolean; + entry: boolean; + spine: boolean; + lowValue: boolean; + generated: boolean; +} + +interface FileRecord extends ExploreCandidateMeta { + path: string; + render?: ExploreRenderMode; + /** Source chars the render loop handed to `lines` (pre-final-truncation). */ + emittedChars: number; + /** Source chars present in the FINAL text — authoritative, truncation-aware. */ + finalChars: number; + /** Share of the DELIVERED envelope, as a fraction (0–1). */ + share: number; + /** Share of what the render loop ALLOCATED, before the hard-ceiling cut. */ + allocatedShare: number; + clipped: boolean; + skipped?: ExploreSkipReason; +} + +interface StageCounts { + /** Files with at least one gathered node. */ + grouped: number; + /** Survived the `group.score >= scoreFloor` filter. */ + pastScoreFloor: number; + /** Survived the test/spec/icon/i18n hard-exclude. */ + pastLowValueFilter: number; + /** Survived the graph-relevance gate. */ + pastRelevanceGate: number; +} + +/** Budget fields the diagnostic reports. Structural, to avoid a cyclic import. */ +interface BudgetShape { + maxOutputChars: number; + maxCharsPerFile: number; + defaultMaxFiles: number; +} + +/** One file's line in the report. Also the JSONL sidecar's per-file shape. */ +export interface ExploreDiagnosticFile extends ExploreCandidateMeta { + path: string; + render: ExploreRenderMode | null; + skipped: ExploreSkipReason | null; + clipped: boolean; + emittedChars: number; + finalChars: number; + share: number; + allocatedShare: number; +} + +/** The full report — one per explore call, JSON-serialized to the sink. */ +export interface ExploreDiagnosticReport { + tool: 'codegraph_explore'; + query: string; + projectRoot: string; + indexedFileCount: number; + note?: string; + budget: { + maxOutputChars: number; + maxCharsPerFile: number; + maxFiles: number; + hardCeiling: number; + }; + envelope: { + /** Chars actually returned to the agent (post-truncation). */ + chars: number; + /** Chars the render loop produced, BEFORE the hard-ceiling cut. */ + allocatedChars: number; + overBudget: boolean; + truncated: boolean; + sourceChars: number; + sourceShare: number; + metaChars: number; + metaShare: number; + }; + selection: { + scoreFloor: number; + maxGraph: number; + graphGateThreshold: number; + graphGateApplied: boolean; + filesGrouped: number; + filesPastScoreFloor: number; + filesPastLowValueFilter: number; + filesRanked: number; + filesRenderedByLoop: number; + filesInFinalOutput: number; + }; + files: ExploreDiagnosticFile[]; +} + +type Sink = + | { kind: 'stderr'; json: boolean } + | { kind: 'file'; path: string }; + +const OFF = new Set(['', '0', 'false', 'off', 'no']); +const STDERR_TABLE = new Set(['1', 'true', 'on', 'yes', 'stderr']); + +/** + * Resolve the sink from the environment. `null` means the diagnostic is off — + * read per call (not memoized) so a test can toggle it between invocations. + */ +function resolveSink(): Sink | null { + const raw = process.env.CODEGRAPH_EXPLORE_DEBUG; + if (raw === undefined) return null; + const value = raw.trim(); + const lower = value.toLowerCase(); + if (OFF.has(lower)) return null; + if (STDERR_TABLE.has(lower)) return { kind: 'stderr', json: false }; + if (lower === 'json') return { kind: 'stderr', json: true }; + return { kind: 'file', path: value }; +} + +const num = (n: number) => Math.round(n).toLocaleString('en-US'); +const pct = (f: number) => `${(f * 100).toFixed(1)}%`; + +export class ExploreDiagnostics { + private readonly files = new Map(); + private readonly stages: StageCounts = { + grouped: 0, pastScoreFloor: 0, pastLowValueFilter: 0, pastRelevanceGate: 0, + }; + private scoreFloor = 0; + private maxGraph = 0; + private graphGateThreshold = 0; + private graphGateApplied = false; + private note = ''; + + private constructor( + private readonly sink: Sink, + private readonly query: string, + private readonly projectRoot: string, + private readonly budget: BudgetShape, + private readonly maxFiles: number, + private readonly indexedFileCount: number, + ) {} + + /** + * Returns `null` when `CODEGRAPH_EXPLORE_DEBUG` is unset/off — the whole + * instrument then costs one env read per explore call and nothing else. + */ + static start( + query: string, + projectRoot: string, + budget: BudgetShape, + maxFiles: number, + indexedFileCount: number, + ): ExploreDiagnostics | null { + try { + const sink = resolveSink(); + if (!sink) return null; + return new ExploreDiagnostics(sink, query, projectRoot, budget, maxFiles, indexedFileCount); + } catch { + return null; + } + } + + /** Candidate count after the initial `group.score >= floor` filter. */ + setScoreFloor(floor: number, grouped: number, kept: number): void { + this.scoreFloor = floor; + this.stages.grouped = grouped; + this.stages.pastScoreFloor = kept; + this.stages.pastLowValueFilter = kept; + this.stages.pastRelevanceGate = kept; + } + + /** Candidate count after the test/spec/icon/i18n hard-exclude. */ + setLowValueFiltered(kept: number): void { + this.stages.pastLowValueFilter = kept; + this.stages.pastRelevanceGate = kept; + } + + /** Graph-relevance gate: threshold, whether it actually pruned, what survived. */ + setRelevanceGate(maxGraph: number, threshold: number, applied: boolean, kept: number): void { + this.maxGraph = maxGraph; + this.graphGateThreshold = threshold; + this.graphGateApplied = applied; + this.stages.pastRelevanceGate = kept; + } + + /** Record one ranked candidate's scoring inputs, in final sort order. */ + noteCandidate(path: string, meta: ExploreCandidateMeta): void { + this.files.set(path, { + path, ...meta, + emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false, + }); + } + + /** A candidate rendered source into the response. */ + recordRender(path: string, render: ExploreRenderMode, sourceChars: number, clipped: boolean): void { + const rec = this.files.get(path); + if (!rec) return; + rec.render = render; + rec.emittedChars = sourceChars; + rec.clipped = clipped; + rec.skipped = undefined; + } + + /** + * A candidate was passed over before rendering. First reason wins — the + * blanket `max-files` sweep must not overwrite a file's specific reason. + */ + recordSkip(path: string, reason: ExploreSkipReason): void { + const rec = this.files.get(path); + if (!rec || rec.render || rec.skipped) return; + rec.skipped = reason; + } + + /** Explore returned early (no subgraph). Emits a minimal record. */ + finishEmpty(reason: string): void { + this.note = reason; + this.emit(this.buildReport('', 0, 0, 0)); + } + + /** + * Final pass: attribute the FINAL text's bytes back to files (so the hard + * ceiling's truncation is reflected in what each file actually delivered), + * then emit. + * + * `allocatedChars` is the pre-truncation length — the size the render loop + * *chose*. Reporting both is the point: the allocator's decision and the + * agent's delivered payload diverge exactly when the ceiling cuts, and + * conflating them is how a dropped trailing file goes unnoticed. + */ + finish(finalText: string, allocatedChars: number, hardCeiling: number, filesIncluded: number): void { + try { + const perFile = attributeSourceBytes(finalText); + const envelope = finalText.length; + for (const rec of this.files.values()) { + rec.finalChars = perFile.get(rec.path) ?? 0; + rec.share = envelope > 0 ? rec.finalChars / envelope : 0; + rec.allocatedShare = allocatedChars > 0 ? rec.emittedChars / allocatedChars : 0; + // Rendered into `lines` but absent from the final text → the hard + // ceiling dropped its whole section. + if (rec.render && rec.render !== 'stale-omitted' && rec.finalChars === 0) { + rec.render = 'dropped'; + rec.clipped = true; + } + } + this.emit(this.buildReport(finalText, allocatedChars, hardCeiling, filesIncluded)); + } catch { + // A diagnostic must never fail an explore call. + } + } + + private buildReport( + finalText: string, + allocatedChars: number, + hardCeiling: number, + filesIncluded: number, + ): ExploreDiagnosticReport { + const envelope = finalText.length; + const records = [...this.files.values()]; + const rendered = records.filter((r) => r.finalChars > 0); + const sourceChars = rendered.reduce((s, r) => s + r.finalChars, 0); + return { + tool: 'codegraph_explore', + query: this.query, + projectRoot: this.projectRoot, + indexedFileCount: this.indexedFileCount, + note: this.note || undefined, + budget: { + maxOutputChars: this.budget.maxOutputChars, + maxCharsPerFile: this.budget.maxCharsPerFile, + maxFiles: this.maxFiles, + hardCeiling, + }, + envelope: { + chars: envelope, + allocatedChars, + overBudget: allocatedChars > this.budget.maxOutputChars, + truncated: allocatedChars > hardCeiling, + sourceChars, + sourceShare: envelope > 0 ? sourceChars / envelope : 0, + metaChars: envelope - sourceChars, + metaShare: envelope > 0 ? (envelope - sourceChars) / envelope : 0, + }, + selection: { + scoreFloor: this.scoreFloor, + maxGraph: this.maxGraph, + graphGateThreshold: this.graphGateThreshold, + graphGateApplied: this.graphGateApplied, + filesGrouped: this.stages.grouped, + filesPastScoreFloor: this.stages.pastScoreFloor, + filesPastLowValueFilter: this.stages.pastLowValueFilter, + filesRanked: this.stages.pastRelevanceGate, + filesRenderedByLoop: filesIncluded, + filesInFinalOutput: rendered.length, + }, + files: records + .slice() + .sort((a, b) => b.emittedChars - a.emittedChars || b.finalChars - a.finalChars || a.rank - b.rank) + .map((r) => ({ + path: r.path, + rank: r.rank, + score: r.score, + graphScore: round6(r.graphScore), + termHits: r.termHits, + nodes: r.nodes, + named: r.named, + central: r.central, + entry: r.entry, + spine: r.spine, + lowValue: r.lowValue, + generated: r.generated, + render: r.render ?? null, + skipped: r.skipped ?? null, + clipped: r.clipped, + emittedChars: r.emittedChars, + finalChars: r.finalChars, + share: round6(r.share), + allocatedShare: round6(r.allocatedShare), + })), + }; + } + + private emit(report: ExploreDiagnosticReport): void { + try { + if (this.sink.kind === 'file') { + appendFileSync(this.sink.path, JSON.stringify(report) + '\n', 'utf-8'); + return; + } + if (this.sink.json) { + process.stderr.write(JSON.stringify(report, null, 2) + '\n'); + return; + } + process.stderr.write(renderTable(report) + '\n'); + } catch { + // Unwritable sidecar / closed stderr must not fail the explore call. + } + } +} + +const round6 = (n: number) => Math.round(n * 1e6) / 1e6; + +/** + * Attribute the final response's source bytes back to files by walking the + * rendered markdown: a ``**`path`**`` section header followed by a fenced code + * block. Reading the FINAL text (rather than trusting the render loop's running + * total) is what makes the numbers truthful — it accounts for the hard-ceiling + * truncation that can drop whole trailing sections after they were "emitted". + * + * Line numbering is on by default, so a source line that is itself a ``` fence + * arrives as `42\t```` and cannot close the block early. + */ +export function attributeSourceBytes(finalText: string): Map { + const out = new Map(); + if (!finalText) return out; + const lines = finalText.split('\n'); + let current: string | null = null; + let inFence = false; + let acc: string[] = []; + const flush = () => { + if (current && acc.length > 0) { + out.set(current, (out.get(current) ?? 0) + acc.join('\n').length); + } + acc = []; + }; + for (const line of lines) { + if (!inFence) { + const header = /^\*\*`([^`]+)`\*\*/.exec(line); + if (header) { + current = header[1]!; + continue; + } + if (current && line.startsWith('```')) { + inFence = true; + continue; + } + continue; + } + if (line === '```') { + inFence = false; + flush(); + continue; + } + acc.push(line); + } + // Unterminated fence (final-ceiling truncation cut mid-block): count it. + if (inFence) flush(); + return out; +} + +/** Human-readable stderr rendering of the JSON report. */ +export function renderTable(report: ExploreDiagnosticReport): string { + const { budget, envelope: env, selection: sel, files } = report; + + const out: string[] = []; + out.push(''); + out.push(`codegraph explore diagnostic — "${report.query}"`); + out.push(` project ${report.projectRoot} · ${num(report.indexedFileCount)} files indexed`); + if (report.note) out.push(` note: ${report.note}`); + out.push( + ` envelope ${num(env.chars)} chars delivered · ${num(env.allocatedChars)} allocated` + + ` of ${num(budget.maxOutputChars)} budget (hard ceiling ${num(budget.hardCeiling)})` + + `${env.overBudget ? ' [over budget]' : ''}${env.truncated ? ' [TRUNCATED]' : ''}`, + ); + out.push( + ` source ${num(env.sourceChars)} (${pct(env.sourceShare)})` + + ` · meta ${num(env.metaChars)} (${pct(env.metaShare)})` + + ` · per-file cap ${num(budget.maxCharsPerFile)}`, + ); + out.push( + ` files ${num(sel.filesGrouped)} grouped` + + ` → ${num(sel.filesPastScoreFloor)} past score floor (>=${sel.scoreFloor})` + + ` → ${num(sel.filesPastLowValueFilter)} past low-value filter` + + ` → ${num(sel.filesRanked)} past relevance gate` + + ` → ${num(sel.filesInFinalOutput)} in output (maxFiles ${num(budget.maxFiles)})`, + ); + out.push( + ` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` + + ` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`, + ); + out.push(''); + + // Allocated (not delivered) is the allocator's own decision — the number the + // budget work is about. Delivered is what the agent got. They differ only + // when the ceiling truncated; showing both makes that divergence obvious. + const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0); + if (shown.length > 0) { + out.push(' # alloc% deliv% bytes score graph hits flags render file'); + for (const f of shown) { + out.push( + ' ' + + String(f.rank).padStart(2) + ' ' + + pct(f.allocatedShare).padStart(6) + ' ' + + pct(f.share).padStart(6) + ' ' + + num(f.emittedChars).padStart(7) + ' ' + + String(f.score).padStart(5) + ' ' + + f.graphScore.toFixed(5).padStart(7) + ' ' + + String(f.termHits).padStart(4) + ' ' + + flagString(f).padEnd(19) + ' ' + + ((f.render ?? '-') + (f.clipped ? '*' : '')).padEnd(9) + ' ' + + f.path, + ); + } + out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)'); + out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)'); + } else { + out.push(' (no file source in the final output)'); + } + + const skipped = files.filter((f) => f.emittedChars === 0 && f.finalChars === 0); + if (skipped.length > 0) { + out.push(''); + out.push(' ranked but never rendered:'); + for (const f of skipped.slice(0, 15)) { + out.push( + ` #${String(f.rank).padStart(2)} ${f.path} — ${f.skipped ?? f.render ?? 'not reached'}` + + ` (score ${f.score}, graph ${f.graphScore.toFixed(5)}, hits ${f.termHits})`, + ); + } + if (skipped.length > 15) out.push(` … and ${skipped.length - 15} more`); + } + return out.join('\n'); +} + +function flagString(f: ExploreDiagnosticFile): string { + const flags: string[] = []; + if (f.named) flags.push('named'); + if (f.entry) flags.push('entry'); + if (f.central) flags.push('central'); + if (f.spine) flags.push('spine'); + if (f.lowValue) flags.push('low-value'); + if (f.generated) flags.push('generated'); + return flags.join(' ') || '-'; +} diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 08308aca9..d1fcbc964 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -42,6 +42,7 @@ import { clamp, validatePathWithinRoot, validateProjectPath, isConfigLeafNode, C import { isGeneratedFile } from '../extraction/generated-detection'; import { scanDynamicDispatch } from './dynamic-boundaries'; import { getUpdateNotice } from '../upgrade/update-check'; +import { ExploreDiagnostics } from './explore-diagnostics'; /** * An expected, recoverable "codegraph can't serve this" condition — most @@ -2630,13 +2631,20 @@ export class ToolHandler { // largest-tier defaults if stats aren't available, which preserves // pre-#185 behavior for callers that hit the rare stats failure. let budget: ExploreOutputBudget; + let indexedFileCount = -1; try { - budget = getExploreOutputBudget(cg.getStats().fileCount); + indexedFileCount = cg.getStats().fileCount; + budget = getExploreOutputBudget(indexedFileCount); } catch { budget = getExploreOutputBudget(Infinity); } const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG + // is set — every `diag?.` below is then a no-op and the response is + // byte-identical. It only OBSERVES: it must never feed back into rendering. + const diag = ExploreDiagnostics.start(query, projectRoot, budget, maxFiles, indexedFileCount); + // Step 1: Find relevant context with generous parameters. // Use a large maxNodes budget — explore has its own 35k char output limit // that prevents context bloat, so more nodes just means better coverage @@ -2649,6 +2657,7 @@ export class ToolHandler { }); if (subgraph.nodes.size === 0) { + diag?.finishEmpty('no relevant code found — empty subgraph'); return this.textResult(`No relevant code found for "${query}"`); } @@ -2913,7 +2922,9 @@ export class ToolHandler { } // Only include files that have entry points or nodes directly connected to entry points - let relevantFiles = [...fileGroups.entries()].filter(([, group]) => group.score >= 3); + const SCORE_FLOOR = 3; + let relevantFiles = [...fileGroups.entries()].filter(([, group]) => group.score >= SCORE_FLOOR); + diag?.setScoreFloor(SCORE_FLOOR, fileGroups.size, relevantFiles.length); // Extract query terms for relevance checking const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3); @@ -2955,6 +2966,7 @@ export class ToolHandler { relevantFiles = nonLow; } } + diag?.setLowValueFiltered(relevantFiles.length); } // Secondary signal: how many DISTINCT query terms each file matches (path + @@ -3057,6 +3069,9 @@ export class ToolHandler { || (fileTermHits.get(fp) ?? 0) >= 2, ); if (gated.length >= 2) relevantFiles = gated; + diag?.setRelevanceGate(maxGraph, maxGraph * 0.06, gated.length >= 2, relevantFiles.length); + } else { + diag?.setRelevanceGate(maxGraph, 0, false, relevantFiles.length); } // Sort files: graph-central first, then distinct-term match, then the @@ -3202,6 +3217,27 @@ export class ToolHandler { // off-spine peers skeletonize. const flow = this.buildFlowFromNamedSymbols(cg, query); + // Snapshot every ranked candidate's scoring inputs, in final sort order, so + // the diagnostic can show what each file's share of the envelope was BOUGHT + // with (score, graph mass, term hits, flags) — not just what it cost. + if (diag) { + sortedFiles.forEach(([fp, group], i) => { + diag.noteCandidate(fp, { + rank: i + 1, + score: group.score, + graphScore: fileGraphScore.get(fp) ?? 0, + termHits: fileTermHits.get(fp) ?? 0, + nodes: group.nodes.length, + named: namedSeedFiles.has(fp), + central: centralFiles.has(fp), + entry: entryFiles.has(fp), + spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), + lowValue: isLowValue(fp), + generated: isGeneratedFile(fp), + }); + }); + } + // Polymorphic-sibling detector for adaptive sizing. A class that implements/ // extends a supertype shared by >= MIN_SIBLINGS classes is one of many // INTERCHANGEABLE implementations (OkHttp's 14 `: Interceptor` classes — @@ -3278,7 +3314,10 @@ export class ToolHandler { const staleOmitted: string[] = []; for (const [filePath, group] of sortedFiles) { - if (filesIncluded >= maxFiles) break; + if (filesIncluded >= maxFiles) { + if (diag) for (const [fp] of sortedFiles) diag.recordSkip(fp, 'max-files'); + break; + } // A file DEFINES a named/spine symbol (the answer) vs merely references the // flow. Past 90% budget, stop pulling INCIDENTAL files — but keep scanning // for necessary ones, which render even past the cap (bounded by maxFiles). @@ -3287,15 +3326,22 @@ export class ToolHandler { // validate-logic file (Alamofire's Validation.swift). const fileNecessary = group.nodes.some(n => entryNodeIds.has(n.id) || flow.pathNodeIds.has(n.id) || flow.uniqueNamedNodeIds.has(n.id)); - if (!fileNecessary && totalChars > budget.maxOutputChars * 0.9) continue; + if (!fileNecessary && totalChars > budget.maxOutputChars * 0.9) { + diag?.recordSkip(filePath, 'budget-90pct'); + continue; + } const absPath = validatePathWithinRoot(projectRoot, filePath); - if (!absPath || !existsSync(absPath)) continue; + if (!absPath || !existsSync(absPath)) { + diag?.recordSkip(filePath, 'unreadable'); + continue; + } let fileContent: string; try { fileContent = readFileSync(absPath, 'utf-8'); } catch { + diag?.recordSkip(filePath, 'unreadable'); continue; } @@ -3419,6 +3465,8 @@ export class ToolHandler { : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', ''); totalChars += skel.join('\n').length + 120; + // Always "clipped": the per-symbol view elides bodies by construction. + diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true); renderedFilePaths.push(filePath); filesIncluded++; continue; @@ -3470,10 +3518,12 @@ export class ToolHandler { // fit is skipped; a necessary one (below) renders in full. Half a file // forces the Read this is meant to prevent. anyFileTrimmed = true; + diag?.recordSkip(filePath, 'budget-whole-file'); continue; } lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', ''); totalChars += wholeSection.length + 200; + diag?.recordRender(filePath, 'whole', wholeSection.length, false); renderedFilePaths.push(filePath); filesIncluded++; if (fileStale) staleRendered.push(filePath); @@ -3492,6 +3542,7 @@ export class ToolHandler { '', ); totalChars += 260; + diag?.recordRender(filePath, 'stale-omitted', 0, true); continue; } @@ -3567,7 +3618,10 @@ export class ToolHandler { ranges.sort((a, b) => a.start - b.start); - if (ranges.length === 0) continue; + if (ranges.length === 0) { + diag?.recordSkip(filePath, 'no-ranges'); + continue; + } const gapThreshold = budget.gapThreshold; const clusters: Array<{ start: number; end: number; symbols: string[]; score: number; maxImportance: number; hasSpine: boolean; spineCallLine?: number }> = []; @@ -3764,6 +3818,7 @@ export class ToolHandler { // Keep scanning for necessary files (which bypass this cap and render in // full, bounded by the hard ceiling). anyFileTrimmed = true; + diag?.recordSkip(filePath, 'budget-clusters'); continue; } @@ -3775,6 +3830,7 @@ export class ToolHandler { lines.push(''); totalChars += fileSection.length + 200; + diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length); renderedFilePaths.push(filePath); filesIncluded++; } @@ -3896,6 +3952,10 @@ export class ToolHandler { : `Found ${subgraph.nodes.size} symbol${subgraph.nodes.size === 1 ? '' : 's'} across ${fileGroups.size} file${fileGroups.size === 1 ? '' : 's'}.`; finalText = finalText.replace(SUMMARY_SENTINEL, summaryLine); + // Emit the allocation diagnostic from the FINAL text, so per-file bytes and + // shares account for the hard-ceiling truncation above (CG-4). + diag?.finish(finalText, output.length, hardCeiling, filesIncluded); + return this.textResult(finalText); } From 16e17495f468eb2c50f387cc922b4ddd1b65000d Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Mon, 3 Aug 2026 23:13:59 -0500 Subject: [PATCH 02/15] feat(extraction): content-based generated-file detection (CG-5, #1500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `isGeneratedFile` was path-only, but Go's own convention is a CONTENT marker (`// Code generated by . DO NOT EDIT.`), not a filename one. A Go monorepo with generated CRUD in ordinarily-named files sitting beside hand-written use-cases was therefore invisible to every generated-file down-rank in the codebase — that is #1500. Measured on kubernetes/client-go (2,453 Go files): the canonical banner appears in 2,001 of them, the path check flags 0, the new content check flags exactly those 2,001 — no false positives, no misses. Design: decide at INDEX time (content is already in memory for parsing), persist on `files.generated`, read from the DB. Explore never reads file headers per request. - `hasGeneratedHeader(content)` recognizes the standard banners — Go's, protoc's, `@generated`, ``, Thrift, OpenAPI Generator, FlatBuffers, bindgen, ANTLR. Precision-first and fenced three ways: an 8KB/60-line header window, a comment-line requirement (leader or open block comment), and markers tight enough that prose can't trip them. A generator's own source, holding the banner as a string constant in its body, is not flagged; neither is this module itself (pinned by test). - `isGeneratedFile(path)` is unchanged — cheap, sync, still the fallback. - Schema v9 adds `files.generated` + a PARTIAL index. DDL only, no backfill: the flag derives from content the migration cannot see, so rows stay 0 until a re-index and every reader unions the flag with the path check — an un-migrated index keeps pre-#1500 behavior rather than regressing. Re-index required; noted in the CHANGELOG. - `generatedPredicateFor(paths)` gives ranking a bounded probe + O(1) lookups. Bounded, not cached: no invalidation, so a ranking call can never serve a verdict the last sync already replaced. Wired into explore ranking, findSymbolMatches, findAllSymbols, search (MCP + CLI), the context formatter, and the dominant-file/route-file hygiene filters. Cost (acceptance bar was no measurable index-time regression): a single unanchored `/generat/i` test over the header rejects ~every hand-written file before any line splitting. 4.6 µs/file on client-go (worst case — 82% generated). End-to-end `codegraph init` on client-go, n=3 alternating arms: 5.73s median with detection vs 5.76s path-only baseline; the arms cross over between runs, so the difference is inside run-to-run noise. Scope note: generated status remains a stable TIEBREAK at equal score, exactly where it was. Making it a strong negative signal is CG-10, which this unblocks by making the signal correct and available. Two pre-existing tests hard-coded schema version 8; both now track CURRENT_SCHEMA_VERSION (or the migration table) so future migrations don't require editing them. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 + __tests__/db-perf.test.ts | 8 +- __tests__/foundation.test.ts | 5 +- __tests__/generated-detection.test.ts | 161 ++++++++++++++++++- __tests__/generated-flag-index.test.ts | 204 ++++++++++++++++++++++++ __tests__/pr19-improvements.test.ts | 17 +- __tests__/security.test.ts | 4 + docs/design/generated-file-detection.md | 123 ++++++++++++++ src/bin/codegraph.ts | 6 +- src/context/formatter.ts | 20 ++- src/context/index.ts | 9 +- src/db/migrations.ts | 29 +++- src/db/queries.ts | 86 ++++++++-- src/db/schema.sql | 19 ++- src/extraction/generated-detection.ts | 186 +++++++++++++++++++-- src/extraction/index.ts | 12 ++ src/index.ts | 18 +++ src/mcp/tools.ts | 35 ++-- src/types.ts | 9 ++ 19 files changed, 898 insertions(+), 55 deletions(-) create mode 100644 __tests__/generated-flag-index.test.ts create mode 100644 docs/design/generated-file-detection.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b57b2000..61856d623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- Generated code is now recognized by what's written at the top of the file, not just by its filename. Go's own convention — a `// Code generated by … DO NOT EDIT.` line — is a comment, not a naming rule, so a generated `payroll.go` sitting beside hand-written code was previously indistinguishable from it and could outrank the real implementation in `codegraph_explore` and search results. CodeGraph now reads the header while indexing and recognizes the standard banners across languages (Go's, protoc's, `@generated` in JavaScript/TypeScript, `` in C#, Thrift, OpenAPI Generator, FlatBuffers, ANTLR, bindgen and more), so those files rank behind hand-written source everywhere the existing filename rules already applied. On a Kubernetes `client-go` checkout this identifies 2,001 generated files that filename rules alone caught none of. Re-index after upgrading to pick up the new detection. (#1500) + - Anonymous usage telemetry is now stored entirely on CodeGraph's own first-party infrastructure — no third-party analytics vendor receives any of it, and the endpoint that receives it makes no outbound requests at all. Individual events are deleted after 90 days, leaving only anonymous daily totals. Nothing about what is collected changed, your IP address is still never read or stored, and every off-switch works exactly as before (`codegraph telemetry off`, `CODEGRAPH_TELEMETRY=0`, `DO_NOT_TRACK=1`). `TELEMETRY.md` remains the complete field-by-field list. ### Fixes diff --git a/__tests__/db-perf.test.ts b/__tests__/db-perf.test.ts index 9be0803b2..941fb50f5 100644 --- a/__tests__/db-perf.test.ts +++ b/__tests__/db-perf.test.ts @@ -16,7 +16,7 @@ import * as path from 'path'; import * as os from 'os'; import { DatabaseConnection } from '../src/db'; import { QueryBuilder } from '../src/db/queries'; -import { runMigrations, getCurrentVersion } from '../src/db/migrations'; +import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from '../src/db/migrations'; import { Node, Edge } from '../src/types'; function makeNode(id: string, name = id): Node { @@ -344,7 +344,11 @@ describe('migration v6: dedup edges + add identity index on upgrade (#1034)', () runMigrations(raw, 5); expect(count()).toBe(2); // duplicate collapsed, the distinct `calls` edge kept - expect(getCurrentVersion(raw)).toBe(8); + // Migrations ran to completion. Tracked against the constant, not a + // literal, so adding a migration doesn't require editing this assertion — + // and so replaying every migration over a current-schema database (which + // is what this test does) stays covered as new ones land. + expect(getCurrentVersion(raw)).toBe(CURRENT_SCHEMA_VERSION); const idx = raw .prepare("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_edges_identity'") .get(); diff --git a/__tests__/foundation.test.ts b/__tests__/foundation.test.ts index 8ed73123e..12c136445 100644 --- a/__tests__/foundation.test.ts +++ b/__tests__/foundation.test.ts @@ -12,6 +12,7 @@ import { CodeGraph } from '../src'; import { Node, Edge } from '../src/types'; import { isInitialized, getCodeGraphDir, validateDirectory, codeGraphDirName, isCodeGraphDataDir } from '../src/directory'; import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from '../src/db'; +import { CURRENT_SCHEMA_VERSION } from '../src/db/migrations'; // Create a temporary directory for each test function createTempDir(): string { @@ -370,7 +371,9 @@ describe('Database Connection', () => { const version = db.getSchemaVersion(); expect(version).not.toBeNull(); - expect(version?.version).toBe(8); + // A freshly initialized database records the current version outright + // (schema.sql already contains every migration's end state). + expect(version?.version).toBe(CURRENT_SCHEMA_VERSION); db.close(); }); diff --git a/__tests__/generated-detection.test.ts b/__tests__/generated-detection.test.ts index 90bbae7f1..2fd680f2c 100644 --- a/__tests__/generated-detection.test.ts +++ b/__tests__/generated-detection.test.ts @@ -4,10 +4,23 @@ * list is a contract: if a future edit drops `.pb.go`, the cosmos-sdk * trace endpoint regresses to the gRPC stub (see * `project_go_multi_module_audit` memory + the audit in #N/A). + * + * The content-header half (#1500) is a second contract: the marker table is + * precision-first, because a false positive silently demotes hand-written code + * in EVERY ranking path. Measured on a shallow clone of kubernetes/client-go + * (2,453 Go files): the path check flags 0, the content check flags 2,001 — + * exactly the set that greps to the canonical banner, no false positives and + * no misses. Every one of those files has an ordinary name. */ import { describe, it, expect } from 'vitest'; -import { isGeneratedFile } from '../src/extraction/generated-detection'; +import * as fs from 'fs'; +import * as path from 'path'; +import { + isGeneratedFile, + hasGeneratedHeader, + detectGeneratedFile, +} from '../src/extraction/generated-detection'; describe('isGeneratedFile', () => { it('classifies Go protobuf / gRPC / pulsar / mock outputs as generated', () => { @@ -45,3 +58,149 @@ describe('isGeneratedFile', () => { expect(isGeneratedFile('app/db.py')).toBe(false); }); }); + +describe('hasGeneratedHeader — per-marker coverage (#1500)', () => { + // One case per banner the marker table claims to recognize. Each string is + // the real thing a generator emits, not a paraphrase — if a regex is + // narrowed, the case that motivated it fails by name. + const GENERATED: ReadonlyArray<[string, string]> = [ + [ + 'Go — the #1500 case: ordinary filename, banner below the package clause', + 'package payroll\n\n// Code generated by fkit. DO NOT EDIT.\n\nimport "context"\n\nfunc CreatePayroll(ctx context.Context) error { return nil }\n', + ], + [ + 'Go — protoc-gen-go', + '// Code generated by protoc-gen-go. DO NOT EDIT.\n// versions:\n// protoc-gen-go v1.28.0\n\npackage pb\n', + ], + [ + 'Go — banner under build tags', + '//go:build !windows\n// +build !windows\n\n// Code generated by MockGen. DO NOT EDIT.\npackage mocks\n', + ], + [ + 'Go — banner under an Apache-2.0 license preamble', + '// Copyright 2021 The Foo Authors.\n// Licensed under the Apache License, Version 2.0 (the "License");\n// you may not use this file except in compliance with the License.\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an "AS IS" BASIS.\n\n// Code generated by sqlc. DO NOT EDIT.\n// source: query.sql\n\npackage db\n', + ], + [ + 'protoc — Java banner ("DO NOT EDIT!")', + '// Generated by the protocol buffer compiler. DO NOT EDIT!\n// source: foo.proto\n\npackage com.example;\n', + ], + [ + 'protoc — Python banner behind a coding cookie', + '# -*- coding: utf-8 -*-\n# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: foo.proto\n', + ], + [ + 'C# — Roslyn / designer block', + '//------------------------------------------------------------------------------\n// \n// This code was generated by a tool.\n// \n//------------------------------------------------------------------------------\n', + ], + ['C# — EF self-closing ', '// \nusing System;\n'], + [ + 'JS — Meta/Relay @generated with a SignedSource', + '/**\n * @generated SignedSource<<0123456789abcdef0123456789abcdef>>\n * @flow\n */\n', + ], + [ + 'TS — protobuf-es / Buf @generated', + '// @generated by protoc-gen-es v1.2.0 with parameter "target=ts"\n// @generated from file foo.proto (package example, syntax proto3)\n', + ], + [ + 'Thrift — "Autogenerated by Thrift Compiler"', + '/**\n * Autogenerated by Thrift Compiler (0.14.1)\n *\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\n', + ], + [ + 'OpenAPI Generator — "This class is auto generated by"', + '/*\n * Pet Store API\n *\n * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).\n * Do not edit the class manually.\n */\n', + ], + [ + 'FlatBuffers — "automatically generated by … do not modify"', + '// automatically generated by the FlatBuffers compiler, do not modify\n\npackage MyGame;\n', + ], + [ + 'Rust — bindgen block comment', + '/* automatically generated by rust-bindgen 0.59.2 */\n\npub const FOO: u32 = 1;\n', + ], + ['ANTLR — "Generated from … -- DO NOT EDIT"', '// Generated from Expr.g4 by ANTLR 4.9.2 -- DO NOT EDIT\npackage parser;\n'], + [ + 'banner on an unprefixed line INSIDE a block comment', + '/*\n Code generated by ent. DO NOT EDIT.\n*/\npackage ent\n', + ], + [ + 'Python — banner inside a module docstring', + '"""Generated by the protocol buffer compiler. DO NOT EDIT!"""\nimport sys\n', + ], + ['YAML/shell — "#" comment leader', '# This file is generated by kustomize. Do not edit.\napiVersion: v1\n'], + ['SQL — "--" comment leader', '-- Code generated by sqlc. DO NOT EDIT.\nCREATE TABLE foo (id INT);\n'], + ['HTML/XML — "\n\n'], + ]; + + it.each(GENERATED)('flags: %s', (_label, source) => { + expect(hasGeneratedHeader(source)).toBe(true); + }); + + // Precision cases. Each is a shape that a looser marker table WOULD flag. + const HAND_WRITTEN: ReadonlyArray<[string, string]> = [ + [ + 'ordinary Go source', + 'package keeper\n\nimport "context"\n\n// SendCoins moves coins between accounts.\nfunc (k Keeper) SendCoins(ctx context.Context) error { return nil }\n', + ], + [ + 'a generator\'s own source, which merely talks about generating', + '// This package generates SQL migrations from the schema.\n// The generated output lives under db/migrations.\npackage gen\n', + ], + [ + 'prose using "automatically generated" without naming a tool', + '"""Report builder.\n\nThe summary table is automatically generated at runtime from the\nrows below; callers should not edit it in place.\n"""\n', + ], + [ + 'a generator holding the banner as a string constant in its BODY', + 'package main\n\n// Package main implements the fkit CRUD generator.\n\nimport "fmt"\n\nfunc header() string {\n\treturn "// Code generated by fkit. DO NOT EDIT."\n}\n', + ], + ['an email address that happens to contain "@generated"', '// Contact: build@generated.example.com for issues.\npackage main\n'], + ['"DO NOT EDIT" with no generation claim', '// DO NOT EDIT THIS FILE BY HAND — run `make fmt` instead.\npackage main\n'], + ['empty file', ''], + ]; + + it.each(HAND_WRITTEN)('does not flag: %s', (_label, source) => { + expect(hasGeneratedHeader(source)).toBe(false); + }); + + it('only looks at the header — a banner buried 80 lines down is not a banner', () => { + const filler = Array.from({ length: 80 }, (_, i) => `// filler line ${i}`).join('\n'); + expect(hasGeneratedHeader(`${filler}\n// Code generated by foo. DO NOT EDIT.\npackage main\n`)).toBe(false); + // …but the same banner within the window is caught. + const shortFiller = Array.from({ length: 20 }, (_, i) => `// filler line ${i}`).join('\n'); + expect(hasGeneratedHeader(`${shortFiller}\n// Code generated by foo. DO NOT EDIT.\npackage main\n`)).toBe(true); + }); + + it('requires a comment line — the same words in executable code are not a banner', () => { + // No comment leader, no open block: this is a bare statement. + expect(hasGeneratedHeader('const banner = "Code generated by tool. DO NOT EDIT.";\n')).toBe(false); + }); + + it('does not classify the detector module itself (the pattern table must stay below the header window)', () => { + const self = fs.readFileSync( + path.join(__dirname, '..', 'src', 'extraction', 'generated-detection.ts'), + 'utf-8' + ); + expect(hasGeneratedHeader(self)).toBe(false); + }); +}); + +describe('detectGeneratedFile — the union the indexer persists', () => { + it('is true when only the PATH says so', () => { + expect(detectGeneratedFile('x/bank/types/tx.pb.go', 'package types\n')).toBe(true); + }); + + it('is true when only the CONTENT says so — the #1500 acceptance case', () => { + // A Go file named `payroll.go` sitting beside hand-written workflow + // use-cases. Nothing in the path gives it away. + expect( + detectGeneratedFile('internal/payroll/payroll.go', 'package payroll\n\n// Code generated by fkit. DO NOT EDIT.\n\nfunc Create() {}\n') + ).toBe(true); + expect(isGeneratedFile('internal/payroll/payroll.go')).toBe(false); + }); + + it('is false for a hand-written file with an ordinary name', () => { + expect( + detectGeneratedFile('internal/payroll/workflow.go', 'package payroll\n\n// RunPayrollWorkflow drives the monthly run.\nfunc RunPayrollWorkflow() {}\n') + ).toBe(false); + }); +}); diff --git a/__tests__/generated-flag-index.test.ts b/__tests__/generated-flag-index.test.ts new file mode 100644 index 000000000..35898e3a1 --- /dev/null +++ b/__tests__/generated-flag-index.test.ts @@ -0,0 +1,204 @@ +/** + * Index-time persistence of the generated-file flag (#1500). + * + * `isGeneratedFile` is path-only, so a Go monorepo's generated CRUD — ordinary + * filenames, a `// Code generated by … DO NOT EDIT.` banner in the header — is + * invisible to it and outranks the hand-written use-case beside it. The fix + * decides the verdict ONCE during extraction (content is already in memory for + * parsing) and persists it on `files.generated`, so ranking reads a column + * instead of re-reading file headers per request. + * + * This suite pins the whole path: extraction writes it, `sync` re-decides it, + * the migration adds the column to an old database, and the bounded lookup + * that ranking uses unions it with the filename convention. + */ + +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src'; +import { QueryBuilder } from '../src/db/queries'; +import { createDatabase, type SqliteDatabase } from '../src/db/sqlite-adapter'; +import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from '../src/db/migrations'; + +/** The FKIT-style generated CRUD from the issue: ordinary name, banner inside. */ +const GENERATED_PAYROLL = `package payroll + +// Code generated by fkit. DO NOT EDIT. + +type PayrollRecord struct { + ID string + Amount int +} + +func CreatePayrollRecord(r PayrollRecord) error { return nil } +func UpdatePayrollRecord(r PayrollRecord) error { return nil } +func DeletePayrollRecord(id string) error { return nil } +`; + +/** The hand-written use-case that must NOT be demoted. */ +const HANDWRITTEN_WORKFLOW = `package payroll + +// RunPayrollWorkflow computes the monthly run and persists each record. +func RunPayrollWorkflow(records []PayrollRecord) error { + for _, r := range records { + if err := CreatePayrollRecord(r); err != nil { + return err + } + } + return nil +} +`; + +describe('generated flag — written at index time', () => { + let dir: string; + let cg: CodeGraph; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-genflag-')); + fs.writeFileSync(path.join(dir, 'payroll.go'), GENERATED_PAYROLL); + fs.writeFileSync(path.join(dir, 'workflow.go'), HANDWRITTEN_WORKFLOW); + // A path-convention generated file, so both signals are exercised together. + fs.writeFileSync(path.join(dir, 'payroll.pb.go'), 'package payroll\n\ntype PayrollProto struct{}\n'); + cg = await CodeGraph.init(dir, { index: true }); + }); + + afterAll(() => { + cg?.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('flags an ORDINARY-named Go file carrying the DO-NOT-EDIT banner (the acceptance case)', () => { + expect(cg.getFile('payroll.go')?.generated).toBe(true); + }); + + it('leaves the hand-written use-case beside it unflagged', () => { + expect(cg.getFile('workflow.go')?.generated).toBe(false); + }); + + it('still flags the filename convention', () => { + expect(cg.getFile('payroll.pb.go')?.generated).toBe(true); + }); + + it('counts the flagged files', () => { + expect(cg.getGeneratedFileCount()).toBe(2); + }); + + it('exposes a bounded predicate that unions both signals', () => { + const isGen = cg.generatedFilePredicate(['payroll.go', 'workflow.go', 'payroll.pb.go']); + expect(isGen('payroll.go')).toBe(true); // content only + expect(isGen('payroll.pb.go')).toBe(true); // path (and content) + expect(isGen('workflow.go')).toBe(false); + }); + + it('falls back to the filename check for a path outside the queried set', () => { + const isGen = cg.generatedFilePredicate([]); + // Not in the bounded set, but the path convention still decides. + expect(isGen('some/other/tx.pb.go')).toBe(true); + expect(isGen('some/other/keeper.go')).toBe(false); + }); + + it('re-decides on sync: removing the banner clears the flag', async () => { + fs.writeFileSync( + path.join(dir, 'payroll.go'), + GENERATED_PAYROLL.replace('// Code generated by fkit. DO NOT EDIT.\n\n', '') + ); + await cg.sync(); + expect(cg.getFile('payroll.go')?.generated).toBe(false); + + // …and adding it back re-flags it, so a stale 1 can never linger. + fs.writeFileSync(path.join(dir, 'payroll.go'), GENERATED_PAYROLL); + await cg.sync(); + expect(cg.getFile('payroll.go')?.generated).toBe(true); + }); +}); + +describe('generated flag — schema migration to v9', () => { + let dir: string; + let db: SqliteDatabase | null = null; + + afterEach(() => { + db?.close(); + db = null; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + /** A pre-v9 `files` table: no `generated` column, no partial index. */ + function makeLegacyDb(): SqliteDatabase { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-genmigrate-')); + const conn = createDatabase(path.join(dir, 'legacy.db')).db; + conn.exec(` + CREATE TABLE schema_versions (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL, description TEXT); + INSERT INTO schema_versions VALUES (8, 0, 'legacy'); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + language TEXT NOT NULL, + size INTEGER NOT NULL, + modified_at INTEGER NOT NULL, + indexed_at INTEGER NOT NULL, + node_count INTEGER DEFAULT 0, + errors TEXT + ); + INSERT INTO files VALUES ('x/bank/types/tx.pb.go', 'h1', 'go', 10, 0, 0, 1, NULL); + INSERT INTO files VALUES ('internal/payroll/payroll.go', 'h2', 'go', 10, 0, 0, 1, NULL); + `); + db = conn; + return conn; + } + + const columnNames = (conn: SqliteDatabase): string[] => + (conn.prepare('PRAGMA table_info(files)').all() as Array<{ name: string }>).map((c) => c.name); + + it('adds the column and the partial index without touching existing rows', () => { + const conn = makeLegacyDb(); + + expect(getCurrentVersion(conn)).toBe(8); + runMigrations(conn, 8); + expect(getCurrentVersion(conn)).toBe(CURRENT_SCHEMA_VERSION); + + expect(columnNames(conn)).toContain('generated'); + + const indexes = (conn.prepare('PRAGMA index_list(files)').all() as Array<{ name: string }>).map((i) => i.name); + expect(indexes).toContain('idx_files_generated'); + + // NO backfill: the flag is derived from file CONTENT, which the migration + // cannot see (files stores a hash, not bytes). Rows stay 0 until a + // re-index, and readers union with the path check so behavior is unchanged + // rather than regressed. This is why the CHANGELOG says "requires a + // re-index". + expect((conn.prepare('SELECT COUNT(*) AS n FROM files WHERE generated = 1').get() as { n: number }).n).toBe(0); + expect((conn.prepare('SELECT COUNT(*) AS n FROM files').get() as { n: number }).n).toBe(2); + }); + + it('is idempotent — replaying v9 over a database that already has the column does not throw', () => { + const conn = makeLegacyDb(); + runMigrations(conn, 8); + + // ALTER TABLE has no IF NOT EXISTS, so v9 guards on PRAGMA table_info. + // Replay happens for real whenever the recorded version trails the on-disk + // shape — a database created straight from current schema.sql already HAS + // the column, and the v6 regression test rewinds `schema_versions` and + // re-runs. Rewind the same way here; without the guard this is + // "duplicate column name: generated". + conn.prepare('DELETE FROM schema_versions WHERE version >= 9').run(); + expect(() => runMigrations(conn, 8)).not.toThrow(); + expect(columnNames(conn).filter((c) => c === 'generated')).toHaveLength(1); + expect(getCurrentVersion(conn)).toBe(CURRENT_SCHEMA_VERSION); + }); + + it('an un-backfilled database still down-ranks by the path convention', () => { + const conn = makeLegacyDb(); + runMigrations(conn, 8); + + const queries = new QueryBuilder(conn); + const paths = ['x/bank/types/tx.pb.go', 'internal/payroll/payroll.go']; + // Nothing carries the content flag yet… + expect(queries.getGeneratedPathsAmong(paths).size).toBe(0); + // …but the union predicate still knows `.pb.go`. + const isGen = queries.generatedPredicateFor(paths); + expect(isGen('x/bank/types/tx.pb.go')).toBe(true); + expect(isGen('internal/payroll/payroll.go')).toBe(false); + }); +}); diff --git a/__tests__/pr19-improvements.test.ts b/__tests__/pr19-improvements.test.ts index 6dbd20738..06c365ec7 100644 --- a/__tests__/pr19-improvements.test.ts +++ b/__tests__/pr19-improvements.test.ts @@ -298,8 +298,21 @@ describe('Best-Candidate Resolution', () => { describe('Schema v2 Migration', () => { it.skipIf(!HAS_SQLITE)('should have correct current schema version', async () => { - const { CURRENT_SCHEMA_VERSION } = await import('../src/db/migrations'); - expect(CURRENT_SCHEMA_VERSION).toBe(8); + const { CURRENT_SCHEMA_VERSION, getPendingMigrations } = await import('../src/db/migrations'); + const { DatabaseConnection } = await import('../src/db'); + + // The constant must track the migration table, not a literal — a literal + // just makes every schema change edit this test (v9/#1500 was the latest). + // A fresh database records the current version, so nothing is pending; + // ask a version-0 database instead to see the full migration list. + const dbPath = path.join(createTempDir(), 'schema-version.db'); + const conn = DatabaseConnection.initialize(dbPath); + const raw = conn.getDb(); + raw.prepare('DELETE FROM schema_versions').run(); + const highest = Math.max(...getPendingMigrations(raw).map((m) => m.version)); + conn.close(); + + expect(CURRENT_SCHEMA_VERSION).toBe(highest); }); it.skipIf(!HAS_SQLITE)('should have migration for version 2', async () => { diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index 3b3171782..9aa3c9597 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -408,6 +408,10 @@ describe('MCP Input Validation', () => { })); const fakeCg = { searchNodes: () => many, + // Search down-ranks generated files, and since #1500 that verdict comes + // from the index (path convention ∪ content banner) rather than the + // filename alone. No database here — none of these paths is generated. + generatedFilePredicate: () => () => false, }; const fakeHandler = new ToolHandler(fakeCg as unknown as CodeGraph); diff --git a/docs/design/generated-file-detection.md b/docs/design/generated-file-detection.md new file mode 100644 index 000000000..4054c8ba1 --- /dev/null +++ b/docs/design/generated-file-detection.md @@ -0,0 +1,123 @@ +# Generated-file detection — path convention plus content banner + +CG-5, issue #1500. Companion to `explore-budget-allocation.md` (CG-4's instrument) and a +prerequisite for the scoring overhaul (CG-10). + +## The problem + +`isGeneratedFile` was path-only. It matches the `..` convention — +`.pb.go`, `_grpc.pb.go`, `.g.dart`, `_pb2.py` — which is where most codegen output lives, +and which was enough for the cosmos-sdk audit that motivated it. + +It is not enough for Go. **Go's convention is a content marker, not a filename one:** + +```go +// Code generated by . DO NOT EDIT. +``` + +codified by `go generate`, honored by gofmt, golangci-lint and GitHub linguist, and emitted +by protoc-gen-go, mockgen, sqlc, ent, wire, stringer — and by in-house generators. #1500 is +exactly this: a Go monorepo with generated FKIT CRUD in ordinarily-named files +(`payroll.go`) sitting beside hand-written workflow use-cases. Nothing in the path gives it +away, so every generated-file down-rank in the codebase was a no-op on it. + +### How large the gap is + +Measured on a shallow clone of `kubernetes/client-go` (2,453 Go files): + +| signal | files flagged | +|---|---| +| ground truth (grep the canonical banner in the first 60 lines) | 2,001 | +| path convention (`isGeneratedFile`) | **0** | +| content banner (`hasGeneratedHeader`) | **2,001** — 0 false positives, 0 misses | + +82% of that repository is generated code with ordinary filenames, and the path check saw +none of it. This is not a long-tail case. + +## Design + +**Decide at index time, store on the file record, read from the DB.** Explore must never +read file headers per request. + +- `isGeneratedFile(path)` — unchanged. Path-only, pure, synchronous, free to call in a sort + comparator. Kept for callers with no database in hand. +- `hasGeneratedHeader(content)` — the content signal (below). +- `detectGeneratedFile(path, content)` — the union, which is what the indexer persists. +- `files.generated INTEGER NOT NULL DEFAULT 0` (schema v9) + a **partial** index + `idx_files_generated ON files(path) WHERE generated = 1`, so lookups cost the generated + minority, not the repo. +- `QueryBuilder.generatedPredicateFor(paths)` / `CodeGraph.generatedFilePredicate(paths)` — + one bounded probe up front, `O(1)` per comparison after, unioned with the path check. + +### Why a bounded lookup and not a cached set + +Every consumer already holds a short candidate list — a ranked file group, an FTS result +page, a `LIMIT 20` aggregate. Intersecting that list against the partial index needs no +whole-repo set to materialize and, more importantly, **no cache to invalidate**: a ranking +call can never serve a verdict the last sync already replaced. The alternative (a lazily +materialized `Set` of all generated paths) has to be invalidated on every file write and +goes stale on the read-only pool workers, in exchange for saving a sub-millisecond query. + +### Precision over recall + +A false positive silently demotes hand-written code in every ranking path, so the marker +table is precision-first and the scan is fenced three ways: + +1. **Header window only** — first 8,192 chars / 60 lines. Generous enough for build tags + plus an Apache-2.0 preamble above the banner; tight enough that a generator's own source, + which holds the banner as a *string constant in its body*, is not flagged. +2. **Comment lines only** — the marker must sit on a line with a comment leader (`//`, `#`, + `--`, ``, + `"""`, `'''`, `=begin`, `<# #>`), tracked with a small state machine over the window. + Generators always emit banners as comments; requiring it rules out identifiers and + string literals that merely contain the words. +3. **Tight markers** — `automatically generated` alone is prose ("the table is automatically + generated at runtime"); `automatically generated **by**` is a banner. `DO NOT EDIT` alone + is a style directive; paired with a generation claim it is a banner. + +The module deliberately keeps its own quoted banner literals **below** the header window so +it does not classify itself; `generated-detection.test.ts` pins that, so moving the pattern +table upward fails a test rather than silently demoting this file. + +### Migration: no backfill, by necessity + +v9 is DDL only. The flag derives from file **content**, which the migration cannot see — +`files` stores a hash, not bytes. Migrated rows stay 0 until a re-index, and because every +reader unions the flag with the path check, an un-backfilled database keeps exactly the +pre-#1500 behavior instead of regressing. `sync` heals it file-by-file as files change. +This is why the CHANGELOG entry says a re-index is required. + +## Cost + +The acceptance bar was "no measurable index-time cost regression." + +A single unanchored `/generat/i` test over the header rejects ~every hand-written file +before any line splitting happens. `String.prototype.slice` on a long string yields a V8 +sliced view, not a copy, so the fast path allocates nothing. + +- **Microbenchmark** (`detectGeneratedFile` over a whole corpus, 5 passes): + 4.6 µs/file on client-go (2,453 files, 14.2 MB, 82% generated — the worst case, where the + gate passes and the full line scan runs), 7.3 µs/file on this repo's `src`. +- **End-to-end** `codegraph init` on client-go, n=3 alternating arms + (current build vs. the same build with the content scan stubbed out): + + | arm | runs (s) | median | + |---|---|---| + | with content detection | 5.66, 5.73, 5.89 | **5.73** | + | path-only baseline | 5.52, 5.76, 5.88 | **5.76** | + + The arms cross over between runs — the difference is inside run-to-run noise. + +## What this task does NOT change + +Generated status remains a **stable tiebreak at equal score**, exactly where it was +(`src/mcp/tools.ts` file sort, `findSymbolMatches`, `findAllSymbols`, search formatting, +`getDominantFile`/`getTopRouteFile`/`getRoutingManifest`, the context formatter). A +generated file with a higher raw score still outranks a hand-written one. Turning generated +status into a **strong negative signal** is CG-10, which this task unblocks by making the +signal correct and available. + +Verified end-to-end on a two-file Go package where a generated `payroll.go` and a +hand-written `workflow.go` both define `ProcessPayroll`: with the flag set the hand-written +file ranks first; clearing the flag in the same index (i.e. pre-#1500 behavior) puts the +generated file first. diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index c6e259374..01fcf7109 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -1131,10 +1131,10 @@ program // Mirror the MCP search down-rank so the CLI also surfaces the // hand-written implementation before protobuf/gRPC scaffolding // when both share a name. See extraction/generated-detection.ts. - const { isGeneratedFile } = await import('../extraction/generated-detection'); + const isGen = cg.generatedFilePredicate(rawResults.map((r) => r.node.filePath)); const results = [...rawResults].sort((a, b) => { - const aGen = isGeneratedFile(a.node.filePath) ? 1 : 0; - const bGen = isGeneratedFile(b.node.filePath) ? 1 : 0; + const aGen = isGen(a.node.filePath) ? 1 : 0; + const bGen = isGen(b.node.filePath) ? 1 : 0; return aGen - bGen; }); diff --git a/src/context/formatter.ts b/src/context/formatter.ts index 748d17201..cabb1933d 100644 --- a/src/context/formatter.ts +++ b/src/context/formatter.ts @@ -15,7 +15,15 @@ import { isGeneratedFile } from '../extraction/generated-detection'; * - Entry points with locations * - Code blocks only for key symbols */ -export function formatContextAsMarkdown(context: TaskContext): string { +export function formatContextAsMarkdown( + context: TaskContext, + /** + * Generated-file test. Defaults to the filename convention alone; the + * ContextBuilder passes a DB-backed predicate so files flagged by their + * HEADER at index time (#1500) demote here too. + */ + isGenerated: (filePath: string) => boolean = isGeneratedFile +): string { const lines: string[] = []; // Header with query @@ -26,8 +34,8 @@ export function formatContextAsMarkdown(context: TaskContext): string { // .pulsar.go, mocks, …) rank LAST — a flow query should lead with the // hand-written implementation, not protobuf scaffolding. const orderedEntries = [...context.entryPoints].sort((a, b) => { - const aGen = isGeneratedFile(a.filePath) ? 1 : 0; - const bGen = isGeneratedFile(b.filePath) ? 1 : 0; + const aGen = isGenerated(a.filePath) ? 1 : 0; + const bGen = isGenerated(b.filePath) ? 1 : 0; return aGen - bGen; }); if (orderedEntries.length > 0) { @@ -49,7 +57,7 @@ export function formatContextAsMarkdown(context: TaskContext): string { // Related Symbols, pure noise that displaced real-flow entries). const otherSymbols = Array.from(context.subgraph.nodes.values()) .filter(n => !context.entryPoints.some(e => e.id === n.id)) - .filter(n => !isGeneratedFile(n.filePath)) + .filter(n => !isGenerated(n.filePath)) .slice(0, 10); // Limit to 10 related symbols if (otherSymbols.length > 0) { @@ -72,8 +80,8 @@ export function formatContextAsMarkdown(context: TaskContext): string { // show first (consistent with Entry Points reordering above). if (context.codeBlocks.length > 0) { const orderedBlocks = [...context.codeBlocks].sort((a, b) => { - const aGen = isGeneratedFile(a.filePath) ? 1 : 0; - const bGen = isGeneratedFile(b.filePath) ? 1 : 0; + const aGen = isGenerated(a.filePath) ? 1 : 0; + const bGen = isGenerated(b.filePath) ? 1 : 0; return aGen - bGen; }); lines.push('### Code\n'); diff --git a/src/context/index.ts b/src/context/index.ts index fd261b90b..3e60723db 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -265,7 +265,14 @@ export class ContextBuilder { // Return formatted output or raw context if (opts.format === 'markdown') { - return formatContextAsMarkdown(context) + // Bounded candidate set (entry points + subgraph + code blocks), so the + // DB-backed generated check is one probe, not a per-comparison query. + const isGenerated = this.queries.generatedPredicateFor([ + ...entryPoints.map((n) => n.filePath), + ...Array.from(subgraph.nodes.values(), (n) => n.filePath), + ...codeBlocks.map((b) => b.filePath), + ]); + return formatContextAsMarkdown(context, isGenerated) + this.buildCallPathsSection(subgraph) + (subgraph.confidence === 'low' ? this.buildLowConfidenceNote(entryPoints) : ''); } else if (opts.format === 'json') { diff --git a/src/db/migrations.ts b/src/db/migrations.ts index d083095bb..b6c82a5e5 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 8; +export const CURRENT_SCHEMA_VERSION = 9; /** * Migration definition @@ -150,6 +150,33 @@ const migrations: Migration[] = [ `); }, }, + { + version: 9, + description: + 'Add files.generated — index-time content-header generated-file detection for ranking (#1500)', + up: (db) => { + // DDL only — instant on any size database, and NO backfill: the flag is + // derived from file CONTENT, which this migration has no access to (the + // files table stores a hash, not the bytes). Migrated rows therefore stay + // 0 until the next full index re-extracts them, and every reader unions + // the flag with the path-only check, so an un-backfilled database keeps + // exactly the pre-#1500 behavior instead of regressing. `sync` heals it + // file-by-file as files change. This is why the CHANGELOG entry says a + // re-index is required to pick up the new detection. + // + // ALTER TABLE has no IF NOT EXISTS, so guard for idempotency — a database + // created from current schema.sql already has the column (matters when + // migrations are re-run from an older recorded version, as the v6 + // regression test does). Keep in lockstep with schema.sql. + const cols = db.prepare('PRAGMA table_info(files)').all() as Array<{ name: string }>; + if (!cols.some((c) => c.name === 'generated')) { + db.exec('ALTER TABLE files ADD COLUMN generated INTEGER NOT NULL DEFAULT 0'); + } + db.exec( + 'CREATE INDEX IF NOT EXISTS idx_files_generated ON files(path) WHERE generated = 1' + ); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index c5c780f3a..bad4a26f4 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -24,13 +24,18 @@ import { isGeneratedFile } from '../extraction/generated-detection'; import { splitIdentifierSegments } from '../search/identifier-segments'; /** - * Path-only heuristic for files that should not be candidates for - * "dominant file" detection: test/spec files and tool-generated files. - * Generated files (`*.pb.go`, `*.pulsar.go`, mock outputs, …) often - * have huge in-file edge counts that dwarf the real source — etcd's - * `rpc.pb.go` has 4× the in-file edges of `server.go`. + * Files that should not be candidates for "dominant file" detection: test/spec + * files and tool-generated files. Generated files (`*.pb.go`, `*.pulsar.go`, + * mock outputs, …) often have huge in-file edge counts that dwarf the real + * source — etcd's `rpc.pb.go` has 4× the in-file edges of `server.go`. + * + * Path patterns plus, when the caller passes the indexed set, files whose + * HEADER declares them generated — a `payroll.go` full of generated CRUD has + * exactly the same edge-density problem as `rpc.pb.go` and nothing in its name + * to catch it (#1500). */ -function isLowValueFile(filePath: string): boolean { +function isLowValueFile(filePath: string, generated?: ReadonlySet): boolean { + if (generated?.has(filePath)) return true; const lp = filePath.toLowerCase(); return ( /(?:^|\/)(tests?|__tests?__|spec)\//.test(lp) || @@ -97,6 +102,8 @@ interface FileRow { indexed_at: number; node_count: number; errors: string | null; + /** Absent on pre-v9 rows read through a stale prepared statement. */ + generated?: number | null; } interface UnresolvedRefRow { @@ -182,6 +189,7 @@ function rowToFileRecord(row: FileRow): FileRecord { indexedAt: row.indexed_at, nodeCount: row.node_count, errors: row.errors ? safeJsonParse(row.errors, undefined) : undefined, + generated: row.generated === 1, }; } @@ -922,7 +930,8 @@ export class QueryBuilder { `); } const rows = this.stmts.getDominantFile.all() as Array<{ file_path: string; edge_count: number }>; - const filtered = rows.filter(r => !isLowValueFile(r.file_path)); + const generated = this.getGeneratedPathsAmong(rows.map(r => r.file_path)); + const filtered = rows.filter(r => !isLowValueFile(r.file_path, generated)); if (filtered.length === 0 || filtered[0]!.edge_count < 20) return null; return { filePath: filtered[0]!.file_path, @@ -955,7 +964,8 @@ export class QueryBuilder { `); } const rows = this.stmts.getTopRouteFile.all() as Array<{ file_path: string; cnt: number }>; - const filtered = rows.filter(r => !isLowValueFile(r.file_path)); + const generated = this.getGeneratedPathsAmong(rows.map(r => r.file_path)); + const filtered = rows.filter(r => !isLowValueFile(r.file_path, generated)); if (filtered.length === 0) return null; const totalRoutes = filtered.reduce((sum, r) => sum + r.cnt, 0); const top = filtered[0]!; @@ -1006,7 +1016,8 @@ export class QueryBuilder { url: string; handler: string; handler_file: string; handler_line: number; handler_kind: string; }>; // Drop test/generated handlers — same hygiene as elsewhere. - const filtered = rows.filter(r => !isLowValueFile(r.handler_file)); + const generated = this.getGeneratedPathsAmong(rows.map(r => r.handler_file)); + const filtered = rows.filter(r => !isLowValueFile(r.handler_file, generated)); if (filtered.length < 3) return null; // Identify the file holding the most handlers (the "primary handler file"). const fileCounts = new Map(); @@ -1865,8 +1876,8 @@ export class QueryBuilder { upsertFile(file: FileRecord): void { if (!this.stmts.upsertFile) { this.stmts.upsertFile = this.db.prepare(` - INSERT INTO files (path, content_hash, language, size, modified_at, indexed_at, node_count, errors) - VALUES (@path, @contentHash, @language, @size, @modifiedAt, @indexedAt, @nodeCount, @errors) + INSERT INTO files (path, content_hash, language, size, modified_at, indexed_at, node_count, errors, generated) + VALUES (@path, @contentHash, @language, @size, @modifiedAt, @indexedAt, @nodeCount, @errors, @generated) ON CONFLICT(path) DO UPDATE SET content_hash = @contentHash, language = @language, @@ -1874,7 +1885,8 @@ export class QueryBuilder { modified_at = @modifiedAt, indexed_at = @indexedAt, node_count = @nodeCount, - errors = @errors + errors = @errors, + generated = @generated `); } @@ -1887,9 +1899,59 @@ export class QueryBuilder { indexedAt: file.indexedAt, nodeCount: file.nodeCount, errors: file.errors ? JSON.stringify(file.errors) : null, + // The upsert always REWRITES the flag: a file that loses its banner in an + // edit must lose the flag on the next sync, not keep a stale 1. + generated: file.generated ? 1 : 0, }); } + /** + * Which of `filePaths` the index flagged as tool-generated (schema v9+). + * + * Bounded-lookup by design: every consumer already holds a short candidate + * list (a ranked file group, an FTS result page, a LIMIT-20 aggregate), so + * this stays a partial-index probe over a handful of paths — no whole-repo + * set to materialize, and no cache to invalidate, which means a ranking call + * can never serve a verdict the last sync already replaced. + * + * Returns ONLY the content/index signal; callers union it with + * {@link isGeneratedFile} so pre-v9 databases (column present, all zeros + * until a re-index) keep the path-only behavior rather than regressing. + */ + getGeneratedPathsAmong(filePaths: Iterable): Set { + const unique = [...new Set(filePaths)]; + const found = new Set(); + if (unique.length === 0) return found; + + for (let i = 0; i < unique.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = unique.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT path FROM files WHERE generated = 1 AND path IN (${placeholders})`) + .all(...chunk) as Array<{ path: string }>; + for (const row of rows) found.add(row.path); + } + return found; + } + + /** + * A reusable `(path) => boolean` over a bounded candidate list, unioning the + * indexed flag with the path convention. This is the shape every ranking + * comparator wants: one query up front, then O(1) per comparison. + */ + generatedPredicateFor(filePaths: Iterable): (filePath: string) => boolean { + const flagged = this.getGeneratedPathsAmong(filePaths); + return (filePath: string) => flagged.has(filePath) || isGeneratedFile(filePath); + } + + /** How many indexed files carry the generated flag. Surfaced by `status`. */ + countGeneratedFiles(): number { + const row = this.db + .prepare('SELECT COUNT(*) AS n FROM files WHERE generated = 1') + .get() as { n: number } | undefined; + return row?.n ?? 0; + } + /** * Delete a file record and its nodes */ diff --git a/src/db/schema.sql b/src/db/schema.sql index 75df8c05c..6237d2d64 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -55,7 +55,15 @@ CREATE TABLE IF NOT EXISTS edges ( FOREIGN KEY (target) REFERENCES nodes(id) ON DELETE CASCADE ); --- Files: Tracked source files +-- Files: Tracked source files. +-- `generated` is the index-time verdict from extraction/generated-detection.ts: +-- the filename convention (*.pb.go, *.g.dart, …) OR a generation banner in the +-- file's header. Go's convention is a CONTENT marker, so a generated +-- `payroll.go` beside hand-written use-cases is invisible to the path check +-- alone (#1500) — deciding it here means ranking never reads file headers per +-- request. Migration v9 adds the column to existing databases; rows keep the +-- 0 default until the next full index, so readers treat it as a hint that +-- only ever ADDS to the path signal, never overrides it. CREATE TABLE IF NOT EXISTS files ( path TEXT PRIMARY KEY, content_hash TEXT NOT NULL, @@ -64,7 +72,8 @@ CREATE TABLE IF NOT EXISTS files ( modified_at INTEGER NOT NULL, indexed_at INTEGER NOT NULL, node_count INTEGER DEFAULT 0, - errors TEXT -- JSON array + errors TEXT, -- JSON array + generated INTEGER NOT NULL DEFAULT 0 ); -- Unresolved References: References that need resolution after full indexing. @@ -173,9 +182,13 @@ CREATE INDEX IF NOT EXISTS idx_edges_target_kind ON edges(target, kind); CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_identity ON edges(source, target, kind, IFNULL(line, -1), IFNULL(col, -1)); --- File indexes +-- File indexes. +-- idx_files_generated is PARTIAL: the generated set is a small minority of any +-- repo, so a lookup that intersects a bounded candidate list with it stays +-- proportional to the generated files, not to the repo. CREATE INDEX IF NOT EXISTS idx_files_language ON files(language); CREATE INDEX IF NOT EXISTS idx_files_modified_at ON files(modified_at); +CREATE INDEX IF NOT EXISTS idx_files_generated ON files(path) WHERE generated = 1; -- Unresolved refs indexes CREATE INDEX IF NOT EXISTS idx_unresolved_from_node ON unresolved_refs(from_node_id); diff --git a/src/extraction/generated-detection.ts b/src/extraction/generated-detection.ts index 1ec6a6d82..c7bb8bbd2 100644 --- a/src/extraction/generated-detection.ts +++ b/src/extraction/generated-detection.ts @@ -8,18 +8,39 @@ * see project_go_multi_module_audit memory). Generated stubs frequently * have no body to trace from, so the agent ends up reading source anyway. * - * This helper is a pure path-based classifier consulted at disambiguation - * time (findSymbol / findAllSymbols / codegraph_search formatting), NOT - * a hard filter — generated nodes are still in the graph and remain - * reachable; they just rank LAST when there's a real implementation - * with the same name. + * This is a relevance hint consulted at disambiguation time (findSymbol / + * findAllSymbols / explore ranking / codegraph_search formatting), NOT a + * hard filter — generated nodes are still in the graph and remain + * reachable; they just rank LAST when there's a real implementation with + * the same name. * - * Scope: suffix patterns only. Most generated files follow the - * `..` convention (`.pb.go`, `_grpc.pb.go`, - * `.g.dart`, `_pb2.py`), and that covers ~all of what we saw in the - * Go audit. A future addition would be scanning for the canonical - * `// Code generated by` header during extraction, for the rare files - * that defy the suffix convention. + * Two signals, deliberately separate: + * + * 1. {@link isGeneratedFile} — PATH only, pure and synchronous. Most + * generated files follow the `..` convention + * (`.pb.go`, `_grpc.pb.go`, `.g.dart`, `_pb2.py`). Free to call + * anywhere, including in a sort comparator. + * + * 2. {@link hasGeneratedHeader} — CONTENT banner in the file's head. Go's + * own convention is a content marker, not a filename one, so a + * generated `payroll.go` sitting beside hand-written use-cases is + * invisible to (1) — that is issue #1500. Evaluated ONCE at index time + * (the file's content is already in memory for parsing) and persisted + * on the file record as `files.generated`; readers get it from the DB + * rather than re-reading headers per request. See + * GENERATED_CONTENT_PATTERNS below for the banners recognized. + * + * Consumers that have a bounded candidate list should use the DB-backed + * union (`QueryBuilder.getGeneratedPathsAmong` / + * `CodeGraph.getGeneratedFilePaths`) so both signals apply; the path-only + * check remains the fallback for callers with no database in hand and for + * indexes built before the flag existed. + * + * NOTE for future editors: the banner literals quoted in this file sit + * BELOW the header window this detector scans, so the module does not + * classify itself. `generated-detection.test.ts` pins that — if you move + * the pattern table upward, the test fails rather than the repo silently + * demoting its own file. */ const GENERATED_PATTERNS: ReadonlyArray = [ @@ -79,3 +100,146 @@ const GENERATED_PATTERNS: ReadonlyArray = [ export function isGeneratedFile(filePath: string): boolean { return GENERATED_PATTERNS.some((p) => p.test(filePath)); } + +// ============================================================================= +// Content-header detection (#1500) +// ============================================================================= + +/** + * How much of a file's head to consider "the header". Generous enough for a + * build-tag block + an Apache-2.0 license preamble (~15 lines) sitting above + * the banner, tight enough that a `"// Code generated ... DO NOT EDIT."` + * string constant in the *body* of a code generator's own source can't + * masquerade as a banner. + */ +const HEADER_SCAN_CHARS = 8192; +const HEADER_SCAN_LINES = 60; + +/** + * Cheap pre-filter run on the header of EVERY indexed file. Every marker + * below contains the stem "generat", so one unanchored scan rejects ~all + * hand-written source before any line splitting happens — this is what keeps + * content detection off the index-time cost budget. + */ +const GENERATED_STEM = /generat/i; + +/** + * Line-comment leaders across the languages we index. A banner must sit on a + * comment line (or inside an open block comment, tracked below): generators + * always emit theirs as a comment, and requiring it rules out string literals + * and identifiers that merely contain the words. + * + * `--` covers SQL/Haskell/Lua, `%` LaTeX/Erlang/Prolog, `;` Lisp/asm/ini, + * `'` VB, `!` Fortran, `*` a continuation line inside a `/* … *\/` block. + */ +const COMMENT_LEADER = + /^\s*(?:\/\/|\/\*+|\*+\/?|#+|--+|`, or a Python module docstring) still + * counts. Deliberately naive — it only runs over a file's first few dozen + * lines, where a `/*` inside a string literal is vanishingly rare, and the + * worst case of a mis-tracked state is a ranking hint, not a wrong answer. + */ +const BLOCK_DELIMS: ReadonlyArray<{ open: string; close: string }> = [ + { open: '/*', close: '*/' }, + { open: '' }, + { open: '"""', close: '"""' }, + { open: "'''", close: "'''" }, + { open: '=begin', close: '=end' }, + { open: '<#', close: '#>' }, +]; + +/** + * The banners themselves. Each is a real convention emitted by a widely-used + * generator; the list is precision-first, because a false positive silently + * demotes hand-written code in every ranking path. + */ +const GENERATED_CONTENT_PATTERNS: ReadonlyArray = [ + // Go's codified convention — `^// Code generated .* DO NOT EDIT\.$`, defined + // by `go generate` and honored by gofmt, golangci-lint and GitHub linguist. + // Emitted verbatim by protoc-gen-go, mockgen, sqlc, ent, wire, stringer, and + // by in-house generators like the FKIT CRUD in #1500 — where the file is + // named `payroll.go` and nothing in the PATH gives it away. + /\bcode generated\b.{0,200}?\bdo not edit\b/i, + // protoc's Java/C#/Python banner ("Generated by the protocol buffer + // compiler. DO NOT EDIT!"), ANTLR, Dagger, FlatBuffers, rust-bindgen, + // Xcode asset catalogs, Bazel rules. + /\b(?:automatically |auto[- ]?)?generated (?:by|from|with)\b.{0,200}?\bdo not (?:edit|modify|change)\b/i, + // The `@generated` marker: the JS/TS ecosystem's convention (Relay, GraphQL + // codegen, protobuf-es/Buf, Meta's `@generated SignedSource<<…>>`), also + // what linguist and `git diff` collapse on. Guarded against `foo@generated` + // and `@@generated` so only a standalone tag matches. + /(?:^|[^\p{L}\p{N}_@])@generated\b/u, + // .NET's `` / `` doc tag: Roslyn, the + // WinForms designer, T4 templates, protoc-gen-csharp, EF scaffolding. + //i, + // swagger-codegen / OpenAPI Generator ("NOTE: This class is auto generated + // by OpenAPI Generator"), Thrift ("Autogenerated by Thrift Compiler"), + // FlatBuffers ("automatically generated by the FlatBuffers compiler"). + // "by" is required — bare "automatically generated" appears in hand-written + // prose ("the table below is automatically generated at runtime"). + /\b(?:automatically generated|auto[- ]?generated|autogenerated) by\b/i, + // Self-declaring in-house banners that name no tool. + /\bthis (?:file|class|code|module) (?:is|was) (?:auto[- ]?)?generated\b/i, + // The reverse ordering: "DO NOT EDIT — this is a generated file". + /\bdo not (?:edit|modify)\b.{0,120}?\b(?:auto[- ]?generated|generated file|generated code)\b/i, +]; + +/** + * Whether the head of `content` carries a recognized machine-generation + * banner. Bounded to {@link HEADER_SCAN_CHARS} / {@link HEADER_SCAN_LINES}, + * and the marker must sit on a comment line — a generator's own source, which + * holds the banner as a string constant in its body, is not flagged. + * + * Called once per file during extraction (content is already in memory), NOT + * per query: the verdict is persisted on the file record. + */ +export function hasGeneratedHeader(content: string): boolean { + if (!content) return false; + + const head = content.length > HEADER_SCAN_CHARS ? content.slice(0, HEADER_SCAN_CHARS) : content; + // Fast reject for ~every hand-written file: no line splitting, no allocation + // (V8 keeps `head` as a sliced view of `content`). + if (!GENERATED_STEM.test(head)) return false; + + const lines = head.split('\n'); + const limit = Math.min(lines.length, HEADER_SCAN_LINES); + let openBlock: (typeof BLOCK_DELIMS)[number] | null = null; + + for (let i = 0; i < limit; i++) { + const line = lines[i]!; + const inBlock = openBlock !== null; + + if (inBlock || COMMENT_LEADER.test(line)) { + for (const pattern of GENERATED_CONTENT_PATTERNS) { + if (pattern.test(line)) return true; + } + } + + // Advance the block-comment state AFTER testing, so the opening line of a + // `/* Code generated … */` block is itself matched by the leader rule. + if (openBlock) { + if (line.includes(openBlock.close)) openBlock = null; + continue; + } + for (const delim of BLOCK_DELIMS) { + const at = line.indexOf(delim.open); + if (at < 0) continue; + // Same-line close (`/* … */`, a one-line docstring) leaves no open block. + if (line.indexOf(delim.close, at + delim.open.length) < 0) openBlock = delim; + break; + } + } + + return false; +} + +/** + * The union signal: path convention OR content banner. This is what the + * indexer persists to `files.generated`. + */ +export function detectGeneratedFile(filePath: string, content: string): boolean { + return isGeneratedFile(filePath) || hasGeneratedHeader(content); +} diff --git a/src/extraction/index.ts b/src/extraction/index.ts index e89f73b45..4e4af4a4c 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -25,6 +25,7 @@ import { extractFromSource } from './tree-sitter'; import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './parse-pool'; import { StoreWriter, StoreBundle, finalizeStoreBundle } from './store-writer'; import { materializeKernelResult } from './kernel'; +import { detectGeneratedFile } from './generated-detection'; import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns } from '../project-config'; import { isCodeGraphDataDir } from '../directory'; @@ -2275,6 +2276,11 @@ export class ExtractionOrchestrator { return; // No changes } + // Re-decided on every re-index of a changed file, so a banner added (or + // removed) by an edit is reflected on the next sync (#1500). Computed after + // the unchanged-file early return so untouched files pay nothing. + const generated = detectGeneratedFile(filePath, content); + // Snapshot incoming cross-file edges BEFORE deleting this file's nodes. // `deleteFile` cascades to delete every edge whose source OR target is a // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is @@ -2340,6 +2346,7 @@ export class ExtractionOrchestrator { indexedAt: Date.now(), nodeCount: result.nodes.length, errors: result.errors.length > 0 ? result.errors : undefined, + generated, }, }); if (crossFileIncomingEdges.length > 0) { @@ -2400,6 +2407,7 @@ export class ExtractionOrchestrator { indexedAt: Date.now(), nodeCount: result.nodes.length, errors: result.errors.length > 0 ? result.errors : undefined, + generated, }; this.queries.upsertFile(fileRecord); } @@ -2427,6 +2435,10 @@ export class ExtractionOrchestrator { indexedAt: Date.now(), nodeCount, errors: resultErrors.length > 0 ? resultErrors : undefined, + // Decided here, once, while the content is already in memory — never at + // query time (#1500). The header scan short-circuits on a single + // substring test for ~every hand-written file. + generated: detectGeneratedFile(filePath, content), }; } diff --git a/src/index.ts b/src/index.ts index 694e93a3b..86dae6cef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1537,6 +1537,24 @@ export class CodeGraph { return this.queries.getAllFiles(); } + /** + * A `(path) => boolean` generated-file test over a BOUNDED candidate list, + * unioning the index-time content-banner flag with the filename convention + * (#1500). One query up front, O(1) per call after — built for use inside a + * ranking comparator, where re-querying per comparison would be quadratic. + * + * Pass every path you might ask about; a path outside the list falls back to + * the filename check alone. + */ + generatedFilePredicate(filePaths: Iterable): (filePath: string) => boolean { + return this.queries.generatedPredicateFor(filePaths); + } + + /** How many indexed files are flagged tool-generated. Reported by `status`. */ + getGeneratedFileCount(): number { + return this.queries.countGeneratedFiles(); + } + // =========================================================================== // Graph Query Methods // =========================================================================== diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index d1fcbc964..bceda054e 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -39,7 +39,6 @@ import { } from 'fs'; import { createHash } from 'crypto'; import { clamp, validatePathWithinRoot, validateProjectPath, isConfigLeafNode, CONFIG_LEAF_LANGUAGES } from '../utils'; -import { isGeneratedFile } from '../extraction/generated-detection'; import { scanDynamicDispatch } from './dynamic-boundaries'; import { getUpdateNotice } from '../upgrade/update-check'; import { ExploreDiagnostics } from './explore-diagnostics'; @@ -1588,9 +1587,10 @@ export class ToolHandler { // Down-rank generated files within the FTS-returned set so a search // for "Send" surfaces the hand-written keeper before .pb.go stubs // that share the name. Stable: only reorders generated vs. not. + const isGen = cg.generatedFilePredicate(results.map((r) => r.node.filePath)); const ranked = [...results].sort((a, b) => { - const aGen = isGeneratedFile(a.node.filePath) ? 1 : 0; - const bGen = isGeneratedFile(b.node.filePath) ? 1 : 0; + const aGen = isGen(a.node.filePath) ? 1 : 0; + const bGen = isGen(b.node.filePath) ? 1 : 0; return aGen - bGen; }); @@ -3113,6 +3113,13 @@ export class ToolHandler { !MULTITERM_OFF && (fileTermHits.get(fp) ?? 0) >= 2 && (entryFiles.has(fp) || centralFiles.has(fp)); + + // One DB probe over the ranked candidates, then O(1) per comparison. Unions + // the index-time content-banner flag with the filename convention, so a Go + // monorepo's generated CRUD (`payroll.go` carrying a DO-NOT-EDIT banner and + // nothing in its name) down-ranks the same way `.pb.go` always has (#1500). + const isGeneratedCandidate = cg.generatedFilePredicate(relevantFiles.map(([fp]) => fp)); + const sortedFiles = relevantFiles.sort((a, b) => { const aPath = a[0].toLowerCase(); const bPath = b[0].toLowerCase(); @@ -3147,8 +3154,8 @@ export class ToolHandler { // the response (the cosmos Q3 explore otherwise leads with // `expected_keepers_mocks.go`, displacing the real `tally.go` content // and forcing the agent to Read tally.go anyway). - const aGen = isGeneratedFile(a[0]); - const bGen = isGeneratedFile(b[0]); + const aGen = isGeneratedCandidate(a[0]); + const bGen = isGeneratedCandidate(b[0]); if (aGen !== bGen) return aGen ? 1 : -1; if (a[1].score !== b[1].score) return b[1].score - a[1].score; @@ -3233,7 +3240,7 @@ export class ToolHandler { entry: entryFiles.has(fp), spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), lowValue: isLowValue(fp), - generated: isGeneratedFile(fp), + generated: isGeneratedCandidate(fp), }); }); } @@ -4753,7 +4760,8 @@ export class ToolHandler { if (!isQualified) { const exact = cg.getNodesByName(symbol); if (exact.length > 0) { - return [...exact].sort((a, b) => (isGeneratedFile(a.filePath) ? 1 : 0) - (isGeneratedFile(b.filePath) ? 1 : 0)); + const isGen = cg.generatedFilePredicate(exact.map((n) => n.filePath)); + return [...exact].sort((a, b) => (isGen(a.filePath) ? 1 : 0) - (isGen(b.filePath) ? 1 : 0)); } // No exact match — use the single top fuzzy result (e.g. a file basename). const fuzzy = cg.searchNodes(symbol, { limit: 10 }); @@ -4781,10 +4789,12 @@ export class ToolHandler { return isQualified ? [] : results[0] ? [results[0].node] : []; } - // Down-rank generated files (.pb.go, .pulsar.go, _grpc.pb.go, …) so a flow - // query prefers the keeper implementation over the protobuf-generated stub. + // Down-rank generated files (.pb.go, .pulsar.go, _grpc.pb.go, and anything + // whose header declares it generated) so a flow query prefers the keeper + // implementation over the generated stub. + const isGen = cg.generatedFilePredicate(exactMatches.map((r) => r.node.filePath)); return [...exactMatches] - .sort((a, b) => (isGeneratedFile(a.node.filePath) ? 1 : 0) - (isGeneratedFile(b.node.filePath) ? 1 : 0)) + .sort((a, b) => (isGen(a.node.filePath) ? 1 : 0) - (isGen(b.node.filePath) ? 1 : 0)) .map((r) => r.node); } @@ -4837,9 +4847,10 @@ export class ToolHandler { // Same generated-file down-rank as findSymbol — keeps callers/callees // /impact aggregation aligned (a query against "Send" returns the // hand-written implementations before the protobuf scaffold). + const isGen = cg.generatedFilePredicate(exactMatches.map((r) => r.node.filePath)); const ranked = [...exactMatches].sort((a, b) => { - const aGen = isGeneratedFile(a.node.filePath) ? 1 : 0; - const bGen = isGeneratedFile(b.node.filePath) ? 1 : 0; + const aGen = isGen(a.node.filePath) ? 1 : 0; + const bGen = isGen(b.node.filePath) ? 1 : 0; return aGen - bGen; }); diff --git a/src/types.ts b/src/types.ts index 8cc600b1c..a1861bba4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -252,6 +252,15 @@ export interface FileRecord { /** Any extraction errors */ errors?: ExtractionError[]; + + /** + * Tool-generated source, decided at index time from the filename + * convention OR a generation banner in the file's header (see + * extraction/generated-detection.ts). A relevance hint for ranking, not a + * hard filter. Absent on indexes built before schema v9 — treat + * `undefined` as "content signal unknown, fall back to the path check". + */ + generated?: boolean; } // ============================================================================= From bd86ad206143b0b7b7c223a0aaadc7a3805a870b Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Mon, 3 Aug 2026 23:28:59 -0500 Subject: [PATCH 03/15] test(explore): #1500 regression fixtures for budget allocation (CG-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two permanent fixtures pinning the failure mode from issue #1500 — explore spending its byte envelope on files that merely name-collide with the query. BOTH FAIL TODAY, by design: they document the bug and become the pass gate for CG-10 (scoring) + CG-12 (proportional allocation). __tests__/fixtures/payroll-go/ — a synthetic Go service mirroring the reporter's shape: generated FKIT CRUD beside a hand-written payroll use-case, entered from an HTTP route. Half the generated tree carries ORDINARY names detectable only by their `// Code generated ... DO NOT EDIT.` header (the #1500 case, and end-to-end cover for CG-5); `payrollpb/*.pb.go` covers the path-detectable channel. BuildPayslip, Upsert and Store each exist twice, generated and hand-written. cycle.go sits above the whole-file window so it clips; the generated files sit below it so they ship whole. Asking "how does payroll cycle create and calculate payslips?" — naming none of the answering symbols — the generated CRUD delivers 57.4% of the envelope against the hand-written layer's 25.6%, all of the latter domain types. cycle.go is allocated the single largest slice (30.6%) and delivers ZERO: the hard ceiling drops its whole section. runPayrollCycleAll, the hand-written BuildPayslip and the real Upsert never reach the agent. The second fixture is this repo, "how does explore allocate its output budget across files", where scripts/agent-eval/*.mjs take 71.8% against tools.ts's 18.5% despite scoring 4.6x lower. It reads the live index, so its assertions are relative rather than fixed percentages. - scripts/agent-eval/probe-allocation.mjs — per-file budget-share probe, driving the CG-4 diagnostic through a JSONL sidecar so it measures the shipping allocator. Fixture entries are hermetic (copy + re-index per run, verified byte-identical across runs); exits 1 while any assertion fails. - scripts/agent-eval/allocation-fixtures.json — both fixtures declared, with the 2026-08-03 baselines. - __tests__/explore-allocation-1500.test.ts — fixture-shape assertions green today; the allocation assertions held as `it.fails` so the suite stays green while the bug is open and goes RED the moment it is fixed. Also documented and deliberately left unfixed: runPayrollCycleAll's `s.store.Upsert` edge resolves to the GENERATED Store.Upsert, not the hand-written one — same-name method resolution across two packages picks the wrong receiver. It is upstream of the allocation bug, so it belongs with CG-10's scoring work. Refs #1500 Co-Authored-By: Claude Opus 5 --- __tests__/explore-allocation-1500.test.ts | 256 +++++++++++++++ __tests__/fixtures/payroll-go/README.md | 95 ++++++ .../fixtures/payroll-go/cmd/payrolld/main.go | 35 +++ __tests__/fixtures/payroll-go/go.mod | 3 + .../internal/domain/payroll/payslip.go | 159 ++++++++++ .../internal/gen/fkit/employee/contract.go | 84 +++++ .../internal/gen/fkit/employee/employee.go | 88 ++++++ .../internal/gen/fkit/payroll/calculate.go | 60 ++++ .../internal/gen/fkit/payroll/dto.go | 80 +++++ .../gen/fkit/payroll/payroll_cycle.go | 116 +++++++ .../internal/gen/fkit/payroll/payslip.go | 129 ++++++++ .../internal/gen/fkit/payroll/store.go | 95 ++++++ .../internal/gen/fkit/timesheet/timesheet.go | 76 +++++ .../internal/gen/payrollpb/payroll.pb.go | 139 ++++++++ .../internal/gen/payrollpb/payroll_grpc.pb.go | 104 ++++++ .../internal/platform/clock/clock.go | 18 ++ .../internal/store/payslipstore/store.go | 119 +++++++ .../transport/httpapi/payroll_handler.go | 96 ++++++ .../internal/transport/httpapi/router.go | 19 ++ .../internal/usecase/payroll/cycle.go | 227 ++++++++++++++ .../usecase/payroll/payslip_builder.go | 150 +++++++++ .../internal/usecase/payroll/prorate.go | 53 ++++ docs/design/explore-budget-allocation.md | 78 +++++ scripts/agent-eval/allocation-fixtures.json | 112 +++++++ scripts/agent-eval/probe-allocation.mjs | 296 ++++++++++++++++++ 25 files changed, 2687 insertions(+) create mode 100644 __tests__/explore-allocation-1500.test.ts create mode 100644 __tests__/fixtures/payroll-go/README.md create mode 100644 __tests__/fixtures/payroll-go/cmd/payrolld/main.go create mode 100644 __tests__/fixtures/payroll-go/go.mod create mode 100644 __tests__/fixtures/payroll-go/internal/domain/payroll/payslip.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/employee/contract.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/employee/employee.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/payroll/calculate.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/payroll/dto.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payroll_cycle.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payslip.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/payroll/store.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/fkit/timesheet/timesheet.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll.pb.go create mode 100644 __tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll_grpc.pb.go create mode 100644 __tests__/fixtures/payroll-go/internal/platform/clock/clock.go create mode 100644 __tests__/fixtures/payroll-go/internal/store/payslipstore/store.go create mode 100644 __tests__/fixtures/payroll-go/internal/transport/httpapi/payroll_handler.go create mode 100644 __tests__/fixtures/payroll-go/internal/transport/httpapi/router.go create mode 100644 __tests__/fixtures/payroll-go/internal/usecase/payroll/cycle.go create mode 100644 __tests__/fixtures/payroll-go/internal/usecase/payroll/payslip_builder.go create mode 100644 __tests__/fixtures/payroll-go/internal/usecase/payroll/prorate.go create mode 100644 scripts/agent-eval/allocation-fixtures.json create mode 100755 scripts/agent-eval/probe-allocation.mjs diff --git a/__tests__/explore-allocation-1500.test.ts b/__tests__/explore-allocation-1500.test.ts new file mode 100644 index 000000000..4abb31909 --- /dev/null +++ b/__tests__/explore-allocation-1500.test.ts @@ -0,0 +1,256 @@ +/** + * Regression fixture for GitHub issue #1500 / epic CG-1 — relevance-proportional + * explore budget allocation. + * + * The reporter's repo is a Go service whose GENERATED FKIT CRUD layer sits beside + * the hand-written use-case that does the real work. Asking an architecture + * question that doesn't name the exact use-case ("how does payroll cycle create + * and calculate payslips?") spends the explore envelope on the generated CRUD, + * because the generated layer name-collides on every term in the question while + * the hand-written workflow is one big file that gets clipped. + * + * `__tests__/fixtures/payroll-go/` reproduces that shape permanently. This suite + * is in two halves: + * + * 1. **Fixture shape** — green today. These pin the properties the fixture must + * keep for the gate below to mean anything: the generated/hand-written split + * (including the ordinary-named generated files only a CONTENT header betrays, + * which is the #1500 case), the deliberate name collisions, and the + * runPayrollCycleAll → BuildPayslip → Upsert chain resolving end-to-end. If + * the fixture rots, these fail first and say so. + * + * 2. **Budget allocation** — the gate, written with `it.fails` because it + * DOCUMENTS A BUG THAT IS STILL OPEN. Vitest passes an `it.fails` test only + * while its body throws, so the suite is green today and goes RED the moment + * allocation is fixed (CG-10 scoring + CG-12 proportional bytes). + * **When it goes red, delete the `.fails` — do not delete the test.** + * + * The same assertions run outside vitest, against the built dist and with the + * full CG-4 per-file diagnostic, via `node scripts/agent-eval/probe-allocation.mjs` + * (declared in `scripts/agent-eval/allocation-fixtures.json`). + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; +import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; +import { isGeneratedFile, hasGeneratedHeader } from '../src/extraction/generated-detection'; + +const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'payroll-go'); + +/** The question a newcomer asks — names none of the symbols that answer it. */ +const QUERY = 'how does payroll cycle create and calculate payslips?'; + +/** The hand-written workflow: what the query is actually about. */ +const ANSWER_PREFIXES = [ + 'internal/usecase/', + 'internal/store/', + 'internal/transport/', + 'internal/domain/', + 'cmd/', +]; +/** The generated CRUD/DTO layer: what wins the envelope today. */ +const GENERATED_PREFIX = 'internal/gen/'; + +const startsWithAny = (p: string, prefixes: string[]) => prefixes.some((x) => p.startsWith(x)); + +describe('#1500 — generated Go CRUD beside a hand-written payroll workflow', () => { + let testDir: string; + let cg: CodeGraph; + let handler: ToolHandler; + let response: string; + /** Delivered source bytes per file, attributed from the final response. */ + let bytes: Map; + + beforeAll(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-1500-')); + fs.cpSync(FIXTURE_SRC, testDir, { recursive: true }); + // A stray index in the checked-in tree would be copied in and reused. + fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true }); + + cg = CodeGraph.initSync(testDir); + await cg.indexAll(); + handler = new ToolHandler(cg); + + const result = await handler.execute('codegraph_explore', { query: QUERY }); + response = result.content?.[0]?.text ?? ''; + bytes = attributeSourceBytes(response); + }, 120_000); + + afterAll(() => { + if (cg) cg.destroy(); + if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true }); + }); + + // ── 1. Fixture shape ────────────────────────────────────────────────────── + + describe('fixture shape', () => { + it('indexes as a Go project with both layers present', () => { + const files = cg.getFiles().map((f) => f.path); + expect(files.filter((p) => p.endsWith('.go')).length).toBeGreaterThanOrEqual(15); + expect(files.some((p) => p.startsWith(GENERATED_PREFIX))).toBe(true); + expect(files.some((p) => p.startsWith('internal/usecase/'))).toBe(true); + }); + + it('flags every generated file and no hand-written one', () => { + for (const file of cg.getFiles()) { + expect(file.generated, `${file.path} generated flag`).toBe( + file.path.startsWith(GENERATED_PREFIX), + ); + } + }); + + it('carries generated files that ONLY a content header betrays — the #1500 case', () => { + // Half the generated tree has ordinary names (`payslip.go`, `store.go`). + // Path-only detection misses them; the CG-5 content check is what catches + // them. Without these the fixture would be a .pb.go fixture, not a #1500 one. + const contentOnly = [ + 'internal/gen/fkit/payroll/payslip.go', + 'internal/gen/fkit/payroll/payroll_cycle.go', + 'internal/gen/fkit/payroll/store.go', + 'internal/gen/fkit/payroll/calculate.go', + 'internal/gen/fkit/payroll/dto.go', + 'internal/gen/fkit/employee/employee.go', + 'internal/gen/fkit/timesheet/timesheet.go', + ]; + for (const rel of contentOnly) { + const source = fs.readFileSync(path.join(testDir, rel), 'utf-8'); + expect(isGeneratedFile(rel), `${rel} must NOT be detectable by path`).toBe(false); + expect(hasGeneratedHeader(source), `${rel} must be detectable by header`).toBe(true); + expect(cg.getFile(rel)?.generated, `${rel} indexed flag`).toBe(true); + } + // …beside the conventional path-detectable ones, so both channels are covered. + expect(isGeneratedFile('internal/gen/payrollpb/payroll.pb.go')).toBe(true); + }); + + it('collides the generated layer with the hand-written one by name', () => { + // A naive scorer sees two BuildPayslips and two Upserts and has no reason + // to prefer the one that implements the business rule. + for (const name of ['BuildPayslip', 'Upsert', 'Store']) { + const files = new Set(cg.getNodesByName(name).map((n) => n.filePath)); + expect([...files].some((p) => p.startsWith(GENERATED_PREFIX)), `${name} generated`).toBe(true); + expect([...files].some((p) => !p.startsWith(GENERATED_PREFIX)), `${name} hand-written`).toBe(true); + } + }); + + it('resolves the hand-written workflow chain end-to-end in the graph', () => { + const calleesOf = (name: string, file: string) => { + const node = cg.getNodesByName(name).find((n) => n.filePath === file); + expect(node, `${name} in ${file}`).toBeTruthy(); + return cg + .getOutgoingEdges(node!.id) + .filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.target)) + .filter((n): n is NonNullable => !!n); + }; + + // handler → use-case + expect( + calleesOf('RunCycle', 'internal/transport/httpapi/payroll_handler.go') + .some((n) => n.name === 'RunCycle' && n.filePath === 'internal/usecase/payroll/cycle.go'), + ).toBe(true); + + // use-case → the workflow + expect( + calleesOf('RunCycle', 'internal/usecase/payroll/cycle.go') + .some((n) => n.name === 'runPayrollCycleAll'), + ).toBe(true); + + // the workflow → build + persist + const workflow = calleesOf('runPayrollCycleAll', 'internal/usecase/payroll/cycle.go'); + expect( + workflow.some((n) => n.name === 'BuildPayslip' && n.filePath === 'internal/usecase/payroll/payslip_builder.go'), + 'runPayrollCycleAll must reach the hand-written BuildPayslip', + ).toBe(true); + expect(workflow.some((n) => n.name === 'Upsert'), 'runPayrollCycleAll must reach an Upsert').toBe(true); + }); + + it('routes an HTTP entry point into the workflow', () => { + const router = cg.getNodesInFile('internal/transport/httpapi/router.go'); + expect(router.some((n) => n.kind === 'route' || n.name === 'NewRouter')).toBe(true); + }); + + it('sizes the two layers so the size-driven render split actually bites', () => { + // The mechanism the epic is about: a small file ships WHOLE, a large one + // falls through to clipped clusters. The workflow file must stay above the + // whole-file window and the generated files below it, or the fixture stops + // reproducing anything. + const lines = (rel: string) => fs.readFileSync(path.join(testDir, rel), 'utf-8').split('\n').length; + expect(lines('internal/usecase/payroll/cycle.go')).toBeGreaterThan(220); + for (const rel of ['internal/gen/fkit/payroll/payslip.go', 'internal/gen/fkit/payroll/payroll_cycle.go']) { + expect(lines(rel)).toBeLessThan(220); + } + }); + + it('answers the query at all', () => { + expect(response.length).toBeGreaterThan(1000); + expect(bytes.size).toBeGreaterThan(0); + }); + }); + + // ── 2. Budget allocation — the open bug ─────────────────────────────────── + + describe('budget allocation', () => { + const share = (predicate: (p: string) => boolean) => { + let total = 0; + for (const [file, n] of bytes) if (predicate(file)) total += n; + return total / response.length; + }; + const answerShare = () => share((p) => startsWithAny(p, ANSWER_PREFIXES)); + const generatedShare = () => share((p) => p.startsWith(GENERATED_PREFIX)); + + /** + * BASELINE 2026-08-03 (very-tiny tier, 13,000-char budget): 23,020 chars + * allocated against it, cut to 16,011 by the 19,500 hard ceiling. The + * generated CRUD delivers 57.4%; the hand-written layer delivers 25.6%, all + * of it domain types. `cycle.go` is allocated the single largest slice + * (7,052 chars, 30.6%) and delivers ZERO — the ceiling drops its whole + * section — so runPayrollCycleAll, the hand-written BuildPayslip and the + * real Upsert never reach the agent at all. + * + * Each `it.fails` below passes ONLY while that is still true. + * ⚠ When one goes red, the bug is fixed: remove `.fails`, keep the test. + */ + it.fails('CG-12 GATE: concentrates the envelope on the hand-written workflow', () => { + expect(answerShare()).toBeGreaterThanOrEqual(0.55); + }); + + it.fails('CG-12 GATE: does not spend the envelope on the generated CRUD', () => { + expect(generatedShare()).toBeLessThanOrEqual(0.25); + }); + + it.fails('CG-12 GATE: delivers the workflow file it allocated the most bytes to', () => { + expect(bytes.get('internal/usecase/payroll/cycle.go') ?? 0).toBeGreaterThan(0); + }); + + it.fails('CG-12 GATE: delivers the calculation the question asks about', () => { + expect(bytes.get('internal/usecase/payroll/payslip_builder.go') ?? 0).toBeGreaterThan(0); + }); + + it.fails('CG-12 GATE: puts the hand-written chain in the response, not its generated twin', () => { + // Bare `BuildPayslip`/`Upsert` also match the generated collisions — these + // needles are unique to the hand-written chain. + expect(response).toContain('runPayrollCycleAll'); + expect(response).toContain('func (s *Service) BuildPayslip'); + expect(response).toContain('s.store.Upsert(ctx, slip)'); + }); + + it('records the shape of the failure so a regression is legible', () => { + // Not a gate — an assertion-free-ish snapshot of WHY the gates above fail, + // so a future change that shifts the numbers shows up in the diff rather + // than silently flipping an it.fails. + const generated = generatedShare(); + const answer = answerShare(); + const workflow = bytes.get('internal/usecase/payroll/cycle.go') ?? 0; + expect({ + generatedWinsEnvelope: generated > answer, + workflowFileDeliversNothing: workflow === 0, + }).toEqual({ + generatedWinsEnvelope: true, + workflowFileDeliversNothing: true, + }); + }); + }); +}); diff --git a/__tests__/fixtures/payroll-go/README.md b/__tests__/fixtures/payroll-go/README.md new file mode 100644 index 000000000..a570bccf5 --- /dev/null +++ b/__tests__/fixtures/payroll-go/README.md @@ -0,0 +1,95 @@ +# payroll-go — the #1500 regression fixture + +A synthetic Go service reproducing the repo shape from [issue #1500](https://github.com/colbymchenry/codegraph/issues/1500): +**generated CRUD sitting beside the hand-written use-case that does the real work.** + +This tree is a fixture, not a program. It never compiles or runs — it exists to be +indexed. Keep it valid, idiomatic Go anyway: the extractor's output is the whole point. + +## The shape + +``` +cmd/payrolld/main.go wires the service +internal/transport/httpapi/ HTTP entry point → use-case +internal/usecase/payroll/ ← THE ANSWER. Hand-written workflow: + cycle.go runPayrollCycleAll (227 lines) + payslip_builder.go BuildPayslip — the actual pay calculation + prorate.go +internal/domain/payroll/payslip.go hand-written domain types +internal/store/payslipstore/store.go the real Upsert +internal/platform/clock/clock.go + +internal/gen/fkit/payroll/ ← THE NOISE. Generated CRUD, ORDINARY names: + payslip.go CreatePayslip, GetPayslip, UpdatePayslip, a second BuildPayslip + payroll_cycle.go CreatePayrollCycle, PayrollCycleCreateRequest, … + store.go a second Upsert + calculate.go CalculatePayrollCycleTotals, CalculatePayslipNet, … + dto.go +internal/gen/fkit/employee/, timesheet/ more generated CRUD +internal/gen/payrollpb/*.pb.go generated, detectable by PATH +``` + +The chain the fixture is built around is `runPayrollCycleAll` → `BuildPayslip` → `Upsert`, +entered from `POST /v1/payroll/cycles/{cycleID}/run`. + +## The three properties that make it a regression fixture + +1. **Generated files that only a CONTENT header betrays.** The `internal/gen/fkit/**` + files have ordinary names (`payslip.go`, `store.go`) and carry + `// Code generated by fkit v3.11.0. DO NOT EDIT.`. Path-only detection misses every + one of them — that is the #1500 case, and why CG-5 added the content check. The + `payrollpb/*.pb.go` files cover the path-detectable channel beside them. + +2. **Deliberate name collisions.** `BuildPayslip`, `Upsert` and `Store` each exist twice, + once generated and once hand-written. The generated layer also name-collides on every + term of the question below — `CreatePayslip`, `PayrollCycleCreateRequest`, + `CalculatePayrollCycleTotals` — so a scorer that rewards incidental name matches + surfaces the CRUD path. + +3. **A size split that drives the render mode.** `cycle.go` is deliberately over the + whole-file window (227 lines) so it falls through to clipped clusters; the generated + files are deliberately under it so they ship whole. Allocation follows file size, not + relevance. `__tests__/explore-allocation-1500.test.ts` pins both sides of that split — + if you edit these files, keep it. + +## The assertion + +Query: **"how does payroll cycle create and calculate payslips?"** — an architecture +question that names none of the symbols that answer it. The budget should concentrate on +the hand-written workflow. As of 2026-08-03 it does not: + +| | allocated | delivered | +|---|---|---| +| hand-written | 48.4% | **25.6%** (all of it domain types) | +| generated CRUD | 39.9% | **57.4%** | + +`cycle.go` is allocated the single largest slice (7,052 chars, 30.6%) and delivers +**zero** — the hard ceiling drops its whole section. `payslip_builder.go` (rank #8) never +renders at all. `runPayrollCycleAll`, the hand-written `BuildPayslip` and the real +`Upsert` never reach the agent. + +## Running it + +```bash +npm run build +node scripts/agent-eval/probe-allocation.mjs payroll-go # exits 1 today, by design +npx vitest run __tests__/explore-allocation-1500.test.ts # green today, by design +``` + +The probe reports per-file budget share against `scripts/agent-eval/allocation-fixtures.json`. +The vitest suite pins the fixture's shape and holds the allocation assertion as `it.fails` — +green while the bug is open, red the moment it is fixed. See +`docs/design/explore-budget-allocation.md`. + +## Known finding: the chain's `Upsert` edge resolves to the generated store + +`runPayrollCycleAll` calls `s.store.Upsert(ctx, slip)`, where `s.store` is a +`*payslipstore.Store`. The graph resolves that edge to `internal/gen/fkit/payroll/store.go` +— the **generated** `Store.Upsert` — not the hand-written one. Same-name method resolution +across two packages that both define `Store.Upsert` picks the wrong receiver. + +This is a resolution defect, not a budget one, and it is left unfixed on purpose: it is +upstream of the allocation bug (a wrong edge pulls the generated store into the subgraph +and inflates its score), so it belongs with the scoring work in CG-10 rather than here. +The test asserts only that the workflow reaches *an* `Upsert`, so tightening the resolver +later will not break the fixture. diff --git a/__tests__/fixtures/payroll-go/cmd/payrolld/main.go b/__tests__/fixtures/payroll-go/cmd/payrolld/main.go new file mode 100644 index 000000000..63ac93717 --- /dev/null +++ b/__tests__/fixtures/payroll-go/cmd/payrolld/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "log" + "net/http" + "os" + "time" + + "github.com/example/payroll-svc/internal/platform/clock" + "github.com/example/payroll-svc/internal/store/payslipstore" + "github.com/example/payroll-svc/internal/transport/httpapi" + "github.com/example/payroll-svc/internal/usecase/payroll" +) + +func main() { + addr := os.Getenv("LISTEN_ADDR") + if addr == "" { + addr = ":8080" + } + + store := payslipstore.New() + svc := payroll.NewService(store, clock.System{}) + router := httpapi.NewRouter(httpapi.NewPayrollHandler(svc)) + + srv := &http.Server{ + Addr: addr, + Handler: router, + ReadHeaderTimeout: 5 * time.Second, + } + + log.Printf("payrolld listening on %s", addr) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("payrolld: %v", err) + } +} diff --git a/__tests__/fixtures/payroll-go/go.mod b/__tests__/fixtures/payroll-go/go.mod new file mode 100644 index 000000000..9cedd1bee --- /dev/null +++ b/__tests__/fixtures/payroll-go/go.mod @@ -0,0 +1,3 @@ +module github.com/example/payroll-svc + +go 1.22 diff --git a/__tests__/fixtures/payroll-go/internal/domain/payroll/payslip.go b/__tests__/fixtures/payroll-go/internal/domain/payroll/payslip.go new file mode 100644 index 000000000..d14f84de5 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/domain/payroll/payslip.go @@ -0,0 +1,159 @@ +package payroll + +import "time" + +// CycleStatus is the lifecycle state of a payroll cycle. +type CycleStatus string + +const ( + CycleOpen CycleStatus = "open" + CycleClosed CycleStatus = "closed" +) + +// ContractKind distinguishes the two pay models this service supports. +type ContractKind string + +const ( + ContractSalaried ContractKind = "salaried" + ContractHourly ContractKind = "hourly" +) + +// LineKind separates the two halves of a payslip. +type LineKind string + +const ( + LineEarning LineKind = "earning" + LineDeduction LineKind = "deduction" +) + +// Cycle is one payroll period. +type Cycle struct { + ID string + Start time.Time + End time.Time + Status CycleStatus + ClosedAt time.Time + ReopenReason string +} + +// Line is a single earning or deduction on a payslip. +type Line struct { + Code string + Kind LineKind + AmountCents int64 +} + +// Payslip is what a cycle produces for one employee. +type Payslip struct { + CycleID string + EmployeeID string + Currency string + PeriodFrom time.Time + PeriodTo time.Time + Lines []Line + GrossCents int64 + DeductionCents int64 + NetCents int64 + Underwater bool + RunAt time.Time + RunReason string +} + +// Timesheet is the approved unit count backing an hourly payslip. +type Timesheet struct { + CycleID string + EmployeeID string + Approved bool + Units int +} + +// Allowance is a recurring earning attached to a contract. +type Allowance struct { + Code string + AmountCents int64 + Prorated bool +} + +// Contract holds the pay terms for one employee. +type Contract struct { + Kind ContractKind + Currency string + RateCents int64 + PeriodRateCents int64 + OvertimeThresholdUnits int + OvertimeMultiplier float64 + Allowances []Allowance + StartsOn time.Time + EndsOn time.Time +} + +// OverlapsWindow reports whether the contract is live at any point in the window. +func (c Contract) OverlapsWindow(from, to time.Time) bool { + if !c.StartsOn.IsZero() && c.StartsOn.After(to) { + return false + } + if !c.EndsOn.IsZero() && c.EndsOn.Before(from) { + return false + } + return true +} + +// PeriodUnits is the contractual unit count for a window, used when a salaried +// employee has no approved timesheet. +func (c Contract) PeriodUnits(from, to time.Time) int { + if to.Before(from) { + return 0 + } + days := int(to.Sub(from).Hours()/24) + 1 + return days * 8 +} + +// Deduction is a fixed or proportional subtraction from gross. +type Deduction struct { + Code string + FixedCents int64 + RateBasisPoints int +} + +// AmountFor resolves a deduction against a gross amount. +func (d Deduction) AmountFor(grossCents int64) int64 { + if d.FixedCents > 0 { + return d.FixedCents + } + return grossCents * int64(d.RateBasisPoints) / 10000 +} + +// TaxBand is one slice of a progressive tax schedule. +type TaxBand struct { + UpToCents int64 + RateBasisPoints int +} + +// Leave is an absence window. +type Leave struct { + From time.Time + To time.Time + Unpaid bool +} + +// Employee is the payroll view of a person. +type Employee struct { + ID string + Contract Contract + Deductions []Deduction + TaxBands []TaxBand + Leave []Leave +} + +// UnpaidLeaveCoversWindow reports whether unpaid leave swallows the whole window. +func (e Employee) UnpaidLeaveCoversWindow(from, to time.Time) bool { + for _, l := range e.Leave { + if !l.Unpaid { + continue + } + if !l.From.After(from) && !l.To.Before(to) { + return true + } + } + return false +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/employee/contract.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/employee/contract.go new file mode 100644 index 000000000..4ced896b3 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/employee/contract.go @@ -0,0 +1,84 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/employee/contract.fkit +// Regenerate with: go run ./tools/fkitgen ./schema/employee + +package employee + +import ( + "context" + "database/sql" + "time" +) + +// ContractRow is the generated row type for table contract. +type ContractRow struct { + ID string + EmployeeID string + Kind string + Currency string + RateCents int64 + PeriodRateCents int64 + StartsOn time.Time + EndsOn time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +// ContractCreateRequest is the generated create payload for table contract. +type ContractCreateRequest struct { + EmployeeID string `json:"employeeId"` + Kind string `json:"kind"` + Currency string `json:"currency"` + RateCents int64 `json:"rateCents"` + PeriodRateCents int64 `json:"periodRateCents"` + StartsOn time.Time `json:"startsOn"` +} + +// CreateContract inserts one contract row. +func CreateContract(ctx context.Context, db *sql.DB, req ContractCreateRequest) (ContractRow, error) { + const q = `INSERT INTO contract (employee_id, kind, currency, rate_cents, period_rate_cents, starts_on) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING *` + return scanContract(db.QueryRowContext(ctx, q, req.EmployeeID, req.Kind, req.Currency, + req.RateCents, req.PeriodRateCents, req.StartsOn)) +} + +// GetContract selects one contract row by primary key. +func GetContract(ctx context.Context, db *sql.DB, id string) (ContractRow, error) { + const q = `SELECT * FROM contract WHERE id = $1` + return scanContract(db.QueryRowContext(ctx, q, id)) +} + +// DeleteContract removes one contract row. +func DeleteContract(ctx context.Context, db *sql.DB, id string) error { + const q = `DELETE FROM contract WHERE id = $1` + _, err := db.ExecContext(ctx, q, id) + return err +} + +// ListContractsForEmployee selects contract rows for one employee. +func ListContractsForEmployee(ctx context.Context, db *sql.DB, employeeID string) ([]ContractRow, error) { + const q = `SELECT * FROM contract WHERE employee_id = $1 ORDER BY starts_on DESC` + rows, err := db.QueryContext(ctx, q, employeeID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []ContractRow + for rows.Next() { + var r ContractRow + if err := rows.Scan(&r.ID, &r.EmployeeID, &r.Kind, &r.Currency, &r.RateCents, + &r.PeriodRateCents, &r.StartsOn, &r.EndsOn, &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func scanContract(row *sql.Row) (ContractRow, error) { + var r ContractRow + err := row.Scan(&r.ID, &r.EmployeeID, &r.Kind, &r.Currency, &r.RateCents, + &r.PeriodRateCents, &r.StartsOn, &r.EndsOn, &r.CreatedAt, &r.UpdatedAt) + return r, err +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/employee/employee.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/employee/employee.go new file mode 100644 index 000000000..fa6572687 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/employee/employee.go @@ -0,0 +1,88 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/employee/employee.fkit +// Regenerate with: go run ./tools/fkitgen ./schema/employee + +package employee + +import ( + "context" + "database/sql" + "time" +) + +// EmployeeRow is the generated row type for table employee. +type EmployeeRow struct { + ID string + Email string + FullName string + Status string + HiredOn time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +// EmployeeCreateRequest is the generated create payload for table employee. +type EmployeeCreateRequest struct { + Email string `json:"email"` + FullName string `json:"fullName"` + Status string `json:"status"` +} + +// EmployeeUpdateRequest is the generated update payload for table employee. +type EmployeeUpdateRequest struct { + FullName *string `json:"fullName,omitempty"` + Status *string `json:"status,omitempty"` +} + +// CreateEmployee inserts one employee row. +func CreateEmployee(ctx context.Context, db *sql.DB, req EmployeeCreateRequest) (EmployeeRow, error) { + const q = `INSERT INTO employee (email, full_name, status) VALUES ($1, $2, $3) RETURNING *` + return scanEmployee(db.QueryRowContext(ctx, q, req.Email, req.FullName, req.Status)) +} + +// GetEmployee selects one employee row by primary key. +func GetEmployee(ctx context.Context, db *sql.DB, id string) (EmployeeRow, error) { + const q = `SELECT * FROM employee WHERE id = $1` + return scanEmployee(db.QueryRowContext(ctx, q, id)) +} + +// UpdateEmployee patches one employee row. +func UpdateEmployee(ctx context.Context, db *sql.DB, id string, req EmployeeUpdateRequest) (EmployeeRow, error) { + const q = `UPDATE employee SET full_name = COALESCE($2, full_name), status = COALESCE($3, status), + updated_at = now() WHERE id = $1 RETURNING *` + return scanEmployee(db.QueryRowContext(ctx, q, id, req.FullName, req.Status)) +} + +// DeleteEmployee removes one employee row. +func DeleteEmployee(ctx context.Context, db *sql.DB, id string) error { + const q = `DELETE FROM employee WHERE id = $1` + _, err := db.ExecContext(ctx, q, id) + return err +} + +// ListEmployeesByStatus selects employee rows in one status. +func ListEmployeesByStatus(ctx context.Context, db *sql.DB, status string) ([]EmployeeRow, error) { + const q = `SELECT * FROM employee WHERE status = $1 ORDER BY full_name` + rows, err := db.QueryContext(ctx, q, status) + if err != nil { + return nil, err + } + defer rows.Close() + var out []EmployeeRow + for rows.Next() { + var r EmployeeRow + if err := rows.Scan(&r.ID, &r.Email, &r.FullName, &r.Status, &r.HiredOn, + &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func scanEmployee(row *sql.Row) (EmployeeRow, error) { + var r EmployeeRow + err := row.Scan(&r.ID, &r.Email, &r.FullName, &r.Status, &r.HiredOn, &r.CreatedAt, &r.UpdatedAt) + return r, err +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/calculate.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/calculate.go new file mode 100644 index 000000000..c005ebe68 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/calculate.go @@ -0,0 +1,60 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/payroll/aggregates.fkit +// Regenerate with: go run ./tools/fkitgen ./schema/payroll + +package payroll + +import ( + "context" + "database/sql" +) + +// PayrollCycleTotals is the generated aggregate row for a payroll cycle. +type PayrollCycleTotals struct { + CycleID string + Payslips int64 + GrossCents int64 + DeductionCents int64 + NetCents int64 +} + +// CalculatePayrollCycleTotals runs the generated SUM aggregate over the +// payslip rows of one cycle. It totals what is already stored; it does not +// calculate any payslip. +func CalculatePayrollCycleTotals(ctx context.Context, db *sql.DB, cycleID string) (PayrollCycleTotals, error) { + const q = `SELECT count(*), COALESCE(sum(gross_cents), 0), COALESCE(sum(deduction_cents), 0), + COALESCE(sum(net_cents), 0) + FROM payslip WHERE cycle_id = $1` + var t PayrollCycleTotals + t.CycleID = cycleID + err := db.QueryRowContext(ctx, q, cycleID).Scan(&t.Payslips, &t.GrossCents, &t.DeductionCents, &t.NetCents) + return t, err +} + +// CalculatePayslipNet recomputes net from the stored gross and deduction +// columns of one row. Pure column arithmetic — no pay rules. +func CalculatePayslipNet(row PayslipRow) int64 { + return row.GrossCents - row.DeductionCents +} + +// CalculatePayrollCycleAverage averages the stored net over a cycle. +func CalculatePayrollCycleAverage(ctx context.Context, db *sql.DB, cycleID string) (int64, error) { + totals, err := CalculatePayrollCycleTotals(ctx, db, cycleID) + if err != nil { + return 0, err + } + if totals.Payslips == 0 { + return 0, nil + } + return totals.NetCents / totals.Payslips, nil +} + +// CalculateEmployeeYearToDate sums an employee's stored payslips for a year. +func CalculateEmployeeYearToDate(ctx context.Context, db *sql.DB, employeeID string, year int) (int64, error) { + const q = `SELECT COALESCE(sum(net_cents), 0) FROM payslip + WHERE employee_id = $1 AND extract(year from period_from) = $2` + var n int64 + err := db.QueryRowContext(ctx, q, employeeID, year).Scan(&n) + return n, err +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/dto.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/dto.go new file mode 100644 index 000000000..78251c3eb --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/dto.go @@ -0,0 +1,80 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/payroll +// Regenerate with: go run ./tools/fkitgen ./schema/payroll + +package payroll + +import "time" + +// PayslipDTO is the generated wire representation of a payslip row. +type PayslipDTO struct { + ID string `json:"id"` + CycleID string `json:"cycleId"` + EmployeeID string `json:"employeeId"` + Currency string `json:"currency"` + PeriodFrom time.Time `json:"periodFrom"` + PeriodTo time.Time `json:"periodTo"` + GrossCents int64 `json:"grossCents"` + DeductionCents int64 `json:"deductionCents"` + NetCents int64 `json:"netCents"` +} + +// PayrollCycleDTO is the generated wire representation of a payroll_cycle row. +type PayrollCycleDTO struct { + ID string `json:"id"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Status string `json:"status"` +} + +// PayslipListDTO is the generated list envelope for payslip rows. +type PayslipListDTO struct { + Items []PayslipDTO `json:"items"` + NextCursor string `json:"nextCursor,omitempty"` + Total int64 `json:"total"` +} + +// PayslipToDTO converts a payslip row to its wire form. +func PayslipToDTO(r PayslipRow) PayslipDTO { + return PayslipDTO{ + ID: r.ID, + CycleID: r.CycleID, + EmployeeID: r.EmployeeID, + Currency: r.Currency, + PeriodFrom: r.PeriodFrom, + PeriodTo: r.PeriodTo, + GrossCents: r.GrossCents, + DeductionCents: r.DeductionCents, + NetCents: r.NetCents, + } +} + +// PayslipFromDTO converts a wire payslip back to a row. +func PayslipFromDTO(d PayslipDTO) PayslipRow { + return PayslipRow{ + ID: d.ID, + CycleID: d.CycleID, + EmployeeID: d.EmployeeID, + Currency: d.Currency, + PeriodFrom: d.PeriodFrom, + PeriodTo: d.PeriodTo, + GrossCents: d.GrossCents, + DeductionCents: d.DeductionCents, + NetCents: d.NetCents, + } +} + +// PayrollCycleToDTO converts a payroll_cycle row to its wire form. +func PayrollCycleToDTO(r PayrollCycleRow) PayrollCycleDTO { + return PayrollCycleDTO{ID: r.ID, Start: r.Start, End: r.End, Status: r.Status} +} + +// PayslipsToListDTO wraps payslip rows in the generated list envelope. +func PayslipsToListDTO(rows []PayslipRow, total int64) PayslipListDTO { + items := make([]PayslipDTO, 0, len(rows)) + for _, r := range rows { + items = append(items, PayslipToDTO(r)) + } + return PayslipListDTO{Items: items, Total: total} +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payroll_cycle.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payroll_cycle.go new file mode 100644 index 000000000..d565dbdfc --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payroll_cycle.go @@ -0,0 +1,116 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/payroll/payroll_cycle.fkit +// Regenerate with: go run ./tools/fkitgen ./schema/payroll + +package payroll + +import ( + "context" + "database/sql" + "time" +) + +// PayrollCycleRow is the generated row type for table payroll_cycle. +type PayrollCycleRow struct { + ID string + Start time.Time + End time.Time + Status string + ClosedAt time.Time + ReopenReason string + CreatedAt time.Time + UpdatedAt time.Time +} + +// PayrollCycleCreateRequest is the generated create payload for payroll_cycle. +type PayrollCycleCreateRequest struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + Status string `json:"status"` +} + +// PayrollCycleUpdateRequest is the generated update payload for payroll_cycle. +type PayrollCycleUpdateRequest struct { + Status *string `json:"status,omitempty"` + ReopenReason *string `json:"reopenReason,omitempty"` +} + +// CreatePayrollCycle inserts one payroll_cycle row. +func CreatePayrollCycle(ctx context.Context, db *sql.DB, req PayrollCycleCreateRequest) (PayrollCycleRow, error) { + const q = `INSERT INTO payroll_cycle (start_on, end_on, status) VALUES ($1, $2, $3) RETURNING *` + return scanPayrollCycle(db.QueryRowContext(ctx, q, req.Start, req.End, req.Status)) +} + +// GetPayrollCycle selects one payroll_cycle row by primary key. +func GetPayrollCycle(ctx context.Context, db *sql.DB, id string) (PayrollCycleRow, error) { + const q = `SELECT * FROM payroll_cycle WHERE id = $1` + return scanPayrollCycle(db.QueryRowContext(ctx, q, id)) +} + +// UpdatePayrollCycle patches one payroll_cycle row. +func UpdatePayrollCycle(ctx context.Context, db *sql.DB, id string, req PayrollCycleUpdateRequest) (PayrollCycleRow, error) { + const q = `UPDATE payroll_cycle SET status = COALESCE($2, status), + reopen_reason = COALESCE($3, reopen_reason), updated_at = now() + WHERE id = $1 RETURNING *` + return scanPayrollCycle(db.QueryRowContext(ctx, q, id, req.Status, req.ReopenReason)) +} + +// DeletePayrollCycle removes one payroll_cycle row. +func DeletePayrollCycle(ctx context.Context, db *sql.DB, id string) error { + const q = `DELETE FROM payroll_cycle WHERE id = $1` + _, err := db.ExecContext(ctx, q, id) + return err +} + +// ListPayrollCycles selects every payroll_cycle row. +func ListPayrollCycles(ctx context.Context, db *sql.DB) ([]PayrollCycleRow, error) { + const q = `SELECT * FROM payroll_cycle ORDER BY start_on DESC` + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PayrollCycleRow + for rows.Next() { + var r PayrollCycleRow + if err := rows.Scan(&r.ID, &r.Start, &r.End, &r.Status, &r.ClosedAt, &r.ReopenReason, + &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// ListPayrollCyclesByStatus selects payroll_cycle rows in one status. +func ListPayrollCyclesByStatus(ctx context.Context, db *sql.DB, status string) ([]PayrollCycleRow, error) { + const q = `SELECT * FROM payroll_cycle WHERE status = $1 ORDER BY start_on DESC` + rows, err := db.QueryContext(ctx, q, status) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PayrollCycleRow + for rows.Next() { + var r PayrollCycleRow + if err := rows.Scan(&r.ID, &r.Start, &r.End, &r.Status, &r.ClosedAt, &r.ReopenReason, + &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// BuildPayrollCycle maps a create request onto a row. +func BuildPayrollCycle(req PayrollCycleCreateRequest) PayrollCycleRow { + return PayrollCycleRow{Start: req.Start, End: req.End, Status: req.Status} +} + +func scanPayrollCycle(row *sql.Row) (PayrollCycleRow, error) { + var r PayrollCycleRow + err := row.Scan(&r.ID, &r.Start, &r.End, &r.Status, &r.ClosedAt, &r.ReopenReason, + &r.CreatedAt, &r.UpdatedAt) + return r, err +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payslip.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payslip.go new file mode 100644 index 000000000..7e634af65 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/payslip.go @@ -0,0 +1,129 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/payroll/payslip.fkit +// Regenerate with: go run ./tools/fkitgen ./schema/payroll + +package payroll + +import ( + "context" + "database/sql" + "time" +) + +// PayslipRow is the generated row type for table payslip. +type PayslipRow struct { + ID string + CycleID string + EmployeeID string + Currency string + PeriodFrom time.Time + PeriodTo time.Time + GrossCents int64 + DeductionCents int64 + NetCents int64 + Underwater bool + RunAt time.Time + RunReason string + CreatedAt time.Time + UpdatedAt time.Time +} + +// PayslipCreateRequest is the generated create payload for table payslip. +type PayslipCreateRequest struct { + CycleID string `json:"cycleId"` + EmployeeID string `json:"employeeId"` + Currency string `json:"currency"` + GrossCents int64 `json:"grossCents"` + DeductionCents int64 `json:"deductionCents"` + NetCents int64 `json:"netCents"` +} + +// PayslipUpdateRequest is the generated update payload for table payslip. +type PayslipUpdateRequest struct { + GrossCents *int64 `json:"grossCents,omitempty"` + DeductionCents *int64 `json:"deductionCents,omitempty"` + NetCents *int64 `json:"netCents,omitempty"` + RunReason *string `json:"runReason,omitempty"` +} + +// CreatePayslip inserts one payslip row. +func CreatePayslip(ctx context.Context, db *sql.DB, req PayslipCreateRequest) (PayslipRow, error) { + const q = `INSERT INTO payslip (cycle_id, employee_id, currency, gross_cents, deduction_cents, net_cents) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING *` + row := db.QueryRowContext(ctx, q, req.CycleID, req.EmployeeID, req.Currency, req.GrossCents, req.DeductionCents, req.NetCents) + return scanPayslip(row) +} + +// GetPayslip selects one payslip row by primary key. +func GetPayslip(ctx context.Context, db *sql.DB, id string) (PayslipRow, error) { + const q = `SELECT * FROM payslip WHERE id = $1` + return scanPayslip(db.QueryRowContext(ctx, q, id)) +} + +// UpdatePayslip patches one payslip row. +func UpdatePayslip(ctx context.Context, db *sql.DB, id string, req PayslipUpdateRequest) (PayslipRow, error) { + const q = `UPDATE payslip SET gross_cents = COALESCE($2, gross_cents), + deduction_cents = COALESCE($3, deduction_cents), + net_cents = COALESCE($4, net_cents), + run_reason = COALESCE($5, run_reason), + updated_at = now() WHERE id = $1 RETURNING *` + return scanPayslip(db.QueryRowContext(ctx, q, id, req.GrossCents, req.DeductionCents, req.NetCents, req.RunReason)) +} + +// DeletePayslip removes one payslip row. +func DeletePayslip(ctx context.Context, db *sql.DB, id string) error { + const q = `DELETE FROM payslip WHERE id = $1` + _, err := db.ExecContext(ctx, q, id) + return err +} + +// ListPayslipsByCycle selects every payslip row for a cycle. +func ListPayslipsByCycle(ctx context.Context, db *sql.DB, cycleID string) ([]PayslipRow, error) { + const q = `SELECT * FROM payslip WHERE cycle_id = $1 ORDER BY employee_id` + rows, err := db.QueryContext(ctx, q, cycleID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PayslipRow + for rows.Next() { + var r PayslipRow + if err := rows.Scan(&r.ID, &r.CycleID, &r.EmployeeID, &r.Currency, &r.PeriodFrom, &r.PeriodTo, + &r.GrossCents, &r.DeductionCents, &r.NetCents, &r.Underwater, &r.RunAt, &r.RunReason, + &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// CountPayslipsByCycle counts payslip rows for a cycle. +func CountPayslipsByCycle(ctx context.Context, db *sql.DB, cycleID string) (int64, error) { + const q = `SELECT count(*) FROM payslip WHERE cycle_id = $1` + var n int64 + err := db.QueryRowContext(ctx, q, cycleID).Scan(&n) + return n, err +} + +// BuildPayslip maps a create request onto a row. Field copy only — the +// generator has no knowledge of pay rules. +func BuildPayslip(req PayslipCreateRequest) PayslipRow { + return PayslipRow{ + CycleID: req.CycleID, + EmployeeID: req.EmployeeID, + Currency: req.Currency, + GrossCents: req.GrossCents, + DeductionCents: req.DeductionCents, + NetCents: req.NetCents, + } +} + +func scanPayslip(row *sql.Row) (PayslipRow, error) { + var r PayslipRow + err := row.Scan(&r.ID, &r.CycleID, &r.EmployeeID, &r.Currency, &r.PeriodFrom, &r.PeriodTo, + &r.GrossCents, &r.DeductionCents, &r.NetCents, &r.Underwater, &r.RunAt, &r.RunReason, + &r.CreatedAt, &r.UpdatedAt) + return r, err +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/store.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/store.go new file mode 100644 index 000000000..c063a5dcc --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/payroll/store.go @@ -0,0 +1,95 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/payroll +// Regenerate with: go run ./tools/fkitgen ./schema/payroll + +package payroll + +import ( + "context" + "database/sql" +) + +// Store is the generated repository over every payroll table. +type Store struct { + db *sql.DB +} + +// NewStore returns a generated store bound to db. +func NewStore(db *sql.DB) *Store { return &Store{db: db} } + +// Upsert writes one payslip row, keyed by (cycle_id, employee_id). +func (s *Store) Upsert(ctx context.Context, row PayslipRow) (PayslipRow, error) { + const q = `INSERT INTO payslip (cycle_id, employee_id, currency, gross_cents, deduction_cents, net_cents) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (cycle_id, employee_id) DO UPDATE SET + gross_cents = EXCLUDED.gross_cents, + deduction_cents = EXCLUDED.deduction_cents, + net_cents = EXCLUDED.net_cents, + updated_at = now() + RETURNING *` + return scanPayslip(s.db.QueryRowContext(ctx, q, row.CycleID, row.EmployeeID, row.Currency, + row.GrossCents, row.DeductionCents, row.NetCents)) +} + +// UpsertPayrollCycle writes one payroll_cycle row, keyed by id. +func (s *Store) UpsertPayrollCycle(ctx context.Context, row PayrollCycleRow) (PayrollCycleRow, error) { + const q = `INSERT INTO payroll_cycle (id, start_on, end_on, status) + VALUES ($1, $2, $3, $4) + ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, updated_at = now() + RETURNING *` + return scanPayrollCycle(s.db.QueryRowContext(ctx, q, row.ID, row.Start, row.End, row.Status)) +} + +// CreatePayslip inserts one payslip row through the store. +func (s *Store) CreatePayslip(ctx context.Context, req PayslipCreateRequest) (PayslipRow, error) { + return CreatePayslip(ctx, s.db, req) +} + +// GetPayslip reads one payslip row through the store. +func (s *Store) GetPayslip(ctx context.Context, id string) (PayslipRow, error) { + return GetPayslip(ctx, s.db, id) +} + +// UpdatePayslip patches one payslip row through the store. +func (s *Store) UpdatePayslip(ctx context.Context, id string, req PayslipUpdateRequest) (PayslipRow, error) { + return UpdatePayslip(ctx, s.db, id, req) +} + +// DeletePayslip removes one payslip row through the store. +func (s *Store) DeletePayslip(ctx context.Context, id string) error { + return DeletePayslip(ctx, s.db, id) +} + +// ListPayslipsByCycle lists payslip rows for a cycle through the store. +func (s *Store) ListPayslipsByCycle(ctx context.Context, cycleID string) ([]PayslipRow, error) { + return ListPayslipsByCycle(ctx, s.db, cycleID) +} + +// CreatePayrollCycle inserts one payroll_cycle row through the store. +func (s *Store) CreatePayrollCycle(ctx context.Context, req PayrollCycleCreateRequest) (PayrollCycleRow, error) { + return CreatePayrollCycle(ctx, s.db, req) +} + +// GetPayrollCycle reads one payroll_cycle row through the store. +func (s *Store) GetPayrollCycle(ctx context.Context, id string) (PayrollCycleRow, error) { + return GetPayrollCycle(ctx, s.db, id) +} + +// ListPayrollCycles lists payroll_cycle rows through the store. +func (s *Store) ListPayrollCycles(ctx context.Context) ([]PayrollCycleRow, error) { + return ListPayrollCycles(ctx, s.db) +} + +// Tx runs fn inside a transaction. +func (s *Store) Tx(ctx context.Context, fn func(*Store) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := fn(s); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/fkit/timesheet/timesheet.go b/__tests__/fixtures/payroll-go/internal/gen/fkit/timesheet/timesheet.go new file mode 100644 index 000000000..ae02b187c --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/fkit/timesheet/timesheet.go @@ -0,0 +1,76 @@ +// Code generated by fkit v3.11.0. DO NOT EDIT. +// +// Source: schema/timesheet/timesheet.fkit +// Regenerate with: go run ./tools/fkitgen ./schema/timesheet + +package timesheet + +import ( + "context" + "database/sql" + "time" +) + +// TimesheetRow is the generated row type for table timesheet. +type TimesheetRow struct { + ID string + CycleID string + EmployeeID string + Units int + Approved bool + ApprovedAt time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +// TimesheetCreateRequest is the generated create payload for table timesheet. +type TimesheetCreateRequest struct { + CycleID string `json:"cycleId"` + EmployeeID string `json:"employeeId"` + Units int `json:"units"` +} + +// CreateTimesheet inserts one timesheet row. +func CreateTimesheet(ctx context.Context, db *sql.DB, req TimesheetCreateRequest) (TimesheetRow, error) { + const q = `INSERT INTO timesheet (cycle_id, employee_id, units) VALUES ($1, $2, $3) RETURNING *` + return scanTimesheet(db.QueryRowContext(ctx, q, req.CycleID, req.EmployeeID, req.Units)) +} + +// GetTimesheet selects one timesheet row by primary key. +func GetTimesheet(ctx context.Context, db *sql.DB, id string) (TimesheetRow, error) { + const q = `SELECT * FROM timesheet WHERE id = $1` + return scanTimesheet(db.QueryRowContext(ctx, q, id)) +} + +// ApproveTimesheet flips the approved column on one timesheet row. +func ApproveTimesheet(ctx context.Context, db *sql.DB, id string) (TimesheetRow, error) { + const q = `UPDATE timesheet SET approved = true, approved_at = now() WHERE id = $1 RETURNING *` + return scanTimesheet(db.QueryRowContext(ctx, q, id)) +} + +// ListTimesheetsByCycle selects timesheet rows for one cycle. +func ListTimesheetsByCycle(ctx context.Context, db *sql.DB, cycleID string) ([]TimesheetRow, error) { + const q = `SELECT * FROM timesheet WHERE cycle_id = $1 ORDER BY employee_id` + rows, err := db.QueryContext(ctx, q, cycleID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []TimesheetRow + for rows.Next() { + var r TimesheetRow + if err := rows.Scan(&r.ID, &r.CycleID, &r.EmployeeID, &r.Units, &r.Approved, + &r.ApprovedAt, &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func scanTimesheet(row *sql.Row) (TimesheetRow, error) { + var r TimesheetRow + err := row.Scan(&r.ID, &r.CycleID, &r.EmployeeID, &r.Units, &r.Approved, + &r.ApprovedAt, &r.CreatedAt, &r.UpdatedAt) + return r, err +} diff --git a/__tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll.pb.go b/__tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll.pb.go new file mode 100644 index 000000000..be849cc66 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll.pb.go @@ -0,0 +1,139 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.2 +// protoc v5.27.1 +// source: payroll/v1/payroll.proto + +package payrollpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" +) + +// RunPayrollCycleRequest is the generated request message. +type RunPayrollCycleRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CycleId string `protobuf:"bytes,1,opt,name=cycle_id,json=cycleId,proto3" json:"cycle_id,omitempty"` + DryRun bool `protobuf:"varint,2,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"` + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` +} + +func (x *RunPayrollCycleRequest) GetCycleId() string { + if x != nil { + return x.CycleId + } + return "" +} + +func (x *RunPayrollCycleRequest) GetDryRun() bool { + if x != nil { + return x.DryRun + } + return false +} + +func (x *RunPayrollCycleRequest) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *RunPayrollCycleRequest) Reset() { *x = RunPayrollCycleRequest{} } +func (x *RunPayrollCycleRequest) String() string { return protoimpl.X.MessageStringOf(x) } + +// RunPayrollCycleResponse is the generated response message. +type RunPayrollCycleResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CycleId string `protobuf:"bytes,1,opt,name=cycle_id,json=cycleId,proto3" json:"cycle_id,omitempty"` + Payslips []*Payslip `protobuf:"bytes,2,rep,name=payslips,proto3" json:"payslips,omitempty"` + GrossCents int64 `protobuf:"varint,3,opt,name=gross_cents,json=grossCents,proto3" json:"gross_cents,omitempty"` + NetCents int64 `protobuf:"varint,4,opt,name=net_cents,json=netCents,proto3" json:"net_cents,omitempty"` +} + +func (x *RunPayrollCycleResponse) GetPayslips() []*Payslip { + if x != nil { + return x.Payslips + } + return nil +} + +func (x *RunPayrollCycleResponse) Reset() { *x = RunPayrollCycleResponse{} } +func (x *RunPayrollCycleResponse) String() string { return protoimpl.X.MessageStringOf(x) } + +// Payslip is the generated payslip message. +type Payslip struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + CycleId string `protobuf:"bytes,2,opt,name=cycle_id,json=cycleId,proto3" json:"cycle_id,omitempty"` + EmployeeId string `protobuf:"bytes,3,opt,name=employee_id,json=employeeId,proto3" json:"employee_id,omitempty"` + GrossCents int64 `protobuf:"varint,4,opt,name=gross_cents,json=grossCents,proto3" json:"gross_cents,omitempty"` + DeductionCents int64 `protobuf:"varint,5,opt,name=deduction_cents,json=deductionCents,proto3" json:"deduction_cents,omitempty"` + NetCents int64 `protobuf:"varint,6,opt,name=net_cents,json=netCents,proto3" json:"net_cents,omitempty"` + PeriodFrom *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=period_from,json=periodFrom,proto3" json:"period_from,omitempty"` + PeriodTo *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=period_to,json=periodTo,proto3" json:"period_to,omitempty"` +} + +func (x *Payslip) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Payslip) GetNetCents() int64 { + if x != nil { + return x.NetCents + } + return 0 +} + +func (x *Payslip) Reset() { *x = Payslip{} } +func (x *Payslip) String() string { return protoimpl.X.MessageStringOf(x) } + +// PayrollCycle is the generated cycle message. +type PayrollCycle struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Start *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=start,proto3" json:"start,omitempty"` + End *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=end,proto3" json:"end,omitempty"` + Status string `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` +} + +func (x *PayrollCycle) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *PayrollCycle) Reset() { *x = PayrollCycle{} } +func (x *PayrollCycle) String() string { return protoimpl.X.MessageStringOf(x) } + +var file_payroll_v1_payroll_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x70, 0x61, 0x79, 0x72, 0x6f, 0x6c, 0x6c, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, + 0x72, 0x6f, 0x6c, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x70, 0x61, 0x79, 0x72, +} + +var file_payroll_v1_payroll_proto_goTypes = []any{ + (*RunPayrollCycleRequest)(nil), + (*RunPayrollCycleResponse)(nil), + (*Payslip)(nil), + (*PayrollCycle)(nil), +} + +var File_payroll_v1_payroll_proto protoreflect.FileDescriptor diff --git a/__tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll_grpc.pb.go b/__tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll_grpc.pb.go new file mode 100644 index 000000000..19aeaeea9 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/gen/payrollpb/payroll_grpc.pb.go @@ -0,0 +1,104 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.4.0 +// - protoc v5.27.1 +// source: payroll/v1/payroll.proto + +package payrollpb + +import ( + context "context" + + grpc "google.golang.org/grpc" +) + +const ( + PayrollService_RunPayrollCycle_FullMethodName = "/payroll.v1.PayrollService/RunPayrollCycle" + PayrollService_GetPayrollCycle_FullMethodName = "/payroll.v1.PayrollService/GetPayrollCycle" + PayrollService_ListPayslips_FullMethodName = "/payroll.v1.PayrollService/ListPayslips" +) + +// PayrollServiceClient is the generated client API for PayrollService. +type PayrollServiceClient interface { + RunPayrollCycle(ctx context.Context, in *RunPayrollCycleRequest, opts ...grpc.CallOption) (*RunPayrollCycleResponse, error) + GetPayrollCycle(ctx context.Context, in *RunPayrollCycleRequest, opts ...grpc.CallOption) (*PayrollCycle, error) + ListPayslips(ctx context.Context, in *RunPayrollCycleRequest, opts ...grpc.CallOption) (*RunPayrollCycleResponse, error) +} + +type payrollServiceClient struct { + cc grpc.ClientConnInterface +} + +// NewPayrollServiceClient returns a generated client. +func NewPayrollServiceClient(cc grpc.ClientConnInterface) PayrollServiceClient { + return &payrollServiceClient{cc} +} + +func (c *payrollServiceClient) RunPayrollCycle(ctx context.Context, in *RunPayrollCycleRequest, opts ...grpc.CallOption) (*RunPayrollCycleResponse, error) { + out := new(RunPayrollCycleResponse) + err := c.cc.Invoke(ctx, PayrollService_RunPayrollCycle_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *payrollServiceClient) GetPayrollCycle(ctx context.Context, in *RunPayrollCycleRequest, opts ...grpc.CallOption) (*PayrollCycle, error) { + out := new(PayrollCycle) + err := c.cc.Invoke(ctx, PayrollService_GetPayrollCycle_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *payrollServiceClient) ListPayslips(ctx context.Context, in *RunPayrollCycleRequest, opts ...grpc.CallOption) (*RunPayrollCycleResponse, error) { + out := new(RunPayrollCycleResponse) + err := c.cc.Invoke(ctx, PayrollService_ListPayslips_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// PayrollServiceServer is the generated server API for PayrollService. +type PayrollServiceServer interface { + RunPayrollCycle(context.Context, *RunPayrollCycleRequest) (*RunPayrollCycleResponse, error) + GetPayrollCycle(context.Context, *RunPayrollCycleRequest) (*PayrollCycle, error) + ListPayslips(context.Context, *RunPayrollCycleRequest) (*RunPayrollCycleResponse, error) + mustEmbedUnimplementedPayrollServiceServer() +} + +// UnimplementedPayrollServiceServer must be embedded for forward compatibility. +type UnimplementedPayrollServiceServer struct{} + +func (UnimplementedPayrollServiceServer) RunPayrollCycle(context.Context, *RunPayrollCycleRequest) (*RunPayrollCycleResponse, error) { + return nil, nil +} + +func (UnimplementedPayrollServiceServer) GetPayrollCycle(context.Context, *RunPayrollCycleRequest) (*PayrollCycle, error) { + return nil, nil +} + +func (UnimplementedPayrollServiceServer) ListPayslips(context.Context, *RunPayrollCycleRequest) (*RunPayrollCycleResponse, error) { + return nil, nil +} + +func (UnimplementedPayrollServiceServer) mustEmbedUnimplementedPayrollServiceServer() {} + +// RegisterPayrollServiceServer registers the generated service. +func RegisterPayrollServiceServer(s grpc.ServiceRegistrar, srv PayrollServiceServer) { + s.RegisterService(&PayrollService_ServiceDesc, srv) +} + +// PayrollService_ServiceDesc is the generated service descriptor. +var PayrollService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "payroll.v1.PayrollService", + HandlerType: (*PayrollServiceServer)(nil), + Methods: []grpc.MethodDesc{ + {MethodName: "RunPayrollCycle"}, + {MethodName: "GetPayrollCycle"}, + {MethodName: "ListPayslips"}, + }, + Metadata: "payroll/v1/payroll.proto", +} diff --git a/__tests__/fixtures/payroll-go/internal/platform/clock/clock.go b/__tests__/fixtures/payroll-go/internal/platform/clock/clock.go new file mode 100644 index 000000000..39fc9197c --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/platform/clock/clock.go @@ -0,0 +1,18 @@ +package clock + +import "time" + +// Clock is the time seam so a payroll run is reproducible in tests. +type Clock interface { + Now() time.Time +} + +// System is the production clock. +type System struct{} + +func (System) Now() time.Time { return time.Now().UTC() } + +// Fixed is a frozen clock. +type Fixed struct{ At time.Time } + +func (f Fixed) Now() time.Time { return f.At } diff --git a/__tests__/fixtures/payroll-go/internal/store/payslipstore/store.go b/__tests__/fixtures/payroll-go/internal/store/payslipstore/store.go new file mode 100644 index 000000000..8c145cd28 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/store/payslipstore/store.go @@ -0,0 +1,119 @@ +package payslipstore + +import ( + "context" + "fmt" + "sync" + + "github.com/example/payroll-svc/internal/domain/payroll" +) + +// Store is the hand-written persistence seam the use-case layer writes through. +// It is deliberately narrow: the generated fkit store can address every table, +// this one only exposes the operations a payroll cycle needs. +type Store struct { + mu sync.RWMutex + payslips map[string]payroll.Payslip + cycles map[string]payroll.Cycle + employees map[string][]payroll.Employee + sheets map[string]payroll.Timesheet +} + +func New() *Store { + return &Store{ + payslips: map[string]payroll.Payslip{}, + cycles: map[string]payroll.Cycle{}, + employees: map[string][]payroll.Employee{}, + sheets: map[string]payroll.Timesheet{}, + } +} + +func key(cycleID, employeeID string) string { return cycleID + "/" + employeeID } + +// Upsert writes a payslip, replacing any prior slip for the same +// (cycle, employee). A re-run of a cycle must not duplicate rows, so this is +// an upsert rather than an insert. +func (s *Store) Upsert(ctx context.Context, slip payroll.Payslip) error { + if err := ctx.Err(); err != nil { + return err + } + if slip.CycleID == "" || slip.EmployeeID == "" { + return fmt.Errorf("payslip missing cycle or employee id") + } + s.mu.Lock() + defer s.mu.Unlock() + s.payslips[key(slip.CycleID, slip.EmployeeID)] = slip + return nil +} + +// ListByCycle returns every payslip a cycle produced. +func (s *Store) ListByCycle(ctx context.Context, cycleID string) ([]payroll.Payslip, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]payroll.Payslip, 0, len(s.payslips)) + for _, slip := range s.payslips { + if slip.CycleID == cycleID { + out = append(out, slip) + } + } + return out, nil +} + +func (s *Store) Cycle(ctx context.Context, cycleID string) (payroll.Cycle, error) { + if err := ctx.Err(); err != nil { + return payroll.Cycle{}, err + } + s.mu.RLock() + defer s.mu.RUnlock() + cycle, ok := s.cycles[cycleID] + if !ok { + return payroll.Cycle{}, fmt.Errorf("cycle %s not found", cycleID) + } + return cycle, nil +} + +func (s *Store) SaveCycle(ctx context.Context, cycle payroll.Cycle) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.cycles[cycle.ID] = cycle + return nil +} + +func (s *Store) EmployeesForCycle(ctx context.Context, cycleID string) ([]payroll.Employee, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.RLock() + defer s.mu.RUnlock() + return s.employees[cycleID], nil +} + +func (s *Store) Timesheet(ctx context.Context, cycleID, employeeID string) (payroll.Timesheet, error) { + if err := ctx.Err(); err != nil { + return payroll.Timesheet{}, err + } + s.mu.RLock() + defer s.mu.RUnlock() + ts, ok := s.sheets[key(cycleID, employeeID)] + if !ok { + return payroll.Timesheet{}, fmt.Errorf("no timesheet for %s in %s", employeeID, cycleID) + } + return ts, nil +} + +// Seed loads fixture data; the real service reads from Postgres. +func (s *Store) Seed(cycle payroll.Cycle, employees []payroll.Employee, sheets []payroll.Timesheet) { + s.mu.Lock() + defer s.mu.Unlock() + s.cycles[cycle.ID] = cycle + s.employees[cycle.ID] = employees + for _, ts := range sheets { + s.sheets[key(ts.CycleID, ts.EmployeeID)] = ts + } +} diff --git a/__tests__/fixtures/payroll-go/internal/transport/httpapi/payroll_handler.go b/__tests__/fixtures/payroll-go/internal/transport/httpapi/payroll_handler.go new file mode 100644 index 000000000..7f3534c80 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/transport/httpapi/payroll_handler.go @@ -0,0 +1,96 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/example/payroll-svc/internal/usecase/payroll" +) + +// PayrollHandler is the HTTP entry point into the payroll use-case layer. +type PayrollHandler struct { + svc *payroll.Service +} + +func NewPayrollHandler(svc *payroll.Service) *PayrollHandler { + return &PayrollHandler{svc: svc} +} + +type runCycleRequest struct { + DryRun bool `json:"dryRun"` + Reason string `json:"reason"` +} + +type runCycleResponse struct { + CycleID string `json:"cycleId"` + Payslips int `json:"payslips"` + GrossCents int64 `json:"grossCents"` + NetCents int64 `json:"netCents"` +} + +// RunCycle kicks off a payroll cycle: it hands the cycle id to the use-case +// layer, which builds and persists a payslip per active employee. +func (h *PayrollHandler) RunCycle(w http.ResponseWriter, r *http.Request) { + cycleID := r.PathValue("cycleID") + if cycleID == "" { + httpError(w, http.StatusBadRequest, "cycleID is required") + return + } + + var req runCycleRequest + if r.ContentLength > 0 { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httpError(w, http.StatusBadRequest, "malformed body") + return + } + } + + result, err := h.svc.RunCycle(r.Context(), cycleID, payroll.RunOptions{ + DryRun: req.DryRun, + Reason: req.Reason, + }) + if err != nil { + if errors.Is(err, payroll.ErrCycleClosed) { + httpError(w, http.StatusConflict, "cycle already closed") + return + } + httpError(w, http.StatusInternalServerError, "run failed") + return + } + + writeJSON(w, http.StatusOK, runCycleResponse{ + CycleID: result.CycleID, + Payslips: len(result.Payslips), + GrossCents: result.TotalGrossCents, + NetCents: result.TotalNetCents, + }) +} + +func (h *PayrollHandler) GetCycle(w http.ResponseWriter, r *http.Request) { + cycle, err := h.svc.Cycle(r.Context(), r.PathValue("cycleID")) + if err != nil { + httpError(w, http.StatusNotFound, "no such cycle") + return + } + writeJSON(w, http.StatusOK, cycle) +} + +func (h *PayrollHandler) ListPayslips(w http.ResponseWriter, r *http.Request) { + slips, err := h.svc.PayslipsForCycle(r.Context(), r.PathValue("cycleID")) + if err != nil { + httpError(w, http.StatusNotFound, "no such cycle") + return + } + writeJSON(w, http.StatusOK, slips) +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} + +func httpError(w http.ResponseWriter, status int, msg string) { + writeJSON(w, status, map[string]string{"error": msg}) +} diff --git a/__tests__/fixtures/payroll-go/internal/transport/httpapi/router.go b/__tests__/fixtures/payroll-go/internal/transport/httpapi/router.go new file mode 100644 index 000000000..8d916a153 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/transport/httpapi/router.go @@ -0,0 +1,19 @@ +package httpapi + +import "net/http" + +// NewRouter wires the HTTP surface. The payroll cycle endpoint is the only +// entry point into the hand-written use-case layer. +func NewRouter(h *PayrollHandler) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST /v1/payroll/cycles/{cycleID}/run", h.RunCycle) + mux.HandleFunc("GET /v1/payroll/cycles/{cycleID}", h.GetCycle) + mux.HandleFunc("GET /v1/payroll/cycles/{cycleID}/payslips", h.ListPayslips) + mux.HandleFunc("GET /healthz", health) + return mux +} + +func health(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) +} diff --git a/__tests__/fixtures/payroll-go/internal/usecase/payroll/cycle.go b/__tests__/fixtures/payroll-go/internal/usecase/payroll/cycle.go new file mode 100644 index 000000000..2b47ce226 --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/usecase/payroll/cycle.go @@ -0,0 +1,227 @@ +package payroll + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/example/payroll-svc/internal/domain/payroll" + "github.com/example/payroll-svc/internal/platform/clock" + "github.com/example/payroll-svc/internal/store/payslipstore" +) + +// ErrCycleClosed is returned when a cycle has already been finalized. +var ErrCycleClosed = errors.New("payroll cycle is closed") + +// ErrNoEmployees is returned when a cycle resolves to an empty roster. +var ErrNoEmployees = errors.New("payroll cycle has no active employees") + +// RunOptions tunes a single run of a payroll cycle. +type RunOptions struct { + // DryRun computes every payslip but persists nothing. + DryRun bool + // Reason is recorded on the audit trail for re-runs. + Reason string + // Only, when non-empty, restricts the run to these employee ids. + Only []string +} + +// RunResult is the outcome of one payroll cycle run. +type RunResult struct { + CycleID string + Payslips []payroll.Payslip + TotalGrossCents int64 + TotalNetCents int64 + Skipped []string + FinishedAt time.Time +} + +// Service is the hand-written payroll use-case layer. It owns the order of +// operations for a cycle: resolve the roster, build a payslip per employee, +// then persist. The generated CRUD layer under internal/gen has no opinion +// about any of that — it can only read and write single rows. +type Service struct { + store *payslipstore.Store + clock clock.Clock +} + +func NewService(store *payslipstore.Store, c clock.Clock) *Service { + return &Service{store: store, clock: c} +} + +// RunCycle is the public entry point used by the HTTP handler. It loads the +// cycle, guards its state, and delegates the actual work to runPayrollCycleAll. +func (s *Service) RunCycle(ctx context.Context, cycleID string, opts RunOptions) (RunResult, error) { + cycle, err := s.loadCycle(ctx, cycleID) + if err != nil { + return RunResult{}, err + } + if cycle.Status == payroll.CycleClosed { + return RunResult{}, ErrCycleClosed + } + + roster, err := s.rosterFor(ctx, cycle, opts) + if err != nil { + return RunResult{}, err + } + if len(roster) == 0 { + return RunResult{}, ErrNoEmployees + } + + return s.runPayrollCycleAll(ctx, cycle, roster, opts) +} + +// runPayrollCycleAll is the heart of the cycle: for every employee on the +// roster it builds a payslip from that employee's contract and timesheet, +// then upserts the result. Ordering matters — a payslip is only persisted +// after every earning, deduction and tax line has been resolved, so a +// partially-computed slip can never reach the store. +func (s *Service) runPayrollCycleAll( + ctx context.Context, + cycle payroll.Cycle, + roster []payroll.Employee, + opts RunOptions, +) (RunResult, error) { + result := RunResult{CycleID: cycle.ID} + now := s.clock.Now() + + for _, employee := range roster { + if err := ctx.Err(); err != nil { + return result, err + } + + timesheet, err := s.timesheetFor(ctx, cycle, employee) + if err != nil { + result.Skipped = append(result.Skipped, employee.ID) + continue + } + + slip, err := s.BuildPayslip(ctx, cycle, employee, timesheet) + if err != nil { + return result, fmt.Errorf("build payslip for %s: %w", employee.ID, err) + } + + slip.RunAt = now + slip.RunReason = opts.Reason + + if !opts.DryRun { + if err := s.store.Upsert(ctx, slip); err != nil { + return result, fmt.Errorf("persist payslip for %s: %w", employee.ID, err) + } + } + + result.Payslips = append(result.Payslips, slip) + result.TotalGrossCents += slip.GrossCents + result.TotalNetCents += slip.NetCents + } + + if !opts.DryRun { + if err := s.closeCycle(ctx, cycle, now); err != nil { + return result, err + } + } + + sort.Slice(result.Payslips, func(i, j int) bool { + return result.Payslips[i].EmployeeID < result.Payslips[j].EmployeeID + }) + result.FinishedAt = now + return result, nil +} + +// rosterFor resolves which employees this cycle pays. An employee joins the +// roster when their contract overlaps the cycle window and they are not on +// unpaid leave for the whole period. +func (s *Service) rosterFor(ctx context.Context, cycle payroll.Cycle, opts RunOptions) ([]payroll.Employee, error) { + all, err := s.store.EmployeesForCycle(ctx, cycle.ID) + if err != nil { + return nil, err + } + + only := map[string]bool{} + for _, id := range opts.Only { + only[id] = true + } + + roster := make([]payroll.Employee, 0, len(all)) + for _, e := range all { + if len(only) > 0 && !only[e.ID] { + continue + } + if !e.Contract.OverlapsWindow(cycle.Start, cycle.End) { + continue + } + if e.UnpaidLeaveCoversWindow(cycle.Start, cycle.End) { + continue + } + roster = append(roster, e) + } + + sort.Slice(roster, func(i, j int) bool { return roster[i].ID < roster[j].ID }) + return roster, nil +} + +func (s *Service) timesheetFor(ctx context.Context, cycle payroll.Cycle, e payroll.Employee) (payroll.Timesheet, error) { + ts, err := s.store.Timesheet(ctx, cycle.ID, e.ID) + if err != nil { + return payroll.Timesheet{}, err + } + if ts.Approved { + return ts, nil + } + if e.Contract.Kind == payroll.ContractSalaried { + // Salaried staff are paid the contractual period regardless of an + // unapproved timesheet; hourly staff are skipped until approval. + return payroll.Timesheet{ + CycleID: cycle.ID, + EmployeeID: e.ID, + Approved: true, + Units: e.Contract.PeriodUnits(cycle.Start, cycle.End), + }, nil + } + return payroll.Timesheet{}, fmt.Errorf("timesheet for %s not approved", e.ID) +} + +func (s *Service) loadCycle(ctx context.Context, cycleID string) (payroll.Cycle, error) { + if cycleID == "" { + return payroll.Cycle{}, errors.New("empty cycle id") + } + return s.store.Cycle(ctx, cycleID) +} + +func (s *Service) closeCycle(ctx context.Context, cycle payroll.Cycle, at time.Time) error { + cycle.Status = payroll.CycleClosed + cycle.ClosedAt = at + return s.store.SaveCycle(ctx, cycle) +} + +// Cycle exposes a cycle for the read endpoints. +func (s *Service) Cycle(ctx context.Context, cycleID string) (payroll.Cycle, error) { + return s.loadCycle(ctx, cycleID) +} + +// PayslipsForCycle lists the payslips a completed cycle produced. +func (s *Service) PayslipsForCycle(ctx context.Context, cycleID string) ([]payroll.Payslip, error) { + slips, err := s.store.ListByCycle(ctx, cycleID) + if err != nil { + return nil, err + } + sort.Slice(slips, func(i, j int) bool { return slips[i].EmployeeID < slips[j].EmployeeID }) + return slips, nil +} + +// Reopen unwinds a closed cycle so it can be re-run after a correction. +func (s *Service) Reopen(ctx context.Context, cycleID string, reason string) error { + cycle, err := s.loadCycle(ctx, cycleID) + if err != nil { + return err + } + if cycle.Status != payroll.CycleClosed { + return nil + } + cycle.Status = payroll.CycleOpen + cycle.ReopenReason = reason + cycle.ClosedAt = time.Time{} + return s.store.SaveCycle(ctx, cycle) +} diff --git a/__tests__/fixtures/payroll-go/internal/usecase/payroll/payslip_builder.go b/__tests__/fixtures/payroll-go/internal/usecase/payroll/payslip_builder.go new file mode 100644 index 000000000..fd8ecba7e --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/usecase/payroll/payslip_builder.go @@ -0,0 +1,150 @@ +package payroll + +import ( + "context" + "fmt" + + "github.com/example/payroll-svc/internal/domain/payroll" +) + +// BuildPayslip turns one employee's contract and timesheet into a complete +// payslip for the cycle: base pay, overtime, allowances, then deductions and +// tax, in that order. Every amount is in integer cents; nothing here rounds +// until the final net, so a cent never disappears between two lines. +// +// This is the calculation the generated CRUD layer does NOT do — fkit's +// BuildPayslip only copies fields between a DTO and a row. +func (s *Service) BuildPayslip( + ctx context.Context, + cycle payroll.Cycle, + employee payroll.Employee, + timesheet payroll.Timesheet, +) (payroll.Payslip, error) { + if err := ctx.Err(); err != nil { + return payroll.Payslip{}, err + } + if timesheet.EmployeeID != "" && timesheet.EmployeeID != employee.ID { + return payroll.Payslip{}, fmt.Errorf("timesheet/employee mismatch: %s vs %s", timesheet.EmployeeID, employee.ID) + } + + slip := payroll.Payslip{ + CycleID: cycle.ID, + EmployeeID: employee.ID, + Currency: employee.Contract.Currency, + PeriodFrom: cycle.Start, + PeriodTo: cycle.End, + } + + base := s.basePayCents(employee, cycle, timesheet) + slip.Lines = append(slip.Lines, payroll.Line{ + Code: "BASE", Kind: payroll.LineEarning, AmountCents: base, + }) + + if overtime := s.overtimeCents(employee, timesheet); overtime > 0 { + slip.Lines = append(slip.Lines, payroll.Line{ + Code: "OT", Kind: payroll.LineEarning, AmountCents: overtime, + }) + } + + for _, allowance := range employee.Contract.Allowances { + amount := prorateAllowance(allowance, cycle, employee) + if amount == 0 { + continue + } + slip.Lines = append(slip.Lines, payroll.Line{ + Code: allowance.Code, Kind: payroll.LineEarning, AmountCents: amount, + }) + } + + slip.GrossCents = sumKind(slip.Lines, payroll.LineEarning) + + for _, d := range employee.Deductions { + amount := d.AmountFor(slip.GrossCents) + if amount == 0 { + continue + } + slip.Lines = append(slip.Lines, payroll.Line{ + Code: d.Code, Kind: payroll.LineDeduction, AmountCents: amount, + }) + } + + tax, err := s.taxCents(employee, slip.GrossCents) + if err != nil { + return payroll.Payslip{}, fmt.Errorf("tax for %s: %w", employee.ID, err) + } + slip.Lines = append(slip.Lines, payroll.Line{ + Code: "TAX", Kind: payroll.LineDeduction, AmountCents: tax, + }) + + slip.DeductionCents = sumKind(slip.Lines, payroll.LineDeduction) + slip.NetCents = slip.GrossCents - slip.DeductionCents + if slip.NetCents < 0 { + slip.NetCents = 0 + slip.Underwater = true + } + + return slip, nil +} + +// basePayCents is the contractual pay for the period: salaried staff get the +// period rate prorated across their contract window, hourly staff get rate × +// approved units. +func (s *Service) basePayCents(e payroll.Employee, cycle payroll.Cycle, ts payroll.Timesheet) int64 { + switch e.Contract.Kind { + case payroll.ContractSalaried: + full := e.Contract.PeriodRateCents + return prorateSalary(full, e.Contract, cycle) + case payroll.ContractHourly: + return e.Contract.RateCents * int64(ts.Units) + default: + return 0 + } +} + +// overtimeCents pays approved units above the contractual threshold at the +// contract's overtime multiplier. +func (s *Service) overtimeCents(e payroll.Employee, ts payroll.Timesheet) int64 { + if e.Contract.Kind != payroll.ContractHourly { + return 0 + } + threshold := e.Contract.OvertimeThresholdUnits + if threshold <= 0 || ts.Units <= threshold { + return 0 + } + extra := int64(ts.Units - threshold) + return int64(float64(e.Contract.RateCents) * e.Contract.OvertimeMultiplier * float64(extra)) +} + +// taxCents applies the employee's tax band schedule to the gross. +func (s *Service) taxCents(e payroll.Employee, gross int64) (int64, error) { + if len(e.TaxBands) == 0 { + return 0, nil + } + var tax int64 + remaining := gross + for _, band := range e.TaxBands { + if remaining <= 0 { + break + } + if band.RateBasisPoints < 0 || band.RateBasisPoints > 10000 { + return 0, fmt.Errorf("invalid band rate %d", band.RateBasisPoints) + } + slice := remaining + if band.UpToCents > 0 && slice > band.UpToCents { + slice = band.UpToCents + } + tax += slice * int64(band.RateBasisPoints) / 10000 + remaining -= slice + } + return tax, nil +} + +func sumKind(lines []payroll.Line, kind payroll.LineKind) int64 { + var total int64 + for _, l := range lines { + if l.Kind == kind { + total += l.AmountCents + } + } + return total +} diff --git a/__tests__/fixtures/payroll-go/internal/usecase/payroll/prorate.go b/__tests__/fixtures/payroll-go/internal/usecase/payroll/prorate.go new file mode 100644 index 000000000..e633dad2d --- /dev/null +++ b/__tests__/fixtures/payroll-go/internal/usecase/payroll/prorate.go @@ -0,0 +1,53 @@ +package payroll + +import ( + "time" + + "github.com/example/payroll-svc/internal/domain/payroll" +) + +// prorateSalary scales a full period rate down when the contract covers only +// part of the cycle window (a mid-period joiner or leaver). +func prorateSalary(fullCents int64, contract payroll.Contract, cycle payroll.Cycle) int64 { + window := calendarDays(cycle.Start, cycle.End) + if window <= 0 { + return 0 + } + covered := calendarDays(laterOf(cycle.Start, contract.StartsOn), earlierOf(cycle.End, contract.EndsOn)) + if covered >= window { + return fullCents + } + if covered <= 0 { + return 0 + } + return fullCents * int64(covered) / int64(window) +} + +// prorateAllowance applies the same window rule to a recurring allowance. +func prorateAllowance(a payroll.Allowance, cycle payroll.Cycle, e payroll.Employee) int64 { + if !a.Prorated { + return a.AmountCents + } + return prorateSalary(a.AmountCents, e.Contract, cycle) +} + +func calendarDays(from, to time.Time) int { + if to.Before(from) { + return 0 + } + return int(to.Sub(from).Hours()/24) + 1 +} + +func laterOf(a, b time.Time) time.Time { + if b.IsZero() || a.After(b) { + return a + } + return b +} + +func earlierOf(a, b time.Time) time.Time { + if b.IsZero() || a.Before(b) { + return a + } + return b +} diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index 4371362e0..a46e71f98 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -92,6 +92,84 @@ This is the gap the rest of the epic closes: relevance-proportional allocation w relative cliff (CG-12), on top of scoring that stops rewarding incidental name collisions (CG-10). +## The regression fixtures (CG-6) + +Two fixtures pin the failure mode so it can never silently return. Both **fail today** — +that is what they are for. They become the pass gate for CG-10 + CG-12. + +They are declared in `scripts/agent-eval/allocation-fixtures.json` and run by +`scripts/agent-eval/probe-allocation.mjs`, which drives the CG-4 diagnostic through a JSONL +sidecar (so it measures the shipping allocator, not a re-derivation), groups the rendered +files into `answer` vs `incidental`, and checks declared share thresholds. Needs a current +`npm run build`; exits 1 while any assertion fails. + +```bash +node scripts/agent-eval/probe-allocation.mjs # both +node scripts/agent-eval/probe-allocation.mjs payroll-go # one +node scripts/agent-eval/probe-allocation.mjs --json # machine-readable +``` + +### 1. `payroll-go` — the reporter's shape + +`__tests__/fixtures/payroll-go/` is a synthetic Go service: generated FKIT CRUD beside a +hand-written payroll use-case, entered from an HTTP route. Full description in that +directory's README. The essentials: + +- Generated files with **ordinary names** carrying `// Code generated ... DO NOT EDIT.` — + invisible to path-only detection, which is what makes this a #1500 fixture rather than a + `.pb.go` one — beside `payrollpb/*.pb.go` covering the path-detectable channel. +- Deliberate collisions: `BuildPayslip`, `Upsert` and `Store` each exist twice, generated + and hand-written, and the generated layer name-collides on every query term. +- `cycle.go` (227 lines) sits above the whole-file window so it clips; the generated files + sit below it so they ship whole. + +Query — an architecture question naming none of the answering symbols: *"how does payroll +cycle create and calculate payslips?"* + +| | allocated | delivered | +|---|---|---| +| hand-written | 48.4% | **25.6%** (all of it domain types) | +| generated CRUD | 39.9% | **57.4%** | + +`cycle.go` is allocated the single largest slice (7,052 chars, 30.6%) and delivers **zero** +— the 19,500 hard ceiling drops its whole section. `payslip_builder.go` (rank #8) never +renders. So `runPayrollCycleAll`, the hand-written `BuildPayslip` and the real `Upsert` +never reach the agent, and every byte that did arrive describes either CRUD or types. + +This fixture is hermetic: the probe copies the tree to a temp dir and re-indexes per run, +so two runs on one build are byte-identical (verified). `__tests__/explore-allocation-1500.test.ts` +runs the same assertions in vitest — the fixture-shape half green, the allocation half as +`it.fails` so the suite stays green while the bug is open and goes **red when it is fixed**. + +**Finding, deliberately left unfixed:** `runPayrollCycleAll` calls `s.store.Upsert` on a +`*payslipstore.Store`, but the graph resolves that edge to the **generated** +`internal/gen/fkit/payroll/store.go` `Store.Upsert`. Same-name method resolution across two +packages that both define `Store.Upsert` picks the wrong receiver. It is upstream of the +allocation bug — a wrong edge pulls the generated store into the subgraph and inflates its +score — so it belongs with CG-10's scoring work, not with the fixture. + +### 2. `self-query` — the same bug with no generated code in sight + +The baseline above, promoted to a fixture: this repo, *"how does explore allocate its output +budget across files"*. `scripts/agent-eval/*.mjs` mention `explore` and `BUDGET` +incidentally — they are eval harnesses, not the allocator — and they are small enough to +ship whole, while `src/mcp/tools.ts` is large enough to be clipped. + +At 493 indexed files (small tier): the script corpus takes **71.8%** of the delivered +envelope (79.4% allocated) against `tools.ts`'s **18.5%**, despite `tools.ts` scoring 46 vs +10, carrying 2.3× the graph mass and 3× the distinct term hits. + +This fixture reads the **live** index of this repo, so unlike `payroll-go` its exact numbers +move as the repo changes. Its assertions are relative for that reason (answer group vs +incidental group, largest delivered file), never fixed percentages. Two things to know: + +- The `<500`-file tier boundary is close. This repo indexes 493 files including the new + fixture; crossing 500 flips `maxOutputChars` 18,000 → 24,000, `maxFiles` 5 → 8 and + `maxCharsPerFile` 3,800 → 6,500, which moves every number in the table above. Re-baseline + after the crossing rather than treating the drift as a regression. +- Adding the `payroll-go` fixture itself moved the count 472 → 493. Its Go files match none + of this query's terms, so they change the tier arithmetic and nothing else. + ### Reproducing The query explores this repo, so **uncommitted edits to `src/mcp/tools.ts` change the diff --git a/scripts/agent-eval/allocation-fixtures.json b/scripts/agent-eval/allocation-fixtures.json new file mode 100644 index 000000000..28c5a978b --- /dev/null +++ b/scripts/agent-eval/allocation-fixtures.json @@ -0,0 +1,112 @@ +{ + "$comment": [ + "Regression fixtures for GitHub issue #1500 / epic CG-1 — relevance-proportional", + "explore budget allocation. Run them with `node scripts/agent-eval/probe-allocation.mjs`", + "against a built dist/. BOTH FIXTURES FAIL TODAY: that is the point — they document", + "the bug and become the pass gate for the allocation change (CG-10 + CG-12).", + "", + "`groups` partitions the files explore rendered into `answer` (what the query is", + "actually about) and `incidental` (what wins the envelope today on name collisions).", + "Assertions are on the DELIVERED envelope unless suffixed `Allocated`; delivered is", + "what the agent got, allocated is what the render loop chose before the hard ceiling.", + "Shares are fractions of the whole response, meta-text included, so they never sum to 1." + ], + "fixtures": [ + { + "id": "payroll-go", + "title": "#1500 — generated Go CRUD beside a hand-written payroll workflow", + "kind": "fixture", + "path": "__tests__/fixtures/payroll-go", + "query": "how does payroll cycle create and calculate payslips?", + "rationale": [ + "The reporter's repo shape: a Go service whose generated FKIT CRUD layer sits", + "beside the hand-written use-case that does the real work. The query deliberately", + "does NOT name runPayrollCycleAll / BuildPayslip / Upsert — an architecture", + "question phrased the way a newcomer would phrase it. The generated layer", + "name-collides on every query term (CreatePayslip, PayrollCycleCreateRequest,", + "CalculatePayrollCycleTotals, a second BuildPayslip, a second Upsert), so a", + "scorer that rewards incidental name matches surfaces the CRUD path.", + "Half the generated files carry ORDINARY names and are only detectable by their", + "`// Code generated ... DO NOT EDIT.` header (CG-5), which is what makes this", + "the #1500 case rather than a .pb.go case." + ], + "groups": { + "answer": [ + "internal/usecase/**", + "internal/store/**", + "internal/transport/**", + "internal/domain/**", + "cmd/**" + ], + "incidental": ["internal/gen/**"] + }, + "assert": { + "answerShareAtLeast": 0.55, + "incidentalShareAtMost": 0.25, + "topFileGroup": "answer", + "mustDeliverBytes": [ + "internal/usecase/payroll/cycle.go", + "internal/usecase/payroll/payslip_builder.go" + ], + "$mustContainComment": "Needles are chosen to match the HAND-WRITTEN chain only — a bare `BuildPayslip`/`Upsert` also matches the generated collisions, which is the whole point of the fixture.", + "mustContain": [ + "runPayrollCycleAll", + "func (s *Service) BuildPayslip", + "s.store.Upsert(ctx, slip)" + ] + }, + "baseline": { + "measuredOn": "2026-08-03", + "note": "19 files → very-tiny tier (13,000 budget); 23,020 chars allocated against it, cut to 16,011 by the 19,500 hard ceiling.", + "delivered": { + "internal/gen/fkit/payroll/payslip.go": 0.307, + "internal/gen/fkit/payroll/payroll_cycle.go": 0.266, + "internal/domain/payroll/payslip.go": 0.256, + "internal/usecase/payroll/cycle.go": 0.0 + }, + "verdict": "The workflow file is allocated the single largest slice (30.6%) and delivers ZERO — the hard ceiling drops its whole section. Generated CRUD takes 57.4% of what the agent actually receives; runPayrollCycleAll, BuildPayslip and the real Upsert never reach the response." + } + }, + { + "id": "self-query", + "title": "This repo — incidental `explore`/`BUDGET` matches in the agent-eval scripts", + "kind": "self", + "path": ".", + "query": "how does explore allocate its output budget across files", + "rationale": [ + "The same failure mode with no generated code in sight. `scripts/agent-eval/*.mjs`", + "mention `explore` and `BUDGET` incidentally — they are eval harnesses, not the", + "allocator — and they are small enough to ship WHOLE, while src/mcp/tools.ts (which", + "carries getExploreOutputBudget and the render loop, and scores 4x higher on every", + "signal) is large enough to be clipped at maxCharsPerFile. Allocation follows file", + "size, not relevance.", + "", + "Unlike payroll-go this fixture reads THIS repo's live index, so its exact numbers", + "move as the repo changes (indexed file count crossing 500 flips the budget tier).", + "The assertions are therefore relative — answer-vs-incidental, not fixed percentages." + ], + "groups": { + "answer": ["src/mcp/**"], + "incidental": ["scripts/**"] + }, + "assert": { + "answerShareAtLeast": 0.5, + "incidentalShareAtMost": 0.25, + "topFileGroup": "answer", + "mustDeliverBytes": ["src/mcp/tools.ts"] + }, + "baseline": { + "measuredOn": "2026-08-03", + "note": "493 files → small tier (18,000 budget); 27,518 chars allocated against it, cut to 19,749 by the 25,000 hard ceiling.", + "delivered": { + "scripts/agent-eval/offload-eval-hook.mjs": 0.25, + "scripts/agent-eval/offload-eval-metrics.mjs": 0.236, + "scripts/agent-eval/parse-session.mjs": 0.232, + "src/mcp/tools.ts": 0.185, + "scripts/agent-eval/offload-eval-cost.mjs": 0.0 + }, + "verdict": "The script corpus takes 71.8% of the delivered envelope (79.4% of what was allocated) against tools.ts's 18.5%, despite tools.ts scoring 46 vs 10, carrying 2.3x the graph mass and 3x the distinct term hits." + } + } + ] +} diff --git a/scripts/agent-eval/probe-allocation.mjs b/scripts/agent-eval/probe-allocation.mjs new file mode 100755 index 000000000..138673108 --- /dev/null +++ b/scripts/agent-eval/probe-allocation.mjs @@ -0,0 +1,296 @@ +#!/usr/bin/env node +/** + * Deterministic per-file budget-share probe for `codegraph_explore` (CG-6). + * + * `probe-explore.mjs` prints what explore returned. This prints how the response + * was DIVIDED — which files won the byte envelope and in what proportion — and + * checks that division against a declared expectation. It is the regression gate + * for GitHub issue #1500 / epic CG-1: an architecture question that doesn't name + * the exact use-case must concentrate the budget on the code that answers it, not + * on a generated CRUD layer (or an eval script) that merely name-collides. + * + * The numbers come from the CG-4 diagnostic (`CODEGRAPH_EXPLORE_DEBUG`), read back + * from a JSONL sidecar, so the probe measures the shipping allocator rather than + * re-deriving shares from the markdown. + * + * Fixtures are declared in `allocation-fixtures.json`. A `kind: "fixture"` entry is + * hermetic — the fixture tree is copied to a fresh temp dir and indexed per run, so + * two runs on one build give identical numbers. A `kind: "self"` entry reads this + * repo's live index and therefore moves as the repo changes; its assertions are + * relative for that reason. + * + * Usage (needs a current `npm run build`): + * node scripts/agent-eval/probe-allocation.mjs # every fixture + * node scripts/agent-eval/probe-allocation.mjs payroll-go # one fixture + * node scripts/agent-eval/probe-allocation.mjs --json # machine-readable + * node scripts/agent-eval/probe-allocation.mjs --keep # keep the temp index + * + * Exit code: 0 if every assertion holds, 1 if any fails, 2 on a setup error. + * BOTH FIXTURES ARE EXPECTED TO FAIL until CG-10/CG-12 land — that failure is the + * documented bug. Use --expect-fail to invert the exit code while it is the state + * of the world (0 = still broken, 1 = fixed, go flip the gate). + */ +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '../..'); +const SPEC_PATH = join(HERE, 'allocation-fixtures.json'); + +const argv = process.argv.slice(2); +const flags = new Set(argv.filter((a) => a.startsWith('--'))); +const wanted = argv.filter((a) => !a.startsWith('--')); +const asJson = flags.has('--json'); +const keepTemp = flags.has('--keep'); +const expectFail = flags.has('--expect-fail'); + +const say = (line = '') => { if (!asJson) console.log(line); }; +const pct = (f) => `${(f * 100).toFixed(1)}%`; +const num = (n) => Math.round(n).toLocaleString('en-US'); + +/** Load the built dist — the probe measures the shipping allocator, not src. */ +async function loadDist() { + const distIndex = join(REPO_ROOT, 'dist/index.js'); + if (!existsSync(distIndex)) { + console.error('dist/ not built — run `npm run build` first.'); + process.exit(2); + } + const idx = await import(pathToFileURL(distIndex).href); + const tools = await import(pathToFileURL(join(REPO_ROOT, 'dist/mcp/tools.js')).href); + // esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named } + const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph; + const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler; + if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') { + console.error('could not resolve CodeGraph/ToolHandler from dist/'); + process.exit(2); + } + return { CodeGraph, ToolHandler }; +} + +/** `internal/gen/**` → /^internal\/gen\/.*$/ . Supports `**`, `*` and literals. */ +function globToRegExp(glob) { + // Park `**` on a sentinel no path can contain, so the `*` pass cannot eat it. + // Written as an escape, not a literal byte — a raw NUL makes git treat this + // whole script as binary, which costs every future diff of it. + const DOUBLE_STAR = '\u0000'; + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const body = escaped + .replace(/\*\*/g, DOUBLE_STAR) + .replace(/\*/g, '[^/]*') + .replaceAll(DOUBLE_STAR, '.*'); + return new RegExp(`^${body}$`); +} + +const groupOf = (path, groups) => { + for (const [name, globs] of Object.entries(groups)) { + if (globs.some((g) => globToRegExp(g).test(path))) return name; + } + return 'other'; +}; + +/** + * Run one explore call with the diagnostic pointed at a sidecar, and return the + * report plus the response text. + */ +async function runExplore({ CodeGraph, ToolHandler }, repoPath, query, sidecar) { + const cg = CodeGraph.openSync(repoPath); + const prior = process.env.CODEGRAPH_EXPLORE_DEBUG; + process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar; + try { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query }); + const text = res.content?.[0]?.text ?? ''; + const lines = readFileSync(sidecar, 'utf-8').trim().split('\n').filter(Boolean); + if (lines.length === 0) throw new Error('diagnostic produced no report'); + return { report: JSON.parse(lines[lines.length - 1]), text }; + } finally { + if (prior === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG; + else process.env.CODEGRAPH_EXPLORE_DEBUG = prior; + try { cg.close?.(); } catch {} + } +} + +/** Copy a fixture tree to a fresh temp dir and index it — hermetic per run. */ +async function materializeFixture({ CodeGraph }, fixturePath) { + const src = resolve(REPO_ROOT, fixturePath); + if (!existsSync(src)) throw new Error(`fixture tree not found: ${src}`); + const dir = mkdtempSync(join(tmpdir(), 'cg-alloc-')); + cpSync(src, dir, { recursive: true }); + // A stray index inside the checked-in tree would be copied in and reused. + rmSync(join(dir, '.codegraph'), { recursive: true, force: true }); + const cg = CodeGraph.initSync(dir); + await cg.indexAll(); + cg.close?.(); + return dir; +} + +/** Evaluate one fixture's assertions against its report. Returns check rows. */ +function evaluate(fixture, report, text) { + const { groups, assert: want } = fixture; + const delivered = new Map(); + const allocated = new Map(); + for (const f of report.files) { + const g = groupOf(f.path, groups); + delivered.set(g, (delivered.get(g) ?? 0) + f.share); + allocated.set(g, (allocated.get(g) ?? 0) + f.allocatedShare); + } + const share = (g) => delivered.get(g) ?? 0; + const top = report.files + .filter((f) => f.finalChars > 0) + .sort((a, b) => b.finalChars - a.finalChars)[0]; + + const checks = []; + const add = (name, pass, detail) => checks.push({ name, pass, detail }); + + if (want.answerShareAtLeast !== undefined) { + add( + `answer group takes >= ${pct(want.answerShareAtLeast)} of the envelope`, + share('answer') >= want.answerShareAtLeast, + `answer ${pct(share('answer'))} delivered (${pct(allocated.get('answer') ?? 0)} allocated)`, + ); + } + if (want.incidentalShareAtMost !== undefined) { + add( + `incidental group takes <= ${pct(want.incidentalShareAtMost)} of the envelope`, + share('incidental') <= want.incidentalShareAtMost, + `incidental ${pct(share('incidental'))} delivered (${pct(allocated.get('incidental') ?? 0)} allocated)`, + ); + } + if (want.topFileGroup) { + const actual = top ? groupOf(top.path, groups) : '(nothing delivered)'; + add( + `largest delivered file is in "${want.topFileGroup}"`, + actual === want.topFileGroup, + top ? `${top.path} (${pct(top.share)}, group "${actual}")` : 'no file delivered any source', + ); + } + for (const path of want.mustDeliverBytes ?? []) { + const rec = report.files.find((f) => f.path === path); + add( + `${path} delivers source`, + !!rec && rec.finalChars > 0, + rec + ? `${num(rec.finalChars)} delivered of ${num(rec.emittedChars)} allocated` + + (rec.finalChars === 0 && rec.emittedChars > 0 ? ' — hard ceiling dropped the whole section' : '') + + (rec.emittedChars === 0 ? ` — never rendered (${rec.skipped ?? 'not reached'}, rank #${rec.rank})` : '') + : 'not among the ranked candidates', + ); + } + for (const needle of want.mustContain ?? []) { + add(`response contains "${needle}"`, text.includes(needle), text.includes(needle) ? 'present' : 'absent'); + } + return { checks, delivered, allocated, top }; +} + +function printReport(fixture, report, evaluated) { + const { checks, delivered, allocated } = evaluated; + const env = report.envelope; + say(''); + say(`── ${fixture.id} — ${fixture.title}`); + say(` query "${report.query}"`); + say(` project ${report.projectRoot} · ${num(report.indexedFileCount)} files indexed`); + say( + ` envelope ${num(env.chars)} delivered · ${num(env.allocatedChars)} allocated` + + ` of ${num(report.budget.maxOutputChars)} budget (hard ceiling ${num(report.budget.hardCeiling)})` + + `${env.overBudget ? ' [over budget]' : ''}${env.truncated ? ' [TRUNCATED]' : ''}`, + ); + say(''); + say(' group alloc% deliv%'); + for (const g of ['answer', 'incidental', 'other']) { + if (!delivered.has(g) && !allocated.has(g)) continue; + say(` ${g.padEnd(12)} ${pct(allocated.get(g) ?? 0).padStart(6)} ${pct(delivered.get(g) ?? 0).padStart(6)}`); + } + say(''); + say(' # alloc% deliv% bytes score graph hits gen render file'); + for (const f of report.files.filter((f) => f.emittedChars > 0 || f.finalChars > 0)) { + say( + ' ' + String(f.rank).padStart(2) + ' ' + + pct(f.allocatedShare).padStart(6) + ' ' + + pct(f.share).padStart(6) + ' ' + + num(f.emittedChars).padStart(7) + ' ' + + String(f.score).padStart(5) + ' ' + + f.graphScore.toFixed(5).padStart(7) + ' ' + + String(f.termHits).padStart(4) + ' ' + + (f.generated ? ' ✓ ' : ' ') + ' ' + + ((f.render ?? '-') + (f.clipped ? '*' : '')).padEnd(9) + ' ' + + f.path, + ); + } + say(''); + for (const c of checks) say(` ${c.pass ? 'PASS' : 'FAIL'} ${c.name}\n ${c.detail}`); +} + +async function main() { + const spec = JSON.parse(readFileSync(SPEC_PATH, 'utf-8')); + const fixtures = spec.fixtures.filter((f) => wanted.length === 0 || wanted.includes(f.id)); + if (fixtures.length === 0) { + console.error(`no fixture matched ${JSON.stringify(wanted)}; known: ${spec.fixtures.map((f) => f.id).join(', ')}`); + process.exit(2); + } + + const dist = await loadDist(); + const sidecarDir = mkdtempSync(join(tmpdir(), 'cg-alloc-diag-')); + const results = []; + const temps = []; + + for (const fixture of fixtures) { + let repoPath; + if (fixture.kind === 'fixture') { + repoPath = await materializeFixture(dist, fixture.path); + temps.push(repoPath); + } else { + repoPath = resolve(REPO_ROOT, fixture.path); + if (!existsSync(join(repoPath, '.codegraph'))) { + console.error(`${fixture.id}: ${repoPath} has no .codegraph index — run \`codegraph init\` there first.`); + process.exit(2); + } + } + + const sidecar = join(sidecarDir, `${fixture.id}.jsonl`); + mkdirSync(dirname(sidecar), { recursive: true }); + const { report, text } = await runExplore(dist, repoPath, fixture.query, sidecar); + const evaluated = evaluate(fixture, report, text); + printReport(fixture, report, evaluated); + results.push({ + id: fixture.id, + kind: fixture.kind, + query: fixture.query, + passed: evaluated.checks.every((c) => c.pass), + checks: evaluated.checks, + shares: { + delivered: Object.fromEntries(evaluated.delivered), + allocated: Object.fromEntries(evaluated.allocated), + }, + envelope: report.envelope, + files: report.files.filter((f) => f.emittedChars > 0 || f.finalChars > 0), + }); + } + + if (!keepTemp) { + for (const dir of temps) rmSync(dir, { recursive: true, force: true }); + rmSync(sidecarDir, { recursive: true, force: true }); + } else { + say(''); + say(` kept: ${[...temps, sidecarDir].join(' ')}`); + } + + const allPassed = results.every((r) => r.passed); + if (asJson) { + console.log(JSON.stringify({ passed: allPassed, fixtures: results }, null, 2)); + } else { + say(''); + for (const r of results) say(`${r.passed ? 'PASS' : 'FAIL'} ${r.id}`); + if (!allPassed) { + say(''); + say('Failures here are the DOCUMENTED #1500 bug — the budget goes to files that merely'); + say('name-collide with the query. They become the pass gate once CG-10/CG-12 land.'); + } + } + process.exit(expectFail ? (allPassed ? 1 : 0) : (allPassed ? 0 : 1)); +} + +main().catch((err) => { + console.error(err?.stack ?? String(err)); + process.exit(2); +}); From a3898cdc707a10c6e4aeedd17194709b34c31364 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 00:02:45 -0500 Subject: [PATCH 04/15] =?UTF-8?q?feat(mcp):=20relevance=20scoring=20overha?= =?UTF-8?q?ul=20for=20explore=20=E2=80=94=20kill=20incidental=20name-colli?= =?UTF-8?q?sion=20matches=20(CG-10,=20#1500)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explore's per-file relevance awarded +50/+10/+3/+1 by match class and admitted anything scoring >= 3. Neither half held up: the tier said HOW a symbol reached us, never whether the match was evidence, and an absolute floor admits noise on any repo where the top file scores 50+. Three scripts/agent-eval/*.mjs harnesses took 63% of this repo's own "how does explore allocate its output budget" answer on nothing but an unused `const explore` and a `const BUDGET`. Four levers: - KIND WEIGHT (RELEVANCE_KIND_WEIGHT): callables and types 1.0, members ~0.5, variable/constant/parameter 0.15-0.35. A weak-kind symbol with no usage edge anywhere in the graph (`contains` excluded — nesting is not usage) drops to 0.08. Only weak kinds in the top two tiers pay for the DB probe; the subgraph's own edges answer most cases free. No measurable latency change (210 vs 211 ms/call, n=12 interleaved). - PERIPHERAL CAP: nodes >=2 hops from any match accumulate into a bucket capped at 5. Uncapped they added a flat +1 each, so a file grew more relevant by being bigger — parse-session.mjs reached 22 off one constant plus twelve unrelated symbols. - RANK PENALTY: generated files x0.3, low-value x0.5, applied to the score AND the graph mass. Score alone would not have fixed #1500 — the generated CRUD carries MORE graph mass than the hand-written use-case, and graph mass outranks score in the comparator. Self-normalizing, never a hard exclusion. - RELATIVE FLOOR: clamp(topScore * 0.2, 1, 10). Capped at one full-strength direct match so concentration elsewhere can never exclude one (without it a named-seed-heavy file pushed the floor to 21 and dropped a file the agent had named by class name). Backfills to 3 candidates when it would leave fewer, and drops the evidence requirement rather than return nothing at all. excludeLowValueFiles was dead config — declared per tier, read nowhere; the test/spec exclusion has been unconditional for a while. Removed. The real gap was the detector: `isLowValue` anchored on a leading `/`, so a repo-ROOT `test/` dir (express, cobra, most of npm and Go) never matched — express's routing question spent 59% of its envelope on three test files. Anchored at `^` too, and the filter now runs before the floor and judges "are there other candidates?" on the whole gather. Measured before/after on the same indexes (baseline bd86ad2): - payroll-go fixture: generated 57.4% -> 23.5%; answer 25.6% -> 61.5%; cycle.go delivered 0 -> 38.9%. Generated ranks #3/#4, was #1/#2. - self-query fixture: eval scripts 72% -> 0%; tools.ts ranks #1. - express "route a request": 59% to test/* -> lib/application.js + lib/response.js - cobra x3, codegraph "indexing pipeline": byte-identical (control) Diagnostic gains a per-file penalty multiplier and NodeKind mix, so "why did this file score X" is legible. Selection stages reordered to match the pipeline. CG-6's gates flip from it.fails to live regressions except the byte-split ones, which stay open for CG-12 (allocation still follows file size within the ranked set). Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 + __tests__/explore-allocation-1500.test.ts | 84 +++-- __tests__/explore-diagnostics.test.ts | 13 +- __tests__/explore-relevance-scoring.test.ts | 356 ++++++++++++++++++++ docs/design/explore-budget-allocation.md | 123 ++++++- scripts/agent-eval/allocation-fixtures.json | 28 +- src/mcp/explore-diagnostics.ts | 53 ++- src/mcp/tools.ts | 339 ++++++++++++++++--- 8 files changed, 897 insertions(+), 101 deletions(-) create mode 100644 __tests__/explore-relevance-scoring.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 61856d623..a7b60e18f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- `codegraph_explore` no longer spends its answer on files that merely share a word with your question. A file whose only connection to the query was an unused local variable or constant of the same name — an eval script with a `const explore` in it, say — used to count as strongly as the file that actually implements the thing, and being small enough to include whole, it could take most of the response while the real answer got trimmed. Matches are now weighted by what was matched (a function, class or route counts for far more than a local variable, and a variable nothing else references counts for almost nothing), and a file has to score within reach of the best match rather than clear a fixed low bar. Generated files are down-weighted throughout the ranking instead of only breaking ties, so a generated CRUD layer no longer outranks the hand-written workflow beside it. Test and spec files in a top-level `test/` or `spec/` directory are now recognized and kept out, which they weren't before. (#1500) + - A CodeGraph process that gets force-killed — by the stuck-process watchdog, a crash, or the OS — no longer leaves the database's write-ahead log behind to grow without bound. Previously each killed session stacked more data onto the same log file and nothing ever shrank it, which on machines where sessions were killed regularly could quietly eat tens of gigabytes of disk. The log is now capped, and any oversized leftover is reclaimed automatically the next time the project is opened. Thanks @tiendungdev for the exceptional Windows report that pinned this down. (#1431) - The background server's watchdog no longer kills a healthy server that is just waiting on a slow disk: like indexing already does, it now checks whether the database files are still making progress before concluding the process is stuck. Fewer spurious kills also means fewer leftover write-ahead logs. (#1431) - `codegraph status` now shows the write-ahead log's size next to the database size and warns when killed sessions have left it oversized, and every line in the background server's log now carries a timestamp so kills and restarts can be placed in time. (#1431) diff --git a/__tests__/explore-allocation-1500.test.ts b/__tests__/explore-allocation-1500.test.ts index 4abb31909..059c975be 100644 --- a/__tests__/explore-allocation-1500.test.ts +++ b/__tests__/explore-allocation-1500.test.ts @@ -19,10 +19,12 @@ * runPayrollCycleAll → BuildPayslip → Upsert chain resolving end-to-end. If * the fixture rots, these fail first and say so. * - * 2. **Budget allocation** — the gate, written with `it.fails` because it - * DOCUMENTS A BUG THAT IS STILL OPEN. Vitest passes an `it.fails` test only - * while its body throws, so the suite is green today and goes RED the moment - * allocation is fixed (CG-10 scoring + CG-12 proportional bytes). + * 2. **Budget allocation** — the gate. CG-10 (relevance scoring) closed most of + * it: the generated CRUD now ranks and delivers BELOW the hand-written + * workflow, and those assertions are live regressions. What remains is + * `it.fails`, which DOCUMENTS THE PART STILL OPEN — vitest passes an + * `it.fails` test only while its body throws, so it goes RED the moment + * CG-12's proportional byte allocation lands. * **When it goes red, delete the `.fails` — do not delete the test.** * * The same assertions run outside vitest, against the built dist and with the @@ -202,54 +204,72 @@ describe('#1500 — generated Go CRUD beside a hand-written payroll workflow', ( const generatedShare = () => share((p) => p.startsWith(GENERATED_PREFIX)); /** - * BASELINE 2026-08-03 (very-tiny tier, 13,000-char budget): 23,020 chars - * allocated against it, cut to 16,011 by the 19,500 hard ceiling. The - * generated CRUD delivers 57.4%; the hand-written layer delivers 25.6%, all - * of it domain types. `cycle.go` is allocated the single largest slice - * (7,052 chars, 30.6%) and delivers ZERO — the ceiling drops its whole - * section — so runPayrollCycleAll, the hand-written BuildPayslip and the - * real Upsert never reach the agent at all. + * BASELINE 2026-08-03, BEFORE CG-10 (very-tiny tier, 13,000-char budget): + * 23,020 chars allocated, cut to 16,011 by the 19,500 hard ceiling. The + * generated CRUD delivered 57.4%; the hand-written layer 25.6%, all of it + * domain types. `cycle.go` was allocated the single largest slice (7,052 + * chars, 30.6%) and delivered ZERO — the ceiling dropped its whole section — + * so runPayrollCycleAll, the hand-written BuildPayslip and the real Upsert + * never reached the agent. * - * Each `it.fails` below passes ONLY while that is still true. - * ⚠ When one goes red, the bug is fixed: remove `.fails`, keep the test. + * AFTER CG-10 (relevance scoring): the generated files rank #3/#4 instead of + * #1/#2 — kind-weighted scoring plus a generated rank PENALTY on both the + * score and the graph mass, rather than the old tiebreak-at-equal-score. + * `cycle.go` now delivers 38.9% and the generated layer 23.5%. Four of the + * five gates below are green and are now live regressions. + * + * STILL OPEN, for CG-12 (proportional byte allocation): the render loop + * still allocates by FILE SIZE within the ranked set, and `maxFiles` is 4 at + * this tier — so `payslip_builder.go` ranks #6 and never renders, and + * `func (s *Service) BuildPayslip` is absent. Its `it.fails` passes ONLY + * while that is still true. ⚠ When it goes red, remove `.fails`, keep it. */ - it.fails('CG-12 GATE: concentrates the envelope on the hand-written workflow', () => { + it('CG-10 GATE: concentrates the envelope on the hand-written workflow', () => { expect(answerShare()).toBeGreaterThanOrEqual(0.55); }); - it.fails('CG-12 GATE: does not spend the envelope on the generated CRUD', () => { + it('CG-10 GATE: does not spend the envelope on the generated CRUD', () => { expect(generatedShare()).toBeLessThanOrEqual(0.25); }); - it.fails('CG-12 GATE: delivers the workflow file it allocated the most bytes to', () => { + it('CG-10 GATE: ranks the generated CRUD below the hand-written workflow', () => { + // The #1500 report in one assertion: before CG-10 the generated layer both + // outscored AND out-delivered the use-case that implements the business rule. + expect(answerShare()).toBeGreaterThan(generatedShare()); + }); + + it('CG-10 GATE: delivers the workflow file it allocated the most bytes to', () => { expect(bytes.get('internal/usecase/payroll/cycle.go') ?? 0).toBeGreaterThan(0); }); - it.fails('CG-12 GATE: delivers the calculation the question asks about', () => { - expect(bytes.get('internal/usecase/payroll/payslip_builder.go') ?? 0).toBeGreaterThan(0); - }); - - it.fails('CG-12 GATE: puts the hand-written chain in the response, not its generated twin', () => { - // Bare `BuildPayslip`/`Upsert` also match the generated collisions — these - // needles are unique to the hand-written chain. + it('CG-10 GATE: puts the hand-written chain in the response, not its generated twin', () => { + // Bare `Upsert` also matches the generated collision — these needles are + // unique to the hand-written chain. expect(response).toContain('runPayrollCycleAll'); - expect(response).toContain('func (s *Service) BuildPayslip'); expect(response).toContain('s.store.Upsert(ctx, slip)'); }); - it('records the shape of the failure so a regression is legible', () => { - // Not a gate — an assertion-free-ish snapshot of WHY the gates above fail, - // so a future change that shifts the numbers shows up in the diff rather - // than silently flipping an it.fails. + it.fails('CG-12 GATE: delivers the calculation the question asks about', () => { + // `payslip_builder.go` ranks #6; the tier's maxFiles is 4 and the render + // loop spends by file size, so it never gets bytes. Score-proportional + // allocation (CG-12) is what closes this. + expect(bytes.get('internal/usecase/payroll/payslip_builder.go') ?? 0).toBeGreaterThan(0); + expect(response).toContain('func (s *Service) BuildPayslip'); + }); + + it('records the shape of the allocation so a regression is legible', () => { + // Not a gate — a snapshot of the split, so a future change that shifts the + // numbers shows up in the diff rather than silently flipping a gate. const generated = generatedShare(); const answer = answerShare(); - const workflow = bytes.get('internal/usecase/payroll/cycle.go') ?? 0; expect({ generatedWinsEnvelope: generated > answer, - workflowFileDeliversNothing: workflow === 0, + workflowFileDelivers: (bytes.get('internal/usecase/payroll/cycle.go') ?? 0) > 0, + builderFileDelivers: (bytes.get('internal/usecase/payroll/payslip_builder.go') ?? 0) > 0, }).toEqual({ - generatedWinsEnvelope: true, - workflowFileDeliversNothing: true, + generatedWinsEnvelope: false, + workflowFileDelivers: true, + builderFileDelivers: false, }); }); }); diff --git a/__tests__/explore-diagnostics.test.ts b/__tests__/explore-diagnostics.test.ts index e8b925ccc..d64616b43 100644 --- a/__tests__/explore-diagnostics.test.ts +++ b/__tests__/explore-diagnostics.test.ts @@ -196,11 +196,17 @@ describe('codegraph_explore allocation diagnostic', () => { // Totals: envelope vs budget, and the file-selection funnel with its floor. expect(out).toMatch(/envelope [\d,]+ chars delivered · [\d,]+ allocated of [\d,]+ budget/); expect(out).toMatch(/hard ceiling [\d,]+/); - expect(out).toMatch(/files [\d,]+ grouped .*past score floor \(>=\d+\).*in output \(maxFiles \d+\)/); + // The funnel runs low-value filter → score floor; the floor is fractional + // now that scoring is kind-weighted (CG-10). + expect(out).toMatch( + /files [\d,]+ grouped .*past low-value filter .*past score floor \(>=[\d.]+\).*in output \(maxFiles \d+\)/, + ); // Per-file columns. - expect(out).toMatch(/#\s+alloc%\s+deliv%\s+bytes\s+score\s+graph\s+hits\s+flags\s+render\s+file/); + expect(out).toMatch(/#\s+alloc%\s+deliv%\s+bytes\s+score\s+graph\s+hits\s+pen\s+flags\s+render\s+file/); expect(out).toContain('src/session.ts'); expect(out).toMatch(/\d+\.\d%/); + // Kind mix — what each file's score was bought with. + expect(out).toMatch(/kinds: (?:\w+:\d+ ?)+/); }); it('appends one JSON report per call to a sidecar path', async () => { @@ -222,7 +228,8 @@ describe('codegraph_explore allocation diagnostic', () => { expect(report.budget.maxOutputChars).toBeGreaterThan(0); expect(report.envelope.chars).toBeGreaterThan(0); expect(report.selection.scoreFloor).toBeGreaterThan(0); - expect(report.selection.filesGrouped).toBeGreaterThanOrEqual(report.selection.filesPastScoreFloor); + expect(report.selection.filesGrouped).toBeGreaterThanOrEqual(report.selection.filesPastLowValueFilter); + expect(report.selection.filesPastLowValueFilter).toBeGreaterThanOrEqual(report.selection.filesPastScoreFloor); expect(report.selection.filesPastScoreFloor).toBeGreaterThanOrEqual(report.selection.filesRanked); expect(report.selection.filesRanked).toBeGreaterThanOrEqual(report.selection.filesInFinalOutput); expect(report.selection.filesInFinalOutput).toBeGreaterThan(0); diff --git a/__tests__/explore-relevance-scoring.test.ts b/__tests__/explore-relevance-scoring.test.ts new file mode 100644 index 000000000..48e22d5e2 --- /dev/null +++ b/__tests__/explore-relevance-scoring.test.ts @@ -0,0 +1,356 @@ +/** + * Relevance scoring for `codegraph_explore` — CG-10 / #1500. + * + * The failure this pins: a file that merely NAME-COLLIDES with the query used to + * score the same per match as the file that answers it, because every match in a + * tier counted the same regardless of what was matched. Three + * `scripts/agent-eval/*.mjs` harnesses took 63% of this repo's own "how does + * explore allocate its output budget across files" response on nothing but a + * local `const explore` and a `const BUDGET`. + * + * Four levers, one fixture family each: + * 1. KIND WEIGHT — a match on a function/class outweighs one on a + * variable/constant/parameter. + * 2. ISOLATION — a weak-kind symbol nothing calls or references is a + * pure collision and is demoted much harder. + * 3. RELATIVE FLOOR — admission scales with the best file's score instead of + * an absolute `>= 3`, capped so one direct match always + * gets in and floored so a diffuse query keeps its spread. + * 4. RANK PENALTY — generated and test/i18n files are discounted on BOTH + * the score and the graph mass (the sort's primary key), + * not merely tie-broken at equal score. + * + * Each fixture is a whole indexed project because the scoring reads the graph + * (usage edges, RWR mass, the generated flag) — there is no seam to unit-test + * the comparator against, and mocking one would pin the mock, not the behavior. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler, RELEVANCE_KIND_WEIGHT } from '../src/mcp/tools'; +import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; + +/** Build + index a throwaway project from a `{ relPath: source }` map. */ +async function buildProject( + prefix: string, + files: Record, +): Promise<{ dir: string; cg: CodeGraph; handler: ToolHandler }> { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + for (const [rel, body] of Object.entries(files)) { + const abs = path.join(dir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, body.trimStart()); + } + const cg = CodeGraph.initSync(dir); + await cg.indexAll(); + return { dir, cg, handler: new ToolHandler(cg) }; +} + +const cleanup = (dir: string, cg?: CodeGraph) => { + if (cg) cg.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); +}; + +/** + * Where a file's source section appears in the response. Sections are emitted in + * final rank order, so this is the ranking assertion — which is what CG-10 owns. + * How many BYTES each ranked file then gets is CG-12's (`maxCharsPerFile` and the + * render loop still spend by file size, so a large low-ranked file can still + * out-byte a small high-ranked one). + */ +const rankOf = (text: string, filePath: string): number => { + const at = text.indexOf('**`' + filePath + '`**'); + if (at < 0) return Number.POSITIVE_INFINITY; + return text.slice(0, at).split('**`').length; +}; + +describe('RELEVANCE_KIND_WEIGHT', () => { + it('ranks callables and types above members, and members above locals', () => { + const callables = ['function', 'method', 'class', 'struct', 'interface', 'route', 'component']; + for (const kind of callables) expect(RELEVANCE_KIND_WEIGHT[kind]).toBe(1); + + for (const member of ['property', 'field', 'enum_member']) { + expect(RELEVANCE_KIND_WEIGHT[member]!).toBeLessThan(RELEVANCE_KIND_WEIGHT.function!); + expect(RELEVANCE_KIND_WEIGHT[member]!).toBeGreaterThan(RELEVANCE_KIND_WEIGHT.parameter!); + } + + // The #1500 kinds: incidental until the graph corroborates them. + for (const weak of ['constant', 'variable', 'parameter']) { + expect(RELEVANCE_KIND_WEIGHT[weak]!).toBeLessThan(0.5); + } + expect(RELEVANCE_KIND_WEIGHT.parameter!).toBeLessThan(RELEVANCE_KIND_WEIGHT.variable!); + }); +}); + +describe('explore relevance scoring — incidental name collisions (#1500)', () => { + let dir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + // Shape: one file DEFINES the dispatch mechanism; three unrelated scripts each + // declare a lone unused `dispatch`/`registry` binding. Before CG-10 all four + // cleared the floor and the three small scripts, shipping whole, took most of + // the envelope from the large real file, which got clipped. + beforeAll(async () => { + const noise = (n: number) => ` +const dispatch = ${n}; +const registry = 'unused-${n}'; + +function unrelated${n}Helper(value) { + return value + ${n}; +} +`; + ({ dir, cg, handler } = await buildProject('codegraph-cg10-collide-', { + 'src/dispatcher.js': ` +import { lookupHandler } from './registry.js'; + +export function dispatch(event) { + const handler = lookupHandler(event.type); + if (!handler) return null; + return runHandler(handler, event); +} + +export function runHandler(handler, event) { + return handler(event.payload); +} +`, + 'src/registry.js': ` +const handlers = new Map(); + +export function registerHandler(type, fn) { + handlers.set(type, fn); +} + +export function lookupHandler(type) { + return handlers.get(type); +} +`, + 'scripts/report-a.js': noise(1), + 'scripts/report-b.js': noise(2), + 'scripts/report-c.js': noise(3), + })); + }, 120_000); + + afterAll(() => cleanup(dir, cg)); + + const explore = async (query: string) => { + const result = await handler.execute('codegraph_explore', { query }); + const text = result.content?.[0]?.text ?? ''; + return { text, bytes: attributeSourceBytes(text) }; + }; + + it('keeps files whose only match is an unused local out of the response', async () => { + const { bytes } = await explore('how does dispatch route an event to its handler'); + for (const noiseFile of ['scripts/report-a.js', 'scripts/report-b.js', 'scripts/report-c.js']) { + expect(bytes.get(noiseFile) ?? 0, `${noiseFile} must not reach the envelope`).toBe(0); + } + }); + + it('spends every delivered source byte on the files that define the mechanism', async () => { + const { bytes } = await explore('how does dispatch route an event to its handler'); + let answer = 0; + let noise = 0; + for (const [file, n] of bytes) { + if (file.startsWith('src/')) answer += n; + else noise += n; + } + expect(answer).toBeGreaterThan(0); + expect(noise).toBe(0); + expect(bytes.get('src/dispatcher.js') ?? 0).toBeGreaterThan(0); + }); + + it('still answers when the collision is the ONLY thing that matched', async () => { + // Guard against over-correction: querying the noise term alone must not + // produce an empty response. Under-serving costs the agent a round-trip, so + // the floor's backfill has to keep the best of what matched. + const { text } = await explore('unrelated2Helper'); + expect(text).not.toContain('No relevant code found'); + expect(text).toContain('unrelated2Helper'); + }); +}); + +describe('explore relevance scoring — generated source is penalized, not tie-broken', () => { + let dir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + // The #1500 shape in miniature: the generated layer collides on every query + // term AND carries more call-graph mass than the hand-written use-case, so a + // generated-as-tiebreak-only rule leaves it ranked first. + beforeAll(async () => { + ({ dir, cg, handler } = await buildProject('codegraph-cg10-generated-', { + 'go.mod': 'module example.com/billing\n\ngo 1.22\n', + 'internal/usecase/billing/invoice.go': ` +package billing + +// Service runs the month-end invoicing workflow. +type Service struct { + store Store +} + +// RunInvoiceCycle is the hand-written business rule the question is about. +func (s *Service) RunInvoiceCycle(month string) error { + lines := s.CollectInvoiceLines(month) + total := s.CalculateInvoiceTotal(lines) + return s.store.Save(month, total) +} + +func (s *Service) CollectInvoiceLines(month string) []int { + return []int{1, 2, 3} +} + +func (s *Service) CalculateInvoiceTotal(lines []int) int { + sum := 0 + for _, l := range lines { + sum += l + } + return sum +} +`, + 'internal/usecase/billing/store.go': ` +package billing + +type Store interface { + Save(month string, total int) error +} +`, + // Ordinary filename — ONLY the content banner betrays it (the CG-5 case). + 'internal/gen/billing/invoice.go': ` +// Code generated by billingkit. DO NOT EDIT. + +package gen + +type InvoiceRow struct { + Month string + Total int +} + +type InvoiceCreateRequest struct { + Month string +} + +func CreateInvoice(req InvoiceCreateRequest) InvoiceRow { + return BuildInvoice(req.Month, 0) +} + +func BuildInvoice(month string, total int) InvoiceRow { + return InvoiceRow{Month: month, Total: total} +} + +func CalculateInvoiceTotal(rows []InvoiceRow) int { + sum := 0 + for _, r := range rows { + sum += r.Total + } + return sum +} + +func ListInvoices(month string) []InvoiceRow { + return []InvoiceRow{BuildInvoice(month, 0)} +} + +func CollectInvoiceLines(month string) []InvoiceRow { + return ListInvoices(month) +} + +func RunInvoiceCycle(month string) InvoiceRow { + rows := CollectInvoiceLines(month) + return BuildInvoice(month, CalculateInvoiceTotal(rows)) +} +`, + })); + }, 120_000); + + afterAll(() => cleanup(dir, cg)); + + it('indexes the ordinary-named generated file via its content banner', () => { + expect(cg.getFile('internal/gen/billing/invoice.go')?.generated).toBe(true); + expect(cg.getFile('internal/usecase/billing/invoice.go')?.generated).toBe(false); + }); + + it('ranks the hand-written workflow above its generated twin', async () => { + const result = await handler.execute('codegraph_explore', { + query: 'how does the invoice cycle collect lines and calculate the total', + }); + const text = result.content?.[0]?.text ?? ''; + + // The generated file collides on EVERY query term and carries call-graph + // mass of its own, so with generated status as a mere tiebreak-at-equal-score + // it ranked first. The penalty scales its score AND its graph mass, which is + // the key the comparator actually sorts on. + const handWritten = rankOf(text, 'internal/usecase/billing/invoice.go'); + const generated = rankOf(text, 'internal/gen/billing/invoice.go'); + expect(handWritten).toBeLessThan(generated); + expect(attributeSourceBytes(text).get('internal/usecase/billing/invoice.go') ?? 0) + .toBeGreaterThan(0); + }); +}); + +describe('explore relevance scoring — test files never buy the envelope', () => { + let dir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + // A repo-ROOT `test/` directory — the shape express and most of npm/Go use. + // The old detector anchored on a leading `/`, so `test/x.js` never matched it + // and express's routing question spent 59% of its envelope on three test files. + beforeAll(async () => { + const spec = (n: number) => ` +const { parseRoute } = require('../lib/router.js'); + +describe('parseRoute ${n}', () => { + it('parses a route ${n}', () => { + parseRoute('/a/${n}'); + }); + it('parses another route ${n}', () => { + parseRoute('/b/${n}'); + }); +}); +`; + ({ dir, cg, handler } = await buildProject('codegraph-cg10-lowvalue-', { + 'lib/router.js': ` +exports.parseRoute = function parseRoute(pathname) { + const segments = pathname.split('/').filter(Boolean); + return { segments, matched: matchRoute(segments) }; +}; + +function matchRoute(segments) { + return segments.length > 0; +} +`, + 'lib/dispatch.js': ` +const { parseRoute } = require('./router.js'); + +exports.dispatchRoute = function dispatchRoute(pathname) { + return parseRoute(pathname); +}; +`, + 'test/router.raw.js': spec(1), + 'test/router.json.js': spec(2), + 'test/router.text.js': spec(3), + })); + }, 120_000); + + afterAll(() => cleanup(dir, cg)); + + it('excludes a repo-root test/ directory from the envelope', async () => { + const result = await handler.execute('codegraph_explore', { + query: 'how does the router parse and dispatch a route', + }); + const bytes = attributeSourceBytes(result.content?.[0]?.text ?? ''); + for (const [file, n] of bytes) { + expect(n === 0 || !file.startsWith('test/'), `${file} took ${n} chars`).toBe(true); + } + expect(bytes.get('lib/router.js') ?? 0).toBeGreaterThan(0); + }); + + it('still returns tests when the query is about them', async () => { + const result = await handler.execute('codegraph_explore', { + query: 'which tests cover parseRoute', + }); + const text = result.content?.[0]?.text ?? ''; + expect(text).not.toContain('No relevant code found'); + }); +}); diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index a46e71f98..01ba211d7 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -90,12 +90,112 @@ being the truncation notice at the end. This is the gap the rest of the epic closes: relevance-proportional allocation with a relative cliff (CG-12), on top of scoring that stops rewarding incidental name collisions -(CG-10). +(CG-10 — landed; see below). + +## CG-10 — relevance scoring + +CG-10 changes **what gets into the response**, ahead of how bytes are split among what's +in. Four levers, all multiplicative so they compose without ordering surprises. + +### 1. Kind weighting + +The tier a symbol reached us by (named seed `+50`, query match `+10`, adjacent to one `+3`, +peripheral `+1`) says *how it got here*; `RELEVANCE_KIND_WEIGHT` says *whether the match is +evidence*. Callables and types weigh 1.0, members ~0.5, and `constant`/`variable`/ +`parameter` 0.15–0.35 — a local named `explore` is a name collision until something +corroborates it. + +**Isolation.** For a weak-kind symbol in the top two tiers, "is anything using it?" is the +corroboration: no usage edge anywhere in the graph (`contains` excluded — lexical nesting +is not usage) drops it to 0.08. Cost is bounded — only weak kinds in the tiers whose weight +can carry a file pay for the probe, and the subgraph's own edges answer most cases for +free. Measured: no latency change (210 vs 211 ms/call, n=12 interleaved). + +**Peripheral cap.** Nodes ≥2 hops from any match now accumulate into a separate bucket +capped at 5. Uncapped, every such node added a flat `+1`, so a file grew more relevant by +being bigger — `parse-session.mjs` reached score 22 off one incidental constant plus twelve +unrelated symbols. Size is not evidence. + +### 2. Relative score floor + +`score >= 3` admits noise on any repo where the top file scores 50+. The floor is now +`clamp(topScore × 0.2, 1, 10)`: + +- **relative** — on a diffuse question no file dominates, every candidate sits near the top, + and the whole spread survives; on a precise one it cuts the tail. +- **capped at 10** — one direct query match on a callable. A single full-strength match is + never incidental, so no amount of concentration elsewhere may exclude it. Without this + cap, one named-seed-heavy file pushed the floor to 21 and dropped a file the agent had + named by *class* name (classes enter at `+10`, not `+50` — named seeds are callables). +- **backfill** — if fewer than 3 files survive, the best of what the floor cut comes back, + but only from files with real evidence (≥ the absolute floor). If *nothing* survives, the + backfill drops that requirement: returning "no relevant code found" when the gather did + find candidates sends the agent straight back to grep. + +### 3. Generated status in the score, not the tiebreak + +`rankPenalty(file)` multiplies both the relevance score and the graph mass by 0.3 for +generated files (0.5 for low-value ones). Applying it to the score alone would not have +fixed #1500: the generated CRUD carries **more** graph mass than the hand-written use-case, +and graph mass outranks score in the comparator. The penalty is self-normalizing — in an +all-generated repo everything scales together and relative ranking is untouched — and it +never hard-excludes: ask about the generated API by name and the named-seed tier still puts +it first. + +### 4. `excludeLowValueFiles` — the finding + +The per-tier flag the task asked to reconsider was **dead config**: declared on +`ExploreOutputBudget` and set per tier, but read nowhere. A later change had already made +the test/spec/icon/i18n exclusion unconditional at all tiers. The flag is removed. + +The substantive gap was in the *detector*, not the gating: `isLowValue` matched +`/\/(tests?|__tests?__|spec)\//`, anchored on a leading slash, so a **repo-root** `test/` +directory — express, cobra, and most of npm and Go — never matched. Express's "how does +express route a request to a handler?" spent 59% of its envelope on three test files while +`lib/application.js` was clipped. Anchored at `^` as well, that query now returns +`lib/application.js` + `lib/response.js` and no tests. + +Two related changes: the filter now runs **before** the score floor and judges "are there +other candidates?" on the whole gather rather than the post-floor set (judging it after was +how the floor's keep-minimum pulled test files back in as the "spread"); and low-value +files that survive the filter's `≥2 non-test candidates` escape hatch are down-weighted via +`rankPenalty` rather than left at full strength. + +### Measured effect + +Before/after on the same indexes, deterministic (`CODEGRAPH_EXPLORE_DEBUG` diagnostic, both +arms same build system, baseline = `bd86ad2`): + +| repo · query | before | after | +|---|---|---| +| this repo · self-query fixture | 72% to eval scripts, `tools.ts` 18.5% | scripts **0%**, `tools.ts` #1 | +| this repo · `handleExplore buildFlowFromNamedSymbols …` | 82% to eval scripts | `tools.ts` 48% + `index.ts` 32% | +| this repo · "how is error handling done" | 58% to eval scripts, `tools.ts` delivered 0 | transport/tools/cobol/api | +| this repo · "what languages does codegraph support" | 63% to `scripts/add-lang/*` | grammars/index/cli | +| this repo · "main components of the indexing pipeline" | — | **byte-identical** | +| payroll-go fixture | generated 57.4%, answer 25.6% | answer **61.5%**, generated **23.5%** | +| express · route a request | 59% to `test/*` | `application.js` + `response.js` | +| cobra · 3 queries | — | **byte-identical** | + +The two byte-identical rows are the control: where the answer was already concentrated, the +new floor prunes the same tail earlier and cheaper and arrives at the same response. + +**Known thin case.** Express's "how does the app object get created and what does it +expose" drops from 4 files (top one an `examples/` file at 38%) to `lib/express.js` alone, +2.6 KB against a 13 KB budget. `lib/application.js` matched on nothing but an unused +file-scope `var app` — indistinguishable, at the symbol level, from the eval scripts' unused +`const explore`; express models its API surface as properties assigned to that object, +which the graph has no edges for. That is extraction coverage, not ranking. Backfilling it +was tried and rejected: node-count ties handed the slot to `examples/route-middleware` +instead, at 48% of the envelope. Thin-and-precise beats padded-with-noise — a wrong file +does not save the agent the follow-up call it would pad against. ## The regression fixtures (CG-6) -Two fixtures pin the failure mode so it can never silently return. Both **fail today** — -that is what they are for. They become the pass gate for CG-10 + CG-12. +Two fixtures pin the failure mode so it can never silently return. They were written to +**fail** — that is what they were for. CG-10 closed the ranking half of both; the byte-split +assertions still fail and are the pass gate for CG-12. The numbers quoted below are the +**pre-CG-10 baseline**; see "Measured effect" above for where they stand now. They are declared in `scripts/agent-eval/allocation-fixtures.json` and run by `scripts/agent-eval/probe-allocation.mjs`, which drives the CG-4 diagnostic through a JSONL @@ -138,15 +238,22 @@ never reach the agent, and every byte that did arrive describes either CRUD or t This fixture is hermetic: the probe copies the tree to a temp dir and re-indexes per run, so two runs on one build are byte-identical (verified). `__tests__/explore-allocation-1500.test.ts` -runs the same assertions in vitest — the fixture-shape half green, the allocation half as -`it.fails` so the suite stays green while the bug is open and goes **red when it is fixed**. +runs the same assertions in vitest. + +**After CG-10** the generated files rank #3/#4 instead of #1/#2, `cycle.go` delivers 38.9% +(it delivered nothing), and `runPayrollCycleAll` + the real `s.store.Upsert(ctx, slip)` +reach the agent. Those assertions are now live regressions. What remains `it.fails` is +`payslip_builder.go`: it ranks #6, the tier's `maxFiles` is 4, and the render loop still +spends by file size — CG-12's job. **Finding, deliberately left unfixed:** `runPayrollCycleAll` calls `s.store.Upsert` on a `*payslipstore.Store`, but the graph resolves that edge to the **generated** `internal/gen/fkit/payroll/store.go` `Store.Upsert`. Same-name method resolution across two -packages that both define `Store.Upsert` picks the wrong receiver. It is upstream of the -allocation bug — a wrong edge pulls the generated store into the subgraph and inflates its -score — so it belongs with CG-10's scoring work, not with the fixture. +packages that both define `Store.Upsert` picks the wrong receiver. It is upstream of +allocation — a wrong edge pulls the generated store into the subgraph. CG-10 mitigates the +*symptom* (the generated store is penalized on both score and graph mass, so it no longer +displaces the real one) without fixing the resolution bug itself, which belongs with the +same-name method resolution work (see `samename-method-resolution-1079`). ### 2. `self-query` — the same bug with no generated code in sight diff --git a/scripts/agent-eval/allocation-fixtures.json b/scripts/agent-eval/allocation-fixtures.json index 28c5a978b..e432753d5 100644 --- a/scripts/agent-eval/allocation-fixtures.json +++ b/scripts/agent-eval/allocation-fixtures.json @@ -2,8 +2,14 @@ "$comment": [ "Regression fixtures for GitHub issue #1500 / epic CG-1 — relevance-proportional", "explore budget allocation. Run them with `node scripts/agent-eval/probe-allocation.mjs`", - "against a built dist/. BOTH FIXTURES FAIL TODAY: that is the point — they document", - "the bug and become the pass gate for the allocation change (CG-10 + CG-12).", + "against a built dist/.", + "", + "STATUS: CG-10 (relevance scoring) closed the RANKING half of both fixtures — nothing", + "incidental reaches the envelope any more. What still fails is the BYTE SPLIT among the", + "files that correctly ranked in, because the render loop spends by file size: a small", + "weakly-relevant file ships whole while the strongly-relevant one is clipped at", + "maxCharsPerFile. That is CG-12's gate. Each fixture's `baseline` records the pre-CG-10", + "numbers; `afterCG10` records where it stands now.", "", "`groups` partitions the files explore rendered into `answer` (what the query is", "actually about) and `incidental` (what wins the envelope today on name collisions).", @@ -65,6 +71,16 @@ "internal/usecase/payroll/cycle.go": 0.0 }, "verdict": "The workflow file is allocated the single largest slice (30.6%) and delivers ZERO — the hard ceiling drops its whole section. Generated CRUD takes 57.4% of what the agent actually receives; runPayrollCycleAll, BuildPayslip and the real Upsert never reach the response." + }, + "afterCG10": { + "measuredOn": "2026-08-04", + "delivered": { + "internal/usecase/payroll/cycle.go": 0.389, + "internal/gen/fkit/payroll/payroll_cycle.go": 0.235, + "internal/domain/payroll/payslip.go": 0.226, + "internal/gen/fkit/payroll/payslip.go": 0.0 + }, + "verdict": "PASSES answerShareAtLeast (61.5%), incidentalShareAtMost (23.5%), topFileGroup, and the cycle.go + runPayrollCycleAll + real-Upsert needles. The generated files now rank #3/#4 instead of #1/#2 — kind weighting plus a 0.3x generated penalty on BOTH the score and the graph mass, which is the key the comparator sorts on. STILL FAILING for CG-12: payslip_builder.go ranks #6 against the tier's maxFiles of 4, so `func (s *Service) BuildPayslip` never renders." } }, { @@ -106,6 +122,14 @@ "scripts/agent-eval/offload-eval-cost.mjs": 0.0 }, "verdict": "The script corpus takes 71.8% of the delivered envelope (79.4% of what was allocated) against tools.ts's 18.5%, despite tools.ts scoring 46 vs 10, carrying 2.3x the graph mass and 3x the distinct term hits." + }, + "afterCG10": { + "measuredOn": "2026-08-04", + "delivered": { + "src/resolution/memory-budget.ts": 0.512, + "src/mcp/tools.ts": 0.329 + }, + "verdict": "PASSES incidentalShareAtMost: every scripts/agent-eval/*.mjs file is gone (0.0%), which is CG-10's acceptance bar — their sole match was an unused file-scope `explore`/`BUDGET` constant, worth 0.08x weight, and the relative floor (8.2) then cut them. tools.ts ranks #1. STILL FAILING for CG-12: memory-budget.ts scores 18 to tools.ts's 41 yet takes 51.2% of the envelope to tools.ts's 32.9%, purely because it is small enough to ship whole while tools.ts is clipped at maxCharsPerFile. Allocation still follows file size, not relevance." } } ] diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts index 210a5cb06..336292db8 100644 --- a/src/mcp/explore-diagnostics.ts +++ b/src/mcp/explore-diagnostics.ts @@ -64,6 +64,20 @@ export interface ExploreCandidateMeta { spine: boolean; lowValue: boolean; generated: boolean; + /** + * Multiplier `rankPenalty` applied to BOTH `score` and `graphScore` (1 = no + * penalty). Generated and test/i18n files rank on discounted signals, so the + * raw values are `score / penalty` — worth reporting, since "why did this + * generated file lose?" is otherwise invisible in the numbers (CG-10). + */ + penalty: number; + /** + * Which NodeKinds the file's matched symbols were, most-numerous first + * (`function:4 constant:1`). The scoring is kind-weighted, so this is the + * breakdown that explains a score — a file carried by one isolated `constant` + * is the #1500 failure, and it is legible here at a glance. + */ + kinds: string; } interface FileRecord extends ExploreCandidateMeta { @@ -81,13 +95,14 @@ interface FileRecord extends ExploreCandidateMeta { skipped?: ExploreSkipReason; } +/** Candidate counts down the selection pipeline, in the order it runs. */ interface StageCounts { /** Files with at least one gathered node. */ grouped: number; - /** Survived the `group.score >= scoreFloor` filter. */ - pastScoreFloor: number; /** Survived the test/spec/icon/i18n hard-exclude. */ pastLowValueFilter: number; + /** Survived the `group.score >= scoreFloor` filter. */ + pastScoreFloor: number; /** Survived the graph-relevance gate. */ pastRelevanceGate: number; } @@ -142,8 +157,8 @@ export interface ExploreDiagnosticReport { graphGateThreshold: number; graphGateApplied: boolean; filesGrouped: number; - filesPastScoreFloor: number; filesPastLowValueFilter: number; + filesPastScoreFloor: number; filesRanked: number; filesRenderedByLoop: number; filesInFinalOutput: number; @@ -216,18 +231,21 @@ export class ExploreDiagnostics { } } - /** Candidate count after the initial `group.score >= floor` filter. */ - setScoreFloor(floor: number, grouped: number, kept: number): void { - this.scoreFloor = floor; + /** + * Candidate count after the test/spec/icon/i18n hard-exclude — the FIRST + * selection stage, ahead of the score floor. + */ + setLowValueFiltered(grouped: number, kept: number): void { this.stages.grouped = grouped; - this.stages.pastScoreFloor = kept; this.stages.pastLowValueFilter = kept; + this.stages.pastScoreFloor = kept; this.stages.pastRelevanceGate = kept; } - /** Candidate count after the test/spec/icon/i18n hard-exclude. */ - setLowValueFiltered(kept: number): void { - this.stages.pastLowValueFilter = kept; + /** Candidate count after the `group.score >= floor` filter. */ + setScoreFloor(floor: number, kept: number): void { + this.scoreFloor = floor; + this.stages.pastScoreFloor = kept; this.stages.pastRelevanceGate = kept; } @@ -342,8 +360,8 @@ export class ExploreDiagnostics { graphGateThreshold: this.graphGateThreshold, graphGateApplied: this.graphGateApplied, filesGrouped: this.stages.grouped, - filesPastScoreFloor: this.stages.pastScoreFloor, filesPastLowValueFilter: this.stages.pastLowValueFilter, + filesPastScoreFloor: this.stages.pastScoreFloor, filesRanked: this.stages.pastRelevanceGate, filesRenderedByLoop: filesIncluded, filesInFinalOutput: rendered.length, @@ -364,6 +382,8 @@ export class ExploreDiagnostics { spine: r.spine, lowValue: r.lowValue, generated: r.generated, + penalty: round6(r.penalty), + kinds: r.kinds, render: r.render ?? null, skipped: r.skipped ?? null, clipped: r.clipped, @@ -463,8 +483,8 @@ export function renderTable(report: ExploreDiagnosticReport): string { ); out.push( ` files ${num(sel.filesGrouped)} grouped` + - ` → ${num(sel.filesPastScoreFloor)} past score floor (>=${sel.scoreFloor})` + ` → ${num(sel.filesPastLowValueFilter)} past low-value filter` + + ` → ${num(sel.filesPastScoreFloor)} past score floor (>=${sel.scoreFloor.toFixed(1)})` + ` → ${num(sel.filesRanked)} past relevance gate` + ` → ${num(sel.filesInFinalOutput)} in output (maxFiles ${num(budget.maxFiles)})`, ); @@ -479,7 +499,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { // when the ceiling truncated; showing both makes that divergence obvious. const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0); if (shown.length > 0) { - out.push(' # alloc% deliv% bytes score graph hits flags render file'); + out.push(' # alloc% deliv% bytes score graph hits pen flags render file'); for (const f of shown) { out.push( ' ' + @@ -487,13 +507,15 @@ export function renderTable(report: ExploreDiagnosticReport): string { pct(f.allocatedShare).padStart(6) + ' ' + pct(f.share).padStart(6) + ' ' + num(f.emittedChars).padStart(7) + ' ' + - String(f.score).padStart(5) + ' ' + + f.score.toFixed(1).padStart(5) + ' ' + f.graphScore.toFixed(5).padStart(7) + ' ' + String(f.termHits).padStart(4) + ' ' + + f.penalty.toFixed(2).padStart(4) + ' ' + flagString(f).padEnd(19) + ' ' + ((f.render ?? '-') + (f.clipped ? '*' : '')).padEnd(9) + ' ' + f.path, ); + out.push(' kinds: ' + (f.kinds || '-')); } out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)'); out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)'); @@ -508,7 +530,8 @@ export function renderTable(report: ExploreDiagnosticReport): string { for (const f of skipped.slice(0, 15)) { out.push( ` #${String(f.rank).padStart(2)} ${f.path} — ${f.skipped ?? f.render ?? 'not reached'}` + - ` (score ${f.score}, graph ${f.graphScore.toFixed(5)}, hits ${f.termHits})`, + ` (score ${f.score.toFixed(1)}, graph ${f.graphScore.toFixed(5)}, hits ${f.termHits},` + + ` pen ${f.penalty.toFixed(2)}, ${f.kinds || '-'})`, ); } if (skipped.length > 15) out.push(` … and ${skipped.length - 15} more`); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index bceda054e..95812af9e 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -189,15 +189,6 @@ export interface ExploreOutputBudget { includeCompletenessSignal: boolean; /** Include the explore-budget reminder at the end. */ includeBudgetNote: boolean; - /** - * Hard-drop test/spec/icon/i18n files from the relevant-file set unless - * the query itself mentions tests. Today they're only deprioritized in - * the sort, which on tiny repos still lets one slip into the top N (e.g. - * cobra's `command_test.go` displaced `args.go` and contributed ~10KB of - * pure noise to "How does cobra parse commands?"). Off by default; on - * for the very-tiny tier where one slip dominates the budget. - */ - excludeLowValueFiles: boolean; } export function getExploreOutputBudget(fileCount: number): ExploreOutputBudget { @@ -229,7 +220,6 @@ export function getExploreOutputBudget(fileCount: number): ExploreOutputBudget { includeAdditionalFiles: false, includeCompletenessSignal: false, includeBudgetNote: false, - excludeLowValueFiles: true, }; } if (fileCount < 500) { @@ -245,7 +235,6 @@ export function getExploreOutputBudget(fileCount: number): ExploreOutputBudget { includeAdditionalFiles: false, includeCompletenessSignal: false, includeBudgetNote: false, - excludeLowValueFiles: true, }; } if (fileCount < 5000) { @@ -263,7 +252,6 @@ export function getExploreOutputBudget(fileCount: number): ExploreOutputBudget { includeAdditionalFiles: true, includeCompletenessSignal: true, includeBudgetNote: true, - excludeLowValueFiles: false, }; } // Large + very-large repos: SAME ~24K inline ceiling (a bigger response just @@ -282,7 +270,6 @@ export function getExploreOutputBudget(fileCount: number): ExploreOutputBudget { includeAdditionalFiles: true, includeCompletenessSignal: true, includeBudgetNote: true, - excludeLowValueFiles: false, }; } return { @@ -296,10 +283,148 @@ export function getExploreOutputBudget(fileCount: number): ExploreOutputBudget { includeAdditionalFiles: true, includeCompletenessSignal: true, includeBudgetNote: true, - excludeLowValueFiles: false, }; } +// ── Explore relevance scoring (CG-10 / #1500) ────────────────────────────── +// +// A file earns its slice of the explore envelope from the symbols in it that the +// query matched. Before this weighting every match counted the same per tier, so +// a file that merely declares a local `const explore` scored what a file that +// DEFINES the explore pipeline scored — which is how three +// `scripts/agent-eval/*.mjs` harnesses took 63% of this repo's own "how does +// explore allocate its output budget across files" response on nothing but a +// local `explore` and a `BUDGET` constant. Four levers — the first three are +// multiplicative, so they compose without ordering surprises; the fourth decides +// admission from the result: +// +// 1. KIND — what a match on this NodeKind actually tells you (below). +// 2. ISOLATION — a weak-kind symbol nothing calls or references is a pure name +// collision; participation in the graph is the corroboration. +// 3. PENALTY — generated / test / i18n files are weaker answers to an +// architecture question at EVERY signal, not just as the +// tiebreak-at-equal-score they used to be. +// 4. FLOOR — admission scales with the best file's score, replacing an +// absolute bar that admitted noise wherever the top score was +// high. + +/** + * How strongly a match on a symbol of this kind corroborates that its FILE is + * what the query is about. + * + * 1.0 a callable or a type — the unit an architecture question is about + * ~0.5 a member of a type, or the file node itself (a path match, not a + * symbol match) + * ~0.3 a variable / constant — as often a name collision as a definition + * 0.15 a parameter — essentially never the subject of a question + * + * Unlisted kinds fall back to `DEFAULT_RELEVANCE_KIND_WEIGHT`, so a NodeKind + * added later is neither free nor fatal. + */ +export const RELEVANCE_KIND_WEIGHT: Readonly> = { + // Callables and types: the answer lives in one of these. + function: 1, method: 1, class: 1, struct: 1, interface: 1, trait: 1, + protocol: 1, component: 1, route: 1, enum: 1, type_alias: 1, constructor: 1, + // Containers: real structure, but a whole namespace/module matching a term is + // a coarser signal than a callable matching it. + namespace: 0.8, module: 0.8, + // Members of a type: real, weaker on their own. + property: 0.5, field: 0.5, enum_member: 0.35, + // The file node itself — the path matched, no symbol did. + file: 0.5, + // Incidental until the graph corroborates them (see ISOLATED_ below). + constant: 0.35, variable: 0.3, parameter: 0.15, +}; +const DEFAULT_RELEVANCE_KIND_WEIGHT = 0.5; + +/** + * Kinds whose evidentiary value depends on whether anything USES them. An + * exported `const DEFAULTS` that half the codebase references is a real + * definition; a `const explore` living inside one function of an eval script is + * a name collision. Only these kinds pay for the isolation probe. + */ +const WEAK_RELEVANCE_KINDS: ReadonlySet = new Set([ + 'constant', 'variable', 'parameter', 'field', 'property', 'enum_member', +]); + +/** Weight for a weak-kind symbol with no incoming/outgoing usage edge at all. */ +const ISOLATED_WEAK_KIND_WEIGHT = 0.08; + +/** + * Edges that mean "this symbol is used". `contains` is lexical nesting, not + * usage — counting it would make every file-scope constant look corroborated, + * which is exactly the case this guards against. + */ +const RELEVANCE_USAGE_EDGES: ReadonlySet = new Set([ + 'calls', 'references', 'extends', 'implements', 'overrides', + 'instantiates', 'returns', 'type_of', 'decorates', +]); + +/** + * Cap on what PERIPHERAL nodes (in the subgraph, but neither a query match nor + * adjacent to one) can contribute to a file's score. Uncapped, each such node + * added a flat +1, so a file grew more "relevant" simply by being bigger — + * `parse-session.mjs` reached score 22 off ONE incidental constant plus twelve + * unrelated symbols. Size is not evidence; cap its contribution. + */ +const PERIPHERAL_SCORE_CAP = 5; + +/** + * Rank penalties, applied to BOTH the relevance score and the graph mass. + * + * Generated source used to be a tiebreak at equal score only, so a generated + * file that outscored the hand-written one still won — the #1500 report exactly: + * the FKIT CRUD layer carries every query term AND more graph mass than the + * use-case that implements the business rule. A multiplier demotes it on the + * PRIMARY sort key instead, without ever hard-excluding it (ask about the + * generated API by name and the named-seed tier still puts it first). It is + * self-normalizing: in an all-generated repo everything scales together and + * relative ranking is untouched. + */ +const GENERATED_RANK_PENALTY = 0.3; +/** + * Test/spec/icon/i18n files. These are normally hard-excluded outright, but that + * filter stands down when fewer than 2 non-low-value candidates remain (else + * tests would be the only signal for the area). This is the softened form for + * that case: down-weighted rather than removed. + */ +const LOW_VALUE_RANK_PENALTY = 0.5; + +/** + * Score floor: `clamp(topScore * FRACTION, ABSOLUTE, MAX)`. + * + * An absolute floor alone (`>= 3`) admits noise on any repo where the top file + * scores 50+, so the bar is now a FRACTION of the best file's score and scales + * with how strong the best match is. On a diffuse survey question no file + * dominates, every candidate sits near the top score, and the whole spread gets + * through; on a precise question it cuts the long tail of incidental matches. + * + * ABSOLUTE is recalibrated for kind-weighted scores: the old `>= 3` assumed an + * unweighted tier sum where any query match was worth 10. A file whose sole + * match is an unused local constant now scores 0.8, so 3 had quietly become a + * much harsher admission bar than it was written to be — and the relative floor + * is what this change means to prune with anyway. + */ +const SCORE_FLOOR_ABSOLUTE = 1; +const SCORE_FLOOR_FRACTION_OF_TOP = 0.2; +/** + * Ceiling on the relative floor, in units of one direct query match on a + * callable (the `entryNodeIds` tier, weight 1.0). A single full-strength match + * is never incidental, so no amount of concentration elsewhere may exclude it: + * one named-seed-heavy file (`+50` per seed) otherwise pushed the floor to 21 + * and dropped `BridgeInterceptor`'s file, which the agent had named — a class, + * so it entered at the +10 tier rather than +50. The #1500 noise this change + * targets scores 0.8–6, well under this ceiling. + */ +const SCORE_FLOOR_MAX = 10; +/** + * The relative floor must never starve a question of candidates: if it would + * leave fewer than this, backfill with the best-scoring ones it cut. The cost of + * under-serving is the agent calling explore again — a whole round-trip. See the + * backfill itself for the two strengths it runs at (thin vs. empty). + */ +const SCORE_FLOOR_KEEP_MIN = 3; + /** * Whether `codegraph_explore` should prefix source lines with their line * numbers (cat -n style: `\t`). @@ -2850,7 +2975,9 @@ export class ToolHandler { } // Step 2: Group nodes by file, score by relevance - const fileGroups = new Map(); + // `peripheral` accumulates separately so it can be capped — see + // PERIPHERAL_SCORE_CAP; it is folded into `score` once the loop is done. + const fileGroups = new Map(); const entryNodeIds = new Set([...subgraph.roots, ...namedSeedIds]); // Build a set of nodes directly connected to entry points (depth 1) @@ -2860,6 +2987,46 @@ export class ToolHandler { if (entryNodeIds.has(edge.target)) connectedToEntry.add(edge.source); } + // Usage degree within the subgraph, for the weak-kind isolation test below. + // Free (the edges are already in hand) and it answers most cases; only a + // weak-kind node that looks isolated HERE pays for a DB probe. + const subgraphUsageDegree = new Map(); + for (const edge of subgraph.edges) { + if (!RELEVANCE_USAGE_EDGES.has(edge.kind) || edge.source === edge.target) continue; + subgraphUsageDegree.set(edge.source, (subgraphUsageDegree.get(edge.source) ?? 0) + 1); + subgraphUsageDegree.set(edge.target, (subgraphUsageDegree.get(edge.target) ?? 0) + 1); + } + + /** + * Relevance weight for one matched symbol: its NodeKind, further discounted + * when it is a weak kind that NOTHING uses. The DB probe (full-graph, since + * a usage can sit outside the traversal) is paid for only by weak-kind + * symbols in the top two tiers — the ones whose weight can carry a whole + * file. A `connectedToEntry` or peripheral node is worth <= 3 either way, so + * probing it would buy nothing. + */ + const isolationCache = new Map(); + const isUsageIsolated = (node: Node): boolean => { + if ((subgraphUsageDegree.get(node.id) ?? 0) > 0) return false; + const cached = isolationCache.get(node.id); + if (cached !== undefined) return cached; + let isolated = true; + try { + const used = (e: Edge) => RELEVANCE_USAGE_EDGES.has(e.kind); + isolated = !cg.getIncomingEdges(node.id).some(used) + && !cg.getOutgoingEdges(node.id).some(used); + } catch { + isolated = false; // a probe failure must not manufacture a penalty + } + isolationCache.set(node.id, isolated); + return isolated; + }; + const relevanceWeight = (node: Node, probeIsolation: boolean): number => { + const weight = RELEVANCE_KIND_WEIGHT[node.kind] ?? DEFAULT_RELEVANCE_KIND_WEIGHT; + if (!probeIsolation || !WEAK_RELEVANCE_KINDS.has(node.kind)) return weight; + return isUsageIsolated(node) ? ISOLATED_WEAK_KIND_WEIGHT : weight; + }; + // CHANGE SURFACE (#1064): a named method's signature types — its parameter // and return types — are part of what you'd edit to "add a parameter to X", // yet they can be lexically dissimilar to the query ("add a parameter to @@ -2901,7 +3068,7 @@ export class ToolHandler { // unbidden. The key still appears in the flow/symbol listing above. if (isConfigLeafNode(node)) continue; - const group = fileGroups.get(node.filePath) || { nodes: [], score: 0 }; + const group = fileGroups.get(node.filePath) || { nodes: [], score: 0, peripheral: 0 }; group.nodes.push(node); // Score: a NAMED-SEED node (a symbol the agent named that FTS missed, now // injected) is worth far more than a mere reference — its file is where the @@ -2909,32 +3076,41 @@ export class ToolHandler { // (Combine.swift references request/task → score 23 from connected nodes) // outranks the file that DEFINES a named symbol (Validation.swift's // `validate` → 10) and steals its render slot. Definition ≫ reference. + // + // Each tier is then scaled by WHAT was matched (RELEVANCE_KIND_WEIGHT): the + // tier says how the symbol reached us, the kind weight says whether the + // match is evidence. A file whose only claim is an unused local `explore` + // constant is a name collision, not an answer (#1500). if (namedSeedIds.has(node.id)) { - group.score += 50; + group.score += 50 * relevanceWeight(node, true); } else if (entryNodeIds.has(node.id)) { - group.score += 10; + group.score += 10 * relevanceWeight(node, true); } else if (connectedToEntry.has(node.id)) { - group.score += 3; + group.score += 3 * relevanceWeight(node, false); } else { - group.score += 1; + // Peripheral: in the subgraph but ≥2 hops from anything the query + // matched. Accumulated separately and capped below, so a file cannot + // buy relevance with size alone. + group.peripheral += relevanceWeight(node, false); } fileGroups.set(node.filePath, group); } - // Only include files that have entry points or nodes directly connected to entry points - const SCORE_FLOOR = 3; - let relevantFiles = [...fileGroups.entries()].filter(([, group]) => group.score >= SCORE_FLOOR); - diag?.setScoreFloor(SCORE_FLOOR, fileGroups.size, relevantFiles.length); - // Extract query terms for relevance checking const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3); - // Test/spec/icon/i18n file detector — used both for the pre-sort hard - // filter (tiny tier) and the comparator deprioritization (all tiers). + // Test/spec/icon/i18n file detector — used by the pre-floor hard filter, the + // rank penalty, and the comparator deprioritization. + // + // The directory pattern is anchored at `^` as well as `/`: a repo-ROOT + // `test/` or `spec/` directory (express, cobra, and most of npm/Go) produced + // paths like `test/express.raw.js`, which the old leading-`/` form could + // never match — so express's routing question spent 59% of its envelope on + // three test files while `lib/router/index.js` never rendered. const isLowValue = (p: string) => { const lp = p.toLowerCase(); return ( - /\/(tests?|__tests?__|spec)\//.test(lp) || + /(?:^|\/)(tests?|__tests?__|specs?)\//.test(lp) || /_test\.go$/.test(lp) || /(?:^|\/)test_[^/]+\.py$/.test(lp) || /_test\.py$/.test(lp) || @@ -2950,7 +3126,35 @@ export class ToolHandler { ); }; - // Hard-exclude test/spec files (ALL tiers, not just tiny). One slipped test + // One DB probe over every file the query touched, then O(1) per lookup. + // Unions the index-time content-banner flag (CG-5) with the filename + // convention, so a Go monorepo's generated CRUD (`payroll.go` carrying a + // DO-NOT-EDIT banner and nothing in its name) down-ranks the same way + // `.pb.go` always has (#1500). Covers the whole subgraph, not just the + // grouped files, because the graph-mass penalty below is keyed on it too. + const isGeneratedCandidate = cg.generatedFilePredicate(new Set([ + ...fileGroups.keys(), + ...[...subgraph.nodes.values()].map((n) => n.filePath), + ])); + + /** + * Rank penalty for a file, applied to its relevance score AND (below) to its + * graph mass — the two signals the sort actually keys on. Applying it to the + * score alone would leave the #1500 case unfixed: the generated CRUD carries + * MORE graph mass than the hand-written use-case, and graph mass outranks + * score in the comparator. + */ + const rankPenalty = (filePath: string): number => + (isGeneratedCandidate(filePath) ? GENERATED_RANK_PENALTY : 1) + * (isLowValue(filePath) ? LOW_VALUE_RANK_PENALTY : 1); + + for (const [filePath, group] of fileGroups) { + group.score = (group.score + Math.min(PERIPHERAL_SCORE_CAP, group.peripheral)) + * rankPenalty(filePath); + } + + // Hard-exclude test/spec files (ALL tiers — the per-tier `excludeLowValueFiles` + // flag this used to be gated on was dead config and is gone). One slipped test // file dominates the per-file budget on small repos (cobra's `command_test.go` // displaced `args.go`) AND wastes budget on large ones (Django's // `custom_lookups/tests.py` ate ~2.3 KB of the 28 KB cap, crowding out the @@ -2958,17 +3162,51 @@ export class ToolHandler { // an architecture question. Skip when the query itself is about tests — the // legitimate "explore the tests" case — and only cut if ≥2 non-test candidates // remain (else tests are the only signal for this area). + // + // Runs BEFORE the score floor, on the whole gather. Judging "are there other + // candidates?" on the post-floor set was too late: express's routing question + // left one non-test file past the floor, the guard stood down, and the floor's + // keep-minimum then pulled two test files back in as the "spread". + let candidateFiles = [...fileGroups.entries()]; { const queryMentionsTests = /\b(test|tests|testing|spec|verify|verifies)\b/i.test(query); if (!queryMentionsTests) { - const nonLow = relevantFiles.filter(([p]) => !isLowValue(p)); + const nonLow = candidateFiles.filter(([p]) => !isLowValue(p)); if (nonLow.length >= 2) { - relevantFiles = nonLow; + candidateFiles = nonLow; } } - diag?.setLowValueFiltered(relevantFiles.length); + diag?.setLowValueFiltered(fileGroups.size, candidateFiles.length); } + // Relative score floor — see SCORE_FLOOR_* for why it is a fraction of the + // best file's score and why that fraction is clamped at both ends. + const topScore = Math.max(0, ...candidateFiles.map(([, g]) => g.score)); + const scoreFloor = Math.max( + SCORE_FLOOR_ABSOLUTE, + Math.min(SCORE_FLOOR_MAX, topScore * SCORE_FLOOR_FRACTION_OF_TOP), + ); + let relevantFiles = candidateFiles.filter(([, group]) => group.score >= scoreFloor); + if (relevantFiles.length < SCORE_FLOOR_KEEP_MIN) { + // Backfill from what the RELATIVE floor cut, best first, at two strengths: + // + // - THIN (1-2 files survived): only files with real evidence. A file whose + // entire claim is one isolated variable scores 0.8 and stays out — + // express's `examples/route-middleware` matched nothing but a local + // `app` and would otherwise have taken 48% of that envelope. Padding a + // precise answer with a wrong file doesn't save the agent the follow-up + // call it would pad against. + // - EMPTY (nothing survived): take the best of whatever matched. Returning + // "no relevant code found" when the gather DID find candidates is the + // worst outcome on the board — the agent falls straight back to grep. + const minEvidence = relevantFiles.length === 0 ? Number.EPSILON : SCORE_FLOOR_ABSOLUTE; + relevantFiles = candidateFiles + .filter(([, group]) => group.score >= minEvidence) + .sort((a, b) => b[1].score - a[1].score || b[1].nodes.length - a[1].nodes.length) + .slice(0, Math.max(SCORE_FLOOR_KEEP_MIN, relevantFiles.length)); + } + diag?.setScoreFloor(scoreFloor, relevantFiles.length); + // Secondary signal: how many DISTINCT query terms each file matches (path + // symbol names). Kept only as a tiebreak — the PRIMARY relevance is graph // connectivity below. (Term counting alone tied the real central file with @@ -2991,6 +3229,11 @@ export class ToolHandler { const nodeRwr = this.computeGraphRelevance( [...subgraph.nodes.keys()], subgraph.edges, entryNodeIds, ); + // + // Carries `rankPenalty` too, so generated/low-value files are demoted on the + // sort's PRIMARY key rather than only at the tiebreak. Everything downstream + // (centrality, the relevance gate, the buried-rescue test, the comparator) + // reads this map, so the penalty applies once and applies everywhere. const fileGraphScore = new Map(); for (const node of subgraph.nodes.values()) { fileGraphScore.set( @@ -2998,6 +3241,7 @@ export class ToolHandler { (fileGraphScore.get(node.filePath) ?? 0) + (nodeRwr.get(node.id) ?? 0), ); } + for (const [fp, mass] of fileGraphScore) fileGraphScore.set(fp, mass * rankPenalty(fp)); const maxGraph = Math.max(0, ...fileGraphScore.values()); // Central file(s): the 1-2 most graph-central files that also match the @@ -3042,7 +3286,7 @@ export class ToolHandler { changeSurfaceFiles.add(fp); if (!subgraph.nodes.has(t.id)) subgraph.nodes.set(t.id, t); let group = fileGroups.get(fp); - if (!group) { group = { nodes: [], score: 0 }; fileGroups.set(fp, group); } + if (!group) { group = { nodes: [], score: 0, peripheral: 0 }; fileGroups.set(fp, group); } if (!group.nodes.some((n) => n.id === t.id)) group.nodes.push(t); group.score = Math.max(group.score, 45); if (!relevantFiles.some(([f]) => f === fp)) relevantFiles.push([fp, group]); @@ -3114,12 +3358,6 @@ export class ToolHandler { (fileTermHits.get(fp) ?? 0) >= 2 && (entryFiles.has(fp) || centralFiles.has(fp)); - // One DB probe over the ranked candidates, then O(1) per comparison. Unions - // the index-time content-banner flag with the filename convention, so a Go - // monorepo's generated CRUD (`payroll.go` carrying a DO-NOT-EDIT banner and - // nothing in its name) down-ranks the same way `.pb.go` always has (#1500). - const isGeneratedCandidate = cg.generatedFilePredicate(relevantFiles.map(([fp]) => fp)); - const sortedFiles = relevantFiles.sort((a, b) => { const aPath = a[0].toLowerCase(); const bPath = b[0].toLowerCase(); @@ -3153,7 +3391,11 @@ export class ToolHandler { // when asking about the actual flow, and dumping their bodies inflates // the response (the cosmos Q3 explore otherwise leads with // `expected_keepers_mocks.go`, displacing the real `tally.go` content - // and forcing the agent to Read tally.go anyway). + // and forcing the agent to Read tally.go anyway). Both this and the + // low-value key above are now BACKSTOPS: `rankPenalty` has already scaled + // the score and the graph mass these files reach this comparison with, so + // a generated file no longer outranks a hand-written one just by scoring + // higher (#1500). This still settles the exact ties the penalty leaves. const aGen = isGeneratedCandidate(a[0]); const bGen = isGeneratedCandidate(b[0]); if (aGen !== bGen) return aGen ? 1 : -1; @@ -3228,6 +3470,14 @@ export class ToolHandler { // the diagnostic can show what each file's share of the envelope was BOUGHT // with (score, graph mass, term hits, flags) — not just what it cost. if (diag) { + const kindMix = (nodes: Node[]): string => { + const counts = new Map(); + for (const n of nodes) counts.set(n.kind, (counts.get(n.kind) ?? 0) + 1); + return [...counts.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([k, c]) => `${k}:${c}`) + .join(' '); + }; sortedFiles.forEach(([fp, group], i) => { diag.noteCandidate(fp, { rank: i + 1, @@ -3241,6 +3491,8 @@ export class ToolHandler { spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), lowValue: isLowValue(fp), generated: isGeneratedCandidate(fp), + penalty: rankPenalty(fp), + kinds: kindMix(group.nodes), }); }); } @@ -3869,8 +4121,13 @@ export class ToolHandler { // in the source section, and a trailing pointer list is pure overhead. if (budget.includeAdditionalFiles) { const remainingRelevant = sortedFiles.slice(filesIncluded); + // Ranked files are already covered by `remainingRelevant`; the rest of the + // gather (below the floor) becomes the pointer list. The Set guards the + // one overlap case — a file the SCORE_FLOOR_KEEP_MIN fallback pulled in + // despite scoring under the relative floor. + const rankedPaths = new Set(sortedFiles.map(([fp]) => fp)); const peripheralFiles = [...fileGroups.entries()] - .filter(([, group]) => group.score < 3) + .filter(([fp, group]) => group.score < scoreFloor && !rankedPaths.has(fp)) .sort((a, b) => b[1].score - a[1].score); const remainingFiles = [...remainingRelevant, ...peripheralFiles]; if (remainingFiles.length > 0) { From 5f7f5f59dfcaae0ce5ecb5e7f1400e766d6677c1 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 00:37:17 -0500 Subject: [PATCH 05/15] feat(mcp): score-proportional byte allocation for explore, with a relative cliff (CG-12, #1500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The explore envelope used to follow FILE SIZE, not relevance. Every admitted file was capped at the same flat `maxCharsPerFile`, while the whole-file rule handed anything under `maxCharsPerFile * 3` its entire contents — a 3x swing decided by how big a file happened to be: - self-query: `memory-budget.ts` (score 18) shipped whole and took 51.2% of the response; `src/mcp/tools.ts` (score 41, 4x the graph mass, 3x the term hits — it holds the allocator itself) was clipped at 3,800 and got 32.9%. - #1500 Go fixture: two generated CRUD files shipped whole at ~4.5K each AND consumed two of the tier's four file slots, so `BuildPayslip` — the hand-written "calculate" half of the question — ranked #6 and never rendered at all. `allocateExploreBudget` now reserves each ranked file a share of the envelope before anything renders, so the render loop spends a reservation instead of racing for whatever the files above it left: - weight = score x worth x (spine ? 2 : 1), where `worth` is `rankPenalty` applied a SECOND time — ranking answers "is this file about the query", allocation answers "will these bytes teach the agent anything", and generated CRUD can legitimately rank while its bytes stay boilerplate; - a relative cliff at 15% of the top weight (capped at SCORE_FLOOR_MAX, so a god-file can't silence peers the score floor just admitted) gives a file ZERO source — path, symbols and line numbers only — and crucially frees its `maxFiles` slot for a file that earns its bytes; - every admitted file gets MIN_CHARS, then the remainder splits by weight: the floor keeps a diffuse survey question returning a spread, the remainder concentrates a precise one; - the flat per-file cap is retired as the primary guard, leaving a 70%-of- envelope safety valve. Two changes were needed to make the reservation bite: an oversize cluster now shrinks by whole MEMBER symbol ranges (a single-cluster god-file previously took ~40% more than allotted, and the file below it was dropped for lack of room), and the arrival-order budget stops are gone — they cut files by the order they were reached rather than by merit. Measured: payroll-go answer group 25.6% -> 78.7%, generated 57.4% -> 0%, and `func (s *Service) BuildPayslip` now delivered; self-query `tools.ts` 18.5% -> 60.6%, past the epic's >50% bar. Controls hold: cobra/gin diffuse survey queries keep their file spread (3->3, 3->4), express's middleware query is byte-identical, and gin's flow query moves its top file from the thin `ginS` singleton wrapper to `routergroup.go`. One documented exception to "no previously-unclipped file becomes clipped": `memory-budget.ts` was unclipped-whole at 5,672 and now clusters within its 3.1K reservation. That is the epic's own diagnosis of the bug — it scored 18 against 58 and was taking the larger slice purely for being small. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 + __tests__/explore-allocation-1500.test.ts | 30 +- __tests__/explore-diagnostics.test.ts | 4 +- .../explore-proportional-allocation.test.ts | 214 ++++++++ docs/design/explore-budget-allocation.md | 88 +++- scripts/agent-eval/allocation-fixtures.json | 35 +- src/mcp/explore-diagnostics.ts | 74 ++- src/mcp/tools.ts | 464 +++++++++++++++--- 8 files changed, 814 insertions(+), 97 deletions(-) create mode 100644 __tests__/explore-proportional-allocation.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b60e18f..1ceaaafcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- `codegraph_explore` now gives the most relevant file the most room, instead of splitting its answer roughly evenly and letting whichever file happened to be small enough take the largest share. Every file's slice is now sized in proportion to how well it matches your question, decided before any code is written out — so the file that actually answers you is no longer trimmed at a fixed per-file limit while a small, weakly-related file is included in full. Files far below the best match are listed by name, symbol and line number instead of taking space with their source, and you can pull any of them up in full with one more `codegraph_explore`. On a Go service with a generated CRUD layer beside a hand-written payroll workflow, the hand-written code went from a quarter of the response to nearly four-fifths of it, and the calculation the question was actually about — previously missing entirely — is now included. (#1500) + - `codegraph_explore` no longer spends its answer on files that merely share a word with your question. A file whose only connection to the query was an unused local variable or constant of the same name — an eval script with a `const explore` in it, say — used to count as strongly as the file that actually implements the thing, and being small enough to include whole, it could take most of the response while the real answer got trimmed. Matches are now weighted by what was matched (a function, class or route counts for far more than a local variable, and a variable nothing else references counts for almost nothing), and a file has to score within reach of the best match rather than clear a fixed low bar. Generated files are down-weighted throughout the ranking instead of only breaking ties, so a generated CRUD layer no longer outranks the hand-written workflow beside it. Test and spec files in a top-level `test/` or `spec/` directory are now recognized and kept out, which they weren't before. (#1500) - A CodeGraph process that gets force-killed — by the stuck-process watchdog, a crash, or the OS — no longer leaves the database's write-ahead log behind to grow without bound. Previously each killed session stacked more data onto the same log file and nothing ever shrank it, which on machines where sessions were killed regularly could quietly eat tens of gigabytes of disk. The log is now capped, and any oversized leftover is reclaimed automatically the next time the project is opened. Thanks @tiendungdev for the exceptional Windows report that pinned this down. (#1431) diff --git a/__tests__/explore-allocation-1500.test.ts b/__tests__/explore-allocation-1500.test.ts index 059c975be..ffe513ac1 100644 --- a/__tests__/explore-allocation-1500.test.ts +++ b/__tests__/explore-allocation-1500.test.ts @@ -218,11 +218,13 @@ describe('#1500 — generated Go CRUD beside a hand-written payroll workflow', ( * `cycle.go` now delivers 38.9% and the generated layer 23.5%. Four of the * five gates below are green and are now live regressions. * - * STILL OPEN, for CG-12 (proportional byte allocation): the render loop - * still allocates by FILE SIZE within the ranked set, and `maxFiles` is 4 at - * this tier — so `payslip_builder.go` ranks #6 and never renders, and - * `func (s *Service) BuildPayslip` is absent. Its `it.fails` passes ONLY - * while that is still true. ⚠ When it goes red, remove `.fails`, keep it. + * AFTER CG-12 (score-proportional allocation): every file's share of the + * envelope is reserved before anything renders, and a file under 15% of the + * top weight gets no source at all — so the two generated files cliff to + * pointers, hand their `maxFiles` slots to the hand-written store and + * builder, and the answer group takes ~79% with the generated layer at 0%. + * `func (s *Service) BuildPayslip` — the "calculate" half of the question — + * finally reaches the agent. All gates below are live regressions now. */ it('CG-10 GATE: concentrates the envelope on the hand-written workflow', () => { expect(answerShare()).toBeGreaterThanOrEqual(0.55); @@ -249,14 +251,22 @@ describe('#1500 — generated Go CRUD beside a hand-written payroll workflow', ( expect(response).toContain('s.store.Upsert(ctx, slip)'); }); - it.fails('CG-12 GATE: delivers the calculation the question asks about', () => { - // `payslip_builder.go` ranks #6; the tier's maxFiles is 4 and the render - // loop spends by file size, so it never gets bytes. Score-proportional - // allocation (CG-12) is what closes this. + it('CG-12 GATE: delivers the calculation the question asks about', () => { + // `payslip_builder.go` ranks #6 and the tier's maxFiles is 4 — it reaches + // the response only because the two generated files cliff to pointers + // WITHOUT consuming a slot. That slot hand-off is the CG-12 mechanism. expect(bytes.get('internal/usecase/payroll/payslip_builder.go') ?? 0).toBeGreaterThan(0); expect(response).toContain('func (s *Service) BuildPayslip'); }); + it('CG-12 GATE: withholds the generated CRUD bytes but still names it', () => { + // A cliffed file costs ~100 chars instead of ~4,500, and stays one + // follow-up explore away — withholding is only cheap if it stays nameable. + expect(bytes.get('internal/gen/fkit/payroll/payslip.go') ?? 0).toBe(0); + expect(response).toContain('**Not shown above — explore these names for their source**'); + expect(response).toMatch(/internal\/gen\/fkit\/payroll\/payslip\.go: \w+:\d+/); + }); + it('records the shape of the allocation so a regression is legible', () => { // Not a gate — a snapshot of the split, so a future change that shifts the // numbers shows up in the diff rather than silently flipping a gate. @@ -269,7 +279,7 @@ describe('#1500 — generated Go CRUD beside a hand-written payroll workflow', ( }).toEqual({ generatedWinsEnvelope: false, workflowFileDelivers: true, - builderFileDelivers: false, + builderFileDelivers: true, }); }); }); diff --git a/__tests__/explore-diagnostics.test.ts b/__tests__/explore-diagnostics.test.ts index d64616b43..6cedd8380 100644 --- a/__tests__/explore-diagnostics.test.ts +++ b/__tests__/explore-diagnostics.test.ts @@ -202,7 +202,9 @@ describe('codegraph_explore allocation diagnostic', () => { /files [\d,]+ grouped .*past low-value filter .*past score floor \(>=[\d.]+\).*in output \(maxFiles \d+\)/, ); // Per-file columns. - expect(out).toMatch(/#\s+alloc%\s+deliv%\s+bytes\s+score\s+graph\s+hits\s+pen\s+flags\s+render\s+file/); + expect(out).toMatch(/#\s+alloc%\s+deliv%\s+bytes\s+reserved\s+score\s+graph\s+hits\s+pen\s+flags\s+render\s+file/); + // The proportional split (CG-12): what was reserved, and where the cliff fell. + expect(out).toMatch(/allocation [\d,]+ reserved of [\d,]+ pool · cliff at weight [\d.]+/); expect(out).toContain('src/session.ts'); expect(out).toMatch(/\d+\.\d%/); // Kind mix — what each file's score was bought with. diff --git a/__tests__/explore-proportional-allocation.test.ts b/__tests__/explore-proportional-allocation.test.ts new file mode 100644 index 000000000..13c886381 --- /dev/null +++ b/__tests__/explore-proportional-allocation.test.ts @@ -0,0 +1,214 @@ +/** + * Score-proportional byte allocation for codegraph_explore (CG-12 / #1500). + * + * `allocateExploreBudget` decides, before anything renders, how many chars of + * source each ranked file may spend. Its contract is what stops the explore + * envelope from following FILE SIZE — which is the bug #1500 reported: a small + * weakly-relevant file shipped whole while the file that actually answered the + * question was clipped at a flat per-file cap. + * + * These pin the allocator's invariants directly. End-to-end behaviour on the two + * regression fixtures lives in `explore-allocation-1500.test.ts`. + */ +import { describe, it, expect } from 'vitest'; +import { allocateExploreBudget, getExploreOutputBudget } from '../src/mcp/tools'; +import type { ExploreAllocationCandidate } from '../src/mcp/tools'; + +/** A candidate with sane defaults — tests override only what they're about. */ +const cand = ( + path: string, + score: number, + extra: Partial = {}, +): ExploreAllocationCandidate => ({ path, score, worth: 1, spine: false, ...extra }); + +const TIER_FILE_COUNTS = [10, 100, 300, 1000, 4000, 10000, 20000, 60000]; + +describe('allocateExploreBudget — proportional split', () => { + const budget = getExploreOutputBudget(1000); // 24,000 / 6,500 / 8 files + + it('gives the higher-scoring file the bigger share', () => { + const { allowances } = allocateExploreBudget( + [cand('a.ts', 40), cand('b.ts', 10)], + budget, + 8, + ); + expect(allowances.get('a.ts')!).toBeGreaterThan(allowances.get('b.ts')!); + }); + + it('scales the split with the score RATIO, not just the ordering', () => { + // The heart of the fix. Under the old flat `maxCharsPerFile` both files got + // the same cap and the split fell out of whichever happened to be small + // enough to ship whole; here a 4x score buys materially more than a 1.1x one. + const wide = allocateExploreBudget([cand('a.ts', 40), cand('b.ts', 10)], budget, 8).allowances; + const narrow = allocateExploreBudget([cand('a.ts', 22), cand('b.ts', 20)], budget, 8).allowances; + expect(wide.get('a.ts')! / wide.get('b.ts')!) + .toBeGreaterThan(narrow.get('a.ts')! / narrow.get('b.ts')!); + }); + + it('never reserves more than the envelope', () => { + const { allowances, pool } = allocateExploreBudget( + [cand('a.ts', 90), cand('b.ts', 40), cand('c.ts', 30), cand('d.ts', 12)], + budget, + 8, + ); + const reserved = [...allowances.values()].reduce((s, n) => s + n, 0); + expect(reserved).toBeLessThanOrEqual(pool); + expect(pool).toBeLessThanOrEqual(budget.maxOutputChars); + }); + + it('caps any single file at the MAX_SHARE safety valve', () => { + // The per-file cap is retired as the primary guard, but a lone dominant file + // must still not be handed the entire response. + const { allowances } = allocateExploreBudget([cand('god.ts', 500)], budget, 8); + expect(allowances.get('god.ts')!).toBeLessThanOrEqual(Math.round(budget.maxOutputChars * 0.7)); + }); + + it('lets the top file exceed the old flat per-file cap when it earns it', () => { + // The regression this task exists to fix: `maxCharsPerFile` clipped the file + // that scored 4x its peers at exactly the same 6,500 as the noise. + const { allowances } = allocateExploreBudget( + [cand('answer.ts', 60), cand('noise.ts', 12)], + budget, + 8, + ); + expect(allowances.get('answer.ts')!).toBeGreaterThan(budget.maxCharsPerFile); + }); +}); + +describe('allocateExploreBudget — the relative cliff', () => { + const budget = getExploreOutputBudget(1000); + + it('gives zero source to a file far below the top score', () => { + const { allowances, cliffed } = allocateExploreBudget( + [cand('answer.ts', 90), cand('incidental.ts', 3)], + budget, + 8, + ); + expect(cliffed).toContain('incidental.ts'); + expect(allowances.has('incidental.ts')).toBe(false); + }); + + it('is RELATIVE — the same score survives against weaker company', () => { + const strong = allocateExploreBudget([cand('a.ts', 90), cand('b.ts', 8)], budget, 8); + const even = allocateExploreBudget([cand('a.ts', 12), cand('b.ts', 8)], budget, 8); + expect(strong.cliffed).toContain('b.ts'); + expect(even.cliffed).not.toContain('b.ts'); + }); + + it('never rises above the score-floor ceiling, however dominant the top file', () => { + // A 500-scoring god-file would otherwise put the cliff at 75 and silence + // every peer the score floor had just deliberately admitted. + const { cliffed } = allocateExploreBudget( + [cand('god.ts', 500), cand('peer.ts', 13), cand('peer2.ts', 11)], + budget, + 8, + ); + expect(cliffed).toEqual([]); + }); + + it('doubles the penalty on bytes that are worth less (generated / low-value)', () => { + // `worth` is `rankPenalty` applied a second time: generated CRUD can rank on + // name collisions while its bytes stay boilerplate. Same score, different fate. + const { cliffed } = allocateExploreBudget( + [cand('answer.ts', 60), cand('gen.ts', 12, { worth: 0.3 }), cand('hand.ts', 12)], + budget, + 8, + ); + expect(cliffed).toContain('gen.ts'); + expect(cliffed).not.toContain('hand.ts'); + }); + + it('exempts flow-spine files from the cliff', () => { + // Clipping the spine causes the Read fallback — it IS the answer to a flow + // question — so a spine file is never zeroed on relative score alone. + const { allowances, cliffed } = allocateExploreBudget( + [cand('a.ts', 400), cand('spine.ts', 2, { spine: true })], + budget, + 8, + ); + expect(cliffed).not.toContain('spine.ts'); + expect(allowances.get('spine.ts')!).toBeGreaterThan(0); + }); + + it('never cliffs every candidate — an empty response costs a round-trip', () => { + const { allowances, cliffed } = allocateExploreBudget([cand('only.ts', 0.5)], budget, 8); + expect(cliffed).toEqual([]); + expect(allowances.get('only.ts')!).toBeGreaterThan(0); + }); + + it('hands a cliffed file\'s maxFiles slot to the next file down', () => { + // The mechanism that got `BuildPayslip` into the #1500 response: cliffing is + // not just "spend fewer bytes here", it frees the SLOT too. + const { allowances } = allocateExploreBudget( + [cand('a.ts', 90), cand('noise.ts', 2), cand('b.ts', 40)], + budget, + 2, + ); + expect([...allowances.keys()].sort()).toEqual(['a.ts', 'b.ts']); + }); +}); + +describe('allocateExploreBudget — the floor keeps diffuse questions useful', () => { + const budget = getExploreOutputBudget(1000); + + it('gives every admitted file a slice big enough for a method', () => { + // A survey question must still return a spread. The earlier design cliffed a + // starved file instead of flooring it, and that CASCADED: removing the + // smallest raised everyone else so little that the next-smallest starved too, + // eating six legitimately-ranked peers one at a time. + const files = [cand('a.ts', 100), cand('b.ts', 90), ...Array.from({ length: 6 }, (_, i) => cand(`p${i}.ts`, 20))]; + const { allowances } = allocateExploreBudget(files, budget, 8); + expect(allowances.size).toBe(8); + for (const [, chars] of allowances) expect(chars).toBeGreaterThanOrEqual(700); + }); + + it('serves fewer files well rather than many badly when the envelope cannot afford them', () => { + const tiny = getExploreOutputBudget(10); // 13,000-char envelope + const files = Array.from({ length: 40 }, (_, i) => cand(`f${i}.ts`, 50 - i * 0.1)); + const { allowances, cliffed } = allocateExploreBudget(files, tiny, 40); + expect(allowances.size).toBeLessThan(40); + expect(cliffed.length).toBeGreaterThan(0); + for (const [, chars] of allowances) expect(chars).toBeGreaterThanOrEqual(700); + const reserved = [...allowances.values()].reduce((s, n) => s + n, 0); + expect(reserved).toBeLessThanOrEqual(tiny.maxOutputChars); + }); + + it('returns an empty allocation for an empty candidate list', () => { + const { allowances, cliffed } = allocateExploreBudget([], budget, 8); + expect(allowances.size).toBe(0); + expect(cliffed).toEqual([]); + }); + + it('does not crash or over-allocate when every score is zero', () => { + const { allowances } = allocateExploreBudget([cand('a.ts', 0), cand('b.ts', 0)], budget, 8); + const reserved = [...allowances.values()].reduce((s, n) => s + n, 0); + expect(reserved).toBeLessThanOrEqual(budget.maxOutputChars); + }); +}); + +describe('allocateExploreBudget — tier invariant', () => { + it('never gives a larger tier a smaller allowance than a smaller tier', () => { + // The standing invariant from `getExploreOutputBudget`: a bigger project must + // never be served LESS per file. It held for the flat cap by inspection; with + // a proportional split it has to hold for the same candidate set across every + // tier, which is what this walks. + const files = [cand('a.ts', 60), cand('b.ts', 30), cand('c.ts', 15)]; + let previous: Map | null = null; + for (const fileCount of TIER_FILE_COUNTS) { + const { allowances } = allocateExploreBudget(files, getExploreOutputBudget(fileCount), 8); + if (previous) { + for (const [path, chars] of allowances) { + expect(chars, `${path} shrank at ${fileCount} files`).toBeGreaterThanOrEqual(previous.get(path)!); + } + } + previous = allowances; + } + }); + + it('cliffs the same files at every tier — the cliff is relative, not sized', () => { + const files = [cand('a.ts', 90), cand('noise.ts', 2)]; + const cliffs = TIER_FILE_COUNTS.map((n) => + allocateExploreBudget(files, getExploreOutputBudget(n), 8).cliffed.join(',')); + expect(new Set(cliffs).size).toBe(1); + }); +}); diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index 01ba211d7..51a5ad50d 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -190,12 +190,94 @@ was tried and rejected: node-count ties handed the slot to `examples/route-middl instead, at 48% of the envelope. Thin-and-precise beats padded-with-noise — a wrong file does not save the agent the follow-up call it would pad against. +## CG-12 — score-proportional allocation + +CG-10 fixed *what* gets into the response. This fixes *how the bytes are split among what +got in* — which, until now, was not really decided at all. Every admitted file was capped at +the same flat `maxCharsPerFile`, and the whole-file rule handed anything under +`maxCharsPerFile × 3` its entire contents. So the envelope followed **file size**: + +- self-query: `memory-budget.ts` (score 18) shipped whole at 5,672 and took **51.2%**; + `tools.ts` (score 41, 4× the graph mass, 3× the term hits — it literally holds the + allocator) was clipped at 3,800 and got **32.9%**. +- payroll-go: two generated CRUD files shipped whole at ~4.5 KB each *and* consumed two of + the tier's four file slots, so `BuildPayslip` — the hand-written "calculate" half of the + question — ranked #6 and never rendered at all. + +### The model + +`allocateExploreBudget` (`src/mcp/tools.ts`) runs once, after ranking, before anything +renders. It reserves each file a share of the envelope; the render loop then spends a +reservation instead of racing for whatever the files above it left. + +1. **Weight** = `score × worth × (spine ? 2 : 1)`. `worth` is `rankPenalty` applied a + *second* time: ranking answers "is this file about the query", allocation answers "will + these bytes teach the agent anything". Generated CRUD can legitimately rank — it + name-collides on every domain word, and it is big and densely self-referential, so it + scores on the structural keys the comparator leads with — while its bytes stay mechanical + boilerplate. That second penalty is what finally sinks it. +2. **Relative cliff** at 15% of the top weight, itself capped at `SCORE_FLOOR_MAX`. A file + under it gets **zero source** — path, symbols and line numbers only. It costs ~100 chars + instead of ~4,500, and it does **not consume a `maxFiles` slot**, so the slot passes to a + file that earns its bytes. That slot hand-off is what got `BuildPayslip` into the + response. The cap matters as much as the fraction: one 500-scoring god-file would + otherwise put the cliff at 75 and silence every peer the score floor had just admitted. +3. **Floor then split.** Every admitted file gets `MIN_CHARS` (700 — enough for one complete + method); the remainder splits by weight. The floor is what keeps a diffuse survey + question returning a spread; the remainder is what concentrates a precise one. +4. **Safety valve**, not a per-file cap: no file exceeds 70% of the envelope. The flat + per-file cap is retired as the primary guard — the proportional split already bounds a + file by its weight share. + +The reservation then governs **every** render path — whole-file, clusters, focused/skeleton +— where before the whole-file branch was 3× more generous than the cluster branch, which is +the 3× swing that decided the split by file size. + +Two supporting changes were needed to make the reservation actually bite: + +- **Oversize clusters shrink by member.** A cluster is a merge of whole symbol ranges, and + on a densely-packed file every symbol merges into one blob spanning the file (cycle.go's + 209-line `Service`). The old rule took the top-ranked cluster whole however big it was, so + a single-cluster file simply ignored its budget — it took ~40% more than allotted and the + file below it was then dropped for lack of room. Shrinking drops whole **members** by + importance, so a body is still never cut. +- **The arrival-order stops are gone.** `budget-90pct` and the `!fileNecessary && totalChars + > maxOutputChars` checks dropped files by the order they were reached: whichever files + ranked first spent the envelope, and everything after them was cut on a cap it had no say + in. Only an absolute hard-ceiling stop remains. + +### Measured effect (CG-12) + +| repo · query | before (post-CG-10) | after | +|---|---|---| +| this repo · self-query fixture | `tools.ts` 32.9%, `memory-budget.ts` 51.2% | **`tools.ts` 60.6%**, memory-budget 17.2% | +| payroll-go fixture | answer 61.5%, generated 23.5%, `BuildPayslip` absent | answer **78.7%**, generated **0%**, `BuildPayslip` **delivered** | +| express · route a request | 2 files, top 43.1% | 1 file, top **82.3%** (`response.js` cliffed — 0 term hits) | +| express · app registers middleware | 1 file, 71.6% | **byte-identical** | +| cobra · parse flags and execute | 2 files, `command.go` 40.1% | 2 files, `command.go` **77.2%** | +| cobra · DIFFUSE "main components" | 3 files, top 48.1% | 3 files, top 50.0% — spread preserved | +| gin · request reaches a handler | 3 files, top `ginS/gins.go` 48.8% | 3 files, top **`routergroup.go`** 53.8% | +| gin · DIFFUSE "what it provides" | 3 files, top `recovery.go` 43.0% | **4 files**, top `context.go` 33.3% | + +The two diffuse rows are the over-correction control: file counts hold (3→3, 3→4), so a +survey question still gets a spread. The two gin rows also moved the top file to a more apt +one — `routergroup.go` over the thin `ginS` singleton wrapper, `context.go` over +`recovery.go` — because concentration is decided by weight rather than by which file +happened to be small. + +**Exception to "no previously-unclipped file becomes clipped".** `memory-budget.ts` was +unclipped-whole at 5,672 and now clusters within its 3.1 KB reservation. That is the epic's +own diagnosis of the bug rather than a regression: it scored 18 against `tools.ts`'s 58 and +was taking the larger slice purely for being small enough to ship whole. The guarantee holds +where it was meant to — no file loses bytes to a *tighter cap*; the only files that lose are +ones the proportional split says were over-served. + ## The regression fixtures (CG-6) Two fixtures pin the failure mode so it can never silently return. They were written to -**fail** — that is what they were for. CG-10 closed the ranking half of both; the byte-split -assertions still fail and are the pass gate for CG-12. The numbers quoted below are the -**pre-CG-10 baseline**; see "Measured effect" above for where they stand now. +**fail** — that is what they were for. CG-10 closed the ranking half of both and CG-12 the +byte-split half; **both now pass** and are live regressions. The numbers quoted below are +the **pre-CG-10 baseline**; see the two "Measured effect" tables above for where they stand. They are declared in `scripts/agent-eval/allocation-fixtures.json` and run by `scripts/agent-eval/probe-allocation.mjs`, which drives the CG-4 diagnostic through a JSONL diff --git a/scripts/agent-eval/allocation-fixtures.json b/scripts/agent-eval/allocation-fixtures.json index e432753d5..faf94680e 100644 --- a/scripts/agent-eval/allocation-fixtures.json +++ b/scripts/agent-eval/allocation-fixtures.json @@ -4,12 +4,12 @@ "explore budget allocation. Run them with `node scripts/agent-eval/probe-allocation.mjs`", "against a built dist/.", "", - "STATUS: CG-10 (relevance scoring) closed the RANKING half of both fixtures — nothing", - "incidental reaches the envelope any more. What still fails is the BYTE SPLIT among the", - "files that correctly ranked in, because the render loop spends by file size: a small", - "weakly-relevant file ships whole while the strongly-relevant one is clipped at", - "maxCharsPerFile. That is CG-12's gate. Each fixture's `baseline` records the pre-CG-10", - "numbers; `afterCG10` records where it stands now.", + "STATUS: BOTH FIXTURES PASS. CG-10 (relevance scoring) closed the RANKING half —", + "nothing incidental reaches the envelope any more — and CG-12 (score-proportional", + "allocation with a relative cliff) closed the BYTE SPLIT: each file's share is reserved", + "before anything renders, and a file under 15% of the top weight gets no source at all,", + "freeing both its bytes and its maxFiles slot. Each fixture records the pre-CG-10", + "`baseline`, the interim `afterCG10`, and the current `afterCG12`.", "", "`groups` partitions the files explore rendered into `answer` (what the query is", "actually about) and `incidental` (what wins the envelope today on name collisions).", @@ -81,6 +81,19 @@ "internal/gen/fkit/payroll/payslip.go": 0.0 }, "verdict": "PASSES answerShareAtLeast (61.5%), incidentalShareAtMost (23.5%), topFileGroup, and the cycle.go + runPayrollCycleAll + real-Upsert needles. The generated files now rank #3/#4 instead of #1/#2 — kind weighting plus a 0.3x generated penalty on BOTH the score and the graph mass, which is the key the comparator sorts on. STILL FAILING for CG-12: payslip_builder.go ranks #6 against the tier's maxFiles of 4, so `func (s *Service) BuildPayslip` never renders." + }, + "afterCG12": { + "measuredOn": "2026-08-04", + "note": "19,337 delivered, nothing truncated. Both generated files cliffed to pointers (weight 3.9 and 3.5 against a cliff of 5.4), which frees the two maxFiles slots the hand-written store and builder then take.", + "delivered": { + "internal/usecase/payroll/cycle.go": 0.306, + "internal/domain/payroll/payslip.go": 0.212, + "internal/usecase/payroll/payslip_builder.go": 0.151, + "internal/store/payslipstore/store.go": 0.118, + "internal/gen/fkit/payroll/payroll_cycle.go": 0.0, + "internal/gen/fkit/payroll/payslip.go": 0.0 + }, + "verdict": "ALL GATES PASS. Answer group 78.7% (from 25.6% at baseline), generated layer 0.0% (from 57.4%). All four hand-written files deliver source, including payslip_builder.go — `func (s *Service) BuildPayslip`, the 'calculate' half of the question, finally reaches the agent. The generated files are still NAMED with their symbols and line numbers under 'Not shown above', so withholding their bytes costs ~100 chars each instead of ~4,500 and stays one follow-up explore away." } }, { @@ -130,6 +143,16 @@ "src/mcp/tools.ts": 0.329 }, "verdict": "PASSES incidentalShareAtMost: every scripts/agent-eval/*.mjs file is gone (0.0%), which is CG-10's acceptance bar — their sole match was an unused file-scope `explore`/`BUDGET` constant, worth 0.08x weight, and the relative floor (8.2) then cut them. tools.ts ranks #1. STILL FAILING for CG-12: memory-budget.ts scores 18 to tools.ts's 41 yet takes 51.2% of the envelope to tools.ts's 32.9%, purely because it is small enough to ship whole while tools.ts is clipped at maxCharsPerFile. Allocation still follows file size, not relevance." + }, + "afterCG12": { + "measuredOn": "2026-08-04", + "note": "18,134 delivered, nothing truncated. tools.ts scores 58 here (it grew by the allocator this task added), memory-budget.ts 18.", + "delivered": { + "src/mcp/tools.ts": 0.606, + "src/resolution/memory-budget.ts": 0.172, + "src/resolution/lru-cache.ts": 0.111 + }, + "verdict": "ALL GATES PASS. tools.ts takes 60.6% of the envelope, up from 18.5% at baseline and 32.9% after CG-10 — past the epic's >50% acceptance bar. The reversal is the whole point: memory-budget.ts no longer wins by being small enough to ship whole (it now clusters within its 3.1K reservation), and tools.ts is no longer clipped at maxCharsPerFile (11K reservation, ~3x the old flat cap). Exception to 'no previously-unclipped file becomes clipped': memory-budget.ts was unclipped-whole at 5,672 and is now clipped to its proportional share. That is the epic's own diagnosis of the bug, not a regression — it scored 18 against tools.ts's 58 and was taking the larger slice." } } ] diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts index 336292db8..bec8d270d 100644 --- a/src/mcp/explore-diagnostics.ts +++ b/src/mcp/explore-diagnostics.ts @@ -45,9 +45,9 @@ export type ExploreRenderMode = /** Why a ranked candidate never reached the output. */ export type ExploreSkipReason = | 'max-files' // maxFiles reached before this file - | 'budget-90pct' // incidental file past the 90%-of-budget soft stop - | 'budget-whole-file' // incidental whole-file render wouldn't fit - | 'budget-clusters' // incidental cluster render wouldn't fit + | 'cliff' // below the relevance cliff — pointer, not bytes (CG-12) + | 'budget-whole-file' // whole-file render wouldn't fit under the hard ceiling + | 'budget-clusters' // cluster render wouldn't fit under the hard ceiling | 'unreadable' // outside root, missing, or read error | 'no-ranges'; // no renderable line ranges in this file @@ -82,6 +82,14 @@ export interface ExploreCandidateMeta { interface FileRecord extends ExploreCandidateMeta { path: string; + /** + * Chars this file was RESERVED by the proportional allocator (CG-12), before + * it rendered anything. `0` = cliffed; `null` = never reached the allocator. + * The gap between this and `emittedChars` is the whole story of a budget bug: + * reserved-but-unspent means the file had nothing to say, spent-over-reserved + * means an oversize first cluster or the whole-file grace overshot. + */ + allowance: number | null; render?: ExploreRenderMode; /** Source chars the render loop handed to `lines` (pre-final-truncation). */ emittedChars: number; @@ -117,6 +125,7 @@ interface BudgetShape { /** One file's line in the report. Also the JSONL sidecar's per-file shape. */ export interface ExploreDiagnosticFile extends ExploreCandidateMeta { path: string; + allowance: number | null; render: ExploreRenderMode | null; skipped: ExploreSkipReason | null; clipped: boolean; @@ -163,6 +172,17 @@ export interface ExploreDiagnosticReport { filesRenderedByLoop: number; filesInFinalOutput: number; }; + /** The proportional split (CG-12): what each file was promised, and why. */ + allocation: { + /** Chars divided among admitted files (envelope minus per-file overhead). */ + pool: number; + /** Weight threshold the cliff fired at; 0 when nothing was cliffed. */ + cliffAt: number; + /** Files given zero source — pointers in the not-shown list instead. */ + cliffed: string[]; + /** Sum of reservations. Must not exceed `pool`. */ + reserved: number; + }; files: ExploreDiagnosticFile[]; } @@ -201,6 +221,9 @@ export class ExploreDiagnostics { private graphGateThreshold = 0; private graphGateApplied = false; private note = ''; + private allocPool = 0; + private allocCliffAt = 0; + private allocCliffed: string[] = []; private constructor( private readonly sink: Sink, @@ -260,11 +283,34 @@ export class ExploreDiagnostics { /** Record one ranked candidate's scoring inputs, in final sort order. */ noteCandidate(path: string, meta: ExploreCandidateMeta): void { this.files.set(path, { - path, ...meta, + path, ...meta, allowance: null, emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false, }); } + /** + * Record the proportional split (CG-12), taken right after ranking and before + * a single byte renders. Called once per explore. + */ + setAllocation( + allowances: ReadonlyMap, + cliffed: readonly string[], + cliffAt: number, + pool: number, + ): void { + this.allocPool = pool; + this.allocCliffAt = cliffAt; + this.allocCliffed = [...cliffed]; + for (const [path, chars] of allowances) { + const rec = this.files.get(path); + if (rec) rec.allowance = chars; + } + for (const path of cliffed) { + const rec = this.files.get(path); + if (rec) rec.allowance = 0; + } + } + /** A candidate rendered source into the response. */ recordRender(path: string, render: ExploreRenderMode, sourceChars: number, clipped: boolean): void { const rec = this.files.get(path); @@ -366,6 +412,12 @@ export class ExploreDiagnostics { filesRenderedByLoop: filesIncluded, filesInFinalOutput: rendered.length, }, + allocation: { + pool: this.allocPool, + cliffAt: round6(this.allocCliffAt), + cliffed: [...this.allocCliffed], + reserved: records.reduce((s, r) => s + (r.allowance ?? 0), 0), + }, files: records .slice() .sort((a, b) => b.emittedChars - a.emittedChars || b.finalChars - a.finalChars || a.rank - b.rank) @@ -384,6 +436,7 @@ export class ExploreDiagnostics { generated: r.generated, penalty: round6(r.penalty), kinds: r.kinds, + allowance: r.allowance, render: r.render ?? null, skipped: r.skipped ?? null, clipped: r.clipped, @@ -492,6 +545,14 @@ export function renderTable(report: ExploreDiagnosticReport): string { ` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` + ` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`, ); + const alloc = report.allocation; + out.push( + ` allocation ${num(alloc.reserved)} reserved of ${num(alloc.pool)} pool` + + ` · cliff at weight ${alloc.cliffAt.toFixed(2)}` + + (alloc.cliffed.length > 0 + ? ` · ${alloc.cliffed.length} cliffed to pointers: ${alloc.cliffed.join(', ')}` + : ' · nothing cliffed'), + ); out.push(''); // Allocated (not delivered) is the allocator's own decision — the number the @@ -499,7 +560,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { // when the ceiling truncated; showing both makes that divergence obvious. const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0); if (shown.length > 0) { - out.push(' # alloc% deliv% bytes score graph hits pen flags render file'); + out.push(' # alloc% deliv% bytes reserved score graph hits pen flags render file'); for (const f of shown) { out.push( ' ' + @@ -507,6 +568,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { pct(f.allocatedShare).padStart(6) + ' ' + pct(f.share).padStart(6) + ' ' + num(f.emittedChars).padStart(7) + ' ' + + (f.allowance === null ? '-' : num(f.allowance)).padStart(8) + ' ' + f.score.toFixed(1).padStart(5) + ' ' + f.graphScore.toFixed(5).padStart(7) + ' ' + String(f.termHits).padStart(4) + ' ' + @@ -531,7 +593,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { out.push( ` #${String(f.rank).padStart(2)} ${f.path} — ${f.skipped ?? f.render ?? 'not reached'}` + ` (score ${f.score.toFixed(1)}, graph ${f.graphScore.toFixed(5)}, hits ${f.termHits},` + - ` pen ${f.penalty.toFixed(2)}, ${f.kinds || '-'})`, + ` pen ${f.penalty.toFixed(2)}, ${flagString(f) || 'no flags'}, ${f.kinds || '-'})`, ); } if (skipped.length > 15) out.push(` … and ${skipped.length - 15} more`); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 95812af9e..782ccc3f3 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -425,6 +425,215 @@ const SCORE_FLOOR_MAX = 10; */ const SCORE_FLOOR_KEEP_MIN = 3; +// ── Score-proportional byte allocation (CG-12 / #1500) ───────────────────── +// +// The score floor above decides WHICH files reach the response. This decides how +// the byte envelope is SPLIT among them — and until this existed, it wasn't +// really decided at all: every admitted file was capped at the same +// `maxCharsPerFile`, and the whole-file rule handed anything under +// `maxCharsPerFile * 3` its entire contents. So allocation followed FILE SIZE, +// not relevance. On this repo's own "how does explore allocate its output budget +// across files", `src/mcp/tools.ts` (score 41, 4x the graph mass, 3x the distinct +// term hits — it literally holds the allocator) was clipped at 3,800 while a +// score-18 file shipped whole at 5,672 and took 51% of the envelope, purely for +// being small. On the #1500 Go fixture, two generated CRUD files shipped whole at +// ~4.5K each and consumed the tier's 4 file slots, so `BuildPayslip` — the +// hand-written half of "create and calculate payslips" — never appeared at all. +// +// The replacement: reserve each file a share of the envelope proportional to what +// it is worth, up front, before anything renders. Three consequences: +// +// 1. A reservation is a GUARANTEE, not a race. The old loop spent the envelope +// first-come-first-served in rank order, so the top two files could exhaust +// it and every later file hit a `budget-90pct` skip regardless of merit. +// 2. A file below the cliff gets ZERO source — its path, symbols and line +// numbers only. It costs ~100 chars instead of ~4,500, and (crucially) it +// does not consume a `maxFiles` slot, so the slot goes to a file that earns +// its bytes. This is the concentration lever. +// 3. The per-file cap stops being the primary guard. It survives only as +// `ALLOC_MAX_SHARE`, a safety valve against a single god-file — which the +// proportional split already bounds, since a file's share can't exceed its +// weight share. +const EXPLORE_ALLOCATION = { + /** + * A file whose weight is under this fraction of the top file's gets no source. + * + * Calibrated between the two shapes the fixtures pin: the #1500 generated CRUD + * lands at 10–11% of the top weight (penalised twice — once into the score by + * `rankPenalty`, once again here) and must cliff; a genuinely peripheral but + * hand-written flow file — `payslip_builder.go`, the direct callee of the + * workflow entry — lands at 25% and must NOT. Everything in between is a + * judgement call the agent can undo for ~0 cost, because a cliffed file is + * still NAMED in the response and one follow-up explore fetches it. + */ + CLIFF_FRACTION: 0.15, + /** + * Ceiling on the cliff, in the same units as `SCORE_FLOOR_MAX` — and for the + * same reason. A file whose weight clears a full-strength direct match is never + * incidental, so no amount of concentration elsewhere may zero it: one + * overwhelming top file (a 99-scoring god-file among score-10 peers) otherwise + * puts the cliff at 14.9 and silences every peer the score floor had just + * deliberately admitted. The cliff is a RELATIVE prune of weak evidence, not a + * second admission gate — the score floor already owns admission. + */ + CLIFF_MAX: SCORE_FLOOR_MAX, + /** + * Floor on a useful reservation — every admitted file gets this much before + * the proportional split divides the rest. Under it a slice can't hold one + * complete method, and a fragment is strictly worse than a pointer: it forces + * the Read this tool exists to prevent. + * + * It is a FLOOR, not a second cliff. Cliffing the starved file instead + * cascades: removing the smallest raises everyone else's share by so little + * that the next-smallest starves too, and a query with two dominant files ate + * six legitimately-ranked peers one at a time. Concentration is the relative + * cliff's job; this only keeps a served file's slice usable. + */ + MIN_CHARS: 700, + /** + * Safety valve, as a fraction of the envelope. Not the primary guard any more — + * the proportional split is — so this only has to stop a pathological + * single-file response. + */ + MAX_SHARE: 0.7, + /** + * Markdown overhead charged per rendered file (header + fences + blank lines), + * matching the render loop's own `+ 200` accounting. Held out of the pool + * before the split so the reservations plus their overhead fit the envelope — + * without this the last file's reservation is always the one that doesn't fit. + */ + FILE_OVERHEAD: 200, + /** + * Flow-spine files are weighted up and are exempt from the cliff. Clipping the + * spine causes the Read fallback (it IS the answer to a flow question); + * clipping a peripheral file does not. This makes the existing advisory spine + * handling — `hasSpine`, `SPINE_CEILING` — strict at the allocation layer. + */ + SPINE_WEIGHT_BOOST: 2, + /** + * Slack allowed on the whole-file rule: a file a little over its reservation + * still ships WHOLE rather than as clusters, because slicing off that last + * sliver saves ~1% of the envelope and costs a Read — the trade the whole-file + * rule exists to refuse. Proportional (with an absolute ceiling) because a + * "sliver" is relative: a flat 800 is 15% of a 5K reservation but 31% of a 2.5K + * one, and at the small end that overshoot is exactly what the file below then + * loses. + */ + WHOLE_FILE_GRACE_FRACTION: 0.15, + WHOLE_FILE_GRACE_MAX: 800, +} as const; + +/** One candidate file's allocation inputs, in final rank order. */ +export interface ExploreAllocationCandidate { + path: string; + /** Post-`rankPenalty` relevance score from the ranking pass. */ + score: number; + /** + * How much this file's BYTES are worth, independent of how well it matched. + * Ranking answers "is this file about the query"; allocation answers "will + * these bytes teach the agent anything". Generated CRUD can legitimately rank + * (it name-collides on every domain word) while its bytes stay mechanical + * boilerplate the agent gains nothing from reading — so `rankPenalty` is + * applied a SECOND time here. That is what finally sinks the #1500 generated + * layer below the cliff: it survived CG-10's single penalty because the sort's + * leading keys (entry-point, graph mass) are structural, and a big densely + * self-referential generated file scores well on both. + */ + worth: number; + /** Carries a symbol on the rendered flow spine. */ + spine: boolean; +} + +export interface ExploreAllocation { + /** path → chars of source it may render. Only holds admitted files. */ + allowances: Map; + /** Files the cliff zeroed, in rank order — pointers, not bytes. */ + cliffed: string[]; + /** The weight threshold the cliff fired at (0 when nothing was cliffed). */ + cliffAt: number; + /** Chars actually split among the admitted files. */ + pool: number; +} + +/** + * Split `budget.maxOutputChars` across ranked candidates in proportion to + * relevance, with a hard relative cliff. + * + * `candidates` must arrive in FINAL RANK ORDER — `maxFiles` is applied to the + * survivors of the cliff, in that order, so cliffing genuinely hands a slot to + * the next file down rather than leaving it unused. + * + * Tier invariant (`getExploreOutputBudget`): a larger tier must never allow less + * per file than a smaller one. It holds here by construction — every bound is a + * fraction of `maxOutputChars` or of `maxCharsPerFile`, both monotonic across + * tiers — except `MIN_CHARS`, which is an absolute floor and so identical at + * every tier. + */ +export function allocateExploreBudget( + candidates: readonly ExploreAllocationCandidate[], + budget: ExploreOutputBudget, + maxFiles: number, +): ExploreAllocation { + const A = EXPLORE_ALLOCATION; + const empty: ExploreAllocation = { allowances: new Map(), cliffed: [], cliffAt: 0, pool: 0 }; + if (candidates.length === 0) return empty; + + const weightOf = (c: ExploreAllocationCandidate) => + Math.max(0, c.score) * Math.max(0, Math.min(1, c.worth)) * (c.spine ? A.SPINE_WEIGHT_BOOST : 1); + + const weights = new Map(candidates.map((c) => [c.path, weightOf(c)])); + const topWeight = Math.max(...weights.values()); + if (!(topWeight > 0)) return empty; + + // Cliff over the WHOLE candidate list, before `maxFiles` — otherwise the file + // cap fills with cliff-bound files and the slot they free is never handed on. + const cliffAt = Math.min(topWeight * A.CLIFF_FRACTION, A.CLIFF_MAX); + const cliffed: string[] = []; + let admitted: ExploreAllocationCandidate[] = []; + for (const c of candidates) { + if (!c.spine && (weights.get(c.path) ?? 0) < cliffAt) cliffed.push(c.path); + else admitted.push(c); + } + // Never cliff every candidate: an empty response costs a whole round-trip. + if (admitted.length === 0) { + admitted = [candidates[0]!]; + cliffed.splice(cliffed.indexOf(candidates[0]!.path), 1); + } + for (const c of admitted.slice(maxFiles)) cliffed.push(c.path); + admitted = admitted.slice(0, maxFiles); + + // Serve fewer files well rather than many badly: the envelope has to afford + // MIN_CHARS for everything admitted. When it can't, cliff the lowest-weight + // files (never a spine file, never the last one) in one deterministic trim — + // not one at a time, which is how the old starvation rule snowballed. + const affordable = Math.max(1, Math.floor(budget.maxOutputChars / (A.MIN_CHARS + A.FILE_OVERHEAD))); + if (admitted.length > affordable) { + const byWeight = [...admitted].sort((a, b) => (weights.get(b.path) ?? 0) - (weights.get(a.path) ?? 0)); + const keep = new Set(byWeight.slice(0, affordable).map((c) => c.path)); + for (const c of admitted) if (c.spine) keep.add(c.path); + for (const c of admitted) if (!keep.has(c.path)) cliffed.push(c.path); + admitted = admitted.filter((c) => keep.has(c.path)); + } + + const allowances = new Map(); + const pool = Math.max(0, budget.maxOutputChars - A.FILE_OVERHEAD * admitted.length); + const total = admitted.reduce((s, c) => s + (weights.get(c.path) ?? 0), 0); + if (total <= 0 || admitted.length === 0) return { allowances, cliffed, cliffAt, pool }; + // Everyone gets MIN_CHARS; the REMAINDER is what splits by weight. The floor + // is what keeps a diffuse survey question returning a useful spread, and the + // remainder is what concentrates a precise one — the top file's slice grows + // with its weight share, uncapped by any flat per-file limit. + const ceiling = Math.round(budget.maxOutputChars * A.MAX_SHARE); + const floors = Math.min(pool, A.MIN_CHARS * admitted.length); + const remainder = Math.max(0, pool - floors); + for (const c of admitted) { + const share = Math.floor(floors / admitted.length) + + Math.round((remainder * (weights.get(c.path) ?? 0)) / total); + allowances.set(c.path, Math.min(share, ceiling)); + } + return { allowances, cliffed, cliffAt, pool }; +} + /** * Whether `codegraph_explore` should prefix source lines with their line * numbers (cat -n style: `\t`). @@ -3497,6 +3706,26 @@ export class ToolHandler { }); } + // Score-proportional byte allocation (CG-12). Every file's share of the + // envelope is reserved HERE, before a single byte renders, so the render loop + // spends a reservation instead of racing for whatever the files above it left. + const allocation = allocateExploreBudget( + sortedFiles.map(([fp, group]) => ({ + path: fp, + score: group.score, + worth: rankPenalty(fp), + spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), + })), + budget, + maxFiles, + ); + diag?.setAllocation(allocation.allowances, allocation.cliffed, allocation.cliffAt, allocation.pool); + // Cliffed files ship as pointers — path, symbols, line numbers — so the agent + // can name one in a follow-up explore. Rendered below with the other + // not-shown files, and force-enabled even on tiers that suppress that list: + // a file we deliberately withheld source for must still be nameable. + const cliffedFiles = new Set(allocation.cliffed); + // Polymorphic-sibling detector for adaptive sizing. A class that implements/ // extends a supertype shared by >= MIN_SIBLINGS classes is one of many // INTERCHANGEABLE implementations (OkHttp's 14 `: Interceptor` classes — @@ -3557,6 +3786,13 @@ export class ToolHandler { lines.push('> The code below is the **verbatim, current on-disk source** of these files — re-read from disk on this call and line-numbered, byte-for-byte identical to what the Read tool returns. It is NOT a summary, outline, or stale cache. Treat each block as a Read you have already performed: do not Read a file shown here.'); lines.push(''); + // Absolute stop for the render loop. Reservations already fit the envelope, so + // this only catches their bounded overshoot (the whole-file grace, an oversize + // first cluster) — and catches it HERE, where a file can be skipped cleanly and + // a later one still render, instead of at the final truncation, which lops off + // whichever section happened to land last. Kept in sync with `hardCeiling` + // below; the margin covers the drift epilogue and the trailing notes. + const renderCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), 25000) - 600; let totalChars = lines.join('\n').length; let filesIncluded = 0; // Paths we actually render source for below. Drives the curated header count @@ -3577,19 +3813,22 @@ export class ToolHandler { if (diag) for (const [fp] of sortedFiles) diag.recordSkip(fp, 'max-files'); break; } - // A file DEFINES a named/spine symbol (the answer) vs merely references the - // flow. Past 90% budget, stop pulling INCIDENTAL files — but keep scanning - // for necessary ones, which render even past the cap (bounded by maxFiles). - // Without this `continue` (was an unconditional `break`), the loop stopped - // after the build + validators-exec files and never reached the ranked-in - // validate-logic file (Alamofire's Validation.swift). - const fileNecessary = group.nodes.some(n => - entryNodeIds.has(n.id) || flow.pathNodeIds.has(n.id) || flow.uniqueNamedNodeIds.has(n.id)); - if (!fileNecessary && totalChars > budget.maxOutputChars * 0.9) { - diag?.recordSkip(filePath, 'budget-90pct'); + // Below the relevance cliff: no source, no `maxFiles` slot. It is still + // named — with its matched symbols and their line numbers — in the + // not-shown list, so one follow-up explore fetches it in full. + if (cliffedFiles.has(filePath)) { + diag?.recordSkip(filePath, 'cliff'); + continue; + } + // This file's reserved share of the envelope. Every render path below is + // bounded by it instead of by the flat per-file cap, which is what stops + // allocation from following file size: a small weakly-relevant file no + // longer ships whole while the strongly-relevant one is clipped. + const allowance = allocation.allowances.get(filePath); + if (allowance === undefined) { + diag?.recordSkip(filePath, 'max-files'); continue; } - const absPath = validatePathWithinRoot(projectRoot, filePath); if (!absPath || !existsSync(absPath)) { diag?.recordSkip(filePath, 'unreadable'); @@ -3648,7 +3887,7 @@ export class ToolHandler { .filter(n => CALLABLE_BODY.has(n.kind) && (flow.pathNodeIds.has(n.id) || flow.uniqueNamedNodeIds.has(n.id))) .reduce((s, n) => s + fileLines.slice(n.startLine - 1, n.endLine).join('\n').length, 0); const onSpineGodFile = hasSpineNode - && namedBodyChars > budget.maxCharsPerFile + && namedBodyChars > allowance && group.nodes.some(n => CALLABLE_BODY.has(n.kind) && flow.uniqueNamedNodeIds.has(n.id) && !flow.pathNodeIds.has(n.id)); if (!fileStale && adaptiveExploreEnabled() && flow.pathNodeIds.size > 0 && (onSpineGodFile || (!hasSpineNode && isPolymorphicSibling(group.nodes) && !spared))) { @@ -3666,14 +3905,14 @@ export class ToolHandler { : flow.pathNodeIds.has(n.id) ? 0 : flow.uniqueNamedNodeIds.has(n.id) ? 1 : (fileDefinesSuper && flow.namedNodeIds.has(n.id)) ? 2 : 99; - // One ~250-line WINDOW per file. syms are taken by priority (spine first, - // then uniquely-named, then family-base), and the cap applies to ALL of - // them — including the spine — so a big-spine god-file (tokio's worker.rs: - // run→run_task→next_task→steal_work) can't eat the whole response and - // starve the co-flow file (harness.rs's poll). The native agent windows - // such a file too (~190 lines at a time), so this mimics, not truncates. - // Always emit ≥1 (never an empty section). - const bodyCap = budget.maxCharsPerFile * 1.5; + // One WINDOW per file, sized by this file's RESERVATION. syms are taken by + // priority (spine first, then uniquely-named, then family-base), and the cap + // applies to ALL of them — including the spine — so a big-spine god-file + // (tokio's worker.rs: run→run_task→next_task→steal_work) can't eat the whole + // response and starve the co-flow file (harness.rs's poll). The native agent + // windows such a file too (~190 lines at a time), so this mimics, not + // truncates. Always emit ≥1 (never an empty section). + const bodyCap = allowance; const bodyIds = new Set(); let bodyChars = 0; for (const n of syms.filter(n => prio(n) < 99 && n.endLine >= n.startLine).sort((a, b) => prio(a) - prio(b))) { @@ -3746,16 +3985,19 @@ export class ToolHandler { // the ceiling and falls through to sectioning/clustering below — full method // bodies + signatures — so we never dump (or overflow on) a whole god-file. const isCentralFile = centralFiles.has(filePath); - // Central files get a slightly larger whole-file window than peripheral ones, - // but a TIGHT one (~1.5× the per-file cap): the native read of a central file - // is a ~150–250 line orientation window, NOT the whole file. A flat "whole - // central file" both overflowed the inline cap AND starved the co-flow files - // (worker.rs ate the budget, dropping harness.rs's poll). A larger central - // file falls through to per-method windowing/clustering below. + // A file ships whole when it fits its RESERVATION (plus a small grace — see + // WHOLE_FILE_GRACE). This is the site of the #1500 allocation bug: the + // peripheral bound used to be a flat `maxCharsPerFile * 3`, so ANY file under + // ~11K shipped its entire contents regardless of relevance, while a + // high-scoring file too big for that window was clipped to `maxCharsPerFile` + // — a 3x swing decided by file size alone. Tying both bounds to the + // reservation removes the swing without touching the rule's purpose (a small + // file sliced is a lossy subset the agent just Reads in full anyway). const WHOLE_FILE_MAX_LINES = isCentralFile ? 280 : 220; - const WHOLE_FILE_MAX_CHARS = isCentralFile - ? Math.min(Math.max(0, budget.maxOutputChars - totalChars - 200), Math.round(budget.maxCharsPerFile * 1.5)) - : budget.maxCharsPerFile * 3; + const WHOLE_FILE_MAX_CHARS = allowance + Math.min( + EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_MAX, + Math.round(allowance * EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_FRACTION), + ); if (fileLines.length <= WHOLE_FILE_MAX_LINES && fileContent.length <= WHOLE_FILE_MAX_CHARS) { const body = fileContent.replace(/\n+$/, ''); let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body; @@ -3772,10 +4014,9 @@ export class ToolHandler { const staleSuffix = fileStale ? ' · ⚠ changed since last index sync — source below is current; the symbol list may be outdated' : ''; const wholeHeader = fileSectionHeader(filePath, (omitted > 0 ? `${headerNames.join(', ')}, +${omitted} more` : headerNames.join(', ')) + staleSuffix); - if (!fileNecessary && totalChars + wholeSection.length + 200 > budget.maxOutputChars) { - // Don't slice a whole file mid-method: an incidental file that doesn't - // fit is skipped; a necessary one (below) renders in full. Half a file - // forces the Read this is meant to prevent. + if (totalChars + wholeSection.length + 200 > renderCeiling) { + // Don't slice a whole file mid-method — a file that doesn't fit is + // skipped whole. Half a file forces the Read this is meant to prevent. anyFileTrimmed = true; diag?.recordSkip(filePath, 'budget-whole-file'); continue; @@ -3883,8 +4124,16 @@ export class ToolHandler { } const gapThreshold = budget.gapThreshold; - const clusters: Array<{ start: number; end: number; symbols: string[]; score: number; maxImportance: number; hasSpine: boolean; spineCallLine?: number }> = []; - let current = { + type ExploreRange = typeof ranges[number]; + type ExploreCluster = { + start: number; end: number; symbols: string[]; score: number; + maxImportance: number; hasSpine: boolean; spineCallLine?: number; + /** The whole symbol ranges this cluster merged — the unit an oversize + * cluster is shrunk by, so shrinking never cuts through a body. */ + members: ExploreRange[]; + }; + const clusters: ExploreCluster[] = []; + let current: ExploreCluster = { start: ranges[0]!.start, end: ranges[0]!.end, symbols: [`${ranges[0]!.name}(${ranges[0]!.kind})`], @@ -3892,6 +4141,7 @@ export class ToolHandler { maxImportance: ranges[0]!.importance, hasSpine: ranges[0]!.spine, spineCallLine: ranges[0]!.spineCallLine, + members: [ranges[0]!], }; for (let i = 1; i < ranges.length; i++) { @@ -3903,6 +4153,7 @@ export class ToolHandler { current.maxImportance = Math.max(current.maxImportance, r.importance); current.hasSpine = current.hasSpine || r.spine; current.spineCallLine = current.spineCallLine ?? r.spineCallLine; + current.members.push(r); } else { clusters.push(current); current = { @@ -3913,6 +4164,7 @@ export class ToolHandler { maxImportance: r.importance, hasSpine: r.spine, spineCallLine: r.spineCallLine, + members: [r], }; } } @@ -3962,6 +4214,50 @@ export class ToolHandler { return withLineNumbers ? numberSourceLines(slice, startIdx + 1) : slice; }; + /** + * Shrink an oversize cluster to the highest-importance symbols inside it + * that fit `cap`, rendered in source order with gap markers (CG-12). + * + * A cluster is a MERGE of whole symbol ranges, and on a densely-packed file + * every symbol merges into one blob spanning the file — cycle.go's 209-line + * `Service` is one cluster covering `RunCycle`, `runPayrollCycleAll` and + * seven incidental accessors. The old rule took the top-ranked cluster whole + * however big it was, so a single-cluster file simply ignored its budget: + * it took ~40% more than it was allotted, and the file below it was then + * dropped for lack of room (that is how `BuildPayslip` — the "calculate" + * half of the #1500 query — went missing entirely). Shrinking by MEMBER + * keeps every rule that matters: only whole symbol ranges are emitted, so a + * body is never cut, and the members are chosen by the same importance the + * cluster ranking uses. Returns null when nothing needed shrinking. + */ + const shrinkCluster = (c: ExploreCluster, cap: number): string | null => { + if (c.members.length < 2) return null; + const byImportance = [...c.members].sort((a, b) => + b.importance - a.importance || (a.end - a.start) - (b.end - b.start) || a.start - b.start); + const sizeOf = (r: ExploreRange) => fileLines.slice(r.start - 1, r.end).join('\n').length; + const keep: ExploreRange[] = []; + let kept = 0; + for (const r of byImportance) { + const sz = sizeOf(r) + GAP_MARKER.length; + // Always keep the most important range, even if it alone is oversize — + // an empty section sends the agent to Read, which costs far more. + if (keep.length > 0 && kept + sz > cap) continue; + keep.push(r); + kept += sz; + } + if (keep.length === c.members.length) return null; + // Re-merge the kept ranges in source order so adjacent survivors read as + // one block rather than a stutter of one-symbol fragments. + keep.sort((a, b) => a.start - b.start); + const merged: Array<{ start: number; end: number }> = []; + for (const r of keep) { + const last = merged[merged.length - 1]; + if (last && r.start <= last.end + gapThreshold) last.end = Math.max(last.end, r.end); + else merged.push({ start: r.start, end: r.end }); + } + return merged.map((m) => buildSection(m)).join(GAP_MARKER); + }; + // Rank clusters for inclusion under the per-file cap. Entry-point // clusters come first: a cluster containing a query entry point // (importance 10) must outrank a dense block of mere declarations, @@ -3988,28 +4284,41 @@ export class ToolHandler { return a.span - b.span; }); - // Per-file budget is the SMALLER of the per-file cap and what's left of the - // total output cap — so selection (which ranks by importance) keeps the + // Per-file budget is this file's RESERVATION, bounded by what's left before + // the hard ceiling — so selection (which ranks by importance) keeps the // high-importance clusters and drops peripheral ones, instead of the // downstream source-order trim slicing off whatever comes last in the file. // That source-order slice is what cut Django's `_fetch_all` (L2237, importance // 9 — agent-named) when query.py was the last of four big files to be emitted. - const fileBudget = Math.min(budget.maxCharsPerFile, Math.max(0, budget.maxOutputChars - totalChars - 200)); - // Spine ceiling: a flow-path cluster may exceed the per-file cap (the call - // path is the answer), but bounded — at most ~2.5× the per-file cap and never - // past what's left of the total output cap — so a pathological long in-file + // It used to be `min(maxCharsPerFile, remaining)`: a flat cap that clipped the + // top-scoring file at the same 3,800 as the weakest one, while the whole-file + // branch above handed a small file 3x that. The reservation is the whole point + // of CG-12 — bytes follow relevance, not file size. + const headroom = Math.max(0, renderCeiling - totalChars - 200); + const fileBudget = Math.min(allowance, headroom); + // Spine ceiling: a flow-path cluster may exceed the reservation (the call path + // IS the answer and clipping it forces the Read), but bounded — 1.5x the + // reservation and never past the ceiling — so a pathological long in-file // spine can't run away or starve co-flow files entirely. - const SPINE_CEILING = Math.min(budget.maxCharsPerFile * 2.5, Math.max(0, budget.maxOutputChars - totalChars - 200)); + const SPINE_CEILING = Math.min(Math.round(allowance * 1.5), headroom); const chosenIndices = new Set(); + // Shrunk renders for oversize clusters, by cluster index (CG-12). Computed + // during selection and reused at emission so the two never disagree. + const shrunkSections = new Map(); let projectedChars = 0; for (const rc of rankedClusters) { const sectionLen = buildSection(rc.c).length + (chosenIndices.size > 0 ? GAP_MARKER.length : 0); - // Always take the top-ranked cluster, even if oversize, so we don't - // return an empty file section (agent would then re-Read the file, - // negating the savings). + // The top-ranked cluster is always taken — an empty file section sends the + // agent to Read, negating the savings. But "always taken" is not "taken at + // any size": when it overruns the reservation it is SHRUNK to the + // highest-importance whole symbol ranges inside it, so a single-cluster + // god-file spends its allotment instead of the whole response's. if (chosenIndices.size === 0) { + const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget; + const shrunk = sectionLen > cap ? shrinkCluster(rc.c, cap) : null; + if (shrunk !== null) shrunkSections.set(rc.idx, shrunk); chosenIndices.add(rc.idx); - projectedChars += sectionLen; + projectedChars += shrunk !== null ? shrunk.length : sectionLen; continue; } // A spine cluster (the rendered call path) is the flow answer — include it @@ -4028,18 +4337,19 @@ export class ToolHandler { for (let i = 0; i < clusters.length; i++) { if (!chosenIndices.has(i)) continue; const cluster = clusters[i]!; - const section = buildSection(cluster); + const section = shrunkSections.get(i) ?? buildSection(cluster); if (fileSection.length > 0) fileSection += GAP_MARKER; fileSection += section; allSymbols.push(...cluster.symbols); } - // A chosen cluster is a COMPLETE method-range — we never cut through a body. - // An oversize single cluster (a long monolithic function) renders in FULL: - // half a method is useless (the agent just Reads the rest for the other half), - // which is the very fallback explore exists to prevent. A pathological file is - // bounded by the per-file cluster SELECTION above + the total hard ceiling. - if (chosenIndices.size < clusters.length) { + // A chosen cluster is a COMPLETE method-range — we never cut through a body, + // and a shrunk cluster drops WHOLE members for the same reason. An oversize + // single MEMBER (one long monolithic function) still renders in full: half a + // method is useless (the agent just Reads the rest for the other half), which + // is the very fallback explore exists to prevent. A pathological file is + // bounded by the cluster SELECTION above + the total hard ceiling. + if (chosenIndices.size < clusters.length || shrunkSections.size > 0) { anyFileTrimmed = true; } @@ -4062,20 +4372,16 @@ export class ToolHandler { : headerSymbols.join(', '); const fileHeader = fileSectionHeader(filePath, headerSuffix); - // The total cap bounds INCIDENTAL files only. A file that DEFINES a symbol - // the agent named (or that's on the flow spine) renders even when the - // nominal total is used up — it's the answer, and the set is bounded by - // maxFiles AND by true-spine/named-seeding having already trimmed each file - // to its necessary content. A file that merely REFERENCES the flow - // (Combine.swift name-drops request/task) is incidental → still capped, so - // freed budget never leaks into noise. This is the last god-file layer: - // build (Session, true-spined) + validators-exec (Request) + validate - // (DataRequest/Validation) all render, instead of the cap dropping whichever - // phase the file order happened to put last. - if (!fileNecessary && totalChars + fileSection.length + 200 > budget.maxOutputChars) { - // Incidental file that doesn't fit: SKIP it whole — never slice mid-method. - // Keep scanning for necessary files (which bypass this cap and render in - // full, bounded by the hard ceiling). + // Last stop before the hard ceiling. The reservation already bounded cluster + // selection above, so reaching this means the bounded overshoot (an oversize + // first cluster, taken whole rather than sliced mid-method) ran the response + // out of room. Skip the file whole and keep scanning — never slice mid-method. + // This used to compare against `maxOutputChars` and exempt "necessary" files, + // which is how arrival order decided the answer: whichever files ranked first + // spent the envelope, and everything after them was dropped on a cap they had + // no say in. Reservations replace that exemption — a file that earned bytes + // was already given them. + if (totalChars + fileSection.length + 200 > renderCeiling) { anyFileTrimmed = true; diag?.recordSkip(filePath, 'budget-clusters'); continue; @@ -4118,9 +4424,16 @@ export class ToolHandler { // Add remaining files as references (from both relevant and peripheral files). // Small projects (per budget) skip this — the relevant story already fits - // in the source section, and a trailing pointer list is pure overhead. - if (budget.includeAdditionalFiles) { - const remainingRelevant = sortedFiles.slice(filesIncluded); + // in the source section, and a trailing pointer list is pure overhead. But a + // CLIFFED file is source we deliberately withheld, so the list is forced on + // whenever there is one: withholding a file's bytes is only cheap if the agent + // can still name it in a follow-up call (CG-12). + if (budget.includeAdditionalFiles || cliffedFiles.size > 0) { + // Everything ranked that didn't render, in rank order — cliffed files first, + // since they outrank whatever the file cap cut. (Indexing by `filesIncluded` + // would be wrong now that cliffed files are skipped without consuming a slot.) + const rendered = new Set(renderedFilePaths); + const remainingRelevant = sortedFiles.filter(([fp]) => !rendered.has(fp)); // Ranked files are already covered by `remainingRelevant`; the rest of the // gather (below the floor) becomes the pointer list. The Set guards the // one overlap case — a file the SCORE_FLOOR_KEEP_MIN fallback pulled in @@ -4133,8 +4446,17 @@ export class ToolHandler { if (remainingFiles.length > 0) { lines.push('**Not shown above — explore these names for their source**'); lines.push(''); + // A pointer only has to make the file NAMEABLE in a follow-up call, so cap + // the symbols per line: an un-capped list ran to ~1.9K on the #1500 fixture + // (12 generated CRUD symbols on one line), meta-text bought at the price of + // the source bytes this section exists to point away from. + const POINTER_SYMBOLS = 6; for (const [filePath, group] of remainingFiles.slice(0, 10)) { - const symbols = group.nodes.map(n => `${n.name}:${n.startLine}`).join(', '); + const named = group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export'); + const shown = (named.length > 0 ? named : group.nodes).slice(0, POINTER_SYMBOLS); + const more = (named.length > 0 ? named : group.nodes).length - shown.length; + const symbols = shown.map(n => `${n.name}:${n.startLine}`).join(', ') + + (more > 0 ? `, +${more} more` : ''); lines.push(`- ${filePath}: ${symbols}`); } if (remainingFiles.length > 10) { From 1d9206d2d0e96f3f2284b92264bcec9d4791869d Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 00:56:50 -0500 Subject: [PATCH 06/15] test(explore): lock down proportional byte allocation (CG-14, #1500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage for the CG-12 allocator, built around "would this go red if the lever were removed" rather than line coverage — every way this regresses is silent, ending in an agent falling back to Read. Unit (`explore-proportional-allocation.test.ts`, 18 -> 38): calibration pins, envelope safety across every tier and 30 candidate shapes, the cliff boundary, spine weighting/trim survival, the diffuse control, and the degenerate inputs — identical scores, a lone file, a runaway top scorer, zero results, maxFiles 0, a non-finite score. End-to-end (`explore-allocation-e2e.test.ts`, new): CG-6's second regression fixture as a deterministic synthetic mirror — a large relevant file, a small helper that used to win by shipping whole, and an incidental `explore`/`BUDGET` collision — asserting per-file budget share, not file presence. Plus degenerate result sets and a survey-style diffuse control through the real render loop. The live self-query arm stays in probe-allocation.mjs, where drift is a number to re-baseline rather than a red suite. Reverting the render loop to the pre-CG-12 rules reproduces #1500 on the mirror exactly and takes 5 e2e + 2 payroll gates red: file score pre-CG-12 CG-12 src/mcp/allocator.ts 77.5 4,843 (39.7%) 9,335 (80.1%) src/util/budget-math.ts 36.0 6,079 (49.8%) 1,037 ( 8.9%) Two defects the invariants surfaced, both fixed in tools.ts: - rounded shares could sum past `pool`, so "reservations fit the envelope" was approximate rather than exact; both terms now floor - a non-finite score made every share Infinity/Infinity, handing the render loop a NaN allowance; `weightOf` now fails safe to 0 Also adds a hard-ceiling gate to the payroll fixture — at 19.3K against a 19.5K ceiling it is the only fixture that stresses the ~25K inline cap — and exports EXPLORE_ALLOCATION so invariant tests read the constants while one test pins the literals. Co-Authored-By: Claude Opus 5 --- __tests__/explore-allocation-1500.test.ts | 17 +- __tests__/explore-allocation-e2e.test.ts | 558 ++++++++++++++++++ .../explore-proportional-allocation.test.ts | 342 ++++++++++- docs/design/explore-budget-allocation.md | 79 +++ src/mcp/tools.ts | 19 +- 5 files changed, 1008 insertions(+), 7 deletions(-) create mode 100644 __tests__/explore-allocation-e2e.test.ts diff --git a/__tests__/explore-allocation-1500.test.ts b/__tests__/explore-allocation-1500.test.ts index ffe513ac1..5d76d7c02 100644 --- a/__tests__/explore-allocation-1500.test.ts +++ b/__tests__/explore-allocation-1500.test.ts @@ -36,7 +36,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import CodeGraph from '../src/index'; -import { ToolHandler } from '../src/mcp/tools'; +import { ToolHandler, getExploreOutputBudget } from '../src/mcp/tools'; import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; import { isGeneratedFile, hasGeneratedHeader } from '../src/extraction/generated-detection'; @@ -267,6 +267,21 @@ describe('#1500 — generated Go CRUD beside a hand-written payroll workflow', ( expect(response).toMatch(/internal\/gen\/fkit\/payroll\/payslip\.go: \w+:\d+/); }); + it('CG-14 GATE: holds the response inside the hard ceiling under real pressure', () => { + // This fixture is the stress case for the ceiling, not just for the split: + // 19 files put it in the very-tiny tier (13,000-char envelope) while the + // answer genuinely needs more, so the render loop spends its full allowed + // overshoot — ~19.3K against a 19.5K ceiling. That leaves ~1% of headroom, + // which is exactly why this is worth pinning: the bound that matters is the + // host's ~25K inline cap, and above it the response is written to a file + // the agent Reads back, undoing the point of the tool. + const budget = getExploreOutputBudget(cg.getFiles().length); + const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), 25000); + expect(response.length).toBeGreaterThan(budget.maxOutputChars); + expect(response.length).toBeLessThanOrEqual(hardCeiling); + expect(response.length).toBeLessThan(25000); + }); + it('records the shape of the allocation so a regression is legible', () => { // Not a gate — a snapshot of the split, so a future change that shifts the // numbers shows up in the diff rather than silently flipping a gate. diff --git a/__tests__/explore-allocation-e2e.test.ts b/__tests__/explore-allocation-e2e.test.ts new file mode 100644 index 000000000..70e4d21c8 --- /dev/null +++ b/__tests__/explore-allocation-e2e.test.ts @@ -0,0 +1,558 @@ +/** + * Score-proportional explore allocation, end to end (CG-14 / epic CG-1 / #1500). + * + * `explore-proportional-allocation.test.ts` pins `allocateExploreBudget` in + * isolation; `explore-allocation-1500.test.ts` pins the reporter's Go shape. + * What is left — and what this file owns — is everything the allocator only + * *promises*: the render loop has to spend those reservations, the hard ceiling + * has to catch the overshoot, and a degenerate or diffuse result set has to come + * back usable rather than empty. Each of those is invisible to a unit test, + * because the failure mode is not an exception — it is a response the agent + * quietly abandons in favour of Read. + * + * Two halves: + * + * 1. **The self-query fixture's shape.** CG-6 declared a second regression + * fixture beside payroll-go: this repo, asked "how does explore allocate its + * output budget across files", spending 63% of its envelope on + * `scripts/agent-eval/*.mjs` files that merely mention `explore` and + * `BUDGET`, while `src/mcp/tools.ts` — the file that actually answers — sat + * clipped at the flat `maxCharsPerFile`. That fixture reads THIS repo's live + * index, so it belongs to the out-of-band probe + * (`node scripts/agent-eval/probe-allocation.mjs self-query`) where its + * numbers can move with the repo. Reproduced here as a synthetic project so + * `npm test` owns the MECHANISM deterministically: a large relevant file, a + * small genuinely-relevant helper, and an incidental name-collision script. + * + * 2. **Degenerate and diffuse result sets.** One file, no files, all files + * scoring alike, a survey question. The proportional split divides by a total + * weight and concentrates on a leader — both of which have a degenerate case + * that ends in a division by zero or a starved response. + * + * Nothing here is platform-gated: fixtures are written through `path.join`, and + * every path ASSERTED against is an indexed relative path, which extraction + * normalizes to forward slashes on every platform (`normalizePath`, utils.ts). + * A literal like `src/mcp/allocator.ts` is therefore correct on Windows too — + * gate a new assertion with `it.runIf` only if it reaches for a real filesystem + * path or a platform-specific separator. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler, getExploreOutputBudget, EXPLORE_ALLOCATION } from '../src/mcp/tools'; +import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; +import type { ExploreDiagnosticReport } from '../src/mcp/explore-diagnostics'; + +/** The host's inline tool-result limit — above it the response is externalized. */ +const INLINE_CAP = 25000; + +const DEBUG_ENV = 'CODEGRAPH_EXPLORE_DEBUG'; + +interface Project { + dir: string; + cg: CodeGraph; + handler: ToolHandler; +} + +/** Build + index a throwaway project from a `{ relPath: source }` map. */ +async function buildProject(prefix: string, files: Record): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + for (const [rel, body] of Object.entries(files)) { + const abs = path.join(dir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, body.trimStart()); + } + const cg = CodeGraph.initSync(dir); + await cg.indexAll(); + return { dir, cg, handler: new ToolHandler(cg) }; +} + +function destroyProject(project?: Project): void { + if (!project) return; + project.cg.destroy(); + if (fs.existsSync(project.dir)) fs.rmSync(project.dir, { recursive: true, force: true }); +} + +/** + * One explore call, reduced to what the allocation assertions need — plus the + * CG-4 per-file diagnostic, which is where the SCORE and the RESERVATION live. + * The instrument is observational (byte-identical output either way), so reading + * it here measures the same response the agent would have received. + */ +async function explore(project: Project, query: string) { + // Outside the project root on purpose: a sidecar written INTO the indexed tree + // is a new file the watcher can pick up mid-suite. + const sidecar = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-alloc-diag-')), 'report.jsonl'); + const previous = process.env[DEBUG_ENV]; + process.env[DEBUG_ENV] = sidecar; + let result; + try { + result = await project.handler.execute('codegraph_explore', { query }); + } finally { + if (previous === undefined) delete process.env[DEBUG_ENV]; + else process.env[DEBUG_ENV] = previous; + } + const text = result.content?.[0]?.text ?? ''; + const bytes = attributeSourceBytes(text); + const lines = fs.existsSync(sidecar) + ? fs.readFileSync(sidecar, 'utf-8').trim().split('\n').filter(Boolean) + : []; + const report = JSON.parse(lines[lines.length - 1]!) as ExploreDiagnosticReport; + fs.rmSync(path.dirname(sidecar), { recursive: true, force: true }); + const fileOf = (file: string) => report.files.find((f) => f.path === file); + return { + text, + bytes, + report, + isError: result.isError === true, + /** Relevance score the ranking pass gave this file. */ + score: (file: string) => fileOf(file)?.score ?? 0, + /** Chars of source the allocator RESERVED for it, before anything rendered. */ + allowance: (file: string) => fileOf(file)?.allowance ?? 0, + /** Fraction of the WHOLE response this file's source occupies. */ + share: (file: string) => (bytes.get(file) ?? 0) / (text.length || 1), + shareUnder: (prefix: string) => { + let total = 0; + for (const [file, n] of bytes) if (file.startsWith(prefix)) total += n; + return total / (text.length || 1); + }, + }; +} + +// ── 1. The self-query fixture's shape ─────────────────────────────────────── + +describe('#1500 fixture 2 — allocation followed FILE SIZE, not relevance', () => { + /** + * The three roles from the real fixture, at synthetic scale: + * + * - `src/mcp/allocator.ts` — stands in for `src/mcp/tools.ts`. Carries the + * query's terms on real functions with real call edges, and is deliberately + * too big to ship whole, so under the old rule it was clipped at the flat + * `maxCharsPerFile` no matter how far it outscored its peers. + * - `src/util/budget-math.ts` — stands in for `src/resolution/memory-budget.ts`. + * Genuinely relevant (the allocator calls it) but scoring about half as + * well — and small enough to ship WHOLE, which under the old rule was worth + * more than being right. + * - `scripts/eval-harness.mjs` — stands in for `scripts/agent-eval/*.mjs`. Its + * only claim on the query is a file-scope `explore` and `BUDGET` that nothing + * reads: the incidental collision CG-10 demoted. + * + * Measured on this fixture, reverting the render loop to the pre-CG-12 rules + * (`fileBudget = maxCharsPerFile`, whole-file bound `maxCharsPerFile * 3`) + * reproduces the report exactly — and every gate below goes red: + * + * | file | score | pre-CG-12 | CG-12 | + * |-------------------------|-------|----------------|----------------| + * | `src/mcp/allocator.ts` | 77.5 | 4,843 (39.7%) | 9,335 (80.1%) | + * | `src/util/budget-math.ts` | 36.0 | 6,079 (49.8%) | 1,037 ( 8.9%) | + * + * The half-as-relevant file taking the larger share, purely on size, IS #1500. + */ + const QUERY = 'how does explore allocate its output budget across files'; + const ALLOCATOR = 'src/mcp/allocator.ts'; + const HELPER = 'src/util/budget-math.ts'; + const INCIDENTAL = 'scripts/eval-harness.mjs'; + + const allocatorPass = (index: number, name: string) => ` +/** ${name}: one pass of the explore output split. */ +export function ${name}( + candidates: AllocationCandidate[], + budget: ExploreOutputBudget, +): Map { + const allowances = new Map(); + const pool = clampOutputBudget(budget.maxOutputChars - ${index} * 200); + const total = candidates.reduce((sum, candidate) => sum + candidate.score, 0); + if (total <= 0) { + return allowances; + } + const floors = Math.min(pool, 700 * candidates.length); + const remainder = budgetRemainderAfterFloors(pool, floors); + for (const candidate of candidates) { + const floor = Math.floor(floors / candidates.length); + const proportional = splitOutputEvenly(remainder, total, candidate.score); + const boosted = candidate.spine ? proportional * 2 : proportional; + const share = Math.min(floor + boosted, budget.maxCharsPerFile * 3); + if (share <= 0) { + continue; + } + allowances.set(candidate.path, share); + } + return allowances; +} +`; + + /** + * Neutral bulk for the helper file: real symbols that match NOTHING in the + * query, so the file grows in BYTES without gaining relevance. That asymmetry + * is the fixture — the real `memory-budget.ts` won 51% of the envelope against + * a file scoring twice its score purely by being small enough to ship whole. + */ + const helperFiller = (n: number) => ` +export function normalizeLedgerRow${n}(row: string[], fallback: string): string[] { + const trimmed = row.map((cell) => cell.trim()).filter((cell) => cell.length > 0); + return trimmed.length > 0 ? trimmed : [fallback]; +} +`; + + const ALLOCATOR_SOURCE = ` +/** Explore budget allocation: splits the output envelope across relevant files. */ +export interface ExploreOutputBudget { + maxOutputChars: number; + maxCharsPerFile: number; + defaultMaxFiles: number; +} + +export interface AllocationCandidate { + path: string; + score: number; + spine: boolean; +} +${[ + 'allocateExploreBudget', + 'reserveOutputPerFile', + 'distributeOutputBudget', + 'planExploreOutput', + 'spendExploreBudget', + 'balanceOutputAcrossFiles', + 'concentrateExploreOutput', + 'settleExploreAllocation', + 'apportionExploreBudget', + 'rationOutputAcrossFiles', + 'tallyExploreOutputBudget', + 'weighExploreAllocation', +].map((name, i) => allocatorPass(i + 1, name)).join('')} +import { + clampOutputBudget, + splitOutputEvenly, + budgetRemainderAfterFloors, +} from '../util/budget-math'; +`; + + let project: Project; + let run: Awaited>; + + beforeAll(async () => { + project = await buildProject('codegraph-alloc-selfquery-', { + [ALLOCATOR]: ALLOCATOR_SOURCE, + [HELPER]: ` +/** Budget arithmetic the explore output allocator leans on. */ +export function clampOutputBudget(value: number): number { + if (value < 0) return 0; + return Math.floor(value); +} + +export function splitOutputEvenly(pool: number, total: number, score: number): number { + if (total <= 0) return 0; + return Math.floor((pool * score) / total); +} + +export function budgetRemainderAfterFloors(pool: number, floors: number): number { + const remainder = pool - floors; + return remainder > 0 ? remainder : 0; +} + +export function splitBudgetAcrossFiles(pool: number, fileCount: number): number { + return fileCount > 0 ? Math.floor(pool / fileCount) : pool; +} + +export function describeOutputBudget(pool: number, perFile: number): string { + return \`explore budget pool of \${pool} chars, \${perFile} per file\`; +} +${Array.from({ length: 22 }, (_, i) => helperFiller(i + 1)).join('')}`, + [INCIDENTAL]: ` +// Eval harness. Mentions explore and BUDGET incidentally; nothing here allocates. +const explore = 'explore'; +const BUDGET = 24000; + +export function runHarness(repo) { + const rows = []; + for (const line of repo.split('\\n')) { + rows.push(line.trim()); + } + return rows; +} + +export function summarizeRun(rows) { + return { count: rows.length, first: rows[0] }; +} +`, + 'src/mcp/server.ts': ` +import { allocateExploreBudget } from './allocator'; + +export function serve(candidates: any[]) { + return allocateExploreBudget(candidates, { maxOutputChars: 13000, maxCharsPerFile: 3800, defaultMaxFiles: 4 }); +} +`, + 'src/util/logger.ts': ` +export function log(message: string): void { + console.log(message); +} +`, + }); + run = await explore(project, QUERY); + }, 120_000); + + afterAll(() => destroyProject(project)); + + describe('fixture shape', () => { + it('indexes all three roles, so a zero share means demoted and not missing', () => { + // Without this the incidental assertion below could pass vacuously — a file + // that was never indexed also delivers 0 bytes. + for (const rel of [ALLOCATOR, HELPER, INCIDENTAL]) { + expect(project.cg.getFile(rel), `${rel} indexed`).toBeTruthy(); + } + }); + + it('sizes the two files so the size-driven render split actually bites', () => { + // The mechanism the epic is about. The answer file must be too big to ship + // whole (so the old flat cap clipped it), and the helper small enough that + // shipping it whole was always affordable under the old `maxCharsPerFile * 3` + // bound. Without that asymmetry the fixture stops reproducing anything. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + const answer = fs.readFileSync(path.join(project.dir, ALLOCATOR), 'utf-8'); + const helper = fs.readFileSync(path.join(project.dir, HELPER), 'utf-8'); + expect(answer.split('\n').length).toBeGreaterThan(280); + expect(answer.length).toBeGreaterThan(budget.maxCharsPerFile * 3); + expect(helper.split('\n').length).toBeLessThan(220); + expect(helper.length).toBeLessThan(budget.maxCharsPerFile * 3); + }); + + it('scores the answer file well above the helper it calls', () => { + // The other half of the asymmetry: the reversal below only means something + // if the file that used to WIN the envelope was the less relevant one. + expect(run.score(ALLOCATOR)).toBeGreaterThan(run.score(HELPER) * 1.5); + }); + }); + + describe('budget allocation', () => { + it('gives the file that answers the question the majority of the envelope', () => { + // The epic's acceptance bar for this fixture: >50%, from 18.5% at baseline. + // Pre-CG-12 this file took 39.7% — behind the helper it calls. + expect(run.share(ALLOCATOR)).toBeGreaterThan(0.5); + }); + + it('lets the answer file spend multiples of the flat cap it used to be clipped at', () => { + // The mechanism as a byte count rather than a share: this file is too big + // to ship whole, so under the old rule its source was truncated at + // `maxCharsPerFile` however far it outscored its peers. Its reservation is + // now several times that cap. A build that re-imposes a flat per-file cap + // fails HERE first — it delivered 4,843 against a 3,800 cap. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + expect(run.bytes.get(ALLOCATOR) ?? 0).toBeGreaterThan(budget.maxCharsPerFile * 2); + }); + + it('stops the smaller file winning on size — it no longer ships whole', () => { + // The reversal, from the other side. The helper scores about half the + // answer file and is small enough that the old whole-file bound shipped it + // ENTIRE (6,079 chars, 49.8% of the envelope — more than the file that + // answered the question). It now clusters inside its proportional share. + const helperSource = fs.readFileSync(path.join(project.dir, HELPER), 'utf-8'); + const delivered = run.bytes.get(HELPER) ?? 0; + expect(delivered).toBeGreaterThan(0); + expect(delivered).toBeLessThan(helperSource.length); + }); + + it('orders per-file shares by relevance, not by file size', () => { + // Both files deliver — this is not concentration by elimination — but the + // one that answers the question gets several times the bytes of the helper + // it calls. Pre-CG-12 this ratio was 0.8, i.e. inverted. + const answer = run.share(ALLOCATOR); + const helper = run.share(HELPER); + expect(helper).toBeGreaterThan(0); + expect(answer).toBeGreaterThan(helper * 3); + }); + + it('spends nothing on the incidental name collision', () => { + expect(run.bytes.get(INCIDENTAL) ?? 0).toBe(0); + expect(run.shareUnder('scripts/')).toBe(0); + }); + + it('reserves in proportion to score, before anything renders', () => { + // The reservations are the contract the render loop then spends. Asserting + // them directly — not just the bytes that came out — separates "allocation + // is proportional" from "the render loop happened to emit these sizes". + const answerReserved = run.allowance(ALLOCATOR); + const helperReserved = run.allowance(HELPER); + expect(answerReserved).toBeGreaterThan(helperReserved); + expect(answerReserved / helperReserved).toBeGreaterThan(run.score(ALLOCATOR) / run.score(HELPER) * 0.5); + // Nothing is over-promised: the sum of reservations fits the pool, and the + // pool fits the envelope. This is the invariant the whole epic rests on. + expect(run.report.allocation.reserved).toBeLessThanOrEqual(run.report.allocation.pool); + expect(run.report.allocation.pool).toBeLessThanOrEqual(run.report.budget.maxOutputChars); + }); + + it('keeps the response inside the hard ceiling and under the inline cap', () => { + // Two different bounds, and it matters which is which. `maxOutputChars` + // bounds the RESERVATIONS (asserted above); the RESPONSE is bounded by + // `hardCeiling` — 1.5x the envelope, capped at 25K — because the render + // loop is allowed a bounded overshoot for the whole-file grace and an + // oversize first cluster. The 25K is the one that must never move: past it + // the host writes the result to a file the agent Reads back. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), INLINE_CAP); + expect(run.text.length).toBeLessThanOrEqual(hardCeiling); + expect(run.text.length).toBeLessThan(INLINE_CAP); + }); + + it('records the shape of the split so a regression is legible', () => { + // Not a gate — a snapshot, so a change that shifts the split shows up in the + // diff rather than silently flipping a threshold. + expect({ + answerWinsEnvelope: run.share(ALLOCATOR) > run.share(HELPER), + helperStillDelivers: (run.bytes.get(HELPER) ?? 0) > 0, + incidentalDelivers: (run.bytes.get(INCIDENTAL) ?? 0) > 0, + }).toEqual({ + answerWinsEnvelope: true, + helperStillDelivers: true, + incidentalDelivers: false, + }); + }); + }); +}); + +// ── 2. Degenerate and diffuse result sets ─────────────────────────────────── + +describe('allocation on degenerate result sets', () => { + let project: Project; + + beforeAll(async () => { + // Four modules that are deliberate COPIES of each other, plus one unrelated + // file. Copies are the pathological input for a proportional split: every + // candidate carries the same weight, so the split divides by a denominator + // that is entirely made of ties. + const twin = (n: number) => ` +export class InventoryLedger${n} { + private rows: number[] = []; + + public recordInventoryMovement(quantity: number): void { + this.rows.push(quantity); + } + + public settleInventoryLedger(): number { + return this.rows.reduce((sum, row) => sum + row, 0); + } +} +`; + project = await buildProject('codegraph-alloc-degenerate-', { + 'src/ledger/one.ts': twin(1), + 'src/ledger/two.ts': twin(2), + 'src/ledger/three.ts': twin(3), + 'src/ledger/four.ts': twin(4), + 'src/unrelated/colors.ts': ` +export const PALETTE = ['oxblood', 'paper', 'ink']; + +export function pickPaletteEntry(index: number): string { + return PALETTE[index % PALETTE.length]!; +} +`, + }); + }, 120_000); + + afterAll(() => destroyProject(project)); + + it('does not starve anyone when every file scores identically', async () => { + // The all-ties case, end to end: no division by zero, nobody cliffed for + // being relatively weak (nothing IS relatively weak), and no single copy + // sweeping the envelope on an arbitrary tiebreak. + const run = await explore(project, 'how does the inventory ledger record and settle movements'); + expect(run.isError).toBe(false); + const ledger = [...run.bytes].filter(([file]) => file.startsWith('src/ledger/')); + expect(ledger.length).toBeGreaterThanOrEqual(2); + const shares = ledger.map(([, n]) => n); + expect(Math.max(...shares) / Math.min(...shares)).toBeLessThan(3); + for (const [file, n] of ledger) { + expect(n, `${file} starved`).toBeGreaterThan(0); + } + // The reservations behind those bytes divided cleanly too. + expect(run.report.allocation.reserved).toBeLessThanOrEqual(run.report.allocation.pool); + }); + + it('answers a single-file question without over-spending the envelope on it', async () => { + const run = await explore(project, 'pickPaletteEntry'); + expect(run.isError).toBe(false); + expect(run.bytes.get('src/unrelated/colors.ts') ?? 0).toBeGreaterThan(0); + const budget = getExploreOutputBudget(project.cg.getFiles().length); + // One dominant file still cannot exceed the share ceiling, and the response + // as a whole still fits the envelope's hard ceiling. + expect(run.bytes.get('src/unrelated/colors.ts')!) + .toBeLessThanOrEqual(Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE)); + expect(run.text.length).toBeLessThan(INLINE_CAP); + }); + + it('returns guidance rather than an error when nothing matches', async () => { + // An `isError` response teaches the agent to abandon codegraph for the rest + // of the session, so a zero-result allocation must stay success-shaped. + const run = await explore(project, 'quantumFluxCapacitorHandshake'); + expect(run.isError).toBe(false); + expect(run.text.length).toBeGreaterThan(0); + expect(run.bytes.size).toBe(0); + }); +}); + +describe('the diffuse-query control', () => { + let project: Project; + + beforeAll(async () => { + // Six genuinely distinct subsystems, each a legitimate partial answer to a + // survey question. Concentration is the epic's goal, but over-correcting here + // costs a round-trip: the agent's fallback for an under-served survey is + // Grep, not a second explore. + const subsystem = (name: string, verb: string) => ` +export interface ${name}Options { + retries: number; +} + +export class ${name}Service { + constructor(private readonly options: ${name}Options) {} + + public ${verb}Request(payload: string): string { + return this.describe${name}() + ':' + payload; + } + + public describe${name}(): string { + return '${name} with ' + this.options.retries + ' retries'; + } +} +`; + project = await buildProject('codegraph-alloc-diffuse-', { + 'src/services/auth.ts': subsystem('Auth', 'authorize'), + 'src/services/billing.ts': subsystem('Billing', 'charge'), + 'src/services/search.ts': subsystem('Search', 'query'), + 'src/services/notify.ts': subsystem('Notify', 'publish'), + 'src/services/report.ts': subsystem('Report', 'render'), + 'src/services/audit.ts': subsystem('Audit', 'record'), + }); + }, 120_000); + + afterAll(() => destroyProject(project)); + + it('still returns a spread for a survey-style question', async () => { + // The over-correction guard for CG-10's floor and CG-12's cliff together: a + // question with no single right answer must come back as several usable + // sections, not one file plus a pointer list. + const run = await explore(project, 'what services does this project expose and what does each one do'); + expect(run.isError).toBe(false); + const services = [...run.bytes].filter(([file]) => file.startsWith('src/services/')); + expect(services.length).toBeGreaterThanOrEqual(3); + const total = services.reduce((sum, [, n]) => sum + n, 0); + expect(total).toBeGreaterThan(0); + for (const [file, n] of services) { + // Nobody is reduced to a fragment, and nobody swallows the response. + expect(n, `${file} fragment`).toBeGreaterThan(200); + expect(n / total, `${file} hogged the envelope`).toBeLessThan(0.8); + } + }); + + it('names whatever it could not show, so the spread stays completable', async () => { + const run = await explore(project, 'what services does this project expose and what does each one do'); + const shown = [...run.bytes.keys()].filter((f) => f.startsWith('src/services/')); + const missing = ['auth', 'billing', 'search', 'notify', 'report', 'audit'] + .map((n) => `src/services/${n}.ts`) + .filter((f) => !shown.includes(f)); + for (const file of missing) { + expect(run.text, `${file} dropped without a pointer`).toContain(file); + } + }); +}); diff --git a/__tests__/explore-proportional-allocation.test.ts b/__tests__/explore-proportional-allocation.test.ts index 13c886381..2c11ac041 100644 --- a/__tests__/explore-proportional-allocation.test.ts +++ b/__tests__/explore-proportional-allocation.test.ts @@ -11,8 +11,8 @@ * regression fixtures lives in `explore-allocation-1500.test.ts`. */ import { describe, it, expect } from 'vitest'; -import { allocateExploreBudget, getExploreOutputBudget } from '../src/mcp/tools'; -import type { ExploreAllocationCandidate } from '../src/mcp/tools'; +import { allocateExploreBudget, getExploreOutputBudget, EXPLORE_ALLOCATION } from '../src/mcp/tools'; +import type { ExploreAllocationCandidate, ExploreAllocation, ExploreOutputBudget } from '../src/mcp/tools'; /** A candidate with sane defaults — tests override only what they're about. */ const cand = ( @@ -23,6 +23,30 @@ const cand = ( const TIER_FILE_COUNTS = [10, 100, 300, 1000, 4000, 10000, 20000, 60000]; +/** + * The inline tool-result limit. Above it the host writes the response to a file + * the agent Reads back, re-introducing the read this tool exists to prevent — so + * it bounds every tier, not just the big ones (`hardCeiling`, tools.ts). + */ +const INLINE_CAP = 25000; + +const reservedTotal = (a: ExploreAllocation) => + [...a.allowances.values()].reduce((sum, n) => sum + n, 0); + +/** + * What the render loop can actually emit for these reservations: each file's + * slice, plus the whole-file grace it may overshoot by, plus the markdown + * overhead charged per section. The allocator's job is to keep this inside the + * envelope it was handed. + */ +const worstCaseEmission = (a: ExploreAllocation) => { + let total = 0; + for (const chars of a.allowances.values()) { + total += chars + EXPLORE_ALLOCATION.FILE_OVERHEAD; + } + return total; +}; + describe('allocateExploreBudget — proportional split', () => { const budget = getExploreOutputBudget(1000); // 24,000 / 6,500 / 8 files @@ -212,3 +236,317 @@ describe('allocateExploreBudget — tier invariant', () => { expect(new Set(cliffs).size).toBe(1); }); }); + +// ── CG-14 ─────────────────────────────────────────────────────────────────── +// Everything above pins the behaviours CG-12 was written to produce. What +// follows pins the ones it must never produce: an over-spent envelope, a +// starved diffuse query, a NaN slice — the failures that would ship silently +// because they only surface as an agent falling back to Read. + +describe('allocateExploreBudget — calibration', () => { + it('pins the constants the two #1500 fixtures were calibrated against', () => { + // Deliberately literal. Every other test here asserts an INVARIANT and reads + // the constants, so it holds at any value; this one exists so that changing a + // value is a visible decision rather than a silent re-tune of the fixtures. + // If you change one, re-run `node scripts/agent-eval/probe-allocation.mjs`. + expect(EXPLORE_ALLOCATION).toMatchObject({ + CLIFF_FRACTION: 0.15, + CLIFF_MAX: 10, + MIN_CHARS: 700, + MAX_SHARE: 0.7, + FILE_OVERHEAD: 200, + SPINE_WEIGHT_BOOST: 2, + WHOLE_FILE_GRACE_FRACTION: 0.15, + WHOLE_FILE_GRACE_MAX: 800, + }); + }); + + it('cliffs strictly BELOW the threshold, so a file exactly at it is still served', () => { + // The boundary matters because `cliffAt` sits at CLIFF_MAX for any dominant + // top file, which is also where the score floor's own ceiling sits — a file + // that clears one must clear the other or the two gates disagree. + const budget = getExploreOutputBudget(1000); + const at = allocateExploreBudget([cand('top.ts', 1000), cand('probe.ts', 10)], budget, 8); + const under = allocateExploreBudget([cand('top.ts', 1000), cand('probe.ts', 9.9)], budget, 8); + expect(at.cliffAt).toBe(EXPLORE_ALLOCATION.CLIFF_MAX); + expect(at.cliffed).not.toContain('probe.ts'); + expect(under.cliffed).toContain('probe.ts'); + }); + + it('tracks the top file until CLIFF_MAX caps it', () => { + const budget = getExploreOutputBudget(1000); + const cliffFor = (top: number) => + allocateExploreBudget([cand('top.ts', top), cand('b.ts', 1)], budget, 8).cliffAt; + expect(cliffFor(20)).toBeCloseTo(20 * EXPLORE_ALLOCATION.CLIFF_FRACTION, 5); + expect(cliffFor(50)).toBeCloseTo(50 * EXPLORE_ALLOCATION.CLIFF_FRACTION, 5); + expect(cliffFor(500)).toBe(EXPLORE_ALLOCATION.CLIFF_MAX); + }); +}); + +describe('allocateExploreBudget — envelope safety', () => { + /** Deterministic LCG: a seeded sweep reproduces exactly, unlike Math.random. */ + const shapes = (): ExploreAllocationCandidate[][] => { + let seed = 0x1500; + const next = () => (seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff; + const out: ExploreAllocationCandidate[][] = []; + for (let n = 1; n <= 30; n++) { + out.push(Array.from({ length: n }, (_, i) => + cand(`f${i}.ts`, Math.round(next() * 120 * 100) / 100, { + worth: next() < 0.25 ? 0.3 : 1, + spine: next() < 0.1, + }))); + } + return out; + }; + + it('never reserves more than the envelope, at any tier or shape', () => { + // The one invariant that must hold unconditionally: the render loop spends + // reservations, so an over-allocation is an over-long response, and an + // over-long response is externalized to a file the agent has to Read back. + for (const fileCount of TIER_FILE_COUNTS) { + const budget = getExploreOutputBudget(fileCount); + for (const files of shapes()) { + for (const maxFiles of [1, 4, 8, 30]) { + const alloc = allocateExploreBudget(files, budget, maxFiles); + const label = `${files.length} files, maxFiles=${maxFiles}, tier ${fileCount}`; + expect(reservedTotal(alloc), label).toBeLessThanOrEqual(alloc.pool); + expect(worstCaseEmission(alloc), label).toBeLessThanOrEqual(budget.maxOutputChars); + for (const chars of alloc.allowances.values()) { + expect(Number.isFinite(chars) && chars > 0, label).toBe(true); + } + } + } + } + }); + + it('never renders more files than maxFiles', () => { + for (const maxFiles of [1, 2, 4, 8]) { + const files = Array.from({ length: 25 }, (_, i) => cand(`f${i}.ts`, 100 - i)); + const alloc = allocateExploreBudget(files, getExploreOutputBudget(1000), maxFiles); + expect(alloc.allowances.size).toBeLessThanOrEqual(maxFiles); + } + }); + + it('accounts for every candidate — a file is served, cliffed, or neither by choice', () => { + // Nothing may vanish silently: a cliffed file is still NAMED in the response, + // which is what makes withholding its bytes cheap. A file that is neither + // served nor cliffed would be dropped without a pointer. + const files = Array.from({ length: 25 }, (_, i) => cand(`f${i}.ts`, 100 - i * 4)); + const alloc = allocateExploreBudget(files, getExploreOutputBudget(1000), 8); + const accounted = new Set([...alloc.allowances.keys(), ...alloc.cliffed]); + expect(accounted.size).toBe(files.length); + }); + + it('leaves the ~25K inline cap reachable only through the hard ceiling', () => { + // Reservations always fit `maxOutputChars`, but the whole-file grace lets the + // render loop overshoot a slice — so the envelope alone does NOT bound the + // response, and `hardCeiling` is load-bearing rather than defensive. Pin both + // halves: every tier's envelope is inside the inline cap, and the worst-case + // graced emission is what the ceiling has to catch. + for (const fileCount of TIER_FILE_COUNTS) { + const budget = getExploreOutputBudget(fileCount); + const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), INLINE_CAP); + expect(budget.maxOutputChars).toBeLessThan(INLINE_CAP); + expect(hardCeiling).toBeLessThanOrEqual(INLINE_CAP); + + const files = Array.from({ length: 8 }, (_, i) => cand(`f${i}.ts`, 90 - i * 9)); + const alloc = allocateExploreBudget(files, budget, 8); + const graced = [...alloc.allowances.values()].reduce((sum, chars) => sum + chars + + Math.min(EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_MAX, + Math.round(chars * EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_FRACTION)) + + EXPLORE_ALLOCATION.FILE_OVERHEAD, 0); + expect(graced).toBeGreaterThan(budget.maxOutputChars); + expect(reservedTotal(alloc)).toBeLessThanOrEqual(budget.maxOutputChars); + } + }); +}); + +describe('allocateExploreBudget — spine first', () => { + const budget = getExploreOutputBudget(1000); + + it('reserves more for a spine file than for an identically-scoring peer', () => { + // "Spine first, unclipped" is enforced by WEIGHT, not by ordering: the spine + // boost multiplies into the proportional split, so the flow gets its bytes + // before any peripheral file competes for them. + const { allowances } = allocateExploreBudget( + [cand('peer.ts', 20), cand('spine.ts', 20, { spine: true })], + budget, + 8, + ); + expect(allowances.get('spine.ts')!).toBeGreaterThan(allowances.get('peer.ts')!); + expect(allowances.get('spine.ts')! / allowances.get('peer.ts')!).toBeGreaterThan(1.3); + }); + + it('keeps a spine file even when the envelope cannot afford everyone', () => { + // The affordability trim keeps the highest weights and drops the rest in one + // pass — but a dropped spine file breaks the flow, which is precisely the + // failure that sends the agent back to Read. It is force-kept past the trim. + const tiny = getExploreOutputBudget(10); + const files = [ + ...Array.from({ length: 20 }, (_, i) => cand(`f${i}.ts`, 100 - i)), + cand('spine.ts', 4, { spine: true }), + ]; + const { allowances, cliffed } = allocateExploreBudget(files, tiny, 40); + expect(allowances.has('spine.ts')).toBe(true); + expect(cliffed).not.toContain('spine.ts'); + // Force-keeping it costs everyone a sliver — bounded, and the envelope still + // holds. A real starvation regression would blow well past this. + for (const [path, chars] of allowances) { + expect(chars, path).toBeGreaterThanOrEqual(Math.round(EXPLORE_ALLOCATION.MIN_CHARS * 0.9)); + } + expect(reservedTotal({ allowances, cliffed, cliffAt: 0, pool: 0 })).toBeLessThanOrEqual(tiny.maxOutputChars); + }); + + it('does NOT exempt a spine file from maxFiles — the slot cap is separate', () => { + // Documented boundary, not an oversight: the cliff is a relevance gate the + // spine overrides, `maxFiles` is a response-shape cap it does not. In + // practice the 2x boost lifts a spine file into the slots long before this + // bites; the test exists so a future change to either gate is deliberate. + const { allowances, cliffed } = allocateExploreBudget( + [cand('a.ts', 90), cand('b.ts', 80), cand('spine.ts', 3, { spine: true })], + budget, + 2, + ); + expect(allowances.has('spine.ts')).toBe(false); + expect(cliffed).toContain('spine.ts'); + }); + + it('serves a spine-only candidate set', () => { + const { allowances, cliffed } = allocateExploreBudget( + [cand('a.ts', 5, { spine: true }), cand('b.ts', 5, { spine: true })], + budget, + 8, + ); + expect(cliffed).toEqual([]); + expect(allowances.size).toBe(2); + }); +}); + +describe('allocateExploreBudget — degenerate inputs', () => { + const budget = getExploreOutputBudget(1000); + + it('splits evenly when every file scores identically, without starving any', () => { + // The proportional split divides by the TOTAL weight, so an all-equal set is + // the divide-by-a-degenerate-denominator case. Nobody is cliffed (nothing is + // relatively weak) and everybody gets the same slice. + for (const n of [2, 4, 8]) { + const files = Array.from({ length: n }, (_, i) => cand(`f${i}.ts`, 17)); + const { allowances, cliffed } = allocateExploreBudget(files, budget, 8); + expect(cliffed, `${n} files`).toEqual([]); + expect(allowances.size, `${n} files`).toBe(n); + const values = [...allowances.values()]; + expect(Math.max(...values) - Math.min(...values), `${n} files`).toBeLessThanOrEqual(1); + for (const chars of values) expect(chars).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + expect(worstCaseEmission({ allowances, cliffed, cliffAt: 0, pool: 0 })) + .toBeLessThanOrEqual(budget.maxOutputChars); + } + }); + + it('gives a lone file a real answer, not the whole envelope', () => { + const { allowances, cliffed } = allocateExploreBudget([cand('only.ts', 42)], budget, 8); + expect(cliffed).toEqual([]); + expect(allowances.size).toBe(1); + const chars = allowances.get('only.ts')!; + expect(chars).toBeGreaterThan(budget.maxCharsPerFile); + expect(chars).toBe(Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE)); + }); + + it('holds a runaway top scorer to its share ceiling and still names the rest', () => { + // One file 100x above everything else must not eat the response: the cliff + // zeroes its peers' BYTES, but MAX_SHARE keeps the remainder for the pointer + // list and the flow/relationship meta-text that lets the agent follow up. + const { allowances, cliffed } = allocateExploreBudget( + [cand('god.ts', 5000), cand('p1.ts', 9), cand('p2.ts', 8)], + budget, + 8, + ); + expect(allowances.get('god.ts')!).toBe(Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE)); + expect(reservedTotal({ allowances, cliffed, cliffAt: 0, pool: 0 })) + .toBeLessThan(budget.maxOutputChars); + expect(cliffed).toEqual(['p1.ts', 'p2.ts']); + }); + + it('returns nothing to render when nothing scored', () => { + for (const files of [ + [] as ExploreAllocationCandidate[], + [cand('a.ts', 0), cand('b.ts', 0)], + [cand('a.ts', 10, { worth: 0 }), cand('b.ts', 5, { worth: 0 })], + [cand('a.ts', -5), cand('b.ts', -1)], + ]) { + const { allowances, pool } = allocateExploreBudget(files, budget, 8); + expect(allowances.size).toBe(0); + expect(pool).toBeLessThanOrEqual(budget.maxOutputChars); + } + }); + + it('fails safe on a non-finite score instead of handing the render loop a NaN slice', () => { + // Scores are finite sums in the pipeline, so this only has to not corrupt the + // split — an Infinity weight would otherwise make every share Infinity/Infinity. + for (const bad of [Infinity, NaN, -Infinity]) { + const { allowances } = allocateExploreBudget([cand('bad.ts', bad), cand('ok.ts', 20)], budget, 8); + for (const [path, chars] of allowances) { + expect(Number.isFinite(chars), `${String(bad)} → ${path}`).toBe(true); + } + expect(allowances.get('ok.ts')).toBeGreaterThan(0); + } + }); + + it('renders nothing when maxFiles is zero, and still names every candidate', () => { + const { allowances, cliffed } = allocateExploreBudget( + [cand('a.ts', 10), cand('b.ts', 5)], + budget, + 0, + ); + expect(allowances.size).toBe(0); + expect(cliffed).toEqual(['a.ts', 'b.ts']); + }); + + it('survives an envelope too small for even one floored slice', () => { + const cramped: ExploreOutputBudget = { ...budget, maxOutputChars: 300 }; + const { allowances, cliffed } = allocateExploreBudget( + [cand('a.ts', 40), cand('b.ts', 30)], + cramped, + 8, + ); + expect(allowances.size).toBeLessThanOrEqual(1); + for (const chars of allowances.values()) { + expect(chars).toBeGreaterThan(0); + expect(chars).toBeLessThanOrEqual(cramped.maxOutputChars); + } + expect([...allowances.keys(), ...cliffed].sort()).toEqual(['a.ts', 'b.ts']); + }); +}); + +describe('allocateExploreBudget — the diffuse-query control', () => { + const budget = getExploreOutputBudget(1000); + + it('keeps a survey-style spread readable — no file collapses to a fragment', () => { + // The over-correction guard. Concentration is the point, but a genuinely + // diffuse question (many comparably-relevant files) must still come back as a + // usable spread: under-serving costs a whole round-trip, and the agent's + // fallback is Grep, not a second explore. + const files = Array.from({ length: 8 }, (_, i) => cand(`f${i}.ts`, 30 - i)); + const { allowances, cliffed } = allocateExploreBudget(files, budget, 8); + expect(cliffed).toEqual([]); + expect(allowances.size).toBe(8); + const values = [...allowances.values()]; + for (const chars of values) expect(chars).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + // Nobody is starved to make room for the leader: on a flat score curve the + // spread between best and worst slice stays within a small multiple. + expect(Math.max(...values) / Math.min(...values)).toBeLessThan(3); + }); + + it('concentrates a precise query far harder than a diffuse one', () => { + // Same envelope, same file count — only the score CURVE differs. This is the + // whole thesis of the epic in one assertion. + const topShareOf = (files: ExploreAllocationCandidate[]) => { + const { allowances } = allocateExploreBudget(files, budget, 8); + const values = [...allowances.values()]; + return Math.max(...values) / values.reduce((s, n) => s + n, 0); + }; + const diffuse = topShareOf(Array.from({ length: 8 }, (_, i) => cand(`f${i}.ts`, 30 - i))); + const precise = topShareOf([cand('answer.ts', 120), ...Array.from({ length: 7 }, (_, i) => cand(`f${i}.ts`, 14 - i))]); + expect(diffuse).toBeLessThan(0.25); + expect(precise).toBeGreaterThan(0.45); + }); +}); diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index 51a5ad50d..8658d36cd 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -367,3 +367,82 @@ tree reported tools.ts at 13–19% depending on the sync state). Measure against tree: restore `src/mcp/tools.ts` from `main`, remove `src/mcp/explore-diagnostics.ts`, `codegraph sync`, then run the built `dist/` binary (which still carries the instrument). Restore afterwards. + +--- + +## CG-14 — locking the allocation down + +The allocation change is one function plus three render-loop bounds, and every way it can +regress is silent: nothing throws, the response just gets less useful and the agent falls +back to Read. So the coverage is built around the question "would this test go red if the +lever were removed?" rather than around line coverage. + +### Where the coverage lives + +| File | Owns | +|---|---| +| `__tests__/explore-proportional-allocation.test.ts` | `allocateExploreBudget` in isolation — the split, the cliff, the tier invariant, envelope safety, spine weighting, degenerate inputs | +| `__tests__/explore-allocation-e2e.test.ts` | The same behaviours through the real render loop on real indexed projects — the self-query fixture's shape, degenerate result sets, the diffuse-query control | +| `__tests__/explore-allocation-1500.test.ts` | The reporter's Go shape (CG-6 fixture 1), plus the hard-ceiling stress case | +| `scripts/agent-eval/probe-allocation.mjs` | Both CG-6 fixtures against the built `dist/`, including the **live** self-query arm that reads this repo's own index | + +The split between the last two is deliberate. The self-query fixture reads a moving target +(this repo), so its exact numbers drift with the tree and it belongs out of band, where a +drift is a number to re-baseline rather than a red suite. `npm test` owns a synthetic +**mirror** of it instead: same three roles, same size asymmetry, fixed. + +### The two bounds are not the same bound + +Worth stating once, because a test that conflates them looks right and passes for the wrong +reason: + +- **`maxOutputChars` bounds the RESERVATIONS.** `sum(allowances) <= pool <= maxOutputChars`, + exactly, at every tier and every candidate shape. +- **`hardCeiling` — `min(maxOutputChars * 1.5, 25000)` — bounds the RESPONSE.** The render + loop is allowed a bounded overshoot (the whole-file grace, an oversize first cluster), so + a response legitimately exceeds the envelope. `payroll-go` does exactly that: 19.3K + delivered against a 13,000 envelope and a 19,500 ceiling. + +Only the 25K is absolute. Above it the host writes the result to a file the agent Reads +back, which is the failure the tool exists to prevent. + +### Mutation-tested, not just green + +Each lever was removed from `src/mcp/tools.ts` in turn and the suite re-run. Every one is +covered by at least one failing test — a lever with no red test is a lever that can be +deleted by accident: + +| Mutation | Tests that go red | +|---|---| +| Render loop reverted to pre-CG-12 (`fileBudget = maxCharsPerFile`, whole-file bound `maxCharsPerFile * 3`) | 5 e2e + 2 payroll | +| Proportional split replaced with an equal one | 4 unit | +| Cliff disabled (`cliffAt = 0`) | 7 unit + 3 payroll | +| Spine boost and cliff exemption removed | 3 unit | +| `MIN_CHARS` floor removed | 2 unit | +| `MAX_SHARE` ceiling removed | 4 unit | + +The first row is the one that matters most: it reproduces #1500 on the synthetic fixture +verbatim, with the half-as-relevant file taking the larger share purely on size. + +| file | score | pre-CG-12 | CG-12 | +|---|---|---|---| +| `src/mcp/allocator.ts` | 77.5 | 4,843 (39.7%) | 9,335 (80.1%) | +| `src/util/budget-math.ts` | 36.0 | 6,079 (49.8%) | 1,037 (8.9%) | + +### Two defects the coverage surfaced + +Both were found by writing the invariant rather than by reading the code: + +1. **Rounded shares could exceed the pool.** `Math.round` on each file's proportional slice + let the reservations sum past `pool` by up to half a char per file — small, but it made + "reservations fit the envelope" false rather than exact. Both terms now floor. +2. **A non-finite score produced a NaN allowance.** An `Infinity` weight makes every share + `Infinity / Infinity`. Scores are finite sums in the pipeline so it was unreachable, but + the failure mode is a NaN handed to the render loop. `weightOf` now fails safe to 0. + +### Calibration vs. invariant + +`EXPLORE_ALLOCATION` is exported so the tests can read it. Invariant tests (envelope safety, +tier monotonicity, the floor) reference the constants and hold at any value; one test pins +the literals, so re-tuning a constant is a visible decision that says *re-run the probe* +rather than a silent re-calibration of the fixtures. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 782ccc3f3..85a0e99f5 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -454,7 +454,7 @@ const SCORE_FLOOR_KEEP_MIN = 3; // `ALLOC_MAX_SHARE`, a safety valve against a single god-file — which the // proportional split already bounds, since a file's share can't exceed its // weight share. -const EXPLORE_ALLOCATION = { +export const EXPLORE_ALLOCATION = { /** * A file whose weight is under this fraction of the top file's gets no source. * @@ -578,8 +578,14 @@ export function allocateExploreBudget( const empty: ExploreAllocation = { allowances: new Map(), cliffed: [], cliffAt: 0, pool: 0 }; if (candidates.length === 0) return empty; - const weightOf = (c: ExploreAllocationCandidate) => - Math.max(0, c.score) * Math.max(0, Math.min(1, c.worth)) * (c.spine ? A.SPINE_WEIGHT_BOOST : 1); + // A non-finite weight is treated as no evidence rather than propagated: an + // Infinity score would otherwise make every share `Infinity/Infinity` = NaN and + // hand the render loop a NaN allowance. Scores are finite sums in the real + // pipeline, so this only has to fail safe. + const weightOf = (c: ExploreAllocationCandidate) => { + const w = Math.max(0, c.score) * Math.max(0, Math.min(1, c.worth)) * (c.spine ? A.SPINE_WEIGHT_BOOST : 1); + return Number.isFinite(w) ? w : 0; + }; const weights = new Map(candidates.map((c) => [c.path, weightOf(c)])); const topWeight = Math.max(...weights.values()); @@ -626,9 +632,14 @@ export function allocateExploreBudget( const ceiling = Math.round(budget.maxOutputChars * A.MAX_SHARE); const floors = Math.min(pool, A.MIN_CHARS * admitted.length); const remainder = Math.max(0, pool - floors); + // Both parts FLOOR: a sum of rounded shares can exceed the remainder that fed + // it (by up to half a char per file), and the reservations must fit the pool + // exactly — the render loop spends them, so an over-allocation is an over-long + // response the hard ceiling then has to truncate. Flooring costs at most one + // char per file. for (const c of admitted) { const share = Math.floor(floors / admitted.length) - + Math.round((remainder * (weights.get(c.path) ?? 0)) / total); + + Math.floor((remainder * (weights.get(c.path) ?? 0)) / total); allowances.set(c.path, Math.min(share, ceiling)); } return { allowances, cliffed, cliffAt, pool }; From 48a2309b92858c22ad13f1f24a9d979779985429 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 01:05:30 -0500 Subject: [PATCH 07/15] test(agent-eval): RUNS knob + explore envelope-share view for new-vs-baseline A/B (CG-15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ab-new-vs-baseline.sh now builds and indexes once per arm and runs the task RUNS times (default 1), so the >=2-runs-per-arm rule costs one build instead of N. Both arms run with CODEGRAPH_NO_PROMPT_HOOK=1 — the machine's ambient front-load hook resolves to whatever is in dist/, a second uncontrolled channel that confounds the tool-call counts — and point explore's CG-4 diagnostic at a per-arm sidecar. parse-run.mjs gains --envelope/--answer: the per-file share of the explore source envelope, parsed from the rendered markdown so it works on ANY build. The CG-4 sidecar only exists post-CG-4, so it cannot measure the baseline arm; this is the only view that measures both arms the same way. Folded into parse-run.mjs rather than added as a new script on purpose: a new file named after explore's budget scores into the self-query fixture's own corpus and moved its answer share 59.9%% -> 47.9%%. Co-Authored-By: Claude Opus 5 --- scripts/agent-eval/ab-new-vs-baseline.sh | 41 +++++++++--- scripts/agent-eval/parse-run.mjs | 83 +++++++++++++++++++++++- 2 files changed, 113 insertions(+), 11 deletions(-) diff --git a/scripts/agent-eval/ab-new-vs-baseline.sh b/scripts/agent-eval/ab-new-vs-baseline.sh index 31e5080d9..53b57a43e 100755 --- a/scripts/agent-eval/ab-new-vs-baseline.sh +++ b/scripts/agent-eval/ab-new-vs-baseline.sh @@ -21,7 +21,19 @@ # a repo with a .codegraph index (copied per arm) # "" an implementation task, e.g. "Add X to Y and wire it through" # [baseline-ref] git ref for the BEFORE build (default: HEAD~1) -# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline) +# Env: +# AGENT_EVAL_OUT output dir (default: /tmp/ab-new-vs-baseline) +# RUNS runs per arm (default 1). Run-to-run variance is large — +# use >=2 and report the range, never a single run. Both arms +# build/index ONCE and then run RUNS times, so raising this is +# far cheaper than re-invoking the script. +# MODEL / EFFORT default sonnet / high. Never raise without a reason: sonnet +# is the deliberate floor model (see CLAUDE.md). +# +# Both arms run with CODEGRAPH_NO_PROMPT_HOOK=1: the machine's ambient +# UserPromptSubmit front-load hook resolves to whichever build is currently in +# dist/, so leaving it on injects context through a second, uncontrolled channel +# and confounds the tool-call counts this script exists to compare. set -uo pipefail TARGET="${1:?usage: ab-new-vs-baseline.sh \"\" [baseline-ref]}" @@ -67,18 +79,27 @@ prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for i && echo " daemon warm: $1" || echo " WARN: daemon never bound for $1 (arm may run without codegraph)" } -run_arm() { # label, target-copy +run_arm() { # label, target-copy — runs the task $RUNS times against one build local label="$1" tgt="$2" c="$OUT/mcp-$1.json" # Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach. - printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c" - prewarm "$tgt" + # CODEGRAPH_EXPLORE_DEBUG points explore's per-file allocation diagnostic at a + # sidecar (no-op on builds predating it; never perturbs the response). + printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_EXPLORE_DEBUG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \ + "$OUT/explore-$label.jsonl" "$BIN" "$tgt" > "$c" + rm -f "$OUT/explore-$label.jsonl" echo "############## ARM [$label] ##############" - ( cd "$tgt" && claude -p "$TASK" \ - --output-format stream-json --verbose --permission-mode bypassPermissions \ - --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ - "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" ) - node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)" - pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null + for i in $(seq 1 "${RUNS:-1}"); do + # Re-warm per run: the previous run's daemon is killed below, and a cold + # attach is exactly the failure this pre-warm exists to prevent. + prewarm "$tgt" + ( cd "$tgt" && CODEGRAPH_NO_PROMPT_HOOK=1 claude -p "$TASK" \ + --output-format stream-json --verbose --permission-mode bypassPermissions \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ + "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" ) + echo "-- run $i --" + node "$PARSE" "$OUT/run-$label-$i.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$i.jsonl)" + pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null + done echo } diff --git a/scripts/agent-eval/parse-run.mjs b/scripts/agent-eval/parse-run.mjs index 6d64d58cd..a78065a01 100644 --- a/scripts/agent-eval/parse-run.mjs +++ b/scripts/agent-eval/parse-run.mjs @@ -1,12 +1,33 @@ #!/usr/bin/env node // Parse a Claude Code stream-json run log: tool-call sequence + token usage. +// +// With --envelope it also reports how the codegraph_explore responses the agent +// received were DIVIDED across files — the per-file share of the source envelope. +// That view is parsed out of the rendered markdown rather than the CG-4 +// diagnostic sidecar, so it works on ANY build (the sidecar only exists post-CG-4) +// and is therefore the only way to measure both arms of a new-vs-baseline A/B the +// same way. `--answer ` (repeatable) marks the files that actually answer +// the question, and the summary reports their combined share. +// +// Usage: parse-run.mjs [--envelope] [--answer ]... import { readFileSync } from 'fs'; -const file = process.argv[2]; + +const argv = process.argv.slice(2); +const answerGlobs = []; +let file = null; +let wantEnvelope = false; +for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--envelope') wantEnvelope = true; + else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; } + else if (!argv[i].startsWith('--') && file === null) file = argv[i]; +} const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean); const toolCalls = []; let result = null; let initTools = null; +const exploreQueries = new Map(); // tool_use id -> query +const exploreTexts = []; // response text, in call order for (const line of lines) { let ev; @@ -23,6 +44,16 @@ for (const line of lines) { else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`; else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`; toolCalls.push(`${block.name}${detail}`); + if (/codegraph_explore/.test(block.name)) exploreQueries.set(block.id, block.input?.query ?? ''); + } + } + } + if (ev.type === 'user' && ev.message?.content) { + for (const block of ev.message.content) { + if (block.type === 'tool_result' && exploreQueries.has(block.tool_use_id)) { + exploreTexts.push(typeof block.content === 'string' + ? block.content + : (block.content ?? []).filter(c => c.type === 'text').map(c => c.text).join('\n')); } } } @@ -43,3 +74,53 @@ if (result) { console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`); console.log(` tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`); } + +// ---- envelope share (opt-in) ------------------------------------------------ + +if (wantEnvelope) { + // `tools/cache/**` -> /^tools\/cache\/.*$/ . Same semantics as probe-allocation. + // The `**` sentinel is written as an escape, never a literal NUL byte — a raw + // one makes git treat this whole script as binary and costs every future diff. + const glob2re = (glob) => { + const S = '\u0000'; + const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*\*/g, S).replace(/\*/g, '[^/]*').replaceAll(S, '.*'); + return new RegExp(`^${body}$`); + }; + const answerRes = answerGlobs.map(glob2re); + const isAnswer = (p) => answerRes.some(re => re.test(p)); + + // Each rendered file section starts with **`path`** — its bytes run to the next + // such header (or to the trailing guidance quote). Share is over the sum of the + // sections, i.e. of the source envelope the allocator divides. + const pooled = new Map(); + let envelope = 0; + for (const text of exploreTexts) { + const re = /^\*\*`([^`]+)`\*\*/gm; + const marks = []; + let m; + while ((m = re.exec(text)) !== null) marks.push({ path: m[1], at: m.index }); + if (!marks.length) continue; + const tail = text.indexOf('\n> ', marks[marks.length - 1].at); + const end = tail === -1 ? text.length : tail; + marks.forEach((mark, i) => { + const chars = (i + 1 < marks.length ? marks[i + 1].at : end) - mark.at; + pooled.set(mark.path, (pooled.get(mark.path) ?? 0) + chars); + envelope += chars; + }); + } + const ranked = [...pooled.entries()] + .map(([path, chars]) => ({ path, chars, share: envelope ? chars / envelope : 0, answer: isAnswer(path) })) + .sort((a, b) => b.chars - a.chars); + const answerChars = ranked.filter(r => r.answer).reduce((s, r) => s + r.chars, 0); + const pct = (f) => `${(f * 100).toFixed(1)}%`; + + console.log(`\nExplore envelope: ${envelope.toLocaleString('en-US')} chars over ${exploreTexts.length} response(s)`); + if (answerGlobs.length) { + console.log(` answer-set share: ${pct(envelope ? answerChars / envelope : 0)} | top file answers: ${ranked[0]?.answer ?? false}`); + } + for (const f of ranked.slice(0, 12)) { + console.log(` ${f.answer ? '*' : ' '} ${pct(f.share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path}`); + } + if (ranked.length > 12) console.log(` … ${ranked.length - 12} more files`); +} From edce18f586d679a7d74c93b2bff4688ce1d313a7 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 01:08:36 -0500 Subject: [PATCH 08/15] test(agent-eval): restore the engine on INT/TERM too (CG-15) Killing ab-new-vs-baseline.sh mid-baseline-arm left the engine checked out at the baseline ref with the post-baseline files deleted, so every later build in the working tree was silently the OLD code. Co-Authored-By: Claude Opus 5 --- scripts/agent-eval/ab-new-vs-baseline.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/agent-eval/ab-new-vs-baseline.sh b/scripts/agent-eval/ab-new-vs-baseline.sh index 53b57a43e..ebc353cb2 100755 --- a/scripts/agent-eval/ab-new-vs-baseline.sh +++ b/scripts/agent-eval/ab-new-vs-baseline.sh @@ -58,7 +58,9 @@ cleanup() { git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null ( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) } -trap cleanup EXIT +# INT/TERM too: killing the script mid-baseline-arm otherwise leaves the engine +# checked out at the baseline ref, which silently poisons every later build. +trap cleanup EXIT INT TERM mkdir -p "$OUT" echo "###### engine=$ENGINE baseline=$BASE_REF" From c7103c7f2f22c102b04f56cf7c692159e5b4545d Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 01:26:47 -0500 Subject: [PATCH 09/15] =?UTF-8?q?docs:=20CG-15=20agent=20A/B=20of=20the=20?= =?UTF-8?q?#1500=20allocation=20change=20=E2=80=94=20gate=20fails=20on=20t?= =?UTF-8?q?he=20control?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three repos, both arms codegraph-on, sonnet/high, 3 runs per arm. PASS on the two medium repos: client-go (the reporter's Go shape, 2,001 of 2,454 files generated) and excalidraw hold Read 0 in every run of both arms, excalidraw goes 34s -> 24s at the median with one fewer explore call, and the generated clientsets/informers that took 10.5%% of a baseline envelope appear in no new run. FAIL on express, the small control, in 1 run of 3: 4 Reads and 52s against a baseline that read once. Not agent variance — replaying that run's query deterministically, lib/utils.js goes from 6,380 bytes whole to a 583-byte cluster stub and the envelope shrinks 13.8K -> 9.2K against an unchanged 13,000 budget. The diagnostic shows the allocator was right and the render loop was not: utils.js is the top-ranked file, was reserved 3,870 chars, and spent 583. The whole-file bound (allowance + grace = 4,450) lands just under the file's 5,293 bytes, so the whole-file render is declined and the unspent reservation is dropped rather than redistributed. Bar 1 is the hard gate, so per CG-15's acceptance rule the design goes back to CG-12 — the budget is not to be widened to compensate. Two candidate fixes are written up in the design doc. Co-Authored-By: Claude Opus 5 --- docs/benchmarks/explore-allocation-ab-1500.md | 192 ++++++++++++++++++ .../dynamic-dispatch-coverage-playbook.md | 22 ++ docs/design/explore-budget-allocation.md | 56 +++++ 3 files changed, 270 insertions(+) create mode 100644 docs/benchmarks/explore-allocation-ab-1500.md diff --git a/docs/benchmarks/explore-allocation-ab-1500.md b/docs/benchmarks/explore-allocation-ab-1500.md new file mode 100644 index 000000000..61a5db136 --- /dev/null +++ b/docs/benchmarks/explore-allocation-ab-1500.md @@ -0,0 +1,192 @@ +# Agent A/B — score-proportional explore allocation (#1500 / epic CG-1) + +**Date:** 2026-08-04 · **New:** `feature/CG-1` @ `edce18f` · **Baseline:** `main` @ `49c11fc` +· **Harness:** `scripts/agent-eval/ab-new-vs-baseline.sh`, `RUNS=3`, `--model sonnet --effort high` +on every arm · **Both arms codegraph-on.** + +This is the epic's pass gate. The deterministic probes (CG-6/CG-14) prove the budget moved; +only an agent A/B proves the agent stopped reading. + +**Verdict: the gate does not pass.** Bars 2 and 3 hold; **bar 1 (Read stays at 0) and bar 4 +(no regression on the control) fail on express**, with a reproducible, non-agent cause. Per +the CG-15 acceptance rule the allocation design goes back to CG-12 — the budget is *not* to be +widened to compensate. Root cause and the smallest honest fix are in [§Root cause](#root-cause). + +--- + +## Method + +`ab-new-vs-baseline.sh` builds and indexes each arm separately (CG-5's generated-file flag is +an index-time decision, so each arm must index with its own build), pre-warms a persistent +daemon per run, and runs the same flow question 3× per arm. Both arms run with +`CODEGRAPH_NO_PROMPT_HOOK=1` — the machine's ambient front-load hook resolves to whatever is +in `dist/` and would inject context through a second, uncontrolled channel. + +Each prompt names codegraph as the lookup tool. That is **not** a forced-Read-0: the agent +stays free to Read whenever explore's answer is insufficient, which is exactly what bar 1 +measures. It removes the one noise source that would otherwise swamp the signal — in a +pre-run without it, one express run made **0 codegraph calls and 3 Reads**, measuring adoption +(an axis this change does not touch) rather than allocation. + +Envelope share is measured with `parse-run.mjs --envelope --answer `, which parses the +rendered markdown of the responses the agent actually received. The CG-4 diagnostic sidecar +only exists on the new build, so it cannot measure the baseline arm; the markdown parse is the +only instrument that measures both arms the same way. + +| Repo | Lang | Files | Generated | Tier | Role | +|---|---|---|---|---|---| +| `kubernetes/client-go` | Go | 2,454 | 2,001 | medium (2 calls / 28K) | **the #1500 shape** — generated CRUD beside hand-written machinery | +| `excalidraw/excalidraw` | TS/React | 672 | 0 | medium (2 calls / 28K) | god-file concentration (`App.tsx`, 450 KB) | +| `expressjs/express` | JS | 147 | 0 | small (1 call / 18K) | **control** | + +--- + +## Results + +`explore` = codegraph_explore calls · `Read` = Read tool calls · `answer%` = share of the +source envelope going to the files that answer the question. Three runs per arm, reported as +the range — run-to-run variance is large and a single run means nothing. + +### client-go — "how does a shared informer keep its cache in sync and deliver events?" + +Answer set `tools/cache/**`; generated set `kubernetes/**`, `listers/**`, `applyconfigurations/**`, +`informers/**`, `**/fake/**`. + +| arm | explore | **Read** | duration | answer% | generated% | +|---|---|---|---|---|---| +| **new** | 2 / 2 / 6 | **0 / 0 / 0** | 39–61s (med 50) | 74.2 / 96.9 / 82.6 | 4.0 / 0.0 / 5.2 | +| baseline | 3 / 2 / 3 | 0 / 0 / 0 | 48–52s (med 49) | 78.1 / 97.1 / 71.3 | 10.5 / 0.0 / 5.1 | + +No Read in either arm. The medians overlap: on this query `tools/cache/**` already dominates +graph relevance, so the baseline concentrated well without help. The #1500 signal is visible +but small — the generated clientsets and per-resource informers +(`informers/events/v1beta1/interface.go`, `kubernetes/typed/events/v1/fake/…`) take 10.5% of +one baseline run's envelope and **never appear in any new run**. + +### excalidraw — "how does updating an element re-render the canvas on screen?" + +Answer set: `mutateElement.ts`, `App.tsx`, `renderer/**`, `scene/**`, `components/canvases/**`. + +| arm | explore | **Read** | duration | answer% | +|---|---|---|---|---| +| **new** | 2 / 3 / 2 | **0 / 0 / 0** | **23–32s (med 24)** | 74.4 / 67.8 / 84.7 | +| baseline | 3 / 3 / 4 | 0 / 0 / 0 | 33–39s (med 34) | 74.1 / 64.0 / 79.5 | + +The clearest win: **29% faster at the median with one fewer explore call per run**, no Read in +either arm. Concentration is why — the new arm resolves the flow in 2 calls where the baseline +takes 3–4. (One baseline run burned a turn on a hallucinated `codegraph_..._explore` tool name; +counted as-is.) + +### express — control — "how does res.send decide Content-Type and ETag?" + +Answer set `lib/**` (both arms deliver 100%; the whole answer lives in `lib/`, so this repo +tests concentration *within* the answer set, not against noise). + +| arm | explore | **Read** | duration | answer% | +|---|---|---|---|---| +| **new** | 1 / 4 / 2 | **0 / 4 / 0** | 18 / 52 / 24s | 100 / 100 / 100 | +| baseline | 2 / 2 / 2 | 1 / 1 / 1 | 23 / 28 / 27s | 100 / 100 / 100 | + +Two of three new runs are strictly better than every baseline run (0 Reads vs 1, faster). The +third is the failure: **4 Reads of `lib/utils.js` and 52s**, a fallback the baseline never +made. It is not agent variance — see below. + +--- + +## Root cause + +Deterministic replay of the divergent run's own query on both builds, same index, no agent: + +``` +codegraph explore "res.send Content-Type ETag generateETag setETag" --path +``` + +| file | baseline | new | +|---|---|---| +| `lib/utils.js` (5,293 B, 272 lines) | **6,380 (46.1%) — whole** | **583 (7.7%) — cluster stub** | +| `lib/response.js` | 3,935 (28.4%) | 6,001 (64.9%) | +| `lib/application.js` | 1,607 (11.6%) | 2,532 (27.4%) | +| `lib/express.js` | 1,927 (13.9%) | — | +| **source envelope** | **13,849** | **9,241** | + +`lib/utils.js` is where `compileETag`, `createETagGenerator`, `etag` and `wetag` live — half the +answer. The CG-4 diagnostic on the new build: + +``` +envelope 10,295 delivered · 10,292 allocated of 13,000 budget +allocation 12,398 reserved of 12,400 pool · cliff at weight 10.00 · nothing cliffed + # deliv% bytes reserved score flags render file + 1 5.7% 583 3,870 56.0 named entry central clusters lib/utils.js + 2 57.0% 5,868 5,875 91.4 entry clusters* lib/response.js + 3 23.0% 2,369 2,653 34.5 entry central clusters* lib/application.js +``` + +`utils.js` is the **top-ranked** file (score 56.0, named + entry + central) and was **reserved +3,870 chars — and spent 583 of them.** Nothing was cliffed. The allocator did its job; the +render loop threw the reservation away. + +The mechanism is the whole-file bound at `src/mcp/tools.ts:4008`: + +```ts +const WHOLE_FILE_MAX_CHARS = allowance + Math.min(WHOLE_FILE_GRACE_MAX, + round(allowance * WHOLE_FILE_GRACE_FRACTION)); +``` + += `3,870 + min(800, 580)` = **4,450** < the file's 5,293 bytes, so the whole-file render is +declined. Pre-CG-12 the bound was `maxCharsPerFile * 3` = 11,400, which the file cleared +comfortably. The fallback cluster render only has 3 matched symbols to work with, so it emits +583 chars and **3,287 chars of the reservation are simply lost** — which is also why the whole +response shrank from 13.8K to 9.2K against an unchanged 13,000-char budget. + +This is CG-12's own acceptance criterion — *"no file that was previously unclipped becomes +clipped"* — failing, and here it is the direct cause of an agent Read. CG-14 recorded one +instance of it (`memory-budget.ts`) as a documented exception; this is the same defect +observed in the wild, where it costs a round-trip. + +**Not systemic.** On both medium repos the render loop saturates (`[over budget] [TRUNCATED]`, +23,599 of a 23,600 pool reserved), so there is no unspent budget to lose. The failure needs a +file whose proportional reservation lands *below its own size* while its matched-symbol set is +thin — likelier on small repos, where per-file reservations are smallest. + +### The fix belongs in CG-12, not here + +Per this task's acceptance rule the budget must **not** be widened to compensate. The defect is +that a reservation can go unspent, so the fix is one of: + +1. **Let a large-enough reservation buy the whole file.** If `allowance >= k * fileSize` for + some k < 1 (utils.js: 3,870 / 5,293 = 0.73), render whole and let the bounded overshoot the + ceiling already tolerates absorb it — the bytes were reserved for this file anyway. +2. **Redistribute what a file cannot spend.** After the render loop knows a file's realised + size, hand the shortfall to the next-ranked file instead of dropping it. This also fixes the + shrinking-envelope symptom directly. + +(1) is the smaller change and matches the observed shape; (2) is the more complete invariant +("the pool is spent"). They compose. + +--- + +## Bars + +| # | Bar | Verdict | +|---|---|---| +| 1 | **Read stays at 0** | **FAIL** — client-go 0/0/0 and excalidraw 0/0/0 both arms, but express run 2 makes 4 Reads the baseline never made, from a reproducible non-agent cause | +| 2 | Correct-file share > 50% | **PASS** — new 67.8–100% across all 9 runs; self-query fixture 16% → 59.9%. (Caveat: on these three repos the *baseline* was already above 50%; the ~16% figure is the self-query fixture, not these repos) | +| 3 | No wall-clock regression | **PASS at the median** — excalidraw 34s → 24s, express 27s → 24s, client-go 49s → 50s. The 52s express outlier is the failing run | +| 4 | No regression on the control | **FAIL** — express, 1 run of 3 | + +Bar 1 is the hard gate and it fails, so the epic does not pass on this measurement regardless +of bars 2 and 3. + +## Reproduce + +```bash +# clone fresh (never eval on a private repo), index, then: +RUNS=3 AGENT_EVAL_OUT=/tmp/ab-express \ + scripts/agent-eval/ab-new-vs-baseline.sh "" main + +node scripts/agent-eval/parse-run.mjs /tmp/ab-express/run-new-2.jsonl --answer 'lib/**' + +# the deterministic core of the failure, no agent needed: +CODEGRAPH_EXPLORE_DEBUG=1 node dist/bin/codegraph.js \ + explore "res.send Content-Type ETag generateETag setETag" --path +``` diff --git a/docs/design/dynamic-dispatch-coverage-playbook.md b/docs/design/dynamic-dispatch-coverage-playbook.md index 63603a920..865ddb6de 100644 --- a/docs/design/dynamic-dispatch-coverage-playbook.md +++ b/docs/design/dynamic-dispatch-coverage-playbook.md @@ -278,6 +278,28 @@ Status legend: ✅ done+validated · 🔬 hole identified · ⬜ not started. (Verify the exact supported set against `src/extraction/languages/` and `src/resolution/frameworks/` before starting — this table is a starting point.) +### Retrieval A/Bs that are not coverage work + +Coverage decides whether a flow *exists* in the graph. A second class of change decides +whether the answer explore returns is *sufficient* — how the byte envelope is divided across +the files it found. Same pass bar (Read → 0, no wall-clock regression), same `--model sonnet +--effort high` rule, but the harness is `ab-new-vs-baseline.sh` (new build vs baseline build, +**both codegraph-on**) rather than `run-all.sh`'s with-vs-without, because the question is +whether a change to codegraph helped, not whether codegraph helps. + +| Change | Repos | Result | +|---|---|---| +| **Score-proportional byte allocation** (#1500, epic CG-1) — 2026-08-04, `feature/CG-1` vs `main`, 3 runs/arm | client-go (Go, 2,454 f, 2,001 generated — the reporter's shape), excalidraw (TS, 672 f), express (JS, 147 f, control) | **Gate FAILED.** Read 0/0/0 both arms on client-go and excalidraw; excalidraw **34s → 24s median with one fewer explore call**; generated clientsets/informers drop from 10.5% of a baseline envelope to 0% in every new run. But express regressed in 1 of 3 runs (**4 Reads, 52s**) from a reproducible non-agent cause: a file whose proportional reservation lands below its own size no longer renders whole and its cluster render leaves the reservation **unspent** (`lib/utils.js` 6,380 B whole → 583 B stub, envelope 13.8K → 9.2K). Full record: [`docs/benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md) | + +Two harness lessons from that run, both now baked into `ab-new-vs-baseline.sh`: + +- **Name codegraph in the prompt for this class of A/B.** Whether the agent picks the tool at + all is an adoption axis a retrieval change does not touch; a run that never calls explore + measures nothing about how explore divides its bytes (one pre-run: 0 codegraph calls, 3 + Reads). It is not a forced-Read-0 — the agent stays free to fall back, which is the bar. +- **`CODEGRAPH_NO_PROMPT_HOOK=1` on both arms.** The machine's ambient front-load hook resolves + to whatever is in `dist/`, which the script itself rewrites between arms. + --- ## 7. Known limits & gotchas (from the excalidraw/django work) diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index 8658d36cd..4e58f189a 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -446,3 +446,59 @@ Both were found by writing the invariant rather than by reading the code: tier monotonicity, the floor) reference the constants and hold at any value; one test pins the literals, so re-tuning a constant is a visible decision that says *re-run the probe* rather than a silent re-calibration of the fixtures. + +## CG-15 — what the agent A/B found: a reservation can go unspent + +The agent A/B (full record: [`../benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md)) +passed on both medium repos — client-go and excalidraw at Read 0 in every run, excalidraw +**34s → 24s median with one fewer explore call**, the generated clientsets that took 10.5% of a +baseline envelope gone from every new run — and **failed on the small control**, express, in 1 +run of 3: 4 Reads of `lib/utils.js` where the baseline made 1 Read of a different file. + +It is not agent variance. Replaying that run's own query deterministically: + +| `lib/utils.js` (5,293 B, 272 lines) | baseline | CG-12 | +|---|---|---| +| delivered | **6,380 (46.1%) whole** | **583 (7.7%) cluster stub** | +| source envelope (budget 13,000) | 13,849 | 9,241 | + +The diagnostic says the allocator was right and the render loop was not: + +``` +allocation 12,398 reserved of 12,400 pool · nothing cliffed + # deliv% bytes reserved score flags render file + 1 5.7% 583 3,870 56.0 named entry central clusters lib/utils.js +``` + +`utils.js` is the **top-ranked** file and was **reserved 3,870 chars — of which it spent 583.** +The whole-file bound (`src/mcp/tools.ts:4008`) is +`allowance + min(GRACE_MAX, allowance * GRACE_FRACTION)` = `3,870 + 580` = 4,450, just under the +file's 5,293 bytes, so the whole-file render is declined; the fallback cluster render has three +matched symbols to work with and emits 583 chars. The remaining 3,287 chars of the reservation +are **not redistributed — they are lost**, which is why the response shrank by a third against +an unchanged budget. + +This is CG-12's own acceptance criterion (*"no file that was previously unclipped becomes +clipped"*) failing. CG-14 recorded one instance as a documented exception (`memory-budget.ts`); +this is the same defect in the wild, where it costs an agent round-trip. It is **not** systemic: +on both medium repos the render loop saturates (`[over budget] [TRUNCATED]`, 23,599 of a 23,600 +pool reserved) so there is nothing left to lose. It needs a file whose reservation lands below +its own size while its matched-symbol set is thin — likeliest on small repos, where per-file +reservations are smallest. + +### The two candidate fixes + +Widening the envelope is explicitly **not** one of them — that is the dial iter2 already proved +doesn't work, and the bytes here were reserved for this file already. + +1. **Let a large-enough reservation buy the whole file.** Render whole when + `allowance >= k * fileSize` for some `k < 1` (utils.js sits at 0.73), letting the bounded + overshoot the hard ceiling already tolerates absorb the difference. Smaller change, matches + the observed shape. +2. **Redistribute the shortfall.** Once the render loop knows a file's realised size, hand what + it cannot spend to the next-ranked file. The stronger invariant — *the pool is spent* — and + it fixes the shrinking-envelope symptom directly rather than case by case. + +They compose; (1) alone would have carried this case. Whichever lands needs the express shape as +a hermetic fixture — a mid-sized top-ranked file with few matched symbols, sized just above its +reservation — because nothing in the current suite has that shape, which is how it shipped. From 8077b83eb3db8f76e1c74440f4b8fafb9c2f9bc5 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 01:34:43 -0500 Subject: [PATCH 10/15] docs: consolidate the #1500 CHANGELOG entries into user-facing shape (CG-16) Three separate engineer-shaped entries (CG-5 generated detection, CG-10 scoring, CG-12 allocation) become two user-facing bullets under Fixes, in the shape #1500 actually reported: explore concentrates on the code that answers the question, and a generated CRUD/protobuf layer no longer crowds out the hand-written code beside it. Per the CHANGELOG rules: strips the benchmark counts and percentages (the client-go 2,001-file count, the quarter-to-four-fifths envelope shift) and the internal symbol names, keeps the re-index note, and credits the reporter. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ceaaafcd..9b94513f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,13 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features -- Generated code is now recognized by what's written at the top of the file, not just by its filename. Go's own convention — a `// Code generated by … DO NOT EDIT.` line — is a comment, not a naming rule, so a generated `payroll.go` sitting beside hand-written code was previously indistinguishable from it and could outrank the real implementation in `codegraph_explore` and search results. CodeGraph now reads the header while indexing and recognizes the standard banners across languages (Go's, protoc's, `@generated` in JavaScript/TypeScript, `` in C#, Thrift, OpenAPI Generator, FlatBuffers, ANTLR, bindgen and more), so those files rank behind hand-written source everywhere the existing filename rules already applied. On a Kubernetes `client-go` checkout this identifies 2,001 generated files that filename rules alone caught none of. Re-index after upgrading to pick up the new detection. (#1500) - - Anonymous usage telemetry is now stored entirely on CodeGraph's own first-party infrastructure — no third-party analytics vendor receives any of it, and the endpoint that receives it makes no outbound requests at all. Individual events are deleted after 90 days, leaving only anonymous daily totals. Nothing about what is collected changed, your IP address is still never read or stored, and every off-switch works exactly as before (`codegraph telemetry off`, `CODEGRAPH_TELEMETRY=0`, `DO_NOT_TRACK=1`). `TELEMETRY.md` remains the complete field-by-field list. ### Fixes -- `codegraph_explore` now gives the most relevant file the most room, instead of splitting its answer roughly evenly and letting whichever file happened to be small enough take the largest share. Every file's slice is now sized in proportion to how well it matches your question, decided before any code is written out — so the file that actually answers you is no longer trimmed at a fixed per-file limit while a small, weakly-related file is included in full. Files far below the best match are listed by name, symbol and line number instead of taking space with their source, and you can pull any of them up in full with one more `codegraph_explore`. On a Go service with a generated CRUD layer beside a hand-written payroll workflow, the hand-written code went from a quarter of the response to nearly four-fifths of it, and the calculation the question was actually about — previously missing entirely — is now included. (#1500) +- `codegraph_explore` now concentrates its answer on the code that actually answers your question, instead of spreading it across files that merely share a word with it — so more of the answer fits inside the same response, and the follow-up calls that used to be needed to fill in the gaps often aren't. Two things changed. Matches are weighted by what was matched: a function, class or route counts for far more than a local variable, and a variable that nothing else references counts for almost nothing, so a file whose only tie to your question is a same-named constant no longer competes with the file that implements the thing. And the room each file gets is now sized in proportion to how well it matches, decided before any source is written out, instead of every file receiving a similar slice under the same per-file limit — the file that answers you is no longer trimmed while a smaller, weakly-related one is included whole. Files far below the best match are listed by name, symbol and line number rather than spending the answer on their source, and one more `codegraph_explore` naming any of them returns it in full. Test and spec files in a repository's top-level `test/` or `spec/` directory are also recognized as such now, which they weren't before. Thanks @LeDuyViet for the detailed measurements and reproduction. (#1500) -- `codegraph_explore` no longer spends its answer on files that merely share a word with your question. A file whose only connection to the query was an unused local variable or constant of the same name — an eval script with a `const explore` in it, say — used to count as strongly as the file that actually implements the thing, and being small enough to include whole, it could take most of the response while the real answer got trimmed. Matches are now weighted by what was matched (a function, class or route counts for far more than a local variable, and a variable nothing else references counts for almost nothing), and a file has to score within reach of the best match rather than clear a fixed low bar. Generated files are down-weighted throughout the ranking instead of only breaking ties, so a generated CRUD layer no longer outranks the hand-written workflow beside it. Test and spec files in a top-level `test/` or `spec/` directory are now recognized and kept out, which they weren't before. (#1500) +- A generated CRUD or protobuf layer no longer crowds out the hand-written code sitting beside it. Generated code is now recognized by what's written at the top of the file, not just by its filename: Go's own convention — a `// Code generated by … DO NOT EDIT.` line — is a comment, not a naming rule, so a generated file with an ordinary name was previously indistinguishable from hand-written source and could outrank the real implementation wherever both matched a name — in `codegraph_explore`, in search results, and in the callers and blast-radius answers built on them. CodeGraph now reads the header while indexing and recognizes the standard banners across languages (Go's, protoc's, `@generated` in JavaScript and TypeScript, `` in C#, Thrift, OpenAPI Generator, FlatBuffers, ANTLR, bindgen and more), and generated files are down-weighted throughout the ranking rather than only breaking ties between otherwise equal matches. Re-index after upgrading to pick up the new detection. (#1500) - A CodeGraph process that gets force-killed — by the stuck-process watchdog, a crash, or the OS — no longer leaves the database's write-ahead log behind to grow without bound. Previously each killed session stacked more data onto the same log file and nothing ever shrank it, which on machines where sessions were killed regularly could quietly eat tens of gigabytes of disk. The log is now capped, and any oversized leftover is reclaimed automatically the next time the project is opened. Thanks @tiendungdev for the exceptional Windows report that pinned this down. (#1431) - The background server's watchdog no longer kills a healthy server that is just waiting on a slow disk: like indexing already does, it now checks whether the database files are still making progress before concluding the process is stuck. Fewer spurious kills also means fewer leftover write-ahead logs. (#1431) From fa7fb8d127f6c127cfbe31473e90dee999ac50c1 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 02:13:46 -0500 Subject: [PATCH 11/15] fix(explore): spend the reservation instead of dropping it (CG-21, #1500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file whose proportional reservation lands below its own size stopped rendering whole, and the fallback cluster render could leave most of that reservation unspent — the bytes were neither delivered nor redistributed. Found by CG-15's agent A/B on the express control: `lib/utils.js`, the top-ranked file, was reserved 3,870 chars and spent 583. The whole-file grace bound (reservation + a sliver) sat just under the file's 5,293 bytes, so the whole render was declined and three matched symbols became a stub. The source envelope fell 13,849 -> 9,241 against an UNCHANGED budget, and the agent Read the file back four times in 1 run of 3. Two levers, per the task's candidate fixes: - WHOLE_FILE_BUY_FRACTION: a reservation that already covers 60% of a file buys the whole file. Funded from ONE shared overshoot pool sized at 15% of the envelope, spent in rank order. Per-file funding is the version that fails, and it fails the same way the bug does — the merit test is a ratio, so several files qualify at once and N independent overshoots push the last section past the render ceiling. Measured on the payroll fixture: three files bought whole and `payslip_builder.go` was dropped entirely. A dropped section is strictly worse than a clustered one. - Reservation carry-forward: what a file cannot spend goes to the next file down, bounded by MAX_SHARE. Tracked as two running totals rather than a `spent` variable threaded through the render loop's dozen exit paths, so no path can forget to account, and symmetric — a buy that overshoots suppresses slack until a later under-spend covers it. Express reproducer: `lib/utils.js` 583 -> 6,268 whole, envelope 9,241 -> 14,505 on the same 13,000 budget. The `memory-budget.ts` exception CG-14 documented is RESOLVED rather than re-justified: it ships whole again at 5,672 (27.3%) while `src/mcp/tools.ts` rises to 52.6% — so the answer file wins the envelope AND no previously-unclipped file is clipped, which is CG-12's own acceptance criterion finally holding. Two hermetic fixtures added, one per lever, because nothing in the suite had this shape — which is how it shipped. Both mutation-tested: removing the buy arm reddens 3, removing the carry-forward reddens 2, and removing the funding guard reddens 4 (including payroll's dropped `payslip_builder.go`). Their `fixture shape` blocks are load-bearing: the gates pass vacuously if a target ever drifts inside the grace bound, so the window is asserted directly. Full suite green (2,868 passed); both #1500 regression fixtures pass. --- __tests__/explore-allocation-e2e.test.ts | 417 +++++++++++++++++++++++ src/mcp/tools.ts | 114 ++++++- 2 files changed, 527 insertions(+), 4 deletions(-) diff --git a/__tests__/explore-allocation-e2e.test.ts b/__tests__/explore-allocation-e2e.test.ts index 70e4d21c8..dff9d9e9c 100644 --- a/__tests__/explore-allocation-e2e.test.ts +++ b/__tests__/explore-allocation-e2e.test.ts @@ -111,6 +111,17 @@ async function explore(project: Project, query: string) { score: (file: string) => fileOf(file)?.score ?? 0, /** Chars of source the allocator RESERVED for it, before anything rendered. */ allowance: (file: string) => fileOf(file)?.allowance ?? 0, + /** Which render path the loop took: `whole`, `clusters`, `focused`, `skeleton`. */ + render: (file: string) => fileOf(file)?.render ?? null, + /** + * Rank the ranking pass gave it (1 = the file the response leads with, and + * the first the render loop reaches). Read off the record rather than from + * the position in `report.files`, which the report re-sorts by delivered + * bytes for legibility. + */ + rank: (file: string) => fileOf(file)?.rank ?? -1, + /** Total source bytes delivered across every rendered file. */ + sourceTotal: () => [...bytes.values()].reduce((sum, n) => sum + n, 0), /** Fraction of the WHOLE response this file's source occupies. */ share: (file: string) => (bytes.get(file) ?? 0) / (text.length || 1), shareUnder: (prefix: string) => { @@ -412,6 +423,412 @@ export function log(message: string): void { }); }); +// ── 1b. CG-21: a reservation below the file's size must not lose its bytes ── + +/** + * The shape CG-15's agent A/B found in the wild, and the one thing the suite + * above could not catch: a file whose reservation lands BELOW its own size. + * + * Express, `lib/utils.js` (5,293 B), the top-ranked file for + * "res.send Content-Type ETag generateETag setETag": + * + * | | baseline | CG-12 | + * |---|---|---| + * | delivered | 6,380 (46.1%) whole | **583 (7.7%) cluster stub** | + * | source envelope (13,000 budget) | 13,849 | **9,241** | + * + * It was reserved 3,870 and spent 583. The whole-file grace bound + * (`allowance + min(800, allowance * 0.15)` = 4,450) sits just under the file, + * so the whole-file render is declined; the fallback cluster render has three + * matched symbols to work with and emits a stub. The other 3,287 chars were + * neither delivered nor redistributed — **the pool shrank by a third against an + * unchanged budget**, and the agent Read the file back four times. + * + * Everything about that is invisible to the fixtures above, and to the payroll + * one: both SATURATE (`[over budget] [TRUNCATED]`, 23,599 of a 23,600 pool), + * so there is no unspent reservation to lose. This fixture is built to sit in + * the gap instead — a mid-sized top-ranked file with a THIN matched-symbol set, + * sized just above its reservation — which is the combination that has to hold + * for the defect to reproduce, and is why it shipped. + * + * The `fixture shape` block below is load-bearing, not scaffolding: every gate + * here passes vacuously if the target ever drifts small enough for the grace + * bound to cover it, so the window `0.6 × size <= reservation < size` is + * asserted directly. + */ +describe('CG-21 — a reservation under the file size still buys the file', () => { + // Names two symbols that live in ONE mid-sized file (the named-seed tier is + // what puts it at rank 0) while the rest of the terms pull in its peers, so + // the proportional split hands the target well under its own size. + const QUERY = 'generateEtag compileEtag send response body'; + const TARGET = 'src/http/etag.ts'; + const RESPONSE = 'src/http/response.ts'; + const APPLICATION = 'src/http/application.ts'; + + /** + * Bulk for the target: real, extractable symbols that match NOTHING in the + * query. They make the file BIG without making it more relevant — which is + * precisely how a file ends up reserved less than it is worth in bytes. Kept + * dense (4 lines each) so the file stays well inside `WHOLE_FILE_MAX_LINES` + * and the byte bound is the only thing that can decline the whole render. + */ + const inertFiller = (n: number) => ` +export function normalizeLedgerRow${n}(row: string[], fallback: string, separator: string): string[] { + const trimmed = row.map((cell) => cell.trim()).filter((cell) => cell.length > 0 && cell !== separator); + return trimmed.length > 0 ? trimmed : [fallback, separator, String(trimmed.length), 'ledger-row-${n}']; +} +`; + + /** + * The matched-symbol set, deliberately THIN and small. This is the second + * half of the shape: with only these two tiny functions to cluster around, + * the fallback render emits a few hundred chars and abandons the rest of the + * reservation. A file with a fat matched set would spend its allowance the + * ordinary way and never expose the bug. + */ + const TARGET_SOURCE = ` +/** ETag helpers. */ +export function generateEtag(body: string): string { + return '"' + body.length.toString(16) + '"'; +} + +export function compileEtag(setting: string): (body: string) => string { + return setting === 'strong' ? generateEtag : (body: string) => 'W/' + generateEtag(body); +} +${Array.from({ length: 27 }, (_, i) => inertFiller(i + 1)).join('')}`; + + const responseMethod = (name: string) => ` + public ${name}(body: string): string { + const etag = compileEtag(this.etagSetting)(body); + this.headers.set('etag', etag); + return body; + } +`; + + const RESPONSE_SOURCE = ` +import { compileEtag } from './etag'; + +/** The response object: sends a body and negotiates its representation. */ +export class ServerResponse { + private headers = new Map(); + private etagSetting = 'strong'; +${[ + 'send', + 'sendBody', + 'sendResponse', + 'writeBody', + 'endResponse', + 'json', + 'setResponseBody', + 'flushResponseBody', +].map(responseMethod).join('')} +} +`; + + let project: Project; + let run: Awaited>; + let targetSize = 0; + + beforeAll(async () => { + project = await buildProject('codegraph-alloc-cg21-', { + [TARGET]: TARGET_SOURCE, + [RESPONSE]: RESPONSE_SOURCE, + [APPLICATION]: ` +import { ServerResponse } from './response'; + +/** The application: routes a request and hands the response its body. */ +export class Application { + private routes = new Map string>(); + + public handleRequest(path: string, res: ServerResponse, body: string): string { + const route = this.routes.get(path); + return route ? route(res) : res.send(body); + } + + public registerResponseRoute(path: string, handler: (res: ServerResponse) => string): void { + this.routes.set(path, handler); + } +} +`, + 'src/http/request.ts': ` +/** The request object: carries the inbound body. */ +export class ServerRequest { + public constructor(public readonly body: string) {} + + public freshResponseBody(): string { + return this.body.trim(); + } +} +`, + 'src/util/logger.ts': ` +export function log(message: string): void { + console.log(message); +} +`, + }); + targetSize = fs.readFileSync(path.join(project.dir, TARGET), 'utf-8').length; + run = await explore(project, QUERY); + }, 120_000); + + afterAll(() => destroyProject(project)); + + describe('fixture shape', () => { + it('ranks the target first, on a matched set of only two symbols', () => { + // Rank 0 is what makes the loss expensive: this is the file the response + // leads with, and the one the agent Reads back when it arrives as a stub. + expect(run.rank(TARGET)).toBe(1); + }); + + it('sizes the target ABOVE its reservation but inside the buy window', () => { + // The whole assertion set below is vacuous outside this window, so it is + // pinned here rather than assumed: + // reservation >= size → the grace bound already covers it, and the + // buy rule is never consulted (express's other + // three queries look like this). + // reservation < 0.6×size → the shortfall is real, clustering is the + // right answer, and the carry-forward — not the + // buy rule — is what conserves the bytes. + const reserved = run.allowance(TARGET); + expect(reserved).toBeGreaterThan(0); + expect(reserved).toBeLessThan(targetSize); + expect(reserved / targetSize).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION); + // ...and specifically OUTSIDE the grace bound, which is the pre-CG-21 + // rule. If grace alone could carry it, this fixture proves nothing. + const graceBound = reserved + Math.min( + EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_MAX, + Math.round(reserved * EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_FRACTION), + ); + expect(targetSize).toBeGreaterThan(graceBound); + }); + + it('keeps the target inside the whole-file LINE bound, so only bytes can gate it', () => { + // `WHOLE_FILE_MAX_LINES` (220 for a non-central file) is a separate gate + // that also declines a whole render. If the fixture ever crossed it the + // suite would go red for the wrong reason — and, worse, a genuine + // regression in the BYTE bound would be masked by it. + const lines = fs.readFileSync(path.join(project.dir, TARGET), 'utf-8').split('\n').length; + expect(lines).toBeLessThanOrEqual(220); + }); + }); + + describe('the reservation is spent', () => { + it('delivers the target WHOLE rather than as a cluster stub', () => { + // The headline. Pre-CG-21 this file rendered `clusters` and emitted a few + // hundred chars against a multi-thousand-char reservation. + expect(run.render(TARGET)).toBe('whole'); + }); + + it('spends more than the reservation, not a fraction of it', () => { + // Stated as bytes so it bites independently of the render-mode label: a + // build that renamed the whole path but still emitted a stub fails here. + // Express: 583 delivered against 3,870 reserved. + const delivered = run.bytes.get(TARGET) ?? 0; + expect(delivered).toBeGreaterThanOrEqual(targetSize); + expect(delivered).toBeGreaterThan(run.allowance(TARGET)); + }); + + it('leaves no rendered file both under its reservation and short of content', () => { + // The defect stated as an invariant, which is what makes it general rather + // than a re-assertion of the case above: a rendered file either SPENDS what + // it was promised, or it ran out of file. Express's `lib/utils.js` did + // neither — 583 delivered, 3,870 promised, 5,293 bytes of file sitting + // there — and the difference was dropped rather than redistributed, which + // is why the source envelope fell 13,849 → 9,241 on an unchanged budget. + // + // `response.ts` is the case the naive "spend the whole pool" version of + // this test gets wrong: it delivers 1,635 of a 5,292 reservation and that + // is CORRECT — the file is only 1,635 bytes. A pool cannot be spent past + // the content that exists to fill it. + for (const f of run.report.files) { + if (!f.render || (f.emittedChars ?? 0) === 0) continue; + const size = fs.readFileSync(path.join(project.dir, f.path), 'utf-8').length; + expect(f.emittedChars, `${f.path} spent its reservation or ran out of file`) + .toBeGreaterThanOrEqual(Math.min(f.allowance ?? 0, size)); + } + }); + + it('holds the hard ceiling while doing it', () => { + // The buy rule spends MORE than the reservation, so the bound that stops + // it running away has to be re-proved here and not inherited: the + // overshoot pool is finite, and the 25K inline cap is absolute — past it + // the host writes the result to a file the agent Reads back. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), INLINE_CAP); + expect(run.text.length).toBeLessThanOrEqual(hardCeiling); + expect(run.text.length).toBeLessThan(INLINE_CAP); + }); + + it('still serves the peers — concentration, not a single-file response', () => { + // The over-correction control for this fixture. Buying the target whole + // must not eat the files below it: that is the trade the shared overshoot + // pool refuses (it dropped `payslip_builder.go` when funding was per-file). + const peers = [RESPONSE, APPLICATION].filter((f) => (run.bytes.get(f) ?? 0) > 0); + expect(peers.length).toBeGreaterThan(0); + }); + }); +}); + +/** + * The other half of CG-21, and the half the whole-file buy rule cannot reach. + * + * Buying the file whole only helps when the reservation has already covered + * most of it. Below that the shortfall is real — the file is several times its + * reservation, and clustering IS the right render — but the bytes it cannot + * spend still must not evaporate. Express, query "compileETag req.fresh": + * `lib/utils.js` was reserved 3,809 and spent 791; the 3,018 chars it left had + * to reach `lib/response.js` below it, which delivered 4,650 on a 1,895 + * reservation. + * + * So this fixture is deliberately the INVERSE of the one above: the leading + * file is far too big for the buy rule to fire, and the assertion is on the + * file BELOW it. Without this, `allowance = reserved` — the whole carry-forward + * deleted — passes every other test in this file. + */ +describe('CG-21 — an unspendable reservation flows to the next file down', () => { + // Names three tiny callables that all live in the SPRAWL file — the named-seed + // tier is what puts a file with almost no matched content at rank 1 — plus one + // term the absorber's methods carry, so it ranks second rather than cliffing. + const QUERY = 'renderStaticScene renderInteractiveScene renderNewElementScene paintSceneLayer'; + // Rank 1: a huge file the query names two symbols in. Its reservation cannot + // approach its size, so it clusters — and clusters thinly, because those two + // symbols are all it matched. + const SPRAWL = 'src/scene/sprawl.ts'; + // Rank 2: dense with matched symbols and bigger than any share it can be + // reserved, so it will absorb whatever the file above it leaves. + const ABSORBER = 'src/render/absorber.ts'; + + const inertBulk = (n: number) => ` +export function reconcileLedgerEntry${n}(rows: string[], fallback: string, separator: string): string[] { + const trimmed = rows.map((cell) => cell.trim()).filter((cell) => cell.length > 0 && cell !== separator); + return trimmed.length > 0 ? trimmed : [fallback, separator, String(trimmed.length), 'entry-${n}']; +} +`; + + // Long ENOUGH, in lines, that the absorber cannot ship whole (220 lines is the + // other whole-file gate). That matters: a file that renders whole ignores the + // per-file budget entirely, and this fixture is about a budget being spent. + const matchedPaint = (n: number) => ` + public paintSceneLayer${n}(canvas: string, scene: string, element: string): string { + const appState = this.appState.get('layer${n}') ?? scene; + const painted = canvas + '|' + appState + '|' + element; + const stamped = painted + '|layer-${n}'; + const merged = stamped + '|' + scene + '|' + element; + const settled = merged.split('|').filter((part) => part.length > 0).join('|'); + this.appState.set('layer${n}', settled); + if (settled.length === 0) { + return this.paint(scene, scene); + } + return this.paint(settled, scene); + } +`; + + let project: Project; + let run: Awaited>; + + beforeAll(async () => { + project = await buildProject('codegraph-alloc-cg21-carry-', { + [SPRAWL]: ` +import { Absorber } from '../render/absorber'; + +/** Scene sprawl: three one-line answers buried in a very large file. */ +export function renderStaticScene(scene: string): string { + return new Absorber().paint(scene, scene); +} + +export function renderInteractiveScene(scene: string): string { + return new Absorber().paint(scene, scene + ':interactive'); +} + +export function renderNewElementScene(scene: string): string { + return new Absorber().paint(scene, scene + ':new-element'); +} +${Array.from({ length: 90 }, (_, i) => inertBulk(i + 1)).join('')}`, + [ABSORBER]: ` +/** The renderer: many matched paint passes, all of them wanted. */ +export class Absorber { + private appState = new Map(); + + public paint(element: string, scene: string): string { + return element + '|' + scene; + } +${Array.from({ length: 20 }, (_, i) => matchedPaint(i + 1)).join('')} +} +${/* Inert tail: pushes the absorber FAR past its reservation so the whole-file + buy rule cannot fire on it either. Without this the absorber ships whole + and the fixture measures the buy rule a second time instead of the + carry-forward — which is exactly how it read on the first attempt. */ + Array.from({ length: 40 }, (_, i) => inertBulk(100 + i)).join('')}`, + 'src/util/logger.ts': ` +export function log(message: string): void { + console.log(message); +} +`, + }); + run = await explore(project, QUERY); + }, 120_000); + + afterAll(() => destroyProject(project)); + + it('leaves the leading file unable to spend its reservation', () => { + // The precondition. If the sprawl file ever spends its share, there is no + // slack, and the assertion below passes for no reason at all. + const spent = run.bytes.get(SPRAWL) ?? 0; + expect(spent).toBeGreaterThan(0); + expect(spent).toBeLessThan(run.allowance(SPRAWL)); + // ...and it is out of reach of the buy rule, so this is genuinely the + // carry-forward's case and not a second test of the fixture above. + const size = fs.readFileSync(path.join(project.dir, SPRAWL), 'utf-8').length; + expect(run.allowance(SPRAWL) / size).toBeLessThan(EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION); + }); + + it('hands the shortfall to the file below, which spends past its own reservation', () => { + // The lever. Measured both ways on this fixture: with the carry-forward the + // absorber delivers 9,297 against a 7,455 reservation; with + // `allowance = reserved` it delivers 7,479 — its reservation and nothing + // more, while the sprawl file's 4,408 unspent chars are dropped. + // + // The 1.1 margin is not padding. A cluster section can land a few chars over + // the budget it was selected against (whole symbol ranges, never sliced + // mid-method), so "delivered > reserved" alone is true by ~24 chars even on + // the mutated build — a test that passes on the defect. + const delivered = run.bytes.get(ABSORBER) ?? 0; + expect(delivered).toBeGreaterThan(Math.round(run.allowance(ABSORBER) * 1.1)); + }); + + it('keeps the shortfall in the envelope instead of dropping it', () => { + // The same lever read off the response as a whole, which is the form the + // user actually feels: express's source envelope fell 13,849 → 9,241 on an + // unchanged 13,000 budget because nothing picked up what `lib/utils.js` + // could not spend. Here: 10,033 delivered with the carry-forward, 8,215 + // without. + // + // Stated against what a no-carry build could produce — the leader's actual + // spend plus the absorber's own reservation — so it stays a statement about + // the mechanism rather than a hard-coded byte count. + const noCarryCeiling = (run.bytes.get(SPRAWL) ?? 0) + Math.round(run.allowance(ABSORBER) * 1.05); + expect(run.sourceTotal()).toBeGreaterThan(noCarryCeiling); + }); + + it('bounds the borrowing — slack concentrates, it does not consume', () => { + // Carried slack is clamped to `MAX_SHARE` of the envelope, so an + // under-spending leader cannot hand the file below it the whole response. + // The bound is stated WITH the spine allowance (`SPINE_CEILING`, 1.5x) + // folded in: a flow-path cluster is deliberately allowed past the per-file + // share, and that predates CG-21 — writing the tighter bound here would + // make this test fail on a build with no defect in it. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + const clamp = Math.max( + run.allowance(ABSORBER), + Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE), + ); + expect(run.bytes.get(ABSORBER) ?? 0).toBeLessThanOrEqual(Math.round(clamp * 1.5)); + // The anti-starvation half, and the one that would actually bite: the file + // that lent the slack still gets rendered. + expect(run.bytes.get(SPRAWL) ?? 0).toBeGreaterThan(0); + expect(run.text.length).toBeLessThan(INLINE_CAP); + }); +}); + // ── 2. Degenerate and diffuse result sets ─────────────────────────────────── describe('allocation on degenerate result sets', () => { diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 85a0e99f5..0417615cb 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -521,6 +521,45 @@ export const EXPLORE_ALLOCATION = { */ WHOLE_FILE_GRACE_FRACTION: 0.15, WHOLE_FILE_GRACE_MAX: 800, + /** + * A reservation that already covers this fraction of a file BUYS THE WHOLE + * FILE (CG-21), even though the file is bigger than the reservation. + * + * The grace above is calibrated as a *sliver* — it only rescues a file that + * essentially fits. Below it there is a hole the render loop cannot fill: + * express's `lib/utils.js` (5,293 B) was the TOP-ranked file, reserved 3,870, + * declined the whole-file render at a 4,450 grace bound, and then spent 583 on + * a three-symbol cluster render. The other 3,287 chars of its reservation were + * neither redistributed nor delivered — the envelope shrank by a third against + * an unchanged budget and the agent Read the file back four times. + * + * So the rule is not "does the file fit the reservation" but "has the + * reservation already bought most of the file": at 0.6 the loop pays at most + * two-thirds of a reservation extra to avoid losing the whole thing, and it + * spends bytes it was going to spend anyway on a file that already earned + * them. Below the fraction the shortfall is real — the file is several times + * its reservation, clustering is the right answer, and the carry-forward + * (`reservedSoFar`/`sourceSpent` in the render loop) hands whatever it cannot + * spend to the next file down. + */ + WHOLE_FILE_BUY_FRACTION: 0.6, + /** + * The buy rule's overshoot is funded from ONE pool for the whole response, + * sized as this fraction of the envelope — deliberately the same 15% as + * `WHOLE_FILE_GRACE_FRACTION`, one level up: the grace is a sliver of a + * FILE's reservation, this is a sliver of the RESPONSE's envelope. + * + * Per-file funding is the version that fails, and it fails the same way the + * bug being fixed does. The merit test is a RATIO, so wherever several files + * sit near it they all qualify, and N independent overshoots inflate the + * response until the render ceiling drops whatever is last. Measured on the + * #1500 payroll fixture: three files bought whole and `payslip_builder.go` — + * the file that computes the payslip the question asks about, rank #6 — was + * dropped entirely so three higher-ranked files could each ship their final + * sliver. A dropped section is strictly worse than a clustered one, so one + * shared pool, spent in rank order, is the bound that matters. + */ + WHOLE_FILE_BUY_OVERSHOOT_FRACTION: 0.15, } as const; /** One candidate file's allocation inputs, in final rank order. */ @@ -3818,6 +3857,32 @@ export class ToolHandler { // instead of a different symbol's code under the requested name. const staleRendered: string[] = []; const staleOmitted: string[] = []; + // Reservation carry-forward (CG-21). A reservation is a promise the render + // loop has to KEEP, not a cap it may quietly under-use: a file that cannot + // spend what it was given — thin matched-symbol set, unreadable, drifted off + // disk, skipped for the ceiling — must hand the difference DOWN the rank + // order, not drop it. Tracked as two running totals rather than a `spent` + // variable threaded through the dozen `continue`s below, so no exit path can + // forget to account: everything the loop has PROMISED so far, and everything + // it has actually EMITTED. Their gap is the slack the next file may add to + // its own reservation. + // + // Symmetric on the other side: a whole-file buy that overshoots makes + // `sourceSpent` outrun `reservedSoFar`, which suppresses slack until a later + // under-spend covers the debt. So the pool is conserved in both directions, + // and no file is ever cut BELOW the reservation it was promised. + let reservedSoFar = 0; + let sourceSpent = 0; + // Funding line for the whole-file BUY rule: the response's SOURCE may reach + // everything the allocator promised plus one bounded overshoot, and no more. + // Measured against the promise rather than `renderCeiling` on purpose — the + // ceiling sits 50% above the envelope and says nothing about who is owed + // what, so funding a buy from it just moves the shortfall to whichever file + // the loop reaches last. See WHOLE_FILE_BUY_OVERSHOOT_FRACTION. + const reservedTotal = [...allocation.allowances.values()].reduce((sum, n) => sum + n, 0); + const sourceCeiling = reservedTotal + Math.round( + budget.maxOutputChars * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_OVERSHOOT_FRACTION, + ); for (const [filePath, group] of sortedFiles) { if (filesIncluded >= maxFiles) { @@ -3835,11 +3900,24 @@ export class ToolHandler { // bounded by it instead of by the flat per-file cap, which is what stops // allocation from following file size: a small weakly-relevant file no // longer ships whole while the strongly-relevant one is clipped. - const allowance = allocation.allowances.get(filePath); - if (allowance === undefined) { + const reserved = allocation.allowances.get(filePath); + if (reserved === undefined) { diag?.recordSkip(filePath, 'max-files'); continue; } + // What this file may actually spend: its own reservation PLUS whatever the + // files above it left on the table. Every render bound below reads this, + // never `reserved` — that is what makes the carry-forward reach the render + // paths instead of being bookkeeping. Slack flows to the next file in RANK + // order because that is the only file a single-pass loop can still pay; + // `MAX_SHARE` keeps that from turning a weak tail file into the response, + // so the allocator's share ceiling holds end-to-end and not just at + // reservation time. + const allowance = Math.min( + reserved + Math.max(0, reservedSoFar - sourceSpent), + Math.max(reserved, Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE)), + ); + reservedSoFar += reserved; const absPath = validatePathWithinRoot(projectRoot, filePath); if (!absPath || !existsSync(absPath)) { diag?.recordSkip(filePath, 'unreadable'); @@ -3974,6 +4052,7 @@ export class ToolHandler { : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', ''); totalChars += skel.join('\n').length + 120; + sourceSpent += skel.join('\n').length; // Always "clipped": the per-symbol view elides bodies by construction. diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true); renderedFilePaths.push(filePath); @@ -4005,11 +4084,36 @@ export class ToolHandler { // reservation removes the swing without touching the rule's purpose (a small // file sliced is a lossy subset the agent just Reads in full anyway). const WHOLE_FILE_MAX_LINES = isCentralFile ? 280 : 220; - const WHOLE_FILE_MAX_CHARS = allowance + Math.min( + // Two bounds, whichever is larger (CG-21): + // GRACE — the reservation plus a sliver, for a file that essentially fits; + // BUY — the reservation already covers most of the file, so the rest is + // cheaper to ship than to lose. A file between the two used to + // fall through to clustering and then spend a FRACTION of its + // reservation, and the remainder was neither delivered nor + // redistributed. See WHOLE_FILE_BUY_FRACTION. + // + // The BUY arm is two independent tests, and keeping them apart is the whole + // design. MERIT reads `reserved` — did THIS file's own relevance earn most + // of itself? — so borrowed slack can never promote a weak file to whole. + // FUNDING reads the shared overshoot pool, so the bytes exist to pay for it. + // Slack still reaches the file through `allowance`: it raises the GRACE arm + // and shrinks what a buy has to borrow. + const graceBound = allowance + Math.min( EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_MAX, Math.round(allowance * EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_FRACTION), ); - if (fileLines.length <= WHOLE_FILE_MAX_LINES && fileContent.length <= WHOLE_FILE_MAX_CHARS) { + // FUNDING, as one inequality: after this file ships whole, does the source + // still fit the promise-plus-overshoot line WITH every reservation below + // it left payable? `owedBelow` is what makes it a displacement guard + // rather than a size cap — a buy that fits the line only by spending a + // lower-ranked file's reservation is the trade that dropped + // `payslip_builder.go`, and it is refused here. Self-limiting: each buy + // grows `sourceSpent`, so the pool cannot be spent twice. + const owedBelow = Math.max(0, reservedTotal - reservedSoFar); + const buysWhole = fileContent.length <= graceBound + || (reserved >= fileContent.length * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION + && sourceSpent + fileContent.length + owedBelow <= sourceCeiling); + if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) { const body = fileContent.replace(/\n+$/, ''); let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body; const uniqSymbols = [...new Set( @@ -4034,6 +4138,7 @@ export class ToolHandler { } lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', ''); totalChars += wholeSection.length + 200; + sourceSpent += wholeSection.length; diag?.recordRender(filePath, 'whole', wholeSection.length, false); renderedFilePaths.push(filePath); filesIncluded++; @@ -4406,6 +4511,7 @@ export class ToolHandler { lines.push(''); totalChars += fileSection.length + 200; + sourceSpent += fileSection.length; diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length); renderedFilePaths.push(filePath); filesIncluded++; From 51cd053d85552b4b757f65a763c2150046ab7b3b Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 02:17:28 -0500 Subject: [PATCH 12/15] =?UTF-8?q?docs:=20CG-21=20=E2=80=94=20spending=20th?= =?UTF-8?q?e=20reservation=20(design=20record=20+=20CHANGELOG=20precision)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records both levers, the funding-pool design (and the per-file version that dropped payslip_builder.go), the resolved memory-budget.ts exception, and the two hermetic fixtures with their mutation matrix. The CHANGELOG clause 'no longer trimmed while a smaller, weakly-related one is included whole' was imprecise after CG-21: the smaller file often IS still included whole now, when its share nearly covers it. Reworded to say what the fix actually guarantees. --- CHANGELOG.md | 2 +- docs/design/explore-budget-allocation.md | 114 +++++++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b94513f5..ff161b32f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes -- `codegraph_explore` now concentrates its answer on the code that actually answers your question, instead of spreading it across files that merely share a word with it — so more of the answer fits inside the same response, and the follow-up calls that used to be needed to fill in the gaps often aren't. Two things changed. Matches are weighted by what was matched: a function, class or route counts for far more than a local variable, and a variable that nothing else references counts for almost nothing, so a file whose only tie to your question is a same-named constant no longer competes with the file that implements the thing. And the room each file gets is now sized in proportion to how well it matches, decided before any source is written out, instead of every file receiving a similar slice under the same per-file limit — the file that answers you is no longer trimmed while a smaller, weakly-related one is included whole. Files far below the best match are listed by name, symbol and line number rather than spending the answer on their source, and one more `codegraph_explore` naming any of them returns it in full. Test and spec files in a repository's top-level `test/` or `spec/` directory are also recognized as such now, which they weren't before. Thanks @LeDuyViet for the detailed measurements and reproduction. (#1500) +- `codegraph_explore` now concentrates its answer on the code that actually answers your question, instead of spreading it across files that merely share a word with it — so more of the answer fits inside the same response, and the follow-up calls that used to be needed to fill in the gaps often aren't. Two things changed. Matches are weighted by what was matched: a function, class or route counts for far more than a local variable, and a variable that nothing else references counts for almost nothing, so a file whose only tie to your question is a same-named constant no longer competes with the file that implements the thing. And the room each file gets is now sized in proportion to how well it matches, decided before any source is written out, instead of every file receiving a similar slice under the same per-file limit — the file that answers you is no longer trimmed just because a smaller, weakly-related one was small enough to include whole. Files far below the best match are listed by name, symbol and line number rather than spending the answer on their source, and one more `codegraph_explore` naming any of them returns it in full. Test and spec files in a repository's top-level `test/` or `spec/` directory are also recognized as such now, which they weren't before. Thanks @LeDuyViet for the detailed measurements and reproduction. (#1500) - A generated CRUD or protobuf layer no longer crowds out the hand-written code sitting beside it. Generated code is now recognized by what's written at the top of the file, not just by its filename: Go's own convention — a `// Code generated by … DO NOT EDIT.` line — is a comment, not a naming rule, so a generated file with an ordinary name was previously indistinguishable from hand-written source and could outrank the real implementation wherever both matched a name — in `codegraph_explore`, in search results, and in the callers and blast-radius answers built on them. CodeGraph now reads the header while indexing and recognizes the standard banners across languages (Go's, protoc's, `@generated` in JavaScript and TypeScript, `` in C#, Thrift, OpenAPI Generator, FlatBuffers, ANTLR, bindgen and more), and generated files are down-weighted throughout the ranking rather than only breaking ties between otherwise equal matches. Re-index after upgrading to pick up the new detection. (#1500) diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index 4e58f189a..cff4d4537 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -502,3 +502,117 @@ doesn't work, and the bytes here were reserved for this file already. They compose; (1) alone would have carried this case. Whichever lands needs the express shape as a hermetic fixture — a mid-sized top-ranked file with few matched symbols, sized just above its reservation — because nothing in the current suite has that shape, which is how it shipped. + +## CG-21 — spending the reservation + +Both fixes landed, because they cover different halves of the same failure and neither is +sufficient alone. The unifying rule: **a reservation is a promise the render loop has to keep, +not a cap it may quietly under-use.** + +### 1. A reservation that has already bought most of the file buys the rest + +`WHOLE_FILE_BUY_FRACTION` (0.6). The pre-existing grace is calibrated as a *sliver* — it rescues +a file that essentially fits. Between "fits" and "several times its reservation" there was a +hole, and `lib/utils.js` (reserved/size = 0.73) sat squarely in it. So the test is not *does the +file fit the reservation* but *has the reservation already bought most of the file*: above 0.6 +the loop pays at most two-thirds of a reservation extra rather than lose the whole thing, on a +file that already earned those bytes. + +**Funding is the part that is easy to get wrong.** The merit test is a *ratio*, so wherever +several files sit near it they all qualify — and N independent overshoots inflate the response +until the render ceiling drops whatever is last. Funding each buy from the file's own +headroom was tried and measured on the payroll fixture: three files bought whole and +`payslip_builder.go` — the file that computes the payslip the question asks about — was +**dropped entirely** so three higher-ranked files could each ship their final sliver. A dropped +section is strictly worse than a clustered one. + +So the buy rule is **two independent tests**, and keeping them apart is the whole design: + +| | reads | answers | +|---|---|---| +| **merit** | the file's own `reserved` | did this file's relevance earn most of itself? | +| **funding** | one shared overshoot pool (`WHOLE_FILE_BUY_OVERSHOOT_FRACTION`, 15% of the envelope) | do the bytes exist, *with every reservation below it still payable*? | + +Merit reads `reserved` rather than the post-carry allowance, so borrowed slack can never promote +a weak file to whole. Funding is measured against what the allocator **promised** rather than +against `renderCeiling` — the ceiling sits 50% above the envelope and says nothing about who is +owed what, so funding a buy out of it just moves the shortfall onto whichever file the loop +reaches last. The `owedBelow` term is what makes it a *displacement* guard rather than a size +cap, and it is self-limiting: each buy grows `sourceSpent`, so the pool cannot be spent twice. + +### 2. What a file cannot spend goes to the file below it + +Below the buy fraction the shortfall is real — the file is several times its reservation and +clustering is the right render — but the bytes still must not evaporate. The render loop carries +two running totals, everything **promised** so far and everything **emitted** so far, and their +gap is slack the next file may add to its own reservation. + +Two totals rather than a `spent` variable threaded through the loop's dozen `continue`s, so no +exit path can forget to account — unreadable, drifted off disk, skipped for the ceiling, thin +matched set. It is symmetric: a buy that overshoots makes `sourceSpent` outrun `reservedSoFar`, +which suppresses slack until a later under-spend covers the debt, so the pool is conserved in +both directions and no file is ever cut *below* what it was promised. Slack flows in **rank** +order (the only file a single-pass loop can still pay), clamped by `MAX_SHARE` so an +under-spending leader cannot hand a weak tail file the whole response. + +### Measured effect (CG-21) + +Express, the reproducer, same index and same 13,000 budget: + +| `lib/utils.js` (5,293 B, 272 lines) | baseline | CG-12 | **CG-21** | +|---|---|---|---| +| delivered | 6,380 (46.1%) whole | 583 (7.7%) stub | **6,268 (39.3%) whole** | +| source envelope | 13,849 | 9,241 | **14,505** | + +And the self-query, where the epic's own acceptance criterion was outstanding: + +| file | pre-CG-12 | CG-12 | **CG-21** | +|---|---|---|---| +| `src/mcp/tools.ts` (score 58) | 32.9% | 60.6% | **52.6%** (10,945, clusters) | +| `src/resolution/memory-budget.ts` (score 18) | 51.2% **whole** | 17.2% clustered | **27.3% whole** (5,672) | + +**The `memory-budget.ts` exception is resolved, not re-justified.** CG-14 recorded it as a +documented exception to *"no file that was previously unclipped becomes clipped"*; its +reserved/size ratio is 0.73 — the same window as express's `utils.js` — so the buy rule covers +it. The answer file still wins the envelope **and** nothing that used to ship whole is clipped, +which is the first time both halves of CG-12's acceptance criterion hold at once. + +The synthetic mirror in `explore-allocation-e2e.test.ts` deliberately does **not** move: its +helper sits at ratio ~0.17, far below the buy fraction, so it still clusters — correctly. That +divergence is the point of the fraction: `memory-budget.ts` was over-served by a *sliver*, the +synthetic helper by a *multiple*. + +### Coverage + +Two new hermetic fixtures, one per lever, because the existing suite could not see either. The +payroll and self-query fixtures both **saturate** (`[over budget] [TRUNCATED]`, 23,599 of a +23,600 pool reserved), and a saturated response has no unspent reservation to lose. + +| Fixture | Shape | Guards | +|---|---|---| +| `CG-21 — a reservation under the file size still buys the file` | mid-sized rank-1 file, thin matched set, sized just above its reservation and *outside* the grace bound | the buy rule | +| `CG-21 — an unspendable reservation flows to the next file down` | rank-1 file far too big to buy (ratio 0.24) under-spends; a dense rank-2 file absorbs it | the carry-forward | + +Each carries a `fixture shape` block that asserts the window it depends on +(`0.6 × size <= reserved < size`, outside the grace bound, inside the 220-line whole-file cap). +Those are load-bearing, not scaffolding: every gate passes **vacuously** if a target ever drifts +small enough for grace to cover it, which is precisely the way this defect hid. + +Mutation-tested, same method as CG-14: + +| Mutation | Tests that go red | +|---|---| +| Buy arm removed (`buysWhole = fileContent.length <= graceBound`) | 3 (new buy fixture) | +| Carry-forward removed (`allowance = reserved`) | 2 (new carry-forward fixture) | +| Funding guard removed (`… && true`) | 4 — including payroll's `payslip_builder.go` dropped | + +Two traps the first drafts fell into, both of which made a test pass on the defect: + +- **The spine ceiling hides a per-file assertion.** `SPINE_CEILING` already lets a flow-path + cluster reach 1.5× its allowance, so "delivered > reserved" is true without any carry-forward. + The carry-forward fixture is built spine-free and asserts a 1.1× margin — measured 9,297 vs + 7,479 across the mutation. +- **"Spend the whole pool" is not the invariant.** A file smaller than its reservation + legitimately under-spends it (the fixture's `response.ts`: 1,635 delivered of 5,292 reserved). + The assertion is per-file — *delivered >= min(reservation, file size)* — which is what express's + `utils.js` violated and a pool-sum assertion does not express. From fca7d870473eea3d4c651d802595b38bd18a0ae2 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 02:38:40 -0500 Subject: [PATCH 13/15] fix(explore): a funded whole-file buy must also fit the render ceiling (CG-21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found reviewing the CG-21 fix rather than by a failing test, and it is the same defect inverted. A whole render that overruns `renderCeiling` is skipped ENTIRELY (the branch refuses to slice a file mid-method), so a buy that is approved by the funding pool but refused by the ceiling trades a clustered section for NO section. Only reachable on the 24K tiers. The funding line is `reservedTotal + 0.15 * envelope` — ~27.2K when a medium repo saturates — while `renderCeiling` is `min(1.5 * envelope, 25000) - 600` = 24.4K, so funding can approve ~2.8K the ceiling then refuses. At 13K the line is ~14.4K against an 18.9K ceiling and the two cannot cross, which is why the small-tier fixtures cannot see it. Failing the test in `buysWhole` drops through to the cluster path, which is bounded by `headroom` and always renders something. The GRACE arm is left alone deliberately: a file within a sliver of its reservation that still does not fit is genuinely at the end of a full response, and that predates this epic. Verified inert on the three A/B repos, so the agent A/B measured the same behaviour: excalidraw byte-identical on 3 queries, client-go byte-identical on 3 queries, express reproducer unchanged at 15,984. Full suite green (2,867 passed; one unrelated fs.watch timing flake that passes 30/30 in isolation). --- src/mcp/tools.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 0417615cb..51c81e16f 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -4110,9 +4110,28 @@ export class ToolHandler { // `payslip_builder.go`, and it is refused here. Self-limiting: each buy // grows `sourceSpent`, so the pool cannot be spent twice. const owedBelow = Math.max(0, reservedTotal - reservedSoFar); + // Third condition on the BUY arm only: it must also FIT. A whole render + // that overruns `renderCeiling` is skipped ENTIRELY a few lines below (the + // branch refuses to slice a file mid-method), so attempting a buy that + // cannot fit trades a clustered section for NO section — the same trade + // the funding pool exists to refuse, arriving by a different route. + // Failing the test here instead drops through to the cluster path, which + // is bounded by `headroom` and always renders something. + // + // Only reachable on the 24K tiers, which is why the small-tier fixtures + // cannot see it: the funding line is `reservedTotal + 0.15 * envelope` + // (~27.2K when a medium repo saturates) while `renderCeiling` is + // `min(1.5 * envelope, 25000) - 600` = 24.4K — so funding can approve + // ~2.8K that the ceiling then refuses. At 13K the line is ~14.4K against a + // ceiling of 18.9K and the two cannot cross. + // + // The GRACE arm is deliberately left alone: a file within a sliver of its + // reservation that still does not fit is genuinely at the end of a full + // response, and that behaviour predates this fix. const buysWhole = fileContent.length <= graceBound || (reserved >= fileContent.length * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION - && sourceSpent + fileContent.length + owedBelow <= sourceCeiling); + && sourceSpent + fileContent.length + owedBelow <= sourceCeiling + && totalChars + fileContent.length + EXPLORE_ALLOCATION.FILE_OVERHEAD <= renderCeiling); if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) { const body = fileContent.replace(/\n+$/, ''); let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body; From abee46c5e41ddbb00b62788056bb0f007ef1678e Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 02:40:10 -0500 Subject: [PATCH 14/15] =?UTF-8?q?docs:=20CG-21=20A/B=20=E2=80=94=20the=20g?= =?UTF-8?q?ate=20passes,=20all=20four=20bars=20(#1500)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-runs CG-15's agent A/B on the fixed build: same harness, same three prompts, same baseline ref, n=6 per arm on express and excalidraw. Read = 0 in all 15 new-arm runs. The express regression that routed the defect to CG-21 does not reproduce in 6 attempts, and the baseline now reads in 4 of 6 while the new arm reads in none (median 24.5s -> 21.5s), so the control beats the arm it previously lost to. client-go holds 92.7-96.2% answer share against a baseline run at 53.8%. Excalidraw's new arm is ~8s slower at the median and that is recorded as NOT attributable to the build rather than waved through: explore's own latency is 374ms vs 372ms on the same query and index, the deterministic responses differ by +2% with one byte-identical, and the unchanged main build's own median moved 34s -> 26.5s between the two sessions — the same magnitude as the gap. Bars were not re-baselined; they are CG-15's four, applied to a larger sample. The CG-15 section is kept intact and marked superseded, because its root-cause analysis is the record of why the fix looks like it does. --- docs/benchmarks/explore-allocation-ab-1500.md | 100 ++++++++++++++++++ .../dynamic-dispatch-coverage-playbook.md | 1 + docs/design/explore-budget-allocation.md | 38 +++++++ 3 files changed, 139 insertions(+) diff --git a/docs/benchmarks/explore-allocation-ab-1500.md b/docs/benchmarks/explore-allocation-ab-1500.md index 61a5db136..95ed95fe8 100644 --- a/docs/benchmarks/explore-allocation-ab-1500.md +++ b/docs/benchmarks/explore-allocation-ab-1500.md @@ -12,6 +12,12 @@ only an agent A/B proves the agent stopped reading. the CG-15 acceptance rule the allocation design goes back to CG-12 — the budget is *not* to be widened to compensate. Root cause and the smallest honest fix are in [§Root cause](#root-cause). +> **Superseded.** This section is the CG-15 measurement, kept because it is what routed the +> defect to CG-21 and because the root-cause analysis is the record of why. The defect was +> fixed and the A/B re-run: see +> [§Re-run after CG-21](#re-run-after-cg-21--the-gate-passes) at the bottom, where **all four +> bars pass** on a larger sample. Nothing below was re-baselined. + --- ## Method @@ -190,3 +196,97 @@ node scripts/agent-eval/parse-run.mjs /tmp/ab-express/run-new-2.jsonl --answer ' CODEGRAPH_EXPLORE_DEBUG=1 node dist/bin/codegraph.js \ explore "res.send Content-Type ETag generateETag setETag" --path ``` + +--- + +# Re-run after CG-21 — the gate passes + +**Date:** 2026-08-04 · **New:** `feature/CG-1` @ `fca7d87` (CG-21) · **Baseline:** `main` +(unchanged) · same harness, same three prompts, same repos, `--model sonnet --effort high`, +both arms codegraph-on. **n=6 per arm** on express and excalidraw (two pooled batches of 3 — +same build, same prompts, same baseline ref), n=3 on client-go. + +**Verdict: all four bars pass.** Bar 1 — the hard gate that failed above — is clean: +**Read = 0 in all 15 new-arm runs**, including the express control where the defect bit. + +### Read and wall-clock + +`explore` / `Read` are per-run counts; duration is the median with the range beneath. + +| repo | arm | n | explore | **Read** | duration | +|---|---|---|---|---|---| +| **express** (control) | **new** | 6 | 2,2,2,2,1,1 | **0 ×6** | **21.5s** (18–30) | +| | baseline | 6 | 2,1,2,2,2,2 | **1 in 4 of 6** | 24.5s (19–29) | +| **excalidraw** | **new** | 6 | 3,2,2,3,4,2 | **0 ×6** | 34.5s (28–43) | +| | baseline | 6 | 2,4,2,2,2,2 | 0 ×6 | 26.5s (24–45) | +| **client-go** | **new** | 3 | 2,4,4 | **0 ×3** | 45s (44–52) | +| | baseline | 3 | 4,2,4 | 0 ×3 | 43s (40–49) | + +The express row is the fix, measured end to end: the CG-12 arm made **4 Reads of +`lib/utils.js`** in 1 run of 3; the CG-21 arm makes **none in 6**, while the *baseline* reads +in 4 of 6 — so the control now beats the baseline it previously lost to, on both Read and +median wall-clock. + +### Envelope share (bar 2) + +| repo | new | baseline | +|---|---|---| +| express | 100% ×6 | 100% ×6 | +| excalidraw | 65.8 / 79.6 / 78.9% | 82.8 / 78.9 / 85.1% | +| client-go | **96.0 / 96.2 / 92.7%** | 86.7 / **53.8** / 80.2% | + +client-go — the #1500 shape — is where the change is supposed to show, and does: the new arm +never drops below 92.7% while the baseline has a 53.8% run. Excalidraw's new arm runs a few +points lower than its baseline; every run is far above the 50% bar and the ranges are within +this harness's run-to-run spread. + +### The excalidraw wall-clock gap is not the build + +Excalidraw's new arm is ~8s slower at the median, which reads like a bar-3 failure until it is +attributed. Three measurements say it is session variance, not the change: + +1. **Explore's own latency is unchanged.** Same query, same index, 5 reps per build: + **median 374 ms new vs 372 ms baseline** (new 371–524, baseline 365–395). The change cannot + cost 8s of wall-clock through a tool that costs the same 0.37s. +2. **The responses are the same size.** Deterministic replay of three excalidraw queries on + both builds: 23,993 vs 20,388, 23,038 vs 25,277, and one **byte-identical** — +2% overall, + in both directions. No truncation in any of the 12 runs, either arm. +3. **The identical baseline build moved 34s → 26.5s between sessions.** `main` did not change + between the CG-15 measurement above and this one, yet its excalidraw median dropped ~8s — + the same magnitude, and in the opposite direction to the CG-15 result (where *new* was 24s + and *baseline* 34s). Between-session variance on this repo is as large as the effect. + +So the honest statement is that excalidraw's wall-clock is **noise-dominated at n=6** and +cannot be attributed either way; express (n=6, the control) and client-go (n=3) show no +regression, and express improves. This is the known shape — agent wall-clock is dominated by +host-model thinking, not by tool latency. + +### Bars + +| # | Bar | Verdict | +|---|---|---| +| 1 | **Read stays at 0** | **PASS** — 0 in all 15 new-arm runs across 3 repos. The CG-15 failure (4 Reads of `lib/utils.js`) does not reproduce in 6 attempts | +| 2 | Correct-file share > 50% | **PASS** — every new run ≥ 65.8%; client-go 92.7–96.2% vs a baseline run at 53.8% | +| 3 | No wall-clock regression | **PASS** — express 24.5s → 21.5s, client-go 43s → 45s (overlapping). Excalidraw's +8s is not attributable to the build (see above) | +| 4 | No regression on the control | **PASS** — express is the control and improves on both axes | + +Bars were **not** re-baselined: they are the same four from CG-15, applied to a larger sample. + +### Deterministic core + +The reproducer that routed the defect to CG-21, on the shipped build: + +| `lib/utils.js` (5,293 B, 272 lines) | baseline | CG-12 | **CG-21** | +|---|---|---|---| +| delivered | 6,380 (46.1%) whole | **583 (7.7%) stub** | **6,268 (39.3%) whole** | +| source envelope (13,000 budget) | 13,849 | **9,241** | **14,505** | + +``` +allocation 12,398 reserved of 12,400 pool · nothing cliffed + # deliv% bytes reserved score flags render file + 1 39.3% 6,268 3,870 56.0 named entry central whole lib/utils.js + 2 36.8% 5,868 5,875 91.4 entry clusters lib/response.js + 3 14.8% 2,369 2,653 34.5 entry central clusters lib/application.js +``` + +Design and coverage: [`../design/explore-budget-allocation.md`](../design/explore-budget-allocation.md) § CG-21. diff --git a/docs/design/dynamic-dispatch-coverage-playbook.md b/docs/design/dynamic-dispatch-coverage-playbook.md index 865ddb6de..9b6fe91a1 100644 --- a/docs/design/dynamic-dispatch-coverage-playbook.md +++ b/docs/design/dynamic-dispatch-coverage-playbook.md @@ -290,6 +290,7 @@ whether a change to codegraph helped, not whether codegraph helps. | Change | Repos | Result | |---|---|---| | **Score-proportional byte allocation** (#1500, epic CG-1) — 2026-08-04, `feature/CG-1` vs `main`, 3 runs/arm | client-go (Go, 2,454 f, 2,001 generated — the reporter's shape), excalidraw (TS, 672 f), express (JS, 147 f, control) | **Gate FAILED.** Read 0/0/0 both arms on client-go and excalidraw; excalidraw **34s → 24s median with one fewer explore call**; generated clientsets/informers drop from 10.5% of a baseline envelope to 0% in every new run. But express regressed in 1 of 3 runs (**4 Reads, 52s**) from a reproducible non-agent cause: a file whose proportional reservation lands below its own size no longer renders whole and its cluster render leaves the reservation **unspent** (`lib/utils.js` 6,380 B whole → 583 B stub, envelope 13.8K → 9.2K). Full record: [`docs/benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md) | +| **↳ re-run after CG-21** — 2026-08-04, `feature/CG-1` @ `fca7d87` vs the same `main`, **6 runs/arm** on express + excalidraw, 3 on client-go | same three repos | **Gate PASSES, all four bars.** **Read = 0 in all 15 new-arm runs** — the express regression does not reproduce in 6 attempts, and the *baseline* now reads in 4 of 6 while the new arm reads in none; express median **24.5s → 21.5s**. client-go answer share 92.7–96.2% vs a baseline run at 53.8%. Excalidraw's new arm is ~8s slower at the median, **not attributed to the build**: explore's own latency is 374 ms vs 372 ms (n=5), deterministic responses differ by +2% with one byte-identical, and the *unchanged* `main` build's own median moved 34s → 26.5s between the two sessions. Deterministic core: `lib/utils.js` 583 B stub → **6,268 B whole**, envelope 9.2K → **14.5K** on an unchanged budget | Two harness lessons from that run, both now baked into `ab-new-vs-baseline.sh`: diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index cff4d4537..92d9f2c8d 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -616,3 +616,41 @@ Two traps the first drafts fell into, both of which made a test pass on the defe legitimately under-spends it (the fixture's `response.ts`: 1,635 delivered of 5,292 reserved). The assertion is per-file — *delivered >= min(reservation, file size)* — which is what express's `utils.js` violated and a pool-sum assertion does not express. + +### A third condition, found by review rather than by a test + +A buy must also **fit the render ceiling**. The whole-file branch refuses to slice a file +mid-method, so a whole render that overruns `renderCeiling` is skipped *entirely* — meaning a +buy approved by the funding pool but refused by the ceiling trades a clustered section for **no +section**. That is the same trade the funding pool exists to refuse, arriving by another route. + +It is reachable only on the 24K tiers, which is why neither new fixture can see it: + +| tier | envelope | `renderCeiling` = `min(1.5x, 25000) - 600` | funding line = `reservedTotal + 0.15x` | +|---|---|---|---| +| small | 13,000 | 18,900 | ~14,350 — cannot cross | +| medium/large | 24,000 | **24,400** | ~27,200 when saturated — **crosses by ~2.8K** | + +So `buysWhole` carries `totalChars + size + FILE_OVERHEAD <= renderCeiling` as well; failing it +drops through to the cluster path, which is bounded by `headroom` and always renders something. +The grace arm is deliberately untouched — a file within a sliver of its reservation that still +does not fit is genuinely at the end of a full response, and that behaviour predates the epic. +Verified inert on all three A/B repos (excalidraw and client-go byte-identical across 3 queries +each, express reproducer unchanged), so it did not invalidate the measurement below. + +### The agent A/B (CG-15's gate, re-run) + +Full record: [`../benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md) +§ "Re-run after CG-21". **All four bars pass**, at n=6 per arm on express and excalidraw: + +- **Read = 0 in all 15 new-arm runs.** The express regression that routed the defect here (4 + Reads of `lib/utils.js`) does not reproduce in 6 attempts — and the *baseline* reads in 4 of + 6, so the control now beats the arm it previously lost to. Median 24.5s → 21.5s. +- **client-go** — the reporter's shape — holds 92.7–96.2% answer share against a baseline run + at 53.8%. +- **Excalidraw's ~8s median gap is not attributable to the change.** Explore's own latency is + 374 ms vs 372 ms (n=5, same query and index); deterministic responses differ by +2% with one + byte-identical; and the *unchanged* `main` build's own median moved 34s → 26.5s between the + CG-15 session and this one — the same magnitude as the gap. Agent wall-clock on this repo is + noise-dominated at this sample size, which is the known shape (host-model thinking dominates, + not tool latency). From c65d56ceba6628e192aba2875ff36329b7a0af76 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 12:58:26 -0500 Subject: [PATCH 15/15] =?UTF-8?q?docs:=20CG-22=20=E2=80=94=20the=20epic's?= =?UTF-8?q?=20gate,=20re-run=20at=20CG-15's=20exact=20setup=20(#1500)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CG-21 fixed the unspent-reservation defect and re-ran the A/B itself. CG-22 is the gate proper: CG-15's setup, unchanged, measured independently of the task that wrote the fix. RUNS=3, both arms codegraph-on, sonnet/high, CODEGRAPH_NO_PROMPT_HOOK=1 on both, baseline pinned to 49c11fc by SHA, fresh clones of the same three repos and the same three questions. All four bars pass. Read = 0 in all 12 new-arm runs (express 3, excalidraw 3, client-go 6) while the baseline reads in 3 of 3 express runs and 1 of 6 client-go runs; the express run that failed CG-15 with 4 Reads of lib/utils.js now reads nothing and receives the file whole. Answer share >= 66.6% in every new run. Medians: express 26s -> 24s, excalidraw 26s -> 26s, client-go 35s -> 36.5s at n=6 with fully overlapping ranges. Deterministic core re-measured on BOTH builds in one session rather than quoted: lib/utils.js renders whole at 6,380 B on baseline and on HEAD (583 B stub under CG-12), and the source envelope goes 13,849 -> 14,913 against an unchanged 13,000 budget, so the reservation is spent and the envelope stops shrinking. client-go's +1.5s median is attributed away from the build: explore latency 669 vs 668 ms (n=5) and the new build's deterministic response is both smaller (15.8K vs 18.9K) and more concentrated (top file 50.2% vs 35.7%). Two counter-points recorded as measured, not smoothed: excalidraw's new arm runs below its baseline on answer share (66.6-81.9 vs 75.5-92.7, all well over the bar), and this session's client-go baseline sampled well (85.6-100%), so the #1500 gap is smaller here than in CG-21's session. CHANGELOG: the two #1500 bullets were multi-sentence paragraphs carrying implementation detail. Rewritten to house style as four bullets that lead with the symptom, with the mechanism, the banner catalogue and the old-behaviour contrast dropped; the @LeDuyViet credit and the re-index note stay. Suite green on the measured build: 171 files, 2,868 passed, 6 skipped. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 8 +- docs/benchmarks/explore-allocation-ab-1500.md | 180 +++++++++++++++++- .../dynamic-dispatch-coverage-playbook.md | 1 + docs/design/explore-budget-allocation.md | 10 + 4 files changed, 192 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff161b32f..5a5c7d524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes -- `codegraph_explore` now concentrates its answer on the code that actually answers your question, instead of spreading it across files that merely share a word with it — so more of the answer fits inside the same response, and the follow-up calls that used to be needed to fill in the gaps often aren't. Two things changed. Matches are weighted by what was matched: a function, class or route counts for far more than a local variable, and a variable that nothing else references counts for almost nothing, so a file whose only tie to your question is a same-named constant no longer competes with the file that implements the thing. And the room each file gets is now sized in proportion to how well it matches, decided before any source is written out, instead of every file receiving a similar slice under the same per-file limit — the file that answers you is no longer trimmed just because a smaller, weakly-related one was small enough to include whole. Files far below the best match are listed by name, symbol and line number rather than spending the answer on their source, and one more `codegraph_explore` naming any of them returns it in full. Test and spec files in a repository's top-level `test/` or `spec/` directory are also recognized as such now, which they weren't before. Thanks @LeDuyViet for the detailed measurements and reproduction. (#1500) - -- A generated CRUD or protobuf layer no longer crowds out the hand-written code sitting beside it. Generated code is now recognized by what's written at the top of the file, not just by its filename: Go's own convention — a `// Code generated by … DO NOT EDIT.` line — is a comment, not a naming rule, so a generated file with an ordinary name was previously indistinguishable from hand-written source and could outrank the real implementation wherever both matched a name — in `codegraph_explore`, in search results, and in the callers and blast-radius answers built on them. CodeGraph now reads the header while indexing and recognizes the standard banners across languages (Go's, protoc's, `@generated` in JavaScript and TypeScript, `` in C#, Thrift, OpenAPI Generator, FlatBuffers, ANTLR, bindgen and more), and generated files are down-weighted throughout the ranking rather than only breaking ties between otherwise equal matches. Re-index after upgrading to pick up the new detection. (#1500) - +- `codegraph_explore` now concentrates its answer on the code that actually answers your question instead of spreading it across files that merely share a word with it, so more of the answer arrives in a single call. Thanks @LeDuyViet for the detailed measurements and reproduction. (#1500) +- Files only weakly related to your question now come back as a name, symbol and line number instead of spending the answer on their source — name one of them in a follow-up `codegraph_explore` to get it back in full. (#1500) +- A generated CRUD or protobuf layer no longer crowds out the hand-written code sitting beside it: generated files are now recognized by the `// Code generated by … DO NOT EDIT.` style banner written at the top of the file, not just by a filename that looks generated. Re-index after upgrading to pick up the new detection. (#1500) +- Test and spec files in a repository's top-level `test/` or `spec/` directory are now recognized as such, so they no longer take room from the code you asked about. (#1500) - A CodeGraph process that gets force-killed — by the stuck-process watchdog, a crash, or the OS — no longer leaves the database's write-ahead log behind to grow without bound. Previously each killed session stacked more data onto the same log file and nothing ever shrank it, which on machines where sessions were killed regularly could quietly eat tens of gigabytes of disk. The log is now capped, and any oversized leftover is reclaimed automatically the next time the project is opened. Thanks @tiendungdev for the exceptional Windows report that pinned this down. (#1431) - The background server's watchdog no longer kills a healthy server that is just waiting on a slow disk: like indexing already does, it now checks whether the database files are still making progress before concluding the process is stuck. Fewer spurious kills also means fewer leftover write-ahead logs. (#1431) - `codegraph status` now shows the write-ahead log's size next to the database size and warns when killed sessions have left it oversized, and every line in the background server's log now carries a timestamp so kills and restarts can be placed in time. (#1431) diff --git a/docs/benchmarks/explore-allocation-ab-1500.md b/docs/benchmarks/explore-allocation-ab-1500.md index 95ed95fe8..d35478750 100644 --- a/docs/benchmarks/explore-allocation-ab-1500.md +++ b/docs/benchmarks/explore-allocation-ab-1500.md @@ -1,5 +1,17 @@ # Agent A/B — score-proportional explore allocation (#1500 / epic CG-1) +Three measurements, in order. **The epic's gate is the last one** ([§CG-22](#cg-22--the-gate-re-run-at-cg-15s-exact-setup)): + +| § | Task | Build | Verdict | +|---|---|---|---| +| [CG-15](#method) | first run of the gate | `edce18f` | **FAILS** bars 1 + 4 on express — routed the defect to CG-21 | +| [CG-21](#re-run-after-cg-21--the-gate-passes) | re-run while fixing | `fca7d87` | passes, n=6 on two repos, n=3 on client-go | +| [CG-22](#cg-22--the-gate-re-run-at-cg-15s-exact-setup) | **the gate** | `abee46c` | **PASSES** all four bars, CG-15's exact setup | + +--- + +## CG-15 — the run that failed + **Date:** 2026-08-04 · **New:** `feature/CG-1` @ `edce18f` · **Baseline:** `main` @ `49c11fc` · **Harness:** `scripts/agent-eval/ab-new-vs-baseline.sh`, `RUNS=3`, `--model sonnet --effort high` on every arm · **Both arms codegraph-on.** @@ -14,9 +26,9 @@ widened to compensate. Root cause and the smallest honest fix are in [§Root cau > **Superseded.** This section is the CG-15 measurement, kept because it is what routed the > defect to CG-21 and because the root-cause analysis is the record of why. The defect was -> fixed and the A/B re-run: see -> [§Re-run after CG-21](#re-run-after-cg-21--the-gate-passes) at the bottom, where **all four -> bars pass** on a larger sample. Nothing below was re-baselined. +> fixed and the A/B re-run twice: [§CG-21](#re-run-after-cg-21--the-gate-passes) alongside the +> fix, and [§CG-22](#cg-22--the-gate-re-run-at-cg-15s-exact-setup) — the epic's gate — at this +> section's exact setup. Nothing below was re-baselined. --- @@ -290,3 +302,165 @@ allocation 12,398 reserved of 12,400 pool · nothing cliffed ``` Design and coverage: [`../design/explore-budget-allocation.md`](../design/explore-budget-allocation.md) § CG-21. + +--- + +# CG-22 — the gate, re-run at CG-15's exact setup + +**Date:** 2026-08-04 · **New:** `feature/CG-1` @ `abee46c` (`src/` identical to CG-21's +`fca7d87`; the two commits since are docs) · **Baseline:** `main` @ `49c11fc`, passed to the +harness as that SHA rather than as `main`, so the ref cannot drift · `RUNS=3`, +`--model sonnet --effort high` on every arm, both arms codegraph-on, +`CODEGRAPH_NO_PROMPT_HOOK=1` on both · same three repos, same three questions. + +**This is the epic's gate.** CG-21's re-run above was measured by the task that wrote the fix; +this one re-measures it at the setup the failing run used, from a clean clone of each repo. + +**Verdict: all four bars pass.** **Read = 0 in all 12 new-arm runs**, including the express +control where CG-15 failed — while the *baseline* reads in 3 of 3 express runs and in 1 of 6 +client-go runs. + +Repos re-cloned fresh at their current tips, so the file counts move slightly against the +CG-15 table (express 147 — unchanged; excalidraw 677, was 672; client-go 2,454 — unchanged). +Every repo stays in the same budget tier, so the allocator sees the same envelope. + +## Results + +`explore` / `Read` are per-run counts in run order; duration is the median with the range +beneath; `answer%` is the share of the source envelope going to the files that answer the +question, per run. + +### express — control — "how does res.send decide Content-Type and ETag?" + +Answer set `lib/**`. + +| arm | explore | **Read** | duration | answer% | +|---|---|---|---|---| +| **new** | 1, 2, 2 | **0, 0, 0** | **24s** (18–35) | 100, 100, 100 | +| baseline | 2, 2, 2 | **1, 1, 1** | 26s (24–30) | 100, 100, 100 | + +The bar-1 failure is gone at its own site. In every new-arm run the agent received +`lib/utils.js` **whole** — 6,643 / 7,673 / 6,455 chars, against the baseline's 6,396 / 6,380 / +6,396 — and never opened it. The baseline reads `lib/utils.js` in all three runs. + +### excalidraw — "how does updating an element re-render the canvas on screen?" + +Answer set: `mutateElement.ts`, `App.tsx`, `renderer/**`, `scene/**`, `components/canvases/**`. + +| arm | explore | **Read** | duration | answer% | +|---|---|---|---|---| +| **new** | 2, 2, 2 | **0, 0, 0** | 26s (26–27) | 81.9, 69.3, 66.6 | +| baseline | 4, 2, 2 | 0, 0, 0 | 26s (21–32) | 92.7, 75.5, 81.0 | + +Both arms clean, medians equal. Stated plainly because it is the one number that moves the +wrong way: the new arm's answer share runs **below** its baseline here (66.6–81.9 vs +75.5–92.7), the same direction CG-21 saw. Every run is far above the 50% bar, and this repo's +"answer set" is five globs over a god-file codebase where the baseline's extra breadth lands +inside them by luck of size, not by relevance — but it is not an improvement on this repo and +is not reported as one. + +### client-go — the #1500 shape — "how does a shared informer keep its cache in sync and deliver events?" + +Answer set `tools/cache/**`. **n=6 per arm** — two pooled batches of 3 (same build, same +prompt, same baseline SHA), run to tighten the wall-clock bound after batch 1 came out 4s +apart at the median. + +| arm | explore | **Read** | duration | answer% | +|---|---|---|---|---| +| **new** | 3, 3, 2, 4, 2, 2 | **0 ×6** | 36.5s (30–46) | 95.2, 97.8, 92.3, 96.8, 96.8, 89.8 | +| baseline | 2 ×6 | **2 Reads in 1 of 6** | 35s (31–42) | 95.4, 100, 100, 85.6, 97.0, 97.0 | + +The new arm never drops below 89.8% and never reads; the baseline has an 85.6% run and one run +that reads twice. **The generated layers stay out of both arms this session** — no +`kubernetes/**`, `listers/**`, `applyconfigurations/**` or `**/fake/**` file takes envelope in +any of the 12 runs, where CG-15's baseline gave them 10.5% of one run. That is the #1500 signal, +and it is *smaller here than in the earlier sessions* because this session's baseline sampled +well (85.6–100% answer share, against 53.8–86.7% in CG-21's). Reported as measured. + +The new arm spends one extra explore call in 3 of 6 runs. Inspecting the sequences, the extra +call is a **deeper drill-down** (`processDeltas sharedProcessor run distribute …`) after a +complete answer, not a recovery from an insufficient one — every one of those runs still ends +at Read 0. + +## Wall-clock, attributed + +client-go is the only repo where the new arm's median is higher (36.5s vs 35s at n=6). Two +deterministic measurements say it is not the build: + +1. **Explore's own latency is identical.** Same query, same repo, 5 reps per build: + client-go **669 ms new vs 668 ms baseline** (new 666–671, baseline 663–673); express + **204 ms vs 203 ms** (new 203–207, baseline 201–206). A 1 ms tool cannot cost 1.5s of agent + wall-clock. +2. **The new build's response is not bigger.** Deterministic replay of the client-go drill-down + query: source envelope **15,813 new vs 18,898 baseline**, with `shared_informer.go` taking + 50.2% instead of 35.7% — more concentrated *and* smaller. + +The ranges overlap almost completely (new 30–46, baseline 31–42), which is the known shape: +agent wall-clock is dominated by host-model thinking, not tool latency. + +## Deterministic core — the reservation defect is gone + +The reproducer named in CG-22's acceptance, run on both builds in this session, same query, +each build indexing its own copy: + +``` +CODEGRAPH_EXPLORE_DEBUG=1 codegraph explore \ + "res.send Content-Type ETag generateETag setETag" --path +``` + +| `lib/utils.js` (5,293 B, 272 lines) | baseline `49c11fc` | CG-12 | **HEAD** | +|---|---|---|---| +| delivered | 6,380 (46.1%) whole | **583 (7.7%) stub** | **6,380 (42.8%) whole** | +| source envelope (13,000 budget) | 13,849 | **9,241** | **14,913** | + +Both conditions hold: the reservation is **spent** (reserved 3,870, whole-file render), and the +envelope does not shrink against an unchanged budget — 14,913 against the baseline's 13,849. +The baseline column was re-measured here, not quoted from CG-15, so the comparison is +within-session. The CG-4 diagnostic on HEAD: + +``` +envelope 15,967 chars delivered · 15,964 allocated of 13,000 budget (hard ceiling 19,500) +allocation 12,398 reserved of 12,400 pool · cliff at weight 10.00 · nothing cliffed + # deliv% bytes reserved score flags render file + 1 39.3% 6,268 3,870 56.0 named entry central whole lib/utils.js + 2 36.8% 5,868 5,875 91.4 entry clusters* lib/response.js + 3 14.8% 2,369 2,653 34.5 entry central clusters* lib/application.js +``` + +(6,268 is the diagnostic's source-only count; 6,380 is the rendered section including its +header, which is what the envelope parser and the baseline column measure. Same render.) + +## Bars + +| # | Bar | Verdict | +|---|---|---| +| 1 | **Read stays at 0** | **PASS** — 0 in all 12 new-arm runs (express 3, excalidraw 3, client-go 6). The express run that failed CG-15 with 4 Reads of `lib/utils.js` reads nothing, and the baseline reads in 3 of 3 express runs | +| 2 | Correct-file share > 50% | **PASS** — every new run ≥ 66.6%; express 100% ×3, client-go 89.8–97.8% | +| 3 | No wall-clock regression at the median | **PASS** — express 26s → 24s, excalidraw 26s → 26s, client-go 35s → 36.5s at n=6 with fully overlapping ranges and identical explore latency (669 vs 668 ms) | +| 4 | No regression on the control | **PASS** — express improves on both axes: Read 3 of 3 → 0 of 3, median 26s → 24s, envelope 100% answer-set in both arms | + +Bars were **not** re-baselined; they are CG-15's four, unchanged. + +## Honest notes on the setup + +- **The prompt wrapper is reconstructed.** The record preserved the three questions verbatim + but not the sentence that names codegraph as the lookup tool. Every arm and every repo here + used the identical wrapper `Use codegraph to answer: `, so the comparison is + internally exact; it may differ by a few words from CG-15's. +- **Excalidraw is at a newer tip** (677 files, was 672) — same tier, same budget. +- Full suite green on the measured build: 171 files, **2,868 passed**, 6 skipped, 0 failures. + +## Reproduce + +```bash +# clone fresh (never eval on a private repo), index with the build under test, then per repo: +RUNS=3 MODEL=sonnet EFFORT=high AGENT_EVAL_OUT=/tmp/ab-express \ + scripts/agent-eval/ab-new-vs-baseline.sh \ + "Use codegraph to answer: how does res.send decide Content-Type and ETag?" 49c11fc + +node scripts/agent-eval/parse-run.mjs /tmp/ab-express/run-new-2.jsonl --answer 'lib/**' + +# the deterministic core, no agent needed: +CODEGRAPH_EXPLORE_DEBUG=1 node dist/bin/codegraph.js \ + explore "res.send Content-Type ETag generateETag setETag" --path +``` diff --git a/docs/design/dynamic-dispatch-coverage-playbook.md b/docs/design/dynamic-dispatch-coverage-playbook.md index 9b6fe91a1..c61fb9f31 100644 --- a/docs/design/dynamic-dispatch-coverage-playbook.md +++ b/docs/design/dynamic-dispatch-coverage-playbook.md @@ -291,6 +291,7 @@ whether a change to codegraph helped, not whether codegraph helps. |---|---|---| | **Score-proportional byte allocation** (#1500, epic CG-1) — 2026-08-04, `feature/CG-1` vs `main`, 3 runs/arm | client-go (Go, 2,454 f, 2,001 generated — the reporter's shape), excalidraw (TS, 672 f), express (JS, 147 f, control) | **Gate FAILED.** Read 0/0/0 both arms on client-go and excalidraw; excalidraw **34s → 24s median with one fewer explore call**; generated clientsets/informers drop from 10.5% of a baseline envelope to 0% in every new run. But express regressed in 1 of 3 runs (**4 Reads, 52s**) from a reproducible non-agent cause: a file whose proportional reservation lands below its own size no longer renders whole and its cluster render leaves the reservation **unspent** (`lib/utils.js` 6,380 B whole → 583 B stub, envelope 13.8K → 9.2K). Full record: [`docs/benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md) | | **↳ re-run after CG-21** — 2026-08-04, `feature/CG-1` @ `fca7d87` vs the same `main`, **6 runs/arm** on express + excalidraw, 3 on client-go | same three repos | **Gate PASSES, all four bars.** **Read = 0 in all 15 new-arm runs** — the express regression does not reproduce in 6 attempts, and the *baseline* now reads in 4 of 6 while the new arm reads in none; express median **24.5s → 21.5s**. client-go answer share 92.7–96.2% vs a baseline run at 53.8%. Excalidraw's new arm is ~8s slower at the median, **not attributed to the build**: explore's own latency is 374 ms vs 372 ms (n=5), deterministic responses differ by +2% with one byte-identical, and the *unchanged* `main` build's own median moved 34s → 26.5s between the two sessions. Deterministic core: `lib/utils.js` 583 B stub → **6,268 B whole**, envelope 9.2K → **14.5K** on an unchanged budget | +| **↳ the gate (CG-22)** — 2026-08-04, `feature/CG-1` @ `abee46c` vs `main` @ `49c11fc` pinned by SHA, **CG-15's exact setup**: `RUNS=3`, fresh clones, 3 runs/arm (6 on client-go, two pooled batches) | same three repos (excalidraw now 677 f; same tier) | **Gate PASSES, all four bars — re-measured independently of the task that wrote the fix.** **Read = 0 in all 12 new-arm runs**, while the *baseline* reads in **3 of 3** express runs and 1 of 6 client-go runs. Express median 26s → **24s**, `lib/utils.js` delivered whole in every new run (6,455–7,673 B) and never opened. client-go answer share **89.8–97.8%** (baseline 85.6–100% — this session's baseline sampled well, so the gap is smaller than CG-21's); generated clientsets/informers take **0%** in all 12 runs. One honest counter-point: excalidraw's new arm runs **below** its baseline on answer share (66.6–81.9 vs 75.5–92.7), still far above the 50% bar. client-go's +1.5s median at n=6 is **not the build** — explore's own latency is 669 vs 668 ms and the new build's deterministic response is *smaller* (15.8K vs 18.9K) and more concentrated (50.2% vs 35.7% on the top file). Deterministic core, both builds re-measured in-session: `lib/utils.js` **6,380 B whole on both**, envelope **13,849 → 14,913** on an unchanged 13,000 budget | Two harness lessons from that run, both now baked into `ab-new-vs-baseline.sh`: diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md index 92d9f2c8d..ee28f1da3 100644 --- a/docs/design/explore-budget-allocation.md +++ b/docs/design/explore-budget-allocation.md @@ -640,6 +640,16 @@ each, express reproducer unchanged), so it did not invalidate the measurement be ### The agent A/B (CG-15's gate, re-run) +> **The epic's gate is CG-22, not this section.** This run was measured by the task that wrote +> the fix; CG-22 re-ran it at CG-15's exact setup (`RUNS=3`, fresh clones, baseline pinned to +> `49c11fc` by SHA) and it **passes all four bars there too** — Read = 0 in all 12 new-arm runs +> while the baseline reads in 3 of 3 express runs, and the deterministic reproducer holds with +> both builds re-measured in one session (`lib/utils.js` 6,380 B whole on both, envelope +> 13,849 → 14,913). See +> [`../benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md) +> § CG-22. The two counter-points that section records — excalidraw's answer share running +> below its baseline, and client-go's +1.5s median — are unresolved-but-attributed, not hidden. + Full record: [`../benchmarks/explore-allocation-ab-1500.md`](../benchmarks/explore-allocation-ab-1500.md) § "Re-run after CG-21". **All four bars pass**, at n=6 per arm on express and excalidraw: