From e0f9c95e2f7e5b8752b5b1bbd1362842c44b1ff3 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Fri, 2 Oct 2026 10:20:23 +0000 Subject: [PATCH] fix(resolution): a busy machine doesn't boot the resolver pool for a small project (#2303) The resolve loop boots the worker pool mid-run when the remaining refs' projected sequential settle (observed per-ref rate x refs left) clears 400 ms. Load inflates the observed rate, so on a loaded 4-core Windows VM a ten-file project (19 refs) projected 506 ms for its last 11 and booted three workers in 122 of 240 runs under 24 parallel inits (0 of 80 at 8). The pool only fans out batches of 1,000 refs, so it resolved nothing; synthesis then waited out its boot (dedupe-merge 6.3 s vs a 6.5 s boot) and its three extra database connections pushed the VM's free commit to 11 MB. Adaptive engagement now also needs 20,000 refs left. tokio-class Rust repos (~55k refs at ~36 us) still engage; a smaller remainder never repays a seconds-long boot. With the floor, the same load booted the pool in 0 of 240 runs. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/resolver-pool-sizing.test.ts | 16 +++++++++++++++- src/resolution/index.ts | 4 ++-- src/resolution/resolver-pool.ts | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b195d0d..066970958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -175,6 +175,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - An agent session that lost its connection to CodeGraph's shared background server, or couldn't reach it at startup, no longer stops that server from running for the rest of the session. It reconnects on its own once the server can start, so other sessions on the same project get auto-sync back instead of running read-only. Thanks @ijbranch for the report. (#2277) - On Windows, `codegraph upgrade` no longer breaks the install while an agent session is using CodeGraph. It used to stop partway and leave CodeGraph unable to start; it now swaps the new files in with CodeGraph running and puts the previous version back if anything fails, and re-running the PowerShell installer works the same way. If an earlier upgrade already broke your install, re-run `irm https://raw.githubusercontent.com/colbymchenry/codegraph/main/install.ps1 | iex` to repair it. Thanks @tippmar-nr for the report. (#2185) - On Windows, `codegraph init`, `codegraph index` and the background server's syncs no longer sometimes crash right after parsing on a busy machine (exit code 3221225477, an access violation). Shutting down the parsing threads could stop one while it was still loading its language grammars, which took the whole process down; they now finish loading first. +- On a busy machine, indexing or syncing a small project no longer starts extra background threads for resolving references that it can't use. Under heavy load the slower pace made a ten-file project look big enough to need them, which added seconds to the run and held extra memory. ## [1.6.1] - 2026-09-29 diff --git a/__tests__/resolver-pool-sizing.test.ts b/__tests__/resolver-pool-sizing.test.ts index cd949a4f0..e70b7a691 100644 --- a/__tests__/resolver-pool-sizing.test.ts +++ b/__tests__/resolver-pool-sizing.test.ts @@ -8,7 +8,7 @@ */ import { describe, it, expect } from 'vitest'; import * as os from 'os'; -import { ResolverPool } from '../src/resolution/resolver-pool'; +import { ResolverPool, ADAPTIVE_ENGAGE_MIN_REFS, shouldEngageAdaptively } from '../src/resolution/resolver-pool'; import { cgroupMemoryAvailable, darwinMemoryAvailable, @@ -106,3 +106,17 @@ describe('memory budget helpers', () => { } ); }); + +describe('adaptive pool engagement needs a real amount of work left', () => { + it('does not boot the pool for a small remainder, however slow the observed rate', () => { + // A busy machine: 11 refs left projected at 506 ms (seen on a loaded 4-core VM). + expect(shouldEngageAdaptively(506, 11, 400)).toBe(false); + expect(shouldEngageAdaptively(60_000, ADAPTIVE_ENGAGE_MIN_REFS - 1, 400)).toBe(false); + }); + + it('boots it for a large remainder whose projected settle clears the bar', () => { + // tokio-class: ~55k Rust refs at ~36 µs each. + expect(shouldEngageAdaptively(1_980, 55_000, 400)).toBe(true); + expect(shouldEngageAdaptively(399, 55_000, 400)).toBe(false); + }); +}); diff --git a/src/resolution/index.ts b/src/resolution/index.ts index f4bb91a23..1eaf14b14 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -27,7 +27,7 @@ import { isCppConstructorRef, matchCppConstructor } from './cpp-constructor'; import { gateSwiftTypeTarget, clearSwiftTypeVisibility, swiftExtendedConformances } from './swift-type-visibility'; import { gateTypeParameter, clearTypeParameterMemos } from './type-parameters'; import { resolveViaImport, resolvePhpImportedStaticCall, resolvePhpQualifiedClassRef, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, isJsPathImportRef, isBoundToOutOfRepoImport, clearImportResolverMemos, resolveImportPath, isExternalImport } from './import-resolver'; -import { ResolverPool, minRefsForPool } from './resolver-pool'; +import { ResolverPool, minRefsForPool, shouldEngageAdaptively } from './resolver-pool'; import { resolveAliasBinding } from './alias-binding'; import { detectFrameworks } from './frameworks'; import { synthesizeCallbackEdges } from './callback-synthesizer'; @@ -2109,7 +2109,7 @@ export class ReferenceResolver { adaptiveSeqRefs += batch.length; const remaining = total - processed - batch.length; const projectedMs = (adaptiveSeqMs / Math.max(1, adaptiveSeqRefs)) * Math.max(0, remaining); - if (projectedMs >= ADAPTIVE_ENGAGE_SETTLE_MS) { + if (shouldEngageAdaptively(projectedMs, remaining, ADAPTIVE_ENGAGE_SETTLE_MS)) { if (process.env.CODEGRAPH_SYNTH_TIMINGS) { console.error(`[pool-timing] adaptive engage: projected ${Math.round(projectedMs)}ms sequential settle over ${remaining} remaining refs`); } diff --git a/src/resolution/resolver-pool.ts b/src/resolution/resolver-pool.ts index 56b9785a6..96f4340d8 100644 --- a/src/resolution/resolver-pool.ts +++ b/src/resolution/resolver-pool.ts @@ -58,6 +58,23 @@ export function minRefsForPool(): number { return 150_000; } +/** + * Fewest refs still to settle for the batch loop to boot the pool mid-run + * from its observed per-ref rate. Booting takes seconds (each worker opens the + * database and warms a resolver), only batches of MIN_PARALLEL_BATCH refs fan + * out, and synthesis waits for a booting pool. A rate measured on a busy + * machine inflates every projection, so with no floor a ten-file project booted + * the pool in half its runs under heavy load and paid seconds for nothing. + * Repos the adaptive bar is for (Rust-class per-ref cost) have tens of + * thousands of refs and still engage. + */ +export const ADAPTIVE_ENGAGE_MIN_REFS = 20_000; + +/** Whether the resolve loop should boot the pool mid-run (see ADAPTIVE_ENGAGE_MIN_REFS). */ +export function shouldEngageAdaptively(projectedMs: number, remainingRefs: number, barMs: number): boolean { + return remainingRefs >= ADAPTIVE_ENGAGE_MIN_REFS && projectedMs >= barMs; +} + export class ResolverPool { private workers: PoolWorker[] = []; private nextId = 0;