mirror of
https://github.com/colbymchenry/codegraph.git
synced 2026-10-18 13:53:01 +08:00
fix(cpp): bound the macro-visibility include walk (#2127)
isVisibleCppMacro walks the call site's translation unit to learn whether a function-like macro is visible there (#1838). A header whose guard the walk cannot decide -- `#define X_H 1` (the OpenSceneGraph / osgEarth idiom, which guardsItself does not read as a guard), or any header reached under an unknown `#if`, where neither its guard nor `#pragma once` settles -- was re-scanned on every inclusion path, and every re-scan pushed its define events onto the cached timeline again. On a layered include graph that is 2^depth scans and 2^depth timeline entries: "Resolving refs" died with "Ineffective mark-compacts near heap limit" at the first ref whose file reached such a tree, at the same ref whatever the heap size. The walk now keeps a version that every observable change bumps (definitions, `#pragma once`, the macro-name set). A re-entry with the same inherited truth, while the version still equals that of an earlier visit which itself changed nothing, is skipped: it would start from the same state, take the same branches and change nothing again, and the events it would push repeat each macro's current state, already its last event. Include cycles cut by the recursion stack are recorded per visit and must still be cut for a skip. A hard budget of 1M directive events per walk bounds what is left; past it the timeline is unknown and nothing at or after the cut line is suppressed, so the budget can only fall back to ordinary resolution, never invent a macro. Answers are unchanged: 20,000 random include graphs (guards bare / valued / pragma once / under flags, #if/#elif/#else, #undef, cycles) give identical isVisibleCppMacro results to the previous walker on 1.25M queries, with about one re-entry in eight skipped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -92,6 +92,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
- A Swift reference to a type now links to the type's own declaration, not to a file that extends it. An `extension View { … }` or `extension Text { … }` used to stand in for SwiftUI's type, so every view, every `Text("…")` and every `Color.red` in an app linked to whichever file happened to extend it. Those files topped the most-depended-on lists and their impact reached the whole app. A bare name like `@State`, `@Test` or `Result<…>` no longer links to some other type's nested `State` or `Result`, and a qualified name like `Build.Id` links to the `Id` it names. Methods declared in an extension of an SDK type still resolve on the types that conform to it, and a protocol's methods declared in any of its extensions now connect to each conforming type's own implementation. Re-index Swift projects after upgrading.
|
||||
- A Vapor route now links to the handler it names. `use: SearchController.show` used to link to whichever controller's `show` came first, so routes with a common handler name, like `show`, `index` or `get`, pointed at another endpoint's code in callers, impact and `codegraph_explore` answers. Nested types like `API.PackageController.get` and handlers declared in an extension of the controller now resolve too, and `use: self.index` resolves to the collection's own `index`. Re-index Vapor projects after upgrading.
|
||||
- A PHP call written without a receiver, such as `redirect($url)`, `view('books.show')`, `auth()` or `basename($path)`, is a function call, and no longer links to a same-named method, field or class elsewhere in the project. These wrong links showed up in callers, impact and `codegraph_explore` answers wherever a Laravel helper or PHP built-in shared its name with a project member. Re-index PHP projects after upgrading.
|
||||
- Indexing a large C or C++ project no longer runs out of memory partway through "Resolving refs". Headers whose include guard is written as `#define X_H 1`, as in OpenSceneGraph and osgEarth, or that are included under a build flag codegraph cannot know, used to be re-read on every include path, which grows exponentially with the depth of the include tree. Each is now read once per include state, and a hard limit stops the check on any include tree that is still too large. (#2127)
|
||||
|
||||
## [1.6.1] - 2026-09-29
|
||||
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* #2127 — the C/C++ macro-visibility walk must stay bounded on deep include
|
||||
* graphs.
|
||||
*
|
||||
* A header whose guard the walk cannot decide — `#define X_H 1` (the OSG /
|
||||
* osgEarth idiom, which the guard reader does not take for a guard), or any
|
||||
* header reached under an unknown `#if` — used to be re-scanned on every
|
||||
* inclusion path: a layered include graph where each header includes the two
|
||||
* of the next layer cost 2^depth header scans and grew the timeline with
|
||||
* every one of them, which is how "Resolving refs" ran out of heap on
|
||||
* osgEarth. The answers must not change: a macro definitely visible at the
|
||||
* call site is still a macro, and an unknown one still is not.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { CodeGraph } from '../src';
|
||||
import { isVisibleCppMacro } from '../src/resolution/cpp-macro-visibility';
|
||||
import type { ResolutionContext, UnresolvedRef } from '../src/resolution/types';
|
||||
|
||||
type Guard = 'value' | 'bare' | 'unknown-flag';
|
||||
|
||||
/** `depth` layers of two headers; each includes both headers of the next layer. */
|
||||
function layeredHeaders(depth: number, guard: Guard): Record<string, string> {
|
||||
const files: Record<string, string> = {};
|
||||
for (let d = 0; d < depth; d++) {
|
||||
for (const side of ['a', 'b']) {
|
||||
const g = `H_${d}_${side}`;
|
||||
const open =
|
||||
guard === 'value' ? `#ifndef ${g}\n#define ${g} 1\n` :
|
||||
guard === 'bare' ? `#ifndef ${g}\n#define ${g}\n` :
|
||||
`#ifdef USE_${d}\n`;
|
||||
const next = d + 1 < depth ? `#include "h${d + 1}_a.h"\n#include "h${d + 1}_b.h"\n` : '';
|
||||
files[`h${d}_${side}.h`] = `${open}${next}#define M_${d}_${side}(x) (x)\nint f_${d}_${side}(int);\n#endif\n`;
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/** A context over in-memory files where every name is both a macro and a function. */
|
||||
function contextOver(files: Record<string, string>): ResolutionContext {
|
||||
return {
|
||||
getNodesByName: (name: string) => [
|
||||
{ kind: 'constant', signature: `#define ${name}(v) (v)` },
|
||||
{ kind: 'function' },
|
||||
],
|
||||
readFile: (f: string) => files[f] ?? null,
|
||||
fileExists: (f: string) => f in files,
|
||||
getAllFiles: () => Object.keys(files),
|
||||
} as unknown as ResolutionContext;
|
||||
}
|
||||
|
||||
const call = (name: string, line: number): UnresolvedRef =>
|
||||
({ language: 'cpp', referenceKind: 'calls', referenceName: name, filePath: 'unit.cpp', line }) as UnresolvedRef;
|
||||
|
||||
describe('#2127 — the macro-visibility include walk is bounded', () => {
|
||||
it.each(['value', 'bare', 'unknown-flag'] as const)(
|
||||
'a 40-layer include graph (2^40 inclusion paths) is walked in well under a second (%s guards)',
|
||||
(guard) => {
|
||||
const files = {
|
||||
...layeredHeaders(40, guard),
|
||||
'trace.h': '#define TRACE(v) ((void)(v))\n',
|
||||
'unit.cpp': '#include "h0_a.h"\n#include "h0_b.h"\n#include "trace.h"\nvoid unit() { TRACE(1); }\n',
|
||||
};
|
||||
const context = contextOver(files);
|
||||
const started = performance.now();
|
||||
// Before the fix this walk never finished (2^40 header scans).
|
||||
expect(isVisibleCppMacro(call('TRACE', 4), context)).toBe(true);
|
||||
expect(performance.now() - started).toBeLessThan(1000);
|
||||
// A header macro under an undecidable guard is not definitely visible,
|
||||
// and a call above the include that defines TRACE is still a call.
|
||||
expect(isVisibleCppMacro(call('M_39_a', 4), context)).toBe(guard === 'bare');
|
||||
expect(isVisibleCppMacro(call('TRACE', 2), context)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it('a walk that cannot be shortened stops at its budget and suppresses nothing past it', () => {
|
||||
// No guards and a known flag toggled on every inclusion: each visit
|
||||
// changes the state, so no re-entry repeats an earlier one and the real
|
||||
// preprocessor would expand all 2^22 paths too. The walk gives up instead:
|
||||
// before the exploding include the macro is still known, after it nothing is.
|
||||
const files: Record<string, string> = {
|
||||
'toggle.h': '#ifdef T\n#undef T\n#else\n#define T\n#endif\n',
|
||||
'unit.cpp': '#define T\n#define TRACE(v) ((void)(v))\nvoid before() { TRACE(1); }\n#include "h0.h"\nvoid after() { TRACE(2); }\n',
|
||||
};
|
||||
for (let d = 0; d < 22; d++) {
|
||||
files[`h${d}.h`] = d + 1 < 22 ? `#include "toggle.h"\n#include "h${d + 1}.h"\n#include "h${d + 1}.h"\n` : '#include "toggle.h"\n';
|
||||
}
|
||||
const context = contextOver(files);
|
||||
const started = performance.now();
|
||||
expect(isVisibleCppMacro(call('TRACE', 3), context)).toBe(true);
|
||||
expect(isVisibleCppMacro(call('TRACE', 5), context)).toBe(false);
|
||||
expect(performance.now() - started).toBeLessThan(10_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2127 — indexing a deep include graph keeps #1838 macro suppression', () => {
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('the macro reached through value-guarded headers still does not bind to the decoy function', async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cpp-walk-'));
|
||||
roots.push(root);
|
||||
const files = {
|
||||
...layeredHeaders(24, 'value'),
|
||||
'trace.h': '#define TRACE(v) ((void)(v))\n',
|
||||
'unit.cpp': '#include "h0_a.h"\n#include "h0_b.h"\n#include "trace.h"\nvoid unit() { TRACE(1); }\n',
|
||||
'decoy.cpp': 'void TRACE(int v) {}\nvoid caller() { TRACE(2); }\n',
|
||||
};
|
||||
for (const [rel, content] of Object.entries(files)) fs.writeFileSync(path.join(root, rel), content);
|
||||
const cg = await CodeGraph.init(root, { index: true });
|
||||
try {
|
||||
const fn = (name: string) => cg.getNodesByKind('function').find((n) => n.name === name)!;
|
||||
const callees = (name: string) =>
|
||||
cg.getCallees(fn(name).id).filter((r) => r.edge.kind === 'calls').map((r) => r.node.filePath);
|
||||
expect(callees('unit')).toEqual([]);
|
||||
expect(callees('caller')).toEqual(['decoy.cpp']);
|
||||
} finally {
|
||||
cg.close();
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -15,6 +15,12 @@
|
||||
* flags remain possible, so only definite macro visibility suppresses a call.
|
||||
* Object-like definitions participate in conditions but never enter the cached
|
||||
* call-site timelines (vendor headers can contain tens of thousands of them).
|
||||
*
|
||||
* A guard the walk cannot decide (`#define X_H 1`, or any header reached under
|
||||
* an unknown `#if`) re-enters its header on every inclusion path, which is
|
||||
* exponential in the include graph's depth (#2127). A re-entry that provably
|
||||
* repeats an earlier visit which changed nothing is skipped, and a hard budget
|
||||
* bounds whatever is left: past it, nothing is known and nothing is suppressed.
|
||||
*/
|
||||
import * as path from 'path';
|
||||
import { maskCppRawStrings } from '../extraction/languages/c-cpp';
|
||||
@@ -29,17 +35,22 @@ type FileEvent =
|
||||
| { kind: 'branch'; op: string; expression: string; guard: boolean; line: number }
|
||||
| { kind: 'once'; line: number };
|
||||
type Event = { line: number; defined: Truth };
|
||||
/** Macro name → events in root-file line order; from `cutoff` on, nothing is known. */
|
||||
type Timeline = { events: Map<string, Event[]>; cutoff: number };
|
||||
type Cache = {
|
||||
summaries: Map<string, FileEvent[]>;
|
||||
includes: Map<string, string | null>;
|
||||
/** Indexed files by basename, for `#include "dir/name.h"` that no include root explains. */
|
||||
byBasename: Map<string, string[]> | null;
|
||||
/** Per root file: macro name → define/undef events in root-file line order. */
|
||||
roots: Map<string, Map<string, Event[]>>;
|
||||
roots: Map<string, Timeline>;
|
||||
};
|
||||
|
||||
const memo = new WeakMap<ResolutionContext, Cache>();
|
||||
const ROOT_TIMELINE_CAP = 32;
|
||||
/** Directive events one translation-unit walk may evaluate (#2127). */
|
||||
const WALK_EVENT_BUDGET = 1_000_000;
|
||||
const NO_CUTS: string[] = [];
|
||||
|
||||
const and = (a: Truth, b: Truth): Truth =>
|
||||
a === false || b === false ? false : a === true && b === true ? true : undefined;
|
||||
@@ -79,7 +90,8 @@ export function isVisibleCppMacro(ref: UnresolvedRef, context: ResolutionContext
|
||||
if (cache.roots.size >= ROOT_TIMELINE_CAP) cache.roots.delete(cache.roots.keys().next().value!);
|
||||
cache.roots.set(rootKey, timeline);
|
||||
}
|
||||
const before = (timeline.get(ref.referenceName) ?? []).filter((e) => e.line <= ref.line);
|
||||
if (ref.line >= timeline.cutoff) return false;
|
||||
const before = (timeline.events.get(ref.referenceName) ?? []).filter((e) => e.line <= ref.line);
|
||||
return before.length > 0 && before[before.length - 1]!.defined === true;
|
||||
}
|
||||
|
||||
@@ -250,12 +262,24 @@ function walkTranslationUnit(
|
||||
language: UnresolvedRef['language'],
|
||||
context: ResolutionContext,
|
||||
cache: Cache
|
||||
): Map<string, Event[]> {
|
||||
): Timeline {
|
||||
const timeline = new Map<string, Event[]>();
|
||||
const definitions = new Map<string, { defined: Truth; value: Truth; macro: Truth }>();
|
||||
const scanning = new Set<string>();
|
||||
const macroNames = new Set<string>();
|
||||
const once = new Map<string, Truth>();
|
||||
// Every change a later directive could observe (definitions, `#pragma once`,
|
||||
// the macro-name set) bumps `version`. A visit that left it unchanged, entered
|
||||
// again with the same inherited truth while it is still unchanged, starts from
|
||||
// the same state, so it would take the same branches and change nothing again;
|
||||
// the events it would push repeat each name's current state, which is already
|
||||
// its last event. Include cycles cut by the recursion stack (`cuts`) must still
|
||||
// be cut for the repeat to hold.
|
||||
let version = 0;
|
||||
const cuts: string[] = [];
|
||||
const visits = new Map<string, { inherited: Truth; start: number; end: number; cuts: string[] }>();
|
||||
let budget = WALK_EVENT_BUDGET;
|
||||
let cutoff = Infinity;
|
||||
const condition = (expression: string): Truth => {
|
||||
const text = expression.trim();
|
||||
if (/^(?:0x[\da-f]+|\d+)[uUlL]*$/i.test(text)) return Number(text.replace(/[uUlL]+$/, '')) !== 0;
|
||||
@@ -267,12 +291,28 @@ function walkTranslationUnit(
|
||||
return /^\w+$/.test(text) ? definitions.get(text)?.value : undefined;
|
||||
};
|
||||
const scan = (file: string, inherited: Truth, includeLine?: number): void => {
|
||||
if (inherited === false || scanning.has(file) || once.get(file) === true) return;
|
||||
if (inherited === false || once.get(file) === true || cutoff !== Infinity) return;
|
||||
if (scanning.has(file)) {
|
||||
cuts.push(file);
|
||||
return;
|
||||
}
|
||||
const seen = visits.get(file);
|
||||
if (
|
||||
seen && seen.inherited === inherited && seen.start === seen.end && seen.end === version &&
|
||||
seen.cuts.every((c) => scanning.has(c))
|
||||
) {
|
||||
cuts.push(...seen.cuts);
|
||||
return;
|
||||
}
|
||||
const start = version;
|
||||
const firstCut = cuts.length;
|
||||
scanning.add(file);
|
||||
let active: Truth = inherited;
|
||||
const frames: Array<{ parent: Truth; taken: Truth }> = [];
|
||||
for (const ev of summarize(file, context, cache)) {
|
||||
const line = includeLine ?? ev.line;
|
||||
if (cutoff === Infinity && --budget < 0) cutoff = line;
|
||||
if (cutoff !== Infinity) break;
|
||||
if (ev.kind === 'branch') {
|
||||
if (ev.op === 'if' || ev.op === 'ifdef' || ev.op === 'ifndef') {
|
||||
const known = definitions.get(ev.expression.trim())?.defined;
|
||||
@@ -294,7 +334,9 @@ function walkTranslationUnit(
|
||||
}
|
||||
if (active === false) continue;
|
||||
if (ev.kind === 'once') {
|
||||
once.set(file, or(once.get(file) ?? false, active));
|
||||
const next = or(once.get(file) ?? false, active);
|
||||
if (next !== (once.get(file) ?? false)) version++;
|
||||
once.set(file, next);
|
||||
continue;
|
||||
}
|
||||
if (ev.kind === 'include') {
|
||||
@@ -309,12 +351,17 @@ function walkTranslationUnit(
|
||||
// A name no directive has touched is unknown, not undefined: the build
|
||||
// can set it on the command line. So an `#undef` under an undecidable
|
||||
// `#if` leaves it unknown (#2069); only a certain one clears it.
|
||||
definitions.set(ev.name, {
|
||||
const next = {
|
||||
defined: defining ? or(prior?.defined, active) : and(prior?.defined, not(active)),
|
||||
value: defining && active === true ? condition(ev.value) : undefined,
|
||||
macro: now,
|
||||
});
|
||||
if (ev.functionLike) macroNames.add(ev.name);
|
||||
};
|
||||
if (next.defined !== prior?.defined || next.value !== prior?.value || next.macro !== prior?.macro) version++;
|
||||
definitions.set(ev.name, next);
|
||||
if (ev.functionLike && !macroNames.has(ev.name)) {
|
||||
macroNames.add(ev.name);
|
||||
version++;
|
||||
}
|
||||
if (macroNames.has(ev.name)) {
|
||||
const events = timeline.get(ev.name) ?? [];
|
||||
events.push({ line, defined: now });
|
||||
@@ -322,8 +369,11 @@ function walkTranslationUnit(
|
||||
}
|
||||
}
|
||||
scanning.delete(file);
|
||||
const own = cuts.length > firstCut ? [...new Set(cuts.splice(firstCut))] : NO_CUTS;
|
||||
cuts.push(...own);
|
||||
visits.set(file, { inherited, start, end: version, cuts: own });
|
||||
};
|
||||
|
||||
scan(rootFile, true);
|
||||
return timeline;
|
||||
return { events: timeline, cutoff };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user