mirror of
https://github.com/colbymchenry/codegraph.git
synced 2026-10-16 20:53:57 +08:00
fix: a minified or bundled script is generated, and generated files are not hubs (#2122)
Found by the README-wide sweep: vendored bundles topped "most depended on"
across many repos — delphimvcframework's bundled `Buffer` (6,162) and four
`n`s (2,386 each), retrofit's docs `main.js` `t`/`e`/`i`, TestBox's
`__webpack_require__` — because only `*.min.js` was recognized as generated.
- detectGeneratedFile (index time, persisted as files.generated) now also
flags a .js/.mjs/.cjs file whose text is mostly lines of 1,000+ chars AND
whose long lines are code (≥3% `;{}(),`) — not an ordinary file carrying
one long base64 string — and any file that defines webpack's
`__webpack_require__` loader, however readable its lines. The path rule
also matches `-min.js` (underscore-min.js).
- Entry points' hubs skip nodes in generated files, as they skip tests.
Scanned every .js file of express, axios, lodash, excalidraw, typeorm, zod,
trpc, hono, bulletproof-react: zero new flags on hand-written code; newly
caught: lodash vendor/underscore/underscore-min.js, retrofit website
main.js (754 KB) + prism.js, TestBox syntaxhighlighter.js (webpack).
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
### Fixes
|
||||
|
||||
- A minified or bundled JavaScript file is now recognized as generated even when it isn't named `.min.js`, for example a docs site's `bundle.js`, a readable webpack build or a vendored `underscore-min.js`. Its one-letter functions now rank below your own code in search and `codegraph_explore` answers.
|
||||
- In C# and VB.NET, a type written in code, like `Type sourceType`, `List<int>`, `new TypeMap()` or `Dictionary<string, Type>`, now links to the type and no longer to a property, method or field that shares its name. `List<…>` used to link to some test class's `List` property, `Type` to an attribute's `Type` property, and a parameter of type `TypeMap` to a `TypeMap` property instead of the `TypeMap` class. Those members showed up with hundreds of made-up dependents. Reading a property or passing a method, like `Builder.Services` or `MapGet("/x", GetItems)`, still links as before. Re-index C# and VB.NET projects after upgrading.
|
||||
- JavaScript and TypeScript calls are no longer linked to a class member that merely shares their name. A call on a namespace import, like zod's `z.string()` (`import * as z from 'zod'`), or on something imported from an npm package, used to link to whichever class in the project had a `string` member, and a bare call like mocha's `it(…)` or `describe(…)` to an interface's `it` property or a class's `describe` field. On zod one test helper's getter had thousands of made-up callers and topped the most-depended-on list. A React hook or component imported from a package, like `useQuery` from `@tanstack/react-query`, no longer links to a same-named one in the project, and a hook nested inside another function is no longer a target at all. Re-index JavaScript and TypeScript projects after upgrading.
|
||||
- A Swift call through a nested type, like `API.PackageController.GetRoute.query(on: db)`, now links to that type's method. Only `query` used to be kept, so the call linked to whichever type's `query` came first; in a Vapor app every route type has one, so callers, impact and `codegraph_explore` answers pointed at another endpoint's code. The same applies to nested types' initializers (`API.PackageController.Model(name:)`), to enum cases (`Gitlab.Error.requestFailed(status:)`), to types declared in an `extension API.PackageController { … }`, and to The Composable Architecture's `Path.State.detail(…)`. A call on a type path that names no type in the project, such as an SDK's `Plot.Node.footer(…)`, is left unlinked. Re-index Swift projects after upgrading.
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
isGeneratedFile,
|
||||
hasGeneratedHeader,
|
||||
detectGeneratedFile,
|
||||
isMinifiedContent,
|
||||
} from '../src/extraction/generated-detection';
|
||||
|
||||
describe('isGeneratedFile', () => {
|
||||
@@ -220,3 +221,37 @@ describe('detectGeneratedFile — the union the indexer persists', () => {
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isMinifiedContent — a bundle not named .min.js', () => {
|
||||
const bundle = Array.from({ length: 300 }, (_, i) => `function f${i}(e){return n(e,${i})+t(e)}`).join(';');
|
||||
|
||||
it('recognizes a minified script by its enormous code lines', () => {
|
||||
expect(isMinifiedContent('docs/assets/bundle.js', bundle)).toBe(true);
|
||||
expect(isMinifiedContent('site/app.mjs', bundle)).toBe(true);
|
||||
expect(detectGeneratedFile('docs/assets/bundle.js', bundle)).toBe(true);
|
||||
});
|
||||
|
||||
it('recognizes a readable webpack build by its module loader, and `-min.js` by name', () => {
|
||||
const webpack = `/*! lib */\n(function(modules) {\n function __webpack_require__(moduleId) {\n return modules[moduleId];\n }\n${' var x = 1;\n'.repeat(400)}})([]);\n`;
|
||||
expect(isMinifiedContent('assets/js/syntaxhighlighter.js', webpack)).toBe(true);
|
||||
expect(isGeneratedFile('vendor/underscore/underscore-min.js')).toBe(true);
|
||||
});
|
||||
|
||||
it('leaves an ordinary file with one long string literal alone', () => {
|
||||
const fixture = [
|
||||
'export function decode(input) {',
|
||||
' return atob(input);',
|
||||
'}',
|
||||
`export const LOGO = "${'QUJD'.repeat(4000)}";`,
|
||||
'export function size() { return LOGO.length; }',
|
||||
].join('\n');
|
||||
expect(isMinifiedContent('src/logo.js', fixture)).toBe(false);
|
||||
});
|
||||
|
||||
it('leaves hand-written code, TypeScript and short files alone', () => {
|
||||
const code = Array.from({ length: 400 }, (_, i) => `export function f${i}(e) {\n return e + ${i};\n}`).join('\n');
|
||||
expect(isMinifiedContent('src/code.js', code)).toBe(false);
|
||||
expect(isMinifiedContent('src/bundle.ts', bundle)).toBe(false);
|
||||
expect(isMinifiedContent('src/tiny.js', 'function n(e){return e};function t(e){return n(e)}')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -391,3 +391,51 @@ describe('splitting a route name', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('entry points skip a vendored bundle', () => {
|
||||
let app: Instance;
|
||||
let payload: WireEntryPoints;
|
||||
|
||||
beforeAll(async () => {
|
||||
resetEntryPointsCache();
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-ui-entry-bundle-'));
|
||||
const root = path.join(dir, 'project');
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
write(
|
||||
root,
|
||||
'src/store.ts',
|
||||
`export function insertNode(name: string): string {
|
||||
return name.trim();
|
||||
}
|
||||
export function readNode(name: string): string {
|
||||
return insertNode(name);
|
||||
}
|
||||
export function warm(): string {
|
||||
return insertNode('a') + readNode('b');
|
||||
}
|
||||
`
|
||||
);
|
||||
// A docs site's bundle, not named .min.js: one-letter functions calling
|
||||
// each other on a few enormous lines.
|
||||
const body = Array.from({ length: 400 }, (_, i) => `function f${i}(e){return n(e,${i})+t(e)}`).join(';');
|
||||
write(root, 'docs/assets/bundle.js', `function n(e,r){return e+r};function t(e){return n(e,1)};${body};\n`);
|
||||
const cg = CodeGraph.initSync(root, {
|
||||
config: { include: ['src/**/*.ts', 'docs/**/*.js'], exclude: [] },
|
||||
});
|
||||
await cg.indexAll();
|
||||
cg.resolveReferences();
|
||||
app = await serve(root, dir, cg);
|
||||
payload = (await getJson(app, '/api/entrypoints')) as WireEntryPoints;
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await stop(app);
|
||||
});
|
||||
|
||||
it('lists the project’s own hubs, not the bundle’s', () => {
|
||||
const names = payload.hubs.items.map((h) => h.name);
|
||||
expect(names).toContain('insertNode');
|
||||
expect(names).not.toContain('n');
|
||||
expect(names).not.toContain('t');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -169,6 +169,8 @@ These describe `codegraph ui` and its screens. They were taken out of `## [Unrel
|
||||
|
||||
## Fixes — Symbols, tests and the viewer
|
||||
|
||||
- **Entry points' most-depended-on list shows your code, not a vendored bundle's.** A minified library or bundled docs script (`n`, `t`, `Buffer` with thousands of callers, all from inside the bundle) could top the list. Symbols in generated files are now left out of it, like test files.
|
||||
|
||||
- **Stopping `codegraph ui` stops the server.** Killing the command by its process id, as a process manager, an editor task or `kill` does, left the server running on its port with no way to reach it, until the machine restarted. The server now notices it has been left behind and shuts down, closing the index first. Ctrl+C was never affected.
|
||||
|
||||
- **A SwiftUI view with a preview is no longer listed as a file that runs something.** A `#Preview { … }` sits at the top level of a view's file, so every view with one showed up under entry points as if it ran code. A preview is Xcode's canvas, not code the app runs: its calls no longer count, and a file whose only top-level code is previews leaves the list.
|
||||
|
||||
@@ -64,7 +64,7 @@ const GENERATED_PATTERNS: ReadonlyArray<RegExp> = [
|
||||
/_grpc_pb\.[jt]s$/,
|
||||
// Minified bundles vendored into a repo (docs sites, examples). Their
|
||||
// single-letter symbols make name-based edges pure noise.
|
||||
/\.min\.m?js$/,
|
||||
/[.-]min\.m?js$/,
|
||||
// Python — protobuf / gRPC / openapi-codegen
|
||||
/_pb2(_grpc)?\.py$/,
|
||||
/_pb2\.pyi$/,
|
||||
@@ -249,5 +249,42 @@ export function hasGeneratedHeader(content: string): boolean {
|
||||
* indexer persists to `files.generated`.
|
||||
*/
|
||||
export function detectGeneratedFile(filePath: string, content: string): boolean {
|
||||
return isGeneratedFile(filePath) || hasGeneratedHeader(content);
|
||||
return isGeneratedFile(filePath) || hasGeneratedHeader(content) || isMinifiedContent(filePath, content);
|
||||
}
|
||||
|
||||
/** Scripts a bundler or minifier writes as a few enormous lines. */
|
||||
const MINIFIABLE_SCRIPT = /\.(?:m?js|cjs)$/i;
|
||||
/** The loader every webpack bundle defines for its modules. */
|
||||
const WEBPACK_RUNTIME = /\bfunction __webpack_require__\s*\(/;
|
||||
/** A line this long is not written by hand. */
|
||||
const MINIFIED_LINE_CHARS = 1000;
|
||||
|
||||
/**
|
||||
* A minified or bundled script that is not NAMED so — a docs site's
|
||||
* `bundle.js`, a sample's vendored `app.js`: most of its text sits on lines of
|
||||
* a thousand characters or more, and that text is code, dense with `;{}(),`,
|
||||
* not one long string literal (a base64 fixture) in an ordinary file. Its
|
||||
* one-letter functions (`n`, `t`, `e`) otherwise topped the most-depended-on
|
||||
* lists of the projects that vendor it.
|
||||
*/
|
||||
export function isMinifiedContent(filePath: string, content: string): boolean {
|
||||
if (!MINIFIABLE_SCRIPT.test(filePath) || content.length < 4 * MINIFIED_LINE_CHARS) return false;
|
||||
// A webpack build carries its module loader, however readable its lines.
|
||||
if (WEBPACK_RUNTIME.test(content)) return true;
|
||||
let long = 0;
|
||||
let punctuation = 0;
|
||||
for (let start = 0; start < content.length; ) {
|
||||
let end = content.indexOf('\n', start);
|
||||
if (end < 0) end = content.length;
|
||||
if (end - start >= MINIFIED_LINE_CHARS) {
|
||||
long += end - start;
|
||||
for (let i = start; i < end; i++) {
|
||||
const c = content.charCodeAt(i);
|
||||
// ; { } ( ) ,
|
||||
if (c === 59 || c === 123 || c === 125 || c === 40 || c === 41 || c === 44) punctuation++;
|
||||
}
|
||||
}
|
||||
start = end + 1;
|
||||
}
|
||||
return long >= content.length * 0.5 && punctuation >= long * 0.03;
|
||||
}
|
||||
|
||||
@@ -396,15 +396,18 @@ function testFiles(cg: CodeGraph, limit: number): WireList<WireEntryTest> {
|
||||
return wireList(items, Math.max(ranked.length, items.length));
|
||||
}
|
||||
|
||||
/** The most depended-on symbols, tests and non-navigable kinds removed. */
|
||||
/** The most depended-on symbols, tests, generated files and non-navigable kinds removed. */
|
||||
function hubs(cg: CodeGraph, limit: number): WireList<WireEntryHub> {
|
||||
const ranked = cg.getTopDependedOn(SCAN_ROWS);
|
||||
const ranked = cg
|
||||
.getTopDependedOn(SCAN_ROWS)
|
||||
.map((row) => ({ row, node: cg.getNode(row.nodeId) }));
|
||||
// A vendored bundle's one-letter functions are depended on by the bundle alone.
|
||||
const generated = cg.generatedFilePredicate(ranked.flatMap(({ node }) => (node ? [node.filePath] : [])));
|
||||
|
||||
const items: WireEntryHub[] = [];
|
||||
let eligible = 0;
|
||||
for (const row of ranked) {
|
||||
const node = cg.getNode(row.nodeId);
|
||||
if (!node || NON_HUB_KINDS.has(node.kind) || isTestFile(node.filePath)) continue;
|
||||
for (const { row, node } of ranked) {
|
||||
if (!node || NON_HUB_KINDS.has(node.kind) || isTestFile(node.filePath) || generated(node.filePath)) continue;
|
||||
eligible += 1;
|
||||
if (items.length >= limit) continue;
|
||||
items.push({ ...toNodeRef(node), dependents: row.dependents });
|
||||
|
||||
Reference in New Issue
Block a user