mirror of
https://github.com/colbymchenry/codegraph.git
synced 2026-10-17 05:07:37 +08:00
Workspace imports could follow excluded dependency archives and decode them without a size guard. Reuse extraction's 1 MiB limit, reject non-files before reading, and cache null results. Add sparse-HAR and real ohpm indexing coverage while preserving legitimate import edges. Co-authored-by: danusha2345 <ewidusoc498@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { CodeGraph } from '../src';
|
||||
import { ReferenceResolver } from '../src/resolution';
|
||||
import type { ResolutionContext } from '../src/resolution/types';
|
||||
import { MAX_SOURCE_FILE_SIZE_BYTES } from '../src/file-limits';
|
||||
|
||||
vi.mock('fs', async (importOriginal) => ({ ...await importOriginal<typeof import('fs')>() }));
|
||||
|
||||
describe('resolution file reads', () => {
|
||||
let root: string;
|
||||
let cg: CodeGraph;
|
||||
let resolver: ReferenceResolver;
|
||||
let context: ResolutionContext;
|
||||
|
||||
beforeEach(() => {
|
||||
root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-resolution-read-'));
|
||||
cg = CodeGraph.initSync(root);
|
||||
resolver = new ReferenceResolver(root, cg.queries);
|
||||
context = resolver.getResolutionContext();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
cg.close();
|
||||
fs.rmSync(root, { recursive: true, force: true, maxRetries: 5 });
|
||||
});
|
||||
|
||||
function sparseArchive(): string {
|
||||
const relative = 'node_modules/example/react_native_openharmony.har';
|
||||
const archive = path.join(root, relative);
|
||||
fs.mkdirSync(path.dirname(archive), { recursive: true });
|
||||
const fd = fs.openSync(archive, 'w');
|
||||
try {
|
||||
fs.writeSync(fd, Buffer.from([0x1f, 0x8b]));
|
||||
fs.ftruncateSync(fd, 2 * 1024 * 1024);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
return relative;
|
||||
}
|
||||
|
||||
it('reads normal source files', () => {
|
||||
fs.writeFileSync(path.join(root, 'small.ts'), 'export const answer = 42;\n');
|
||||
expect(context.readFile('small.ts')).toBe('export const answer = 42;\n');
|
||||
});
|
||||
|
||||
it('rejects an oversized package archive before decoding and caches the rejection', () => {
|
||||
const relative = sparseArchive();
|
||||
const read = vi.spyOn(fs, 'readFileSync');
|
||||
const stat = vi.spyOn(fs, 'statSync');
|
||||
expect(context.readFile(relative)).toBeNull();
|
||||
expect(context.readFile(relative)).toBeNull();
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
expect(stat).toHaveBeenCalledTimes(1);
|
||||
fs.writeFileSync(path.join(root, relative), 'export const repaired = true;');
|
||||
resolver.clearCaches();
|
||||
expect(context.readFile(relative)).toBe('export const repaired = true;');
|
||||
});
|
||||
|
||||
it('accepts exactly the byte limit and rejects one byte more', () => {
|
||||
const content = 'a'.repeat(MAX_SOURCE_FILE_SIZE_BYTES);
|
||||
fs.writeFileSync(path.join(root, 'boundary.ts'), content);
|
||||
expect(context.readFile('boundary.ts')).toBe(content);
|
||||
fs.appendFileSync(path.join(root, 'boundary.ts'), 'a');
|
||||
resolver.clearCaches();
|
||||
const read = vi.spyOn(fs, 'readFileSync');
|
||||
expect(context.readFile('boundary.ts')).toBeNull();
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects directories before reading and caches missing files', () => {
|
||||
fs.mkdirSync(path.join(root, 'directory'));
|
||||
const read = vi.spyOn(fs, 'readFileSync');
|
||||
const stat = vi.spyOn(fs, 'statSync');
|
||||
for (const name of ['directory', 'missing.ts']) {
|
||||
expect(context.readFile(name)).toBeNull();
|
||||
expect(context.readFile(name)).toBeNull();
|
||||
}
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
expect(stat).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('skips excluded HAR imports during indexing while preserving source workspace imports', async () => {
|
||||
const relative = sparseArchive();
|
||||
fs.mkdirSync(path.join(root, 'data'));
|
||||
fs.writeFileSync(path.join(root, 'data/oh-package.json5'), JSON.stringify({ name: 'data', main: 'Index.ets' }));
|
||||
fs.writeFileSync(path.join(root, 'data/Index.ets'), 'export class Repository {}\n');
|
||||
fs.writeFileSync(path.join(root, 'oh-package.json5'), JSON.stringify({ dependencies: {
|
||||
archive: `file:./${relative}`, data: 'file:./data',
|
||||
} }));
|
||||
fs.writeFileSync(path.join(root, 'MainAbility.ets'),
|
||||
"import { RNAbility } from 'archive';\nimport { Repository } from 'data';\n" +
|
||||
'export class MainAbility extends RNAbility {}\nexport class Local extends Repository {}\n');
|
||||
const read = vi.spyOn(fs, 'readFileSync');
|
||||
const stat = vi.spyOn(fs, 'statSync');
|
||||
await cg.indexAll();
|
||||
const archive = path.join(root, relative);
|
||||
expect(stat.mock.calls.some(([file]) => String(file) === archive)).toBe(true);
|
||||
expect(read.mock.calls.some(([file]) => String(file) === archive)).toBe(false);
|
||||
const repository = cg.getNodesByKind('class').find(n => n.name === 'Repository');
|
||||
const local = cg.getNodesByKind('class').find(n => n.name === 'Local');
|
||||
expect(repository).toBeDefined();
|
||||
expect(local).toBeDefined();
|
||||
expect(cg.getOutgoingEdges(local!.id)).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ kind: 'extends', target: repository!.id }),
|
||||
]));
|
||||
});
|
||||
});
|
||||
+6
-12
@@ -35,6 +35,7 @@ import ignore, { Ignore } from 'ignore';
|
||||
import { detectFrameworks } from '../resolution/frameworks';
|
||||
import type { ResolutionContext } from '../resolution/types';
|
||||
import { createYielder, type MaybeYield } from '../resolution/cooperative-yield';
|
||||
import { MAX_SOURCE_FILE_SIZE_BYTES } from '../file-limits';
|
||||
|
||||
/**
|
||||
* Number of files to read in parallel during indexing.
|
||||
@@ -151,13 +152,6 @@ export function hashContent(content: string): string {
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Skip files larger than this (bytes). Generated bundles, minified JS, and
|
||||
* vendored blobs blow the WASM heap and the worker-recycle budget for no useful
|
||||
* symbols. 1 MB covers essentially all hand-written source.
|
||||
*/
|
||||
const MAX_FILE_SIZE = 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Directory names that are dependency, build, cache, or tooling output across the
|
||||
* languages/frameworks CodeGraph supports — curated from the canonical
|
||||
@@ -2287,18 +2281,18 @@ export class ExtractionOrchestrator {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Honour MAX_FILE_SIZE. Without this check, vendored generated
|
||||
// Honour MAX_SOURCE_FILE_SIZE_BYTES. Without this check, vendored generated
|
||||
// headers, minified bundles, and other multi-MB files get indexed,
|
||||
// wasting WASM heap and the worker recycle budget on inputs with no
|
||||
// useful symbols. The single-file extractFile path already enforces
|
||||
// this; the bulk path used to silently skip the check.
|
||||
if (stats.size > MAX_FILE_SIZE) {
|
||||
if (stats.size > MAX_SOURCE_FILE_SIZE_BYTES) {
|
||||
await storeResult(filePath, content, stats, {
|
||||
nodes: [],
|
||||
edges: [],
|
||||
unresolvedReferences: [],
|
||||
errors: [{
|
||||
message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
|
||||
message: `File exceeds max size (${stats.size} > ${MAX_SOURCE_FILE_SIZE_BYTES})`,
|
||||
filePath,
|
||||
severity: 'warning',
|
||||
code: 'size_exceeded',
|
||||
@@ -2628,14 +2622,14 @@ export class ExtractionOrchestrator {
|
||||
const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir));
|
||||
|
||||
// Check file size
|
||||
if (stats.size > MAX_FILE_SIZE) {
|
||||
if (stats.size > MAX_SOURCE_FILE_SIZE_BYTES) {
|
||||
const result: ExtractionResult = {
|
||||
nodes: [],
|
||||
edges: [],
|
||||
unresolvedReferences: [],
|
||||
errors: [
|
||||
{
|
||||
message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
|
||||
message: `File exceeds max size (${stats.size} > ${MAX_SOURCE_FILE_SIZE_BYTES})`,
|
||||
filePath: relativePath,
|
||||
severity: 'warning',
|
||||
code: 'size_exceeded',
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
/**
|
||||
* Largest source file CodeGraph will parse or read during resolution. Generated
|
||||
* bundles, minified sources, and dependency archives above this limit provide no
|
||||
* useful symbols; 1 MB covers essentially all hand-written source.
|
||||
*/
|
||||
export const MAX_SOURCE_FILE_SIZE_BYTES = 1024 * 1024;
|
||||
@@ -26,6 +26,7 @@ import { resolveAliasBinding } from './alias-binding';
|
||||
import { detectFrameworks } from './frameworks';
|
||||
import { synthesizeCallbackEdges } from './callback-synthesizer';
|
||||
import { createYielder, type MaybeYield } from './cooperative-yield';
|
||||
import { MAX_SOURCE_FILE_SIZE_BYTES } from '../file-limits';
|
||||
import { loadProjectAliases, type AliasMap } from './path-aliases';
|
||||
import { loadGoModule, type GoModule } from './go-module';
|
||||
import { loadWorkspacePackages, type WorkspacePackages } from './workspace-packages';
|
||||
@@ -422,6 +423,14 @@ export class ReferenceResolver {
|
||||
}
|
||||
const fullPath = path.join(this.projectRoot, filePath);
|
||||
try {
|
||||
// Import resolvers may follow package metadata to an archive (`file:*.har`,
|
||||
// for example). Reject anything extraction would not accept before UTF-8
|
||||
// decoding can multiply a large binary blob into gigabytes of V8 heap.
|
||||
const stats = fs.statSync(fullPath);
|
||||
if (!stats.isFile() || stats.size > MAX_SOURCE_FILE_SIZE_BYTES) {
|
||||
this.fileCache.set(filePath, null);
|
||||
return null;
|
||||
}
|
||||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||||
this.fileCache.set(filePath, content);
|
||||
return content;
|
||||
|
||||
Reference in New Issue
Block a user