codegraph/__tests__/integration/full-pipeline.test.ts

/**
 * End-to-end pipeline integration tests
 *
 * Exercises the full happy path that unit tests cover in isolation:
 *   init → indexAll → resolveReferences → searchNodes/getCallers/buildContext → sync
 *
 * Also covers two error paths that were previously uncovered:
 *   - Indexing a file that contains a syntactically invalid snippet
 *     (parse errors must not abort the batch).
 *   - Sync correctly applies adds + modifies + removes in a single pass.
 *
 * A synthetic ~120-file project is generated per test (5k files would
 * dwarf the test runner; 120 files of varied TS shape is enough to
 * stress the resolver and graph layers without slowing the suite to a
 * crawl).
 */

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import CodeGraph from '../../src/index';

function createTempDir(prefix = 'codegraph-int-'): string {
  return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
}

function cleanupTempDir(dir: string): void {
  if (fs.existsSync(dir)) {
    fs.rmSync(dir, { recursive: true, force: true });
  }
}

/**
 * Generate a synthetic TypeScript project with the given module count.
 * Each module exports a function that calls the previous module's
 * function so that the resolver has real import edges + call edges to
 * resolve. The first module is a leaf; the last is the root.
 */
function generateSyntheticProject(root: string, moduleCount: number): void {
  const srcDir = path.join(root, 'src');
  fs.mkdirSync(srcDir, { recursive: true });

  // Leaf module — no imports.
  fs.writeFileSync(
    path.join(srcDir, `mod0.ts`),
    `export function fn0(x: number): number { return x + 1; }\n` +
      `export class Mod0 { ping(): string { return 'mod0'; } }\n`
  );

  for (let i = 1; i < moduleCount; i++) {
    const prev = i - 1;
    fs.writeFileSync(
      path.join(srcDir, `mod${i}.ts`),
      `import { fn${prev}, Mod${prev} } from './mod${prev}';\n` +
        `export function fn${i}(x: number): number { return fn${prev}(x) + 1; }\n` +
        `export class Mod${i} extends Mod${prev} {\n` +
        `  call${i}(): number { return fn${i}(${i}); }\n` +
        `}\n`
    );
  }

  // Entry point file.
  fs.writeFileSync(
    path.join(srcDir, 'index.ts'),
    `import { fn${moduleCount - 1}, Mod${moduleCount - 1} } from './mod${moduleCount - 1}';\n` +
      `export function entry(): number {\n` +
      `  const m = new Mod${moduleCount - 1}();\n` +
      `  return fn${moduleCount - 1}(0) + m.call${moduleCount - 1}();\n` +
      `}\n`
  );
}

describe('Integration: full pipeline', () => {
  let tempDir: string;

  beforeEach(() => {
    tempDir = createTempDir();
  });

  afterEach(() => {
    cleanupTempDir(tempDir);
  });

  it('runs init → index → resolve → search → callers → context → sync', async () => {
    const MODULE_COUNT = 120;
    generateSyntheticProject(tempDir, MODULE_COUNT);

    // ── init ──────────────────────────────────────────────────────
    const cg = await CodeGraph.init(tempDir, {
      config: { include: ['**/*.ts'], exclude: [] },
    });

    try {
      // ── indexAll ────────────────────────────────────────────────
      const indexResult = await cg.indexAll();
      // Synthetic project: MODULE_COUNT mod files + 1 index file.
      expect(indexResult.filesIndexed).toBeGreaterThanOrEqual(MODULE_COUNT);

      const statsAfterIndex = cg.getStats();
      expect(statsAfterIndex.fileCount).toBeGreaterThanOrEqual(MODULE_COUNT);
      expect(statsAfterIndex.nodeCount).toBeGreaterThan(MODULE_COUNT * 2);

      // ── resolveReferences ────────────────────────────────────────
      // Many call-site edges are wired up during extraction itself, so
      // the unresolved-reference queue may already be drained by the
      // time we get here. We assert that resolve completes cleanly and
      // returns a well-formed result; downstream callers/callees
      // assertions verify the graph is actually populated.
      cg.reinitializeResolver();
      const resolution = cg.resolveReferences();
      expect(resolution).toBeDefined();
      expect(resolution.stats).toBeDefined();
      expect(typeof resolution.stats.total).toBe('number');
      expect(typeof resolution.stats.resolved).toBe('number');

      // ── searchNodes ──────────────────────────────────────────────
      const entryResults = cg.searchNodes('entry', { limit: 10 });
      expect(entryResults.length).toBeGreaterThan(0);
      const entryNode = entryResults.find((r) => r.node.name === 'entry');
      expect(entryNode).toBeDefined();

      const midResults = cg.searchNodes(`fn50`, { limit: 10 });
      expect(midResults.find((r) => r.node.name === 'fn50')).toBeDefined();

      // ── getCallers / getCallees ──────────────────────────────────
      const fn0Results = cg.searchNodes('fn0', { limit: 5 });
      const fn0Node = fn0Results.find((r) => r.node.name === 'fn0');
      expect(fn0Node).toBeDefined();
      const callers = cg.getCallers(fn0Node!.node.id);
      // fn0 is called by fn1 (at least). After resolution this should
      // be wired up.
      expect(Array.isArray(callers)).toBe(true);

      // ── buildContext ─────────────────────────────────────────────
      const context = await cg.buildContext('entry function chain', {
        maxNodes: 10,
        format: 'markdown',
      });
      expect(typeof context).toBe('string');
      expect((context as string).length).toBeGreaterThan(0);

      // ── sync (add + modify + remove in one pass) ─────────────────
      // Add: a new file referencing entry().
      fs.writeFileSync(
        path.join(tempDir, 'src', 'consumer.ts'),
        `import { entry } from './index';\nexport const result = entry();\n`
      );
      // Modify: change mod0.
      fs.writeFileSync(
        path.join(tempDir, 'src', 'mod0.ts'),
        `export function fn0(x: number): number { return x + 2; }\n` +
          `export function newHelper(): string { return 'new'; }\n` +
          `export class Mod0 { ping(): string { return 'mod0v2'; } }\n`
      );
      // Remove: drop mod1 — note this will leave dangling imports in
      // mod2, which the resolver should tolerate.
      fs.unlinkSync(path.join(tempDir, 'src', 'mod1.ts'));

      const syncResult = await cg.sync();
      expect(syncResult.filesAdded).toBeGreaterThanOrEqual(1);
      expect(syncResult.filesModified).toBeGreaterThanOrEqual(1);
      expect(syncResult.filesRemoved).toBeGreaterThanOrEqual(1);

      // New symbol must now be findable; removed file's symbols gone.
      expect(cg.searchNodes('newHelper').length).toBeGreaterThan(0);

      // Removed file should no longer appear in the indexed file list.
      // (FTS prefix matching makes name-based assertions unreliable here —
      // Mod10/Mod11/… all start with "Mod1" — so we check the file set
      // instead.)
      const filesAfterSync = cg.getNodesInFile('src/mod1.ts');
      expect(filesAfterSync).toHaveLength(0);
    } finally {
      cg.destroy();
    }
  }, 60_000);

  it('keeps indexing files when one file has a parse error', async () => {
    const srcDir = path.join(tempDir, 'src');
    fs.mkdirSync(srcDir, { recursive: true });

    // Valid files
    fs.writeFileSync(
      path.join(srcDir, 'good1.ts'),
      `export function good1(): number { return 1; }\n`
    );
    fs.writeFileSync(
      path.join(srcDir, 'good2.ts'),
      `export function good2(): number { return 2; }\n`
    );
    // Intentionally broken file — unclosed brace, stray tokens.
    fs.writeFileSync(
      path.join(srcDir, 'broken.ts'),
      `export function broken(\n  this is { not valid typescript at all\n`
    );

    const cg = await CodeGraph.init(tempDir, {
      config: { include: ['**/*.ts'], exclude: [] },
    });

    try {
      const result = await cg.indexAll();
      // The two good files must still be indexed regardless of the
      // broken one. Tree-sitter is error-tolerant so it may still
      // extract a partial AST from broken.ts — but the test only
      // requires that the batch completes and finds the good symbols.
      expect(result.filesIndexed).toBeGreaterThanOrEqual(2);

      const good1 = cg.searchNodes('good1');
      const good2 = cg.searchNodes('good2');
      expect(good1.find((r) => r.node.name === 'good1')).toBeDefined();
      expect(good2.find((r) => r.node.name === 'good2')).toBeDefined();
    } finally {
      cg.destroy();
    }
  }, 30_000);

  it('handles repeated sync calls when nothing has changed', async () => {
    generateSyntheticProject(tempDir, 10);

    const cg = await CodeGraph.init(tempDir, {
      config: { include: ['**/*.ts'], exclude: [] },
    });

    try {
      await cg.indexAll();
      const statsBefore = cg.getStats();

      const first = await cg.sync();
      const second = await cg.sync();

      // Subsequent sync with no changes should be a no-op.
      expect(first.filesAdded + first.filesModified + first.filesRemoved).toBe(0);
      expect(second.filesAdded + second.filesModified + second.filesRemoved).toBe(0);

      const statsAfter = cg.getStats();
      expect(statsAfter.fileCount).toBe(statsBefore.fileCount);
      expect(statsAfter.nodeCount).toBe(statsBefore.nodeCount);
    } finally {
      cg.destroy();
    }
  }, 30_000);

  it('reports edgesCreated including resolution + synthesizer phases', async () => {
    // The synthetic project has cross-file imports, calls, and extends —
    // all wired up in the resolution phase, AFTER the orchestrator's
    // per-file extraction counter is done. The CLI summary used to read
    // only the extraction-phase counter and undercount the graph; this
    // test pins the counter to the true DB totals across all phases.
    generateSyntheticProject(tempDir, 30);

    const cg = await CodeGraph.init(tempDir, {
      config: { include: ['**/*.ts'], exclude: [] },
    });

    try {
      const result = await cg.indexAll();
      const stats = cg.getStats();

      expect(result.success).toBe(true);
      expect(result.nodesCreated).toBe(stats.nodeCount);
      expect(result.edgesCreated).toBe(stats.edgeCount);
      // Sanity: cross-file resolution had something to do — calls/extends
      // edges should exist beyond the bare extraction-time contains edges.
      const containsOnly = stats.edgesByKind.contains ?? 0;
      expect(stats.edgeCount).toBeGreaterThan(containsOnly);
    } finally {
      cg.destroy();
    }
  }, 30_000);

  it('reports durationMs for the whole run, resolution and linking included (#2334)', async () => {
    // The CLI summary printed the orchestrator's time — extraction alone —
    // beside the totals of the whole run, so slow resolution never showed.
    generateSyntheticProject(tempDir, 10);
    const cg = await CodeGraph.init(tempDir, {
      config: { include: ['**/*.ts'], exclude: [] },
    });
    const internals = cg as unknown as {
      orchestrator: Record<'indexAll' | 'sync', (...args: unknown[]) => Promise<{ durationMs: number }>>;
      resolver: { resolveDeferredThisMemberRefs: () => Promise<number> };
    };
    // What the orchestrator measured; and a linking pass that takes a known while.
    let extraction = -1;
    const measure = (method: 'indexAll' | 'sync') => {
      const original = internals.orchestrator[method].bind(internals.orchestrator);
      vi.spyOn(internals.orchestrator, method).mockImplementation(async (...args: unknown[]) => {
        const result = await original(...args);
        extraction = result.durationMs;
        return result;
      });
    };
    const LINKING_MS = 250;
    const linking = internals.resolver.resolveDeferredThisMemberRefs.bind(internals.resolver);
    vi.spyOn(internals.resolver, 'resolveDeferredThisMemberRefs').mockImplementation(async () => {
      const until = Date.now() + LINKING_MS;
      while (Date.now() < until) await new Promise((r) => setTimeout(r, 10));
      return linking();
    });

    try {
      measure('indexAll');
      const indexed = await cg.indexAll();
      expect(extraction).toBeGreaterThanOrEqual(0);
      expect(indexed.durationMs).toBeGreaterThanOrEqual(extraction + LINKING_MS);

      // Likewise a sync that re-resolves a changed file.
      fs.appendFileSync(path.join(tempDir, 'src', 'index.ts'), 'export function again(): number { return entry(); }\n');
      extraction = -1;
      measure('sync');
      const synced = await cg.sync();
      expect(synced.filesModified).toBe(1);
      expect(extraction).toBeGreaterThanOrEqual(0);
      expect(synced.durationMs).toBeGreaterThanOrEqual(extraction + LINKING_MS);
    } finally {
      vi.restoreAllMocks();
      cg.destroy();
    }
  }, 30_000);
});