diff --git a/CHANGELOG.md b/CHANGELOG.md index de15f43c2..5a5b87f38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Claude Code now shows the model the full text of `codegraph_status` and of explore's stale-index refusal. Both results also carried `structuredContent`, and Claude Code shows a result's `structuredContent` in place of its text, so the model saw only a freshness JSON object: no refusal message, no retry guidance and no status report. Both results are now text only. Thanks @bompus. (#2088) - When an agent asks CodeGraph about a separate git repository nested inside an indexed project, one the project's index leaves out (for example because the parent's `.gitignore` excludes it), it now gets the usual "isn't indexed" guidance instead of answers from the parent project's code. Nested repositories the parent does index, such as submodules, work as before. Thanks @wstczyw for the report. (#2110) - Editing a file that defines the same name more than once, like two classes that each have an `execute` method or a method's overloads, no longer moves every caller from other files onto one of them during `codegraph sync`. A sync interrupted partway through a file also no longer loses those callers until a full re-index. Thanks @ijbranch for the report. (#2276) +- Delphi and Free Pascal include files (`.inc`) are now indexed as Pascal. Before, they were read as PHP and came up empty. A `.inc` file with a ` { + describe('detectLanguage', () => { + it('reads a directive-only include as Pascal', () => { + expect(detectLanguage('src/defs.inc', DIRECTIVES)).toBe('pascal'); + expect(detectLanguage('src/defs.inc', '{$I other.inc}\n')).toBe('pascal'); + expect(detectLanguage('src/defs.inc', '{$R-}\n{$Q-}\n')).toBe('pascal'); + expect(detectLanguage('src/DEFS.INC', '{$define FOO}\n')).toBe('pascal'); + // Lazarus opens its includes with an IDE marker naming the owning unit. + expect(detectLanguage('lcl/include/control.inc', '{%MainUnit ../controls.pp}\n{ comment }\n')).toBe('pascal'); + }); + + it('skips a leading UTF-8 BOM and CRLF line endings', () => { + expect(detectLanguage('defs.inc', '\uFEFF{$IFDEF WIN32}\r\n{$DEFINE X}\r\n{$ENDIF}\r\n')).toBe('pascal'); + expect(detectLanguage('defs.inc', '\uFEFFprocedure Foo;\r\n')).toBe('pascal'); + }); + + it('reads a declaration fragment as Pascal', () => { + expect(detectLanguage('r.inc', ROUTINES)).toBe('pascal'); + expect(detectLanguage('r.inc', 'procedure Foo;\nfunction Bar(A: Integer): Integer;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'procedure TForm1.Button1Click(Sender: TObject);\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'const\n X = 1;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'const // ids\n{ Internal functions }\n fpc_in_lo = 1;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'const Max: Integer = 10;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'type\n TColor = (Red, Green);\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'type TFoo = class(TObject)\nend;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'var\n G, H: Integer;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'uses\n SysUtils, Classes;\n')).toBe('pascal'); + expect(detectLanguage('r.inc', 'unit Foo;\ninterface\nimplementation\nend.\n')).toBe('pascal'); + }); + + it('keeps an include with a PHP open tag as PHP', () => { + expect(detectLanguage('page.inc', '\n')).toBe('php'); + // A tag anywhere wins over Pascal-looking text elsewhere in the file. + expect(detectLanguage('page.inc', '

\nprocedure Foo;\n

\n\n')).toBe('php'); + }); + + it('leaves non-Pascal, untagged includes on the PHP mapping', () => { + // C / C++ preprocessor and X-macro tables. + expect(detectLanguage('tbl.inc', '#define X 1\n#define Y 2\nconst int z = 3;\n')).toBe('php'); + expect(detectLanguage('tbl.inc', 'OPCODE(ADD, 0x01)\nOPCODE(SUB, 0x02)\n')).toBe('php'); + // C++ out-of-line static member definition. + expect(detectLanguage('Path.inc', 'const file_t::value_type file_t::Invalid = nullptr;\n')).toBe('php'); + // SourcePawn typeset members: `function ();`. + expect( + detectLanguage('menus.inc', 'typeset MenuHandler\n{\n\tfunction int (Menu menu, MenuAction action);\n\tfunction void (Menu menu);\n};\n'), + ).toBe('php'); + // NASM / MASM. + expect(detectLanguage('m.inc', '%define STACK 0x1000\n; comment\nfoo equ 1\nmov eax, 1\n')).toBe('php'); + // POV-Ray. + expect(detectLanguage('s.inc', '#declare R = 1;\n#macro M(a)\n#end\n')).toBe('php'); + // Classic ASP / VBScript server-side include. + expect(detectLanguage('a.inc', 'Const X = 1\nFunction Foo(a)\n Foo = a\nEnd Function\n')).toBe('php'); + // Makefile include. + expect(detectLanguage('config.inc', 'CC := gcc\nCFLAGS := -O2\n')).toBe('php'); + // HTML fragment with inline script, and a Smarty-style `{$var}` line. + expect( + detectLanguage('f.inc', '\n{$title}\n'), + ).toBe('php'); + // Prose that happens to put Pascal keywords on their own lines. + expect(detectLanguage('f.inc', '
  • \n Implementation\n
    • \n
  • \n Begin\n
    • \n')).toBe('php'); + }); + + it('keeps the path-only answer (no source) on PHP', () => { + expect(detectLanguage('defs.inc')).toBe('php'); + expect(detectLanguage('defs.inc', '')).toBe('php'); + }); + + it('lets an explicit codegraph.json mapping for .inc win', () => { + expect(detectLanguage('defs.inc', DIRECTIVES, { '.inc': 'php' })).toBe('php'); + expect(detectLanguage('defs.inc', '#define X 1\n', { '.inc': 'cpp' })).toBe('cpp'); + expect(detectLanguage('page.inc', ' { + it('loads the Pascal grammar when an .inc file is in the set', () => { + expect(preloadLanguagesForFiles(['defs.inc'])).toEqual(expect.arrayContaining(['php', 'pascal'])); + }); + + it('adds nothing for a PHP project without .inc files, or when .inc is mapped explicitly', () => { + expect(preloadLanguagesForFiles(['index.php', 'a.module'])).not.toContain('pascal'); + expect(preloadLanguagesForFiles(['defs.inc'], { '.inc': 'php' })).not.toContain('pascal'); + }); + }); + + describe('index and sync', () => { + let dir: string; + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-inc-')); + clearProjectConfigCache(); + }); + afterEach(() => { + clearProjectConfigCache(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + const write = (rel: string, body: string) => { + const p = path.join(dir, rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, body); + }; + const languageOf = (cg: CodeGraph, rel: string) => cg.getFiles().find((f) => f.path === rel)?.language; + const symbolsIn = (cg: CodeGraph, rel: string) => + cg.getNodesInFile(rel).filter((n) => n.kind !== 'file').map((n) => `${n.kind}:${n.name}:${n.language}`).sort(); + + // Runs first, before anything in this file has loaded the Pascal grammar + // through a `.pas` file: an include-only tree must still get a parser. + it('indexes a tree of Pascal includes with no .pas file', async () => { + write('inc/defs.inc', DIRECTIVES); + write('inc/routines.inc', ROUTINES); + const cg = await CodeGraph.init(dir, { index: true }); + try { + expect(languageOf(cg, 'inc/defs.inc')).toBe('pascal'); + expect(languageOf(cg, 'inc/routines.inc')).toBe('pascal'); + expect(symbolsIn(cg, 'inc/routines.inc')).toEqual(['function:Bar:pascal', 'function:Foo:pascal']); + expect(cg.getFiles().find((f) => f.path === 'inc/routines.inc')?.errors ?? []).toEqual([]); + } finally { + cg.close(); + } + }); + + it('classifies each include by content, and sync keeps the language', async () => { + write('U.pas', 'unit U;\n\ninterface\n\n{$I defs.inc}\n\nprocedure Foo;\n\nimplementation\n\n{$I routines.inc}\n\nend.\n'); + write('defs.inc', DIRECTIVES); + write('routines.inc', ROUTINES); + write('drupal/mymodule.inc', ' f.path === 'defs.inc')?.errors ?? []).toEqual([]); + expect(symbolsIn(cg, 'routines.inc')).toEqual(['function:Bar:pascal', 'function:Foo:pascal']); + expect(symbolsIn(cg, 'drupal/mymodule.inc')).toEqual(['function:mymodule_helper:php']); + + write('defs.inc', DIRECTIVES + '{$DEFINE USE_FAST_MM}\n'); + write('routines.inc', ROUTINES + 'procedure Baz;\nbegin\nend;\n'); + await cg.sync(); + + expect(languageOf(cg, 'defs.inc')).toBe('pascal'); + expect(languageOf(cg, 'routines.inc')).toBe('pascal'); + expect(symbolsIn(cg, 'routines.inc')).toEqual(['function:Bar:pascal', 'function:Baz:pascal', 'function:Foo:pascal']); + expect(languageOf(cg, 'drupal/mymodule.inc')).toBe('php'); + } finally { + cg.close(); + } + }); + + it('honors a codegraph.json .inc mapping through index and sync', async () => { + write('codegraph.json', JSON.stringify({ extensions: { '.inc': 'php' } })); + write('defs.inc', DIRECTIVES); + const cg = await CodeGraph.init(dir, { index: true }); + try { + expect(languageOf(cg, 'defs.inc')).toBe('php'); + write('defs.inc', DIRECTIVES + '{$DEFINE MORE}\n'); + await cg.sync(); + expect(languageOf(cg, 'defs.inc')).toBe('php'); + } finally { + cg.close(); + } + }); + }); +}); diff --git a/scripts/kernel-parity.mjs b/scripts/kernel-parity.mjs index d02ab6d6c..5209369d8 100644 --- a/scripts/kernel-parity.mjs +++ b/scripts/kernel-parity.mjs @@ -198,8 +198,12 @@ process.env.CODEGRAPH_KERNEL_LANGS = 'all'; for (const { file, lang: extLang } of files) { const source = fs.readFileSync(file, 'utf8'); const rel = path.relative(ROOT, file); - // `.h` resolves C vs C++ by content — the same call the indexer makes. - const lang = extLang === 'detect' || extLang === 'javascript' || extLang === 'jsx' ? detectLanguage(rel, source) : extLang; + // `.h` resolves C vs C++ by content — the same call the indexer makes; so + // does `.inc` (PHP vs Pascal, #2279 — a Pascal include is not a kernel file). + const lang = + extLang === 'detect' || extLang === 'javascript' || extLang === 'jsx' || path.extname(file).toLowerCase() === '.inc' + ? detectLanguage(rel, source) + : extLang; if (!KERNEL_LANGS.has(lang)) continue; if (langFilter && !langFilter.has(lang)) continue; processed++; diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 5913349f3..6a05b96c0 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -561,9 +561,101 @@ export function detectLanguage(filePath: string, source?: string, overrides?: Re if (looksLikeObjc(source)) return 'objc'; } + // `.inc` is PHP's include extension (Drupal) and Pascal/Delphi's too + // (`{$I defs.inc}`: directive blocks, declaration fragments), so it is + // decided per file by content (#2279). An explicit codegraph.json mapping + // for `.inc` is the user's answer and is not second-guessed. + if (lang === 'php' && ext === '.inc' && source && !(overrides && overrides[ext]) && looksLikePascalInclude(source)) { + return 'pascal'; + } + return lang; } +/** A PHP open tag: `\n]*>)?`; +const PAS_QNAME = String.raw`[a-z_][\w.]*(?:<[^>\n]*>)?`; +const PAS_COMMENT = String.raw`(?:\/\/[^\n]*|\{[^$}\n][^}\n]*\}[ \t]*)?`; +const PAS_SECTION_BREAK = String.raw`[ \t]*${PAS_COMMENT}(?:\r?\n[ \t]*${PAS_COMMENT})+`; +const pascalLine = (shape: string): RegExp => new RegExp(PAS_LINE + shape, 'im'); + +/** + * Line shapes only Pascal writes, any one of which makes an untagged `.inc` + * Pascal. Each leans on Pascal's own punctuation, so the dialects that share a + * keyword with it stay out: JavaScript `const x = 1;` / `function f() {`, + * SourcePawn `function void (int client);`, C++ `const T X::Y = …`, VBScript + * `Const X = 1` / `Function F(a)`, Smarty `{$var}`, Makefile `X := y`, prose + * with `Begin` on a line of its own. Every pattern stays linear on a long + * whitespace run: no run can be split two ways between neighbouring + * quantifiers. + */ +const PASCAL_INCLUDE_SIGNALS: readonly RegExp[] = [ + // A compiler directive: a name and an argument (`{$IFDEF X}`, `{$DEFINE X}`, + // `{$I file.inc}`, `{$WARN X OFF}`), a bare `{$ELSE}` / `{$ENDIF}` / + // `{$IFEND}`, or a switch (`{$R-}`, `{$A+,B-}`) — or the `{%MainUnit x.pp}` + // line Lazarus opens its include files with. + pascalLine(String.raw`\{(?:\$(?:[a-z]\w*[ \t]+[^\s}]|(?:else|endif|ifend)[ \t]*\}|[a-z][+-][,}])|%MainUnit\b)`), + // A routine header, closed by `;`: `procedure Foo;`, `procedure TForm1.Click(Sender: TObject);`, + // `class constructor Create;` — and a function's result type after its + // parameters (`function Bar(A: Integer): string;`; bare `function Bar;` is + // the implementation-section short form). The parameter list stops at any + // parenthesis, so a file of unclosed `procedure X(` lines stays linear. + pascalLine( + String.raw`(?:class[ \t]+)?(?:(?:procedure|constructor|destructor)[ \t]+${PAS_QNAME}[ \t]*(?:\([^()]*\)[ \t]*)?` + + String.raw`|function[ \t]+${PAS_QNAME}[ \t]*(?:(?:\([^()]*\)[ \t]*)?:[ \t]*[\w.]+(?:<[^>\n]*>)?[ \t]*)?);` + ), + // `unit Foo;` and a `uses A, B;` clause. + pascalLine(String.raw`unit[ \t]+[a-z_][\w.]*[ \t]*;`), + pascalLine(String.raw`uses\s+[a-z_][\w.]*(?:\s*,\s*[a-z_][\w.]*)*\s*;`), + // A `const` section, then `X = …` / `X: T = …`; or a typed constant on one + // line (`const Max: Integer = 10;`). The type never holds a `:`. + pascalLine( + String.raw`(?:const|resourcestring)(?:${PAS_SECTION_BREAK}[a-z_]\w*[ \t]*(?::[^=;:\n]+)?=|[ \t]+[a-z_]\w*[ \t]*:[^=;:\n]+=)` + ), + // A `var` section: `G, H: Integer;`, on the keyword's line or below it. + pascalLine( + String.raw`(?:var|threadvar)(?:${PAS_SECTION_BREAK}|[ \t]+)[a-z_]\w*(?:[ \t]*,[ \t]*[a-z_]\w*)*[ \t]*:(?!:)[^;\n]*;` + ), + // A `type` section, then `TFoo =`; or `type TFoo = class…` (record / + // interface / set of / array / procedure type) on one line. + pascalLine( + String.raw`type(?:${PAS_SECTION_BREAK}${PAS_NAME}[ \t]*=|[ \t]+${PAS_NAME}[ \t]*=[ \t]*(?:packed[ \t]+)?` + + String.raw`(?:class|record|object|interface|dispinterface|set[ \t]+of|array|reference[ \t]+to|procedure|function)\b)` + ), +]; + +/** A `begin` … `end;` block: both halves needed, so neither alone flips a file. */ +const PASCAL_BEGIN_RE = pascalLine(String.raw`begin\b`); +const PASCAL_END_RE = pascalLine(String.raw`end[ \t]*[;.][ \t]*$`); + +/** + * Whether an `.inc` file is a Pascal include rather than a PHP one (#2279). + * + * A PHP include always opens a PHP tag somewhere, so a tag anywhere keeps the + * file PHP. Without one, a Pascal-only line shape (`PASCAL_INCLUDE_SIGNALS`, + * or a `begin` … `end;` pair) makes it Pascal. Anything else keeps the PHP + * mapping — untagged text is inline HTML to PHP, so nothing is extracted — + * rather than handing a C / assembly / POV-Ray / ASP `.inc` to the Pascal + * grammar's error recovery. + * + * Deliberately per file, not "does this project have `.pas` files": the + * answer depends only on the file's own bytes, so a full index, a sync of one + * edited include, and a fresh re-index always agree — a project-level gate + * would flip an untouched include whenever the last `.pas` file came or went. + */ +function looksLikePascalInclude(source: string): boolean { + if (PHP_OPEN_TAG_RE.test(source)) return false; + if (PASCAL_INCLUDE_SIGNALS.some((re) => re.test(source))) return true; + return PASCAL_BEGIN_RE.test(source) && PASCAL_END_RE.test(source); +} + /** Whether a JavaScript file's leading comments carry Flow's `@flow` pragma (and not `@noflow`). */ export function hasFlowPragma(source: string): boolean { const head = source.slice(0, 4096).replace(/^#![^\n]*\n/, ''); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index f87eeacd4..0ca3145be 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -808,6 +808,15 @@ export function preloadLanguagesForFiles( if (!languages.includes(ambiguous)) languages.push(ambiguous); } } + // An `.inc` path-detects as PHP but may read as Pascal (#2279) — unless + // codegraph.json maps `.inc` explicitly, which detectLanguage never overrides. + if ( + !languages.includes('pascal') && + !(overrides && overrides['.inc']) && + files.some((f) => f.toLowerCase().endsWith('.inc')) + ) { + languages.push('pascal'); + } return languages; }