fix(pascal): Delphi routines with a parameter attribute like const [ref] are indexed (#2778)

The Pascal grammar's parameter rule takes no RTTI attribute, so a
declaration like `procedure WithRef(const [ref] AValue: Integer; ...)` or
DelphiMVCFramework's `function CreatePerson([MVCFromBody] const Person:
TPerson)` broke the parse at the `[`. The method was lost, the parameter
after the attribute became a field of the class, every member declared
after it went missing until the next class, and in an implementation the
routine's locals became unit variables and its calls were credited to the
file node.

The Pascal extractor now has an offset-preserving preParse that blanks an
attribute group at the start of a routine parameter, before or after its
modifier (const/var/out/constref), in any routine header, procedural type
or anonymous method (`operator` counts only after `class`, since Delphi
doesn't reserve it). It is a linear hand-written scanner that skips
strings, comments and directives, reads generic and qualified routine
names, and leaves every other bracket alone (set defaults, array types,
member and method attributes, property index lists, code). A new
LanguageExtractor flag, signatureAsWritten, lets Pascal signatures read
the source as written, so `(const [MVCFromBody] Person: TPerson)` keeps
its attribute instead of showing blank space. Only Pascal sets the flag.

Validation:
- New __tests__/pascal-param-attributes.test.ts (8 tests, the issue's unit
  included): 6 fail on main's wiring, all pass with the fix.
- A/B against main 5a6917b4, both arms built from their own sources:
  SynEdit, HeidiSQL and DWScript (2,497 files) are byte-identical, as is
  mORMot2 (its two `[ref] const` signatures keep the attribute).
  DelphiMVCFramework (32 files changed by the preParse, 150 attributes):
  4 nodes recovered (2 methods, a function and a constant), 10 removed,
  all misparses (9 method locals hoisted to unit variables and a
  parameter read as a field), 18 methods get a correct signature, end
  line and return type; edges -39 +90: 29 calls move from the file node
  or the wrong method to the method making them, 60 new edges come from
  bodies and declarations main lost, 10 contains edges went with the
  misparsed nodes. Every changed edge was triaged.
- Controls leveldb (C++, kernel) and SDWebImage (Objective-C, wasm with
  its own preParse) are byte-identical in dump-graph.
- Cost: the preParse takes about 4 ms per MB (95 ms on mORMot2's 25 MB);
  index time is within noise (3 interleaved rounds, medians: HeidiSQL
  6.53 to 6.47 s, mORMot2 16.7 to 17.2 s, DWScript 16.3 to 16.1 s wall).
- The shipped scanner matches the prototype used for triage on all 4,578
  Pascal files of the five repos, with lengths and newlines kept.

Fixes #2545

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/extraction/languages/pascal-preparse.ts

@@ -0,0 +1,260 @@
+/**
+ * Offset-preserving source fixes the Pascal grammar needs before it parses a
+ * Delphi unit (the extractor's `preParse`). Each one blanks syntax the grammar
+ * has no rule for with spaces, keeping every newline, so node positions and
+ * text stay where they are.
+ *
+ * - An attribute on a routine parameter, before or after its modifier:
+ *   `const [ref] AValue: Integer`, `[ref] const Source: TVarData`,
+ *   `const [MVCFromBody] Person: TPerson`, `[MVCFromQueryString('q', '')] Q: string`.
+ *   The grammar's parameter rule takes no attribute, so the parse fell apart
+ *   at the `[` and every member declared after it in the class was lost.
+ *   The extractor reads nothing from an attribute, and a signature still
+ *   shows it: Pascal signatures read the source as written.
+ */
+
+const MODIFIERS: ReadonlySet<string> = new Set(['const', 'var', 'out', 'constref']);
+const ROUTINE_KEYWORDS: ReadonlySet<string> = new Set(['procedure', 'function', 'constructor', 'destructor', 'operator']);
+/** An attribute group longer than this is not one (and keeps the scan linear). */
+const MAX_ATTRIBUTE_LENGTH = 2000;
+
+const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_SPACE = 32, CH_AMP = 38, CH_QUOTE = 39;
+const CH_LPAREN = 40, CH_RPAREN = 41, CH_STAR = 42, CH_DOT = 46, CH_SLASH = 47;
+const CH_SEMI = 59, CH_LT = 60, CH_GT = 62, CH_LBRACKET = 91, CH_RBRACKET = 93, CH_LBRACE = 123;
+
+/** Identifier characters, `&` (Delphi's keyword escape) and any non-ASCII letter. */
+function isIdentStart(c: number): boolean {
+  return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || c === CH_AMP || c >= 128;
+}
+function isIdentPart(c: number): boolean {
+  return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || (c >= 48 && c <= 57) || c >= 128;
+}
+
+function wordEnd(s: string, i: number): number {
+  while (i < s.length && isIdentPart(s.charCodeAt(i))) i++;
+  return i;
+}
+
+/**
+ * The index after the comment or string that starts at `i`, or -1 when none
+ * does. Pascal comments are `{…}` (compiler directives included), `(*…*)` and
+ * `//…`; a string is `'…'` with `''` for a quote and ends at its line's end
+ * when unterminated.
+ */
+function commentOrStringEnd(s: string, i: number): number {
+  const c = s.charCodeAt(i);
+  if (c === CH_LBRACE) {
+    const e = s.indexOf('}', i + 1);
+    return e < 0 ? s.length : e + 1;
+  }
+  if (c === CH_LPAREN && s.charCodeAt(i + 1) === CH_STAR) {
+    const e = s.indexOf('*)', i + 2);
+    return e < 0 ? s.length : e + 2;
+  }
+  if (c === CH_SLASH && s.charCodeAt(i + 1) === CH_SLASH) {
+    const e = s.indexOf('\n', i + 2);
+    return e < 0 ? s.length : e;
+  }
+  if (c === CH_QUOTE) {
+    let j = i + 1;
+    while (j < s.length) {
+      const d = s.charCodeAt(j);
+      if (d === CH_QUOTE) {
+        if (s.charCodeAt(j + 1) !== CH_QUOTE) return j + 1;
+        j += 2;
+        continue;
+      }
+      if (d === CH_LF) return j;
+      j++;
+    }
+    return j;
+  }
+  return -1;
+}
+
+/** The index of the first character from `i` that is neither whitespace nor in a comment. */
+function skipTrivia(s: string, i: number): number {
+  while (i < s.length) {
+    const c = s.charCodeAt(i);
+    if (c === CH_SPACE || c === CH_TAB || c === CH_LF || c === CH_CR) {
+      i++;
+    } else if (c === CH_LBRACE || (c === CH_LPAREN && s.charCodeAt(i + 1) === CH_STAR) || (c === CH_SLASH && s.charCodeAt(i + 1) === CH_SLASH)) {
+      i = commentOrStringEnd(s, i);
+    } else {
+      break;
+    }
+  }
+  return i;
+}
+
+/**
+ * The index after the `]` closing the attribute group whose `[` is at `i`,
+ * or -1 when the group doesn't close the way an attribute does: nested
+ * brackets and parentheses balance, strings and comments are skipped, and an
+ * attribute's arguments hold no `;`.
+ */
+function attributeEnd(s: string, i: number): number {
+  const limit = Math.min(s.length, i + MAX_ATTRIBUTE_LENGTH);
+  let depth = 0;
+  let j = i;
+  while (j < limit) {
+    const k = commentOrStringEnd(s, j);
+    if (k >= 0) {
+      j = k;
+      continue;
+    }
+    const c = s.charCodeAt(j);
+    if (c === CH_LBRACKET || c === CH_LPAREN) {
+      depth++;
+    } else if (c === CH_RBRACKET || c === CH_RPAREN) {
+      depth--;
+      if (depth === 0) return c === CH_RBRACKET ? j + 1 : -1;
+    } else if (c === CH_SEMI) {
+      return -1;
+    }
+    j++;
+  }
+  return -1;
+}
+
+/**
+ * The index after a routine's name — `Name`, `TFoo.Bar`, `TFoo<T>.Bar<U>`,
+ * `Get<T: class, constructor>` — read from `i`, or `i` itself (trivia
+ * skipped) when no name starts there (`procedure(…) of object`).
+ */
+function routineNameEnd(s: string, i: number): number {
+  for (;;) {
+    i = skipTrivia(s, i);
+    if (!isIdentStart(s.charCodeAt(i))) return i;
+    i = wordEnd(s, i + 1);
+    let j = skipTrivia(s, i);
+    if (s.charCodeAt(j) === CH_LT) {
+      let depth = 0;
+      for (; j < s.length; j++) {
+        const c = s.charCodeAt(j);
+        if (c === CH_LT) {
+          depth++;
+        } else if (c === CH_GT) {
+          if (--depth === 0) {
+            j++;
+            break;
+          }
+        } else if (c === CH_SEMI || c === CH_LPAREN || c === CH_RPAREN || c === CH_LF) {
+          return i; // not a generic parameter list
+        }
+      }
+      i = j;
+      j = skipTrivia(s, i);
+    }
+    if (s.charCodeAt(j) !== CH_DOT) return i;
+    i = j + 1;
+  }
+}
+
+/**
+ * Walk the parameter list whose `(` is at `open`, pushing the `[from, to)`
+ * range of each attribute group a parameter starts with — before or after its
+ * modifier — onto `ranges`. Returns the index after the list's `)`.
+ */
+function scanParameterList(s: string, open: number, ranges: number[]): number {
+  let depth = 0;
+  let i = open;
+  let atParameterStart = false;
+  while (i < s.length) {
+    if (atParameterStart) {
+      atParameterStart = false;
+      for (;;) {
+        i = skipTrivia(s, i);
+        const c = s.charCodeAt(i);
+        if (c === CH_LBRACKET) {
+          const end = attributeEnd(s, i);
+          if (end < 0) break;
+          ranges.push(i, end);
+          i = end;
+        } else if (isIdentStart(c)) {
+          const e = wordEnd(s, i + 1);
+          if (!MODIFIERS.has(s.slice(i, e).toLowerCase())) break;
+          i = e;
+        } else {
+          break;
+        }
+      }
+      continue;
+    }
+    const k = commentOrStringEnd(s, i);
+    if (k >= 0) {
+      i = k;
+      continue;
+    }
+    const c = s.charCodeAt(i);
+    if (c === CH_LPAREN || c === CH_LBRACKET) {
+      if (++depth === 1) atParameterStart = true;
+    } else if (c === CH_RPAREN || c === CH_RBRACKET) {
+      if (--depth <= 0) return i + 1;
+    } else if (c === CH_SEMI && depth === 1) {
+      atParameterStart = true;
+    }
+    i++;
+  }
+  return i;
+}
+
+/** `s` with every `[from, to)` range in `ranges` blanked to spaces, newlines kept. */
+function blankRanges(s: string, ranges: number[]): string {
+  if (ranges.length === 0) return s;
+  let out = '';
+  let last = 0;
+  for (let r = 0; r < ranges.length; r += 2) {
+    const from = ranges[r]!;
+    const to = ranges[r + 1]!;
+    out += s.slice(last, from) + s.slice(from, to).replace(/[^\r\n]/g, ' ');
+    last = to;
+  }
+  return out + s.slice(last);
+}
+
+/** Whether the word before index `i`, past whitespace, is `class`. */
+function followsClass(s: string, i: number): boolean {
+  let j = i - 1;
+  while (j >= 0 && (s.charCodeAt(j) === CH_SPACE || s.charCodeAt(j) === CH_TAB || s.charCodeAt(j) === CH_LF || s.charCodeAt(j) === CH_CR)) j--;
+  return j >= 4 && s.slice(j - 4, j + 1).toLowerCase() === 'class' && (j < 5 || !isIdentPart(s.charCodeAt(j - 5)));
+}
+
+/**
+ * Blank what the Pascal grammar can't parse (see the file comment). Returns
+ * the source unchanged when there is nothing to blank.
+ */
+export function preParsePascal(source: string): string {
+  if (source.indexOf('[') < 0) return source;
+  const ranges: number[] = [];
+  const n = source.length;
+  let i = 0;
+  while (i < n) {
+    const k = commentOrStringEnd(source, i);
+    if (k >= 0) {
+      i = k;
+      continue;
+    }
+    const c = source.charCodeAt(i);
+    if (isIdentStart(c)) {
+      const e = wordEnd(source, i + 1);
+      const len = e - i;
+      const word = len >= 8 && len <= 11 ? source.slice(i, e).toLowerCase() : '';
+      // `operator` is no reserved word in Delphi (a call `Operator([1, 2])` is
+      // code), so it starts a header only after `class`.
+      if (ROUTINE_KEYWORDS.has(word) && (word !== 'operator' || followsClass(source, i))) {
+        const open = skipTrivia(source, routineNameEnd(source, e));
+        if (source.charCodeAt(open) === CH_LPAREN) {
+          i = scanParameterList(source, open, ranges);
+          continue;
+        }
+      }
+      i = e;
+    } else if (c >= 48 && c <= 57) {
+      i = wordEnd(source, i + 1); // a number
+    } else {
+      i++;
+    }
+  }
+  return blankRanges(source, ranges);
+}

src/extraction/languages/pascal.ts

@@ -1,8 +1,14 @@
 import type { Node as SyntaxNode } from 'web-tree-sitter';
 import { getNodeText, getChildByField } from '../tree-sitter-helpers';
 import type { LanguageExtractor } from '../tree-sitter-types';
+import { preParsePascal } from './pascal-preparse';
 
 export const pascalExtractor: LanguageExtractor = {
+  // Blank Delphi syntax the grammar has no rule for, such as an attribute on a
+  // parameter (`const [ref] AValue: Integer`), offsets kept (#2545). Signatures
+  // still read the attribute from the source as written.
+  preParse: preParsePascal,
+  signatureAsWritten: true,
   functionTypes: ['declProc'],
   classTypes: ['declClass'],
   methodTypes: ['declProc'],

src/extraction/tree-sitter-types.ts

@@ -97,6 +97,13 @@ export interface LanguageExtractor {
    * language id serves several dialects (C++ also parses `.metal` shaders).
    */
   preParse?: (source: string, filePath?: string) => string;
+  /**
+   * Read `getSignature` from the source as the file has it, not from
+   * `preParse`'s output, when the preParse blanks syntax a reader of the
+   * signature still wants: a Delphi parameter's attribute, `const [ref] AValue`
+   * or `const [MVCFromBody] Person` (#2545). Offsets are the same in both.
+   */
+  signatureAsWritten?: boolean;
 
   // --- Node type mappings ---
 

src/extraction/tree-sitter.ts

@@ -747,6 +747,9 @@ export class TreeSitterExtractor {
   // point (this instance is the wasm fallback for a kernel-deferred file) —
   // don't blank it a second time.
   private sourceIsPreParsed = false;
+  // The source as the file has it, kept when this instance's preParse changed
+  // it, for an extractor whose signatures read it (`signatureAsWritten`).
+  private sourceAsWritten: string | null = null;
 
   constructor(
     filePath: string,
@@ -762,6 +765,11 @@ export class TreeSitterExtractor {
     this.sourceIsPreParsed = options?.sourceIsPreParsed === true;
   }
 
+  /** The text `getSignature` reads: the source as written when the extractor asks for it. */
+  private get signatureSource(): string {
+    return this.sourceAsWritten ?? this.source;
+  }
+
   /**
    * Parse and extract from the source code
    */
@@ -811,7 +819,9 @@ export class TreeSitterExtractor {
       // saw (identical outside the blanked directive lines). Skipped when the
       // kernel route point already applied it (sourceIsPreParsed).
       if (this.extractor?.preParse && !this.sourceIsPreParsed) {
-        this.source = this.extractor.preParse(this.source, this.filePath);
+        const preParsed = this.extractor.preParse(this.source, this.filePath);
+        if (preParsed !== this.source && this.extractor.signatureAsWritten) this.sourceAsWritten = this.source;
+        this.source = preParsed;
       }
       this.tree = parser.parse(this.source) ?? null;
       if (!this.tree) {
@@ -2160,7 +2170,7 @@ export class TreeSitterExtractor {
     }
 
     const docstring = this.docstringFor(node);
-    const signature = this.extractor.getSignature?.(node, this.source);
+    const signature = this.extractor.getSignature?.(node, this.signatureSource);
     const visibility = this.extractor.getVisibility?.(node);
     const isExported = commonJsExport || this.extractor.isExported?.(node, this.source);
     const isAsync = this.extractor.isAsync?.(node);
@@ -2521,7 +2531,7 @@ export class TreeSitterExtractor {
     }
 
     const docstring = this.docstringFor(node);
-    const signature = this.extractor.getSignature?.(node, this.source);
+    const signature = this.extractor.getSignature?.(node, this.signatureSource);
     const visibility = this.extractor.getVisibility?.(node);
     const isAsync = this.extractor.isAsync?.(node);
     const isStatic = this.extractor.isStatic?.(node);
@@ -4088,7 +4098,7 @@ export class TreeSitterExtractor {
     docstring?: string
   ): void {
     if (!this.extractor) return;
-    const signature = this.extractor.getSignature?.(node, this.source);
+    const signature = this.extractor.getSignature?.(node, this.signatureSource);
     const extra: Partial<Node> = { docstring, signature };
     if (receiver) extra.qualifiedName = this.composeReceiverQualifiedName(receiver, name);
     else extra.isExported = this.extractor.isExported?.(node, this.source);
@@ -4291,7 +4301,7 @@ export class TreeSitterExtractor {
       const mname = getNodeText(nameNode, this.source);
       if (mname) {
         const methodNode = this.createNode('method', mname, m, {
-          signature: this.extractor?.getSignature?.(m, this.source),
+          signature: this.extractor?.getSignature?.(m, this.signatureSource),
         });
         if (methodNode) {
           for (const type of goInterfaceMethodTypeNames(m, decl, this.source)) {
@@ -8525,7 +8535,7 @@ export class TreeSitterExtractor {
     // free routines, so there's no duplicate.
     if (!parentId && !fullName.includes('.')) {
       const fnNode = this.createNode('function', fullName, declProc, {
-        signature: this.extractor?.getSignature?.(declProc, this.source),
+        signature: this.extractor?.getSignature?.(declProc, this.signatureSource),
         visibility: this.extractor?.getVisibility?.(declProc),
       });
       if (fnNode) {