From 0b74e94ee7a951b0d3e7db0b72b5298fcffa8ecb Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Wed, 30 Sep 2026 15:36:22 +0000 Subject: [PATCH] fix(kotlin): no automatic semicolon before a same-line infix e-word (#2173) tree-sitter-kotlin 0.3.8's external scanner inserted an automatic semicolon on the same line before any word starting with `e` other than `else`, so Exposed's `Users.id eq id1` ended its statement early and error recovery could swallow the class around it (r2dbc UpsertTests became one loose function nesting every test). Patch scanner.c (same line: never a semicolon before an e-word) in both the vendored kernel C and the wasm build; provenance, shas and rebuild recipe in docs/grammars/tree-sitter-kotlin.md. Exposed files with a parse error 229 -> 71 of 1,004; koin, nowinandroid, moshi, okio, okhttp and kotlinx.coroutines unchanged (one okhttp file fixed, none broken); kernel-parity 0 diffs. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/kotlin-infix-e-parse.test.ts | 55 ++++++++++++++++++++ codegraph-kernel/build.rs | 3 ++ codegraph-kernel/grammars/kotlin/scanner.c | 6 ++- docs/grammars/tree-sitter-kotlin.md | 49 +++++++++++++++++ docs/grammars/tree-sitter-kotlin.patch | 16 ++++++ src/extraction/grammars.ts | 2 + src/extraction/wasm/tree-sitter-kotlin.wasm | Bin 4052313 -> 4052234 bytes 8 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 __tests__/kotlin-infix-e-parse.test.ts create mode 100644 docs/grammars/tree-sitter-kotlin.md create mode 100644 docs/grammars/tree-sitter-kotlin.patch diff --git a/CHANGELOG.md b/CHANGELOG.md index 768f490f..667fef45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- Kotlin code using an infix function whose name starts with `e`, like Exposed's `Users.id eq id1`, now parses cleanly. Before, the rest of such a line was a parse error, and in a file with many of them the whole class around it could be lost. On Exposed, files with a parse error dropped from 229 to 71. Re-index after upgrading. - In Kotlin, a call with no receiver now reaches a method of another class only when that class is in reach. That means a class, an anonymous `object : Base()` or an extension function's receiver around the call, what those inherit, or the receiver of a DSL lambda (koin's `single { get() }` runs on a `Scope`, Exposed's `withTables { … }` on a transaction). An object member the file imports is in reach too. Kotlin's own `error(…)`, `require(x >= 0) { … }` and `check(…)` no longer land on a project class's method of that name. On koin, `module { … }` in one test used to land on another test class's private `module` over 260 times, and inside a lambda `get()` landed on the wrong class's `get`. Re-index after upgrading. - In C#, Java and Kotlin, a call through a field, property or parameter now lands on the method of the type it is declared with. That includes a type inherited from a base class with its type argument filled in, a generic or `for`-each declaration, a C# `using` alias, and a type parameter's bound. A call no longer falls back to a guess at some project class's same-named method, and a type from outside the project gives no link. On Newtonsoft.Json, `_innerWriter.WriteValue(…)` inside a writer wrapper used to land on the wrapper's own `WriteValue`, and a `TextWriter`'s `Write(…)` on a test writer's. On commons-lang, a `DateFormat` field's `parse(…)` used to land on the project's `DateParser` 284 times. - C# members of an interface are now public and types declared without a modifier are now internal, as the language defines. A call on an interface-typed field now reaches the interface's method, where it used to find nothing. A guess from a receiver's name now weighs only the owner type's own name, not its namespace. Standard .NET names like `ToString`, `Add` and `Write` need a receiver named after the owner. In every language, a guess from production code no longer lands on a mock or helper defined in a test suite. Re-index after upgrading. diff --git a/__tests__/kotlin-infix-e-parse.test.ts b/__tests__/kotlin-infix-e-parse.test.ts new file mode 100644 index 00000000..35a19ead --- /dev/null +++ b/__tests__/kotlin-infix-e-parse.test.ts @@ -0,0 +1,55 @@ +/** + * The Kotlin grammar's scanner inserted an automatic semicolon on the same + * line before any word starting with `e` that was not `else` — so an infix + * call named like Exposed's `eq` (`Users.id eq id1`) ended its statement + * early and the rest became a parse error. Error recovery could then swallow + * the class around it: Exposed's r2dbc `UpsertTests` came out as one loose + * function nesting all its tests. The vendored scanner (wasm and kernel) is + * patched; see docs/grammars/tree-sitter-kotlin.md. + */ +import { describe, it, expect, afterAll, beforeAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars'; + +let root = ''; +let cg: CodeGraph; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-kotlin-infix-')); + fs.mkdirSync(path.join(root, 'src'), { recursive: true }); + fs.writeFileSync(path.join(root, 'src/Ops.kt'), `package app + +infix fun Int.eq(other: Int): Boolean = this == other + +class Ops { + fun same(a: Int, b: Int): Boolean = a eq b + + fun found(id1: Int): Int = listOf(1).filter { it eq id1 }.single() +} +`); + cg = await CodeGraph.init(root, { index: true }); +}); + +afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('a Kotlin infix call named with a leading `e`', () => { + it('parses clean', async () => { + await initGrammars(); + await loadGrammarsForLanguages(['kotlin']); + const parser = getParser('kotlin')!; + for (const src of ['fun f() { val c = a eq b }', 'fun f() { w { t.id eq id1 }.single() }', 'fun f() { if (a) b else c }']) { + expect(parser.parse(src)!.rootNode.hasError, src).toBe(false); + } + }); + + it('leaves the class around it and its methods intact', () => { + const methods = cg.getNodesInFile('src/Ops.kt').filter((n) => n.qualifiedName.startsWith('app::Ops::')).map((n) => n.name).sort(); + expect(methods).toEqual(['found', 'same']); + }); +}); diff --git a/codegraph-kernel/build.rs b/codegraph-kernel/build.rs index 8e620ee9..e52d8209 100644 --- a/codegraph-kernel/build.rs +++ b/codegraph-kernel/build.rs @@ -9,6 +9,9 @@ fn main() { // against the crates.io tarball (kotlin checklist §Grammar prep): // parser.c 54104a7ef1555c265b746c790e0f8bb953cc17806e9df0c3af82f7f62c06a70a // scanner.c 27f73337ec357fc341fa57538f34c14277b0346980c3405dc30beab6202ec6d0 + // scanner.c carries docs/grammars/tree-sitter-kotlin.patch (same-line + // `e`-word semicolon fix), shared with the wasm build: + // scanner.c 2ca842dd04b60b0a6df59a03e9ad01209d8c5641f079a79a84d65708669b6cb6 (patched) // Flags crib the tarball's own bindings/rust/build.rs. let mut c = cc::Build::new(); c.include("grammars/kotlin"); diff --git a/codegraph-kernel/grammars/kotlin/scanner.c b/codegraph-kernel/grammars/kotlin/scanner.c index a9979ade..aa3c60a5 100644 --- a/codegraph-kernel/grammars/kotlin/scanner.c +++ b/codegraph-kernel/grammars/kotlin/scanner.c @@ -285,9 +285,11 @@ static bool scan_automatic_semicolon(TSLexer *lexer) { if (sameline) { switch (lexer->lookahead) { - // Don't insert a semicolon before an else + // Don't insert a semicolon before an else — nor before any other word + // starting with `e` on the same line: that is an infix call + // (`Users.id eq id1`), which continues the expression. (codegraph patch) case 'e': - return !scan_for_word(lexer, "lse", 3); + return false; case 'i': return scan_for_word(lexer, "mport", 5); diff --git a/docs/grammars/tree-sitter-kotlin.md b/docs/grammars/tree-sitter-kotlin.md new file mode 100644 index 00000000..6bec8c2c --- /dev/null +++ b/docs/grammars/tree-sitter-kotlin.md @@ -0,0 +1,49 @@ +# tree-sitter-kotlin.wasm — provenance & rebuild + +`src/extraction/wasm/tree-sitter-kotlin.wasm` and the kernel's vendored C +(`codegraph-kernel/grammars/kotlin/`) are both built from +[fwcd/tree-sitter-kotlin](https://github.com/fwcd/tree-sitter-kotlin) (MIT) +tag `0.3.8` (commit `e1a2d5ad1f61f5740677183cd4125bb071cd2f30`), from the +tag's checked-in `src/parser.c` — never `tree-sitter generate` — with the +patch in `tree-sitter-kotlin.patch` applied to `src/scanner.c`. + +| file | upstream sha256 | patched sha256 | +|---|---|---| +| `src/parser.c` | `54104a7ef1555c265b746c790e0f8bb953cc17806e9df0c3af82f7f62c06a70a` | unchanged | +| `src/scanner.c` | `27f73337ec357fc341fa57538f34c14277b0346980c3405dc30beab6202ec6d0` | `2ca842dd04b60b0a6df59a03e9ad01209d8c5641f079a79a84d65708669b6cb6` | +| `tree-sitter-kotlin.wasm` | `c80c88867a589a1a0959bcea89de84b7e9684b3693b2cdb2944812458e62ff48` | `e2ef7c0c83e5fd4bece547d2acd41b9f89da2fed5644d74be139b9765d4c74af` | + +## What the patch fixes + +The external scanner decides where to insert an automatic semicolon. On the +same line as the end of an expression it inserted one before any word +starting with `e` that was not `else` — so an infix call with such a name +(Exposed's `Users.id eq id1`, used in nearly every Exposed query) ended the +statement early and the rest became a parse error. On a file with many of +them, error recovery could swallow the whole class: Exposed's r2dbc +`UpsertTests` came out as one loose function nesting every test. + +On the same line nothing but `;` or `import` starts a new statement, so the +patched scanner inserts no semicolon before any `e` word. Measured on +Exposed, files with a parse error went from 229 to 71 of 1,004; koin, +nowinandroid, moshi, okio, okhttp and kotlinx.coroutines parsed exactly as +before (one okhttp file fixed, none broken). + +## Rebuild + +``` +git clone --depth 1 --branch 0.3.8 https://github.com/fwcd/tree-sitter-kotlin +cd tree-sitter-kotlin +git apply /path/to/codegraph/docs/grammars/tree-sitter-kotlin.patch +# the 0.3.8 tag predates tree-sitter.json, which cli 0.25.10 requires — +# add the METADATA-ONLY shim (grammar name/scope; nothing regenerated): +# {"grammars":[{"name":"kotlin","scope":"source.kotlin","path":".", +# "file-types":["kt","kts"]}],"metadata":{"version":"0.3.8","license":"MIT"}} +npx -y tree-sitter-cli@0.25.10 build --wasm -o tree-sitter-kotlin.wasm . +cp src/scanner.c /path/to/codegraph/codegraph-kernel/grammars/kotlin/scanner.c +``` + +Built without the patch, the same command reproduces the upstream wasm above +byte for byte. After updating either side, rebuild the kernel +(`npm run build:kernel`) and check `scripts/kernel-parity.mjs` on a Kotlin +repo: the wasm and the kernel must parse the same. diff --git a/docs/grammars/tree-sitter-kotlin.patch b/docs/grammars/tree-sitter-kotlin.patch new file mode 100644 index 00000000..45bf4762 --- /dev/null +++ b/docs/grammars/tree-sitter-kotlin.patch @@ -0,0 +1,16 @@ +--- a/src/scanner.c 2026-09-30 10:25:12 ++++ b/src/scanner.c 2026-09-30 10:25:12 +@@ -285,9 +285,11 @@ + + if (sameline) { + switch (lexer->lookahead) { +- // Don't insert a semicolon before an else ++ // Don't insert a semicolon before an else — nor before any other word ++ // starting with `e` on the same line: that is an infix call ++ // (`Users.id eq id1`), which continues the expression. (codegraph patch) + case 'e': +- return !scan_for_word(lexer, "lse", 3); ++ return false; + + case 'i': + return scan_for_word(lexer, "mport", 5); diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index f0702a91..5913349f 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -390,6 +390,8 @@ const VENDORED_WASM_LANGS: ReadonlySet = new Set([ // crate is UNUSABLE by the kernel (pins tree-sitter <0.23) and // tree-sitter-kotlin-ng is a different grammar — the kernel compiles the // same vendored C sources instead (codegraph-kernel/grammars/kotlin). + // Both carry docs/grammars/tree-sitter-kotlin.patch (scanner: no automatic + // semicolon before a same-line `e` word, e.g. an `eq` infix call). 'kotlin', // R7b batch 4 (Dart kernel port prep): the byte-copied tree-sitter-wasms // 0.1.13 artifact (sha256 7f5364e4…, built from UserNobody14/ diff --git a/src/extraction/wasm/tree-sitter-kotlin.wasm b/src/extraction/wasm/tree-sitter-kotlin.wasm index 8188ae26bacc0a8963ee103ee9e559ea60e2ba86..75ce09a9165238b1a349fc262e2edbff625e2712 100755 GIT binary patch delta 304 zcmWm7J5IuI0Knn?TaZF6D)MS6P>W&(dHE=hVq)XKxr!JP)*gV}iKK%h9D-GjzzJCW zIehbX_&$jhHgh5G*YQGlhWLtirY7>XaAnC+0)Lixx0T>1&2%+#tdA(P3!kC23@rf} z3KB@-3@NBcBLfZR(2>OjE^&n%u8~IpMU+s60TUJce^t~_hm9LJXy6uiXyP6&9?(J? fkLaL_9{L#I3D0=JD?AJ_!Wa|ynEr&;+wb-d%?Myr delta 365 zcmW;FyG{Z@7{&4VW?+F`7ex`dtU7=fRJ^b%UeJVz1@D4yp(Z9aCT?%6txPC1@ii>e z#>!`~^$$7uRp%t zI2m`@f+*9UteWEP%R`$Qwv`%$;X_nRej-ojg5