mirror of
https://github.com/colbymchenry/codegraph.git
synced 2026-10-16 20:53:57 +08:00
fix(kotlin): no automatic semicolon before a same-line infix e-word (#2173)
tree-sitter-kotlin 0.3.8's external scanner inserted an automatic semicolon on the same line before any word starting with `e` other than `else`, so Exposed's `Users.id eq id1` ended its statement early and error recovery could swallow the class around it (r2dbc UpsertTests became one loose function nesting every test). Patch scanner.c (same line: never a semicolon before an e-word) in both the vendored kernel C and the wasm build; provenance, shas and rebuild recipe in docs/grammars/tree-sitter-kotlin.md. Exposed files with a parse error 229 -> 71 of 1,004; koin, nowinandroid, moshi, okio, okhttp and kotlinx.coroutines unchanged (one okhttp file fixed, none broken); kernel-parity 0 diffs. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
### Fixes
|
||||
|
||||
- Kotlin code using an infix function whose name starts with `e`, like Exposed's `Users.id eq id1`, now parses cleanly. Before, the rest of such a line was a parse error, and in a file with many of them the whole class around it could be lost. On Exposed, files with a parse error dropped from 229 to 71. Re-index after upgrading.
|
||||
- In Kotlin, a call with no receiver now reaches a method of another class only when that class is in reach. That means a class, an anonymous `object : Base()` or an extension function's receiver around the call, what those inherit, or the receiver of a DSL lambda (koin's `single { get() }` runs on a `Scope`, Exposed's `withTables { … }` on a transaction). An object member the file imports is in reach too. Kotlin's own `error(…)`, `require(x >= 0) { … }` and `check(…)` no longer land on a project class's method of that name. On koin, `module { … }` in one test used to land on another test class's private `module` over 260 times, and inside a lambda `get()` landed on the wrong class's `get`. Re-index after upgrading.
|
||||
- In C#, Java and Kotlin, a call through a field, property or parameter now lands on the method of the type it is declared with. That includes a type inherited from a base class with its type argument filled in, a generic or `for`-each declaration, a C# `using` alias, and a type parameter's bound. A call no longer falls back to a guess at some project class's same-named method, and a type from outside the project gives no link. On Newtonsoft.Json, `_innerWriter.WriteValue(…)` inside a writer wrapper used to land on the wrapper's own `WriteValue`, and a `TextWriter`'s `Write(…)` on a test writer's. On commons-lang, a `DateFormat` field's `parse(…)` used to land on the project's `DateParser` 284 times.
|
||||
- C# members of an interface are now public and types declared without a modifier are now internal, as the language defines. A call on an interface-typed field now reaches the interface's method, where it used to find nothing. A guess from a receiver's name now weighs only the owner type's own name, not its namespace. Standard .NET names like `ToString`, `Add` and `Write` need a receiver named after the owner. In every language, a guess from production code no longer lands on a mock or helper defined in a test suite. Re-index after upgrading.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* The Kotlin grammar's scanner inserted an automatic semicolon on the same
|
||||
* line before any word starting with `e` that was not `else` — so an infix
|
||||
* call named like Exposed's `eq` (`Users.id eq id1`) ended its statement
|
||||
* early and the rest became a parse error. Error recovery could then swallow
|
||||
* the class around it: Exposed's r2dbc `UpsertTests` came out as one loose
|
||||
* function nesting all its tests. The vendored scanner (wasm and kernel) is
|
||||
* patched; see docs/grammars/tree-sitter-kotlin.md.
|
||||
*/
|
||||
import { describe, it, expect, afterAll, beforeAll } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { CodeGraph } from '../src';
|
||||
import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars';
|
||||
|
||||
let root = '';
|
||||
let cg: CodeGraph;
|
||||
|
||||
beforeAll(async () => {
|
||||
root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-kotlin-infix-'));
|
||||
fs.mkdirSync(path.join(root, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'src/Ops.kt'), `package app
|
||||
|
||||
infix fun Int.eq(other: Int): Boolean = this == other
|
||||
|
||||
class Ops {
|
||||
fun same(a: Int, b: Int): Boolean = a eq b
|
||||
|
||||
fun found(id1: Int): Int = listOf(1).filter { it eq id1 }.single()
|
||||
}
|
||||
`);
|
||||
cg = await CodeGraph.init(root, { index: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cg?.close();
|
||||
if (root) fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('a Kotlin infix call named with a leading `e`', () => {
|
||||
it('parses clean', async () => {
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(['kotlin']);
|
||||
const parser = getParser('kotlin')!;
|
||||
for (const src of ['fun f() { val c = a eq b }', 'fun f() { w { t.id eq id1 }.single() }', 'fun f() { if (a) b else c }']) {
|
||||
expect(parser.parse(src)!.rootNode.hasError, src).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves the class around it and its methods intact', () => {
|
||||
const methods = cg.getNodesInFile('src/Ops.kt').filter((n) => n.qualifiedName.startsWith('app::Ops::')).map((n) => n.name).sort();
|
||||
expect(methods).toEqual(['found', 'same']);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,9 @@ fn main() {
|
||||
// against the crates.io tarball (kotlin checklist §Grammar prep):
|
||||
// parser.c 54104a7ef1555c265b746c790e0f8bb953cc17806e9df0c3af82f7f62c06a70a
|
||||
// scanner.c 27f73337ec357fc341fa57538f34c14277b0346980c3405dc30beab6202ec6d0
|
||||
// scanner.c carries docs/grammars/tree-sitter-kotlin.patch (same-line
|
||||
// `e`-word semicolon fix), shared with the wasm build:
|
||||
// scanner.c 2ca842dd04b60b0a6df59a03e9ad01209d8c5641f079a79a84d65708669b6cb6 (patched)
|
||||
// Flags crib the tarball's own bindings/rust/build.rs.
|
||||
let mut c = cc::Build::new();
|
||||
c.include("grammars/kotlin");
|
||||
|
||||
@@ -285,9 +285,11 @@ static bool scan_automatic_semicolon(TSLexer *lexer) {
|
||||
|
||||
if (sameline) {
|
||||
switch (lexer->lookahead) {
|
||||
// Don't insert a semicolon before an else
|
||||
// Don't insert a semicolon before an else — nor before any other word
|
||||
// starting with `e` on the same line: that is an infix call
|
||||
// (`Users.id eq id1`), which continues the expression. (codegraph patch)
|
||||
case 'e':
|
||||
return !scan_for_word(lexer, "lse", 3);
|
||||
return false;
|
||||
|
||||
case 'i':
|
||||
return scan_for_word(lexer, "mport", 5);
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# tree-sitter-kotlin.wasm — provenance & rebuild
|
||||
|
||||
`src/extraction/wasm/tree-sitter-kotlin.wasm` and the kernel's vendored C
|
||||
(`codegraph-kernel/grammars/kotlin/`) are both built from
|
||||
[fwcd/tree-sitter-kotlin](https://github.com/fwcd/tree-sitter-kotlin) (MIT)
|
||||
tag `0.3.8` (commit `e1a2d5ad1f61f5740677183cd4125bb071cd2f30`), from the
|
||||
tag's checked-in `src/parser.c` — never `tree-sitter generate` — with the
|
||||
patch in `tree-sitter-kotlin.patch` applied to `src/scanner.c`.
|
||||
|
||||
| file | upstream sha256 | patched sha256 |
|
||||
|---|---|---|
|
||||
| `src/parser.c` | `54104a7ef1555c265b746c790e0f8bb953cc17806e9df0c3af82f7f62c06a70a` | unchanged |
|
||||
| `src/scanner.c` | `27f73337ec357fc341fa57538f34c14277b0346980c3405dc30beab6202ec6d0` | `2ca842dd04b60b0a6df59a03e9ad01209d8c5641f079a79a84d65708669b6cb6` |
|
||||
| `tree-sitter-kotlin.wasm` | `c80c88867a589a1a0959bcea89de84b7e9684b3693b2cdb2944812458e62ff48` | `e2ef7c0c83e5fd4bece547d2acd41b9f89da2fed5644d74be139b9765d4c74af` |
|
||||
|
||||
## What the patch fixes
|
||||
|
||||
The external scanner decides where to insert an automatic semicolon. On the
|
||||
same line as the end of an expression it inserted one before any word
|
||||
starting with `e` that was not `else` — so an infix call with such a name
|
||||
(Exposed's `Users.id eq id1`, used in nearly every Exposed query) ended the
|
||||
statement early and the rest became a parse error. On a file with many of
|
||||
them, error recovery could swallow the whole class: Exposed's r2dbc
|
||||
`UpsertTests` came out as one loose function nesting every test.
|
||||
|
||||
On the same line nothing but `;` or `import` starts a new statement, so the
|
||||
patched scanner inserts no semicolon before any `e` word. Measured on
|
||||
Exposed, files with a parse error went from 229 to 71 of 1,004; koin,
|
||||
nowinandroid, moshi, okio, okhttp and kotlinx.coroutines parsed exactly as
|
||||
before (one okhttp file fixed, none broken).
|
||||
|
||||
## Rebuild
|
||||
|
||||
```
|
||||
git clone --depth 1 --branch 0.3.8 https://github.com/fwcd/tree-sitter-kotlin
|
||||
cd tree-sitter-kotlin
|
||||
git apply /path/to/codegraph/docs/grammars/tree-sitter-kotlin.patch
|
||||
# the 0.3.8 tag predates tree-sitter.json, which cli 0.25.10 requires —
|
||||
# add the METADATA-ONLY shim (grammar name/scope; nothing regenerated):
|
||||
# {"grammars":[{"name":"kotlin","scope":"source.kotlin","path":".",
|
||||
# "file-types":["kt","kts"]}],"metadata":{"version":"0.3.8","license":"MIT"}}
|
||||
npx -y tree-sitter-cli@0.25.10 build --wasm -o tree-sitter-kotlin.wasm .
|
||||
cp src/scanner.c /path/to/codegraph/codegraph-kernel/grammars/kotlin/scanner.c
|
||||
```
|
||||
|
||||
Built without the patch, the same command reproduces the upstream wasm above
|
||||
byte for byte. After updating either side, rebuild the kernel
|
||||
(`npm run build:kernel`) and check `scripts/kernel-parity.mjs` on a Kotlin
|
||||
repo: the wasm and the kernel must parse the same.
|
||||
@@ -0,0 +1,16 @@
|
||||
--- a/src/scanner.c 2026-09-30 10:25:12
|
||||
+++ b/src/scanner.c 2026-09-30 10:25:12
|
||||
@@ -285,9 +285,11 @@
|
||||
|
||||
if (sameline) {
|
||||
switch (lexer->lookahead) {
|
||||
- // Don't insert a semicolon before an else
|
||||
+ // Don't insert a semicolon before an else — nor before any other word
|
||||
+ // starting with `e` on the same line: that is an infix call
|
||||
+ // (`Users.id eq id1`), which continues the expression. (codegraph patch)
|
||||
case 'e':
|
||||
- return !scan_for_word(lexer, "lse", 3);
|
||||
+ return false;
|
||||
|
||||
case 'i':
|
||||
return scan_for_word(lexer, "mport", 5);
|
||||
@@ -390,6 +390,8 @@ const VENDORED_WASM_LANGS: ReadonlySet<GrammarLanguage> = new Set([
|
||||
// crate is UNUSABLE by the kernel (pins tree-sitter <0.23) and
|
||||
// tree-sitter-kotlin-ng is a different grammar — the kernel compiles the
|
||||
// same vendored C sources instead (codegraph-kernel/grammars/kotlin).
|
||||
// Both carry docs/grammars/tree-sitter-kotlin.patch (scanner: no automatic
|
||||
// semicolon before a same-line `e` word, e.g. an `eq` infix call).
|
||||
'kotlin',
|
||||
// R7b batch 4 (Dart kernel port prep): the byte-copied tree-sitter-wasms
|
||||
// 0.1.13 artifact (sha256 7f5364e4…, built from UserNobody14/
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user