diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f354ff0..58371fc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- C++ classes written around macros and preprocessor conditionals now keep their members. These include an access specifier spelled as a macro (`JSON_PRIVATE_UNLESS_TESTED:`), an attribute macro after a template head or a doc comment, and an `#if` in the middle of a declaration, such as inside a constructor's initializer list. The first branch of a mid-declaration `#if` is the one indexed. nlohmann/json's `basic_json` class used to be lost whole and now has its roughly 390 members. Files with a parse error dropped on redis, curl, pybind11 and leveldb, and no file got worse. Re-index after upgrading. - Kotlin files using newer syntax now keep their classes intact. That covers `when` guard conditions (`is X if cond ->`), `..<` ranges, multi-dollar strings (`$$"…"`) and nullable receivers in function types (`T?.() -> R`). Before, the class around such code could come out as a scatter of loose functions. On Exposed, files with a parse error dropped from 65 to 43, and on kotlinx.coroutines from 50 to 46. Re-index after upgrading. - A call through a class name, like `THorse.Get(…)`, `MyView.as_view()` or `ColorChoices.values()`, now reaches the class method that class inherits. The inheritance is read from the class declarations themselves, in Pascal, Python, Ruby, PHP, TypeScript/JavaScript and the Java family. Before, such a call was resolved by guessing from the name. On Horse, `THorse.Get('/ping', …)` used to land on a route-group interface's `Get` 120 times. Re-index after upgrading. - In Objective-C, `[super init]` and other messages to `super` now reach the superclass's method, never the sending class's own override. This also works where the index lost the `@implementation`'s range. On SDWebImage, `NSObject` subclasses' `[super init]` used to land on `SDDiskCache`'s `init`, and `[self class]` in tests on a proxy class's `class`. On AFNetworking, a serializer's `[super encodeWithCoder:]` now reaches its superclass. Re-index after upgrading. diff --git a/__tests__/cpp-conditional-class.test.ts b/__tests__/cpp-conditional-class.test.ts new file mode 100644 index 00000000..69c690b6 --- /dev/null +++ b/__tests__/cpp-conditional-class.test.ts @@ -0,0 +1,118 @@ +/** + * nlohmann/json's `basic_json` — the class most C++ projects vendor — was lost + * whole: tree-sitter-cpp cannot read + * + * - an access specifier spelled as a macro (`JSON_PRIVATE_UNLESS_TESTED:`), + * - a lone attribute macro after a template head or after a `/// @sa + * https://…/` doc comment (both ends looked like an operator continuing the + * line, so the macro wasn't blanked), + * - a preprocessor conditional in the middle of a declaration: inside a + * member-initializer list, or between a template head and its function. + * + * The macros are rewritten or blanked, and a mid-declaration conditional keeps + * its first branch (directives and later branches blanked, offsets intact). + * An enum member after `A, /* comment *\/` is still an enum member. + */ +import { describe, it, expect, afterAll, beforeAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { blankLoneMacroLines, flattenMidStatementConditionals } from '../src/extraction/languages/c-cpp'; + +const HEADER = `#pragma once +#define JSON_PRIVATE_UNLESS_TESTED private +#define JSON_HEDLEY_RETURNS_NON_NULL +#define JSON_HEDLEY_WARN_UNUSED_RESULT + +template +class basic_json +{ + JSON_PRIVATE_UNLESS_TESTED: + using lexer = int; + + template + JSON_HEDLEY_RETURNS_NON_NULL + static U* create(Args&& ... args) + { + return nullptr; + } + + class nesting_depth_guard + { + public: + nesting_depth_guard() noexcept +#ifdef JSON_NO_THREAD_LOCAL + : m_okay(false) +#else + : m_okay(depth() < 10) +#endif + { + } + private: + bool m_okay; + }; + + public: + /// @brief returns version information on the library + /// @sa https://json.nlohmann.me/api/basic_json/meta/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json meta() + { + return basic_json(); + } + + template < typename ValueType > +#if defined(JSON_HAS_CPP_14) + constexpr +#endif + auto get() const -> ValueType + { + return ValueType(); + } + + void dump() const {} +}; +`; + +let root = ''; +let cg: CodeGraph; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-cpp-conditional-')); + fs.mkdirSync(path.join(root, 'include'), { recursive: true }); + fs.writeFileSync(path.join(root, 'include/json.hpp'), HEADER); + cg = await CodeGraph.init(root, { index: true }); +}); + +afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('a C++ class written around macros and conditionals', () => { + it('keeps its members', () => { + const members = cg.getNodesInFile('include/json.hpp') + .filter((n) => n.qualifiedName.startsWith('basic_json::') && (n.kind === 'method' || n.kind === 'class')) + .map((n) => n.name) + .sort(); + expect(members).toEqual(expect.arrayContaining(['create', 'dump', 'get', 'meta', 'nesting_depth_guard'])); + }); + + it('keeps the first branch of a mid-declaration conditional, offsets intact', () => { + const src = 'A() noexcept\n#ifdef X\n : a(1)\n#else\n : a(2)\n#endif\n{}\n'; + const out = flattenMidStatementConditionals(src); + expect(out.length).toBe(src.length); + expect(out).toContain(' : a(1)'); + expect(out).not.toContain('a(2)'); + expect(out).not.toContain('#'); + // Between whole declarations a conditional stays. + const top = 'int a;\n#ifdef X\nint b;\n#endif\n'; + expect(flattenMidStatementConditionals(top)).toBe(top); + }); + + it('does not blank an enum member after a commented line', () => { + const src = 'typedef enum {\n CURLE_OK,\n CURLE_TOO_LARGE, /* 100 */\n CURL_LAST /* never use! */\n} CURLcode;\n'; + expect(blankLoneMacroLines(src)).toBe(src); + }); +}); diff --git a/src/extraction/languages/c-cpp.ts b/src/extraction/languages/c-cpp.ts index c4aa31a4..ac238978 100644 --- a/src/extraction/languages/c-cpp.ts +++ b/src/extraction/languages/c-cpp.ts @@ -701,7 +701,14 @@ export function blankLoneMacroLines(source: string): string { if (!(m[1] as string).includes('_')) continue; let prev = i - 1; while (prev >= 0 && content(lines[prev] as string) === '') prev--; - if (prev >= 0 && LONE_MACRO_CONTINUATION_END_RE.test(content(lines[prev] as string))) continue; + // Judged on the line's code: a comment — `/// @sa https://…/meta/`, a + // block comment's `*/` — ends in whatever it likes but continues nothing, + // while `CURLE_TOO_LARGE, /* 100 */` continues with its comma. A template + // head (`template`) before a declaration + // continues nothing either. + const prevText = prev >= 0 ? content(lines[prev] as string) : ''; + const prevCode = /^(?:\/\/|\/\*|\*)/.test(prevText) ? '' : prevText.replace(/\/\*.*?\*\/|\/\/.*$/g, '').trim(); + if (prevCode !== '' && !/^template\s* { + const keyword = /PRIVATE/.test(name) ? 'private' : /PROTECTED/.test(name) ? 'protected' + : /PUBLIC|^Q_SIGNALS$|^Q_SLOTS$/.test(name) ? 'public' : null; + if (!keyword || keyword.length > name.length + gap.length) return m; + return `${indent}${keyword}${' '.repeat(name.length + gap.length - keyword.length)}:`; + }); +} + +/** + * A preprocessor conditional in the middle of a declaration or expression — + * inside a constructor's member-initializer list (nlohmann/json's + * `noexcept\n#ifdef JSON_NO_THREAD_LOCAL\n : m_okay(false)\n#else\n : + * m_okay(…)\n#endif`), between a template head and its function + * (`template <…>\n#if defined(JSON_HAS_CPP_14)\n constexpr\n#endif\n auto + * get()`), inside an argument list — is an error to tree-sitter-cpp, and + * json.hpp's recovery took the whole `basic_json` class with it. Where the + * code before the `#if` has not ended a statement or declaration (`;`, `{`, + * `}`, a label's `:`), its directives are blanked and so is every branch + * after the first: one consistent configuration, every newline and offset + * kept. A conditional between whole declarations is left alone. + */ +export function flattenMidStatementConditionals(source: string): string { + if (!source.includes('#')) return source; + const lines = source.split('\n'); + const directive = (i: number) => /^[ \t]*#[ \t]*(\w+)/.exec(lines[i] ?? '')?.[1] ?? null; + const code = (i: number) => (lines[i] ?? '').replace(/\/\/.*$/, '').replace(/\/\*.*?\*\//g, '').trim(); + let changed = false; + const blank = (i: number) => { + lines[i] = (lines[i] ?? '').replace(/[^\r]/g, ' '); + changed = true; + }; + let lastCode = ''; // the last non-directive code line, trimmed + let inBlockComment = false; + let region = 0; // #if depth inside a flattened conditional (0 = none) + let skipFrom = 0; // the depth whose later branches are blanked (0 = none) + let continued = false; // inside a `\`-continued directive (a multi-line #define) + for (let i = 0; i < lines.length; i++) { + if (continued) { + continued = /\\\s*$/.test(lines[i] ?? ''); + continue; + } + const d = directive(i); + if (d !== null && !inBlockComment) { + // A directive continued with `\` spans lines; its body is not code. + if (/\\\s*$/.test(lines[i]!)) { + continued = true; + if (region === 0) continue; + } + const opens = /^if/.test(d); + if (region === 0) { + // Not after a statement macro written without its `;` + // (`RAPIDJSON_DIAG_OFF(padded)`), either. + const mid = opens && lastCode !== '' && !/[;{}]$|^[\w\s]*:$|\*\/$/.test(lastCode) && + !/^(?:public|private|protected)\s*:$/.test(lastCode) && !/^[A-Z][A-Z0-9_]*(?:\s*\(.*\))?$/.test(lastCode); + if (!mid) continue; + region = 1; + blank(i); + continue; + } + if (opens) region++; + else if ((d === 'else' || d === 'elif') && skipFrom === 0) skipFrom = region; + else if (d === 'endif') { + if (skipFrom === region) skipFrom = 0; + region--; + } + if (/^(?:if|ifdef|ifndef|else|elif|elifdef|elifndef|endif)$/.test(d)) blank(i); + continue; + } + const text = lines[i] ?? ''; + if (region > 0 && skipFrom > 0) { + if (text.trim() !== '') blank(i); + continue; + } + // Track block comments so a `#` or `;` inside one is not read as code. + const trimmed = code(i); + if (inBlockComment) { + if (text.includes('*/')) inBlockComment = false; + continue; + } + if (/\/\*(?!.*\*\/)/.test(text.replace(/\/\/.*$/, ''))) { + inBlockComment = true; + const before = trimmed.split('/*')[0]!.trim(); + if (before !== '') lastCode = before; + continue; + } + if (trimmed !== '') lastCode = trimmed; + } + return changed ? lines.join('\n') : source; +} + function preParseCppSource(source: string, filePath?: string): string { const rawStrings = maskCppRawStrings(source); - source = rawStrings.source; + source = flattenMidStatementConditionals(rewriteCppAccessMacros(rawStrings.source)); // blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked // `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape // it matches (the _API token is already spaces by then).