diff --git a/CHANGELOG.md b/CHANGELOG.md index b12da8fd8..d5001316b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -182,6 +182,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In C and C++, a struct, class, union or enum defined in a variable declaration, like `struct Foo { … } foo;`, is now indexed with its methods and enum values, next to the variable. Before, at most the variable was, so the type was missing from search and `codegraph_explore`, a class deriving from it had no base, and calls to its methods linked to nothing: in redis, the Sentinel's `sentinelState` and `redis-cli`'s `config` struct were missing. A struct, union or enum with no name of its own now takes the name of the variable it declares, so redis's `static struct { … } SPT;` is found as `SPT`, and one declared inside a function is named after its variable instead of ``. The comment above the declaration becomes the type's description. Re-index C and C++ projects after upgrading. - In Objective-C, a struct, union or enum defined in a variable declaration, like `static struct { Class isa; } placeholder;` or `enum { stateType, stateParamName } state;`, is now indexed with its enum values, next to the variable, as in C and C++. Before, only the variable was, so the type was missing from search and `codegraph_explore`. The same went for a `typedef enum { … } Name;` or `typedef struct … Name;` written after a line holding only a macro, like `OF_ASSUME_NONNULL_BEGIN` or `RCT_EXTERN_C_BEGIN`: ObjFW's `OFZIPArchiveEntryCompressionMethod` and every value in it were missing, and so was React Native's `RCTMethodInfo`. A struct, union or enum with no name of its own takes the name of the variable it declares, so ObjFW's class-cluster placeholders are found as `placeholder`, one declared inside a function or method is named after its variable instead of ``, and the comment above the declaration becomes the type's description. As in C, a type that a `.m` file defines links only from that file, so a call elsewhere that just shares its name, like ObjFW's calls of AmigaOS's `Lock(path, mode)`, doesn't link to the runtime's private `struct Lock`. Re-index Objective-C projects after upgrading. - In Objective-C, a macro written alone on its line, like `NS_ASSUME_NONNULL_BEGIN`, `OF_SUBCLASSING_RESTRICTED` or React Native's `RCT_EXTERN_C_BEGIN`, no longer hides the declaration below it. Before, the parser read such a macro as the start of a declaration, so an `@interface` under it could come out as a function and a `typedef` under it as two variables: many of ObjFW's classes, with their properties and methods and the classes they extend and protocols they adopt, were missing, and so were classes and protocols in React Native and KSCrash headers. A message to `super` from a class whose `@interface` follows an attribute macro like `NS_SWIFT_NAME(…)` still reaches the superclass. Re-index Objective-C projects after upgrading. +- In Objective-C, and in the C and C++ headers that sit beside it, an enum declared with Foundation's macros, like `typedef NS_ENUM(NSInteger, SDImageCacheType) { … };`, `NS_OPTIONS`, `NS_CLOSED_ENUM`, `NS_ERROR_ENUM` or CoreFoundation's `CF_ENUM` and `CF_OPTIONS`, is now indexed with its values. Before, the parser read the macro as a broken typedef, so these enums were missing (SDWebImage had none at all) and a type alias named after the first value stood in their place. A Swift annotation naming such an enum, like KSCrash's `let unwindMethods: KSBacktraceUnwindMethods`, now links to it. Re-index Objective-C, C and C++ projects after upgrading. - In TypeScript React code, a styled-components or emotion component declared with a type argument, like `const Wrapper = styled.div` followed by its CSS, or `styled(Base)`, is now indexed as a component, as one written without a type argument already was. Before, it was indexed as a plain constant, so `codegraph callers` and impact found none of the code that renders it, and its tags could link to another file's component of the same name instead: in outline/outline, `NudeButton` and `HStack` had no callers, and `` tags linked to a different `Text` than the one each file imports. Re-index TypeScript React projects after upgrading. - `codegraph sync` and the file watcher now move a React Router route to its page when the router file loads that page lazily into a value of its own, like `const Docs = lazy(() => import('./pages/Docs'))` rendered by `} />`, and the page's file is added after the router was indexed, comes back after being deleted, or gets or changes its default export in a later edit. Before, the route kept pointing at the `Docs` declaration, or at the page's old component, until the router file itself changed or the project was re-indexed, unless the page's component happened to share the declaration's name. The same now holds for a `lazy: () => import('./pages/Docs')` route whose page switches its default export to another component. Re-index React projects after upgrading to correct routes an earlier version left behind. - `codegraph sync` and the file watcher now link a component to the JSX that renders it, like ``, when the component is added in a file with no JSX of its own, as in `export default function Team() { return null; }`. Before, the link waited for an unrelated change to a file with JSX, or for a re-index, so the new component looked unused. diff --git a/__tests__/foundation-enum-macros.test.ts b/__tests__/foundation-enum-macros.test.ts new file mode 100644 index 000000000..f6412dca7 --- /dev/null +++ b/__tests__/foundation-enum-macros.test.ts @@ -0,0 +1,174 @@ +/** + * Foundation's enum macros declare most enums in Objective-C code and in the + * C and C++ headers that live beside it: + * + * typedef NS_ENUM(NSInteger, SDImageCacheType) { SDImageCacheTypeNone, … }; + * typedef NS_OPTIONS(NSUInteger, SDWebImageOptions) { … }; + * typedef CF_ENUM(CFIndex, KSCrashMonitorType) { … }; + * + * The parser reads the macro as a broken typedef, so the enum and its values + * were never indexed (SDWebImage had no enum at all), and a junk type alias + * named after the first value took its place. Objective-C, C and C++ now + * rewrite the macro as `enum SDImageCacheType {` before parsing, at the same + * offsets. Runs the C and C++ cases against the native kernel (when built) + * and the wasm extractor, which must agree. + */ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel'; +import { rewriteFoundationEnumMacros } from '../src/extraction/languages/c-cpp'; +import type { ExtractionResult, Language } from '../src/types'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelAvailable = fs.existsSync(KERNEL_PATH) || process.env.CODEGRAPH_KERNEL_EXPECT === '1'; + +const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const; + +/** ` ` for every symbol, file and imports aside. */ +function symbols(result: ExtractionResult): string[] { + return result.nodes + .filter((n) => n.kind !== 'file' && n.kind !== 'import') + .map((n) => `${n.kind} ${n.qualifiedName}`); +} + +describe('Foundation enum macros', () => { + let savedEnv: Record = {}; + + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['objc', 'c', 'cpp']); + }); + + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + resetKernelForTests(); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + resetKernelForTests(); + }); + + function extract(backend: 'kernel' | 'wasm', file: string, source: string, language: Language): ExtractionResult { + if (backend === 'wasm') { + process.env.CODEGRAPH_KERNEL = '0'; + return extractFromSource(file, source, language); + } + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + const result = tryKernelExtract(file, source, language); + expect(result, `kernel extraction of ${file}`).not.toBeNull(); + return result!; + } + + const backends = kernelAvailable ? (['kernel', 'wasm'] as const) : (['wasm'] as const); + + for (const crlf of [false, true]) { + const eol = (s: string) => (crlf ? s.replace(/\n/g, '\r\n') : s); + const label = crlf ? ' (CRLF)' : ''; + + it(`declare enums and their values in Objective-C${label}`, () => { + const source = [ + '#import ', + '', + '/// Where an image came from.', + 'typedef NS_ENUM(NSInteger, SDImageCacheType) {', + ' SDImageCacheTypeNone,', + ' SDImageCacheTypeDisk,', + '};', + '', + 'typedef NS_OPTIONS(NSUInteger, SDWebImageOptions) {', + ' SDWebImageRetryFailed = 1 << 0,', + '};', + '', + 'typedef NS_ERROR_ENUM(SDWebImageErrorDomain, SDWebImageError) {', + ' SDWebImageErrorInvalidURL = 1000,', + '};', + '', + 'typedef NS_ENUM(NSInteger, SDForwardDeclared);', + '', + '@interface SDImageCache : NSObject', + '@end', + '', + ].join('\n'); + const result = extractFromSource('src/SDImageCacheDefine.h', eol(source), 'objc'); + expect(symbols(result)).toEqual([ + 'enum SDImageCacheType', + 'enum_member SDImageCacheType::SDImageCacheTypeNone', + 'enum_member SDImageCacheType::SDImageCacheTypeDisk', + 'enum SDWebImageOptions', + 'enum_member SDWebImageOptions::SDWebImageRetryFailed', + 'enum SDWebImageError', + 'enum_member SDWebImageError::SDWebImageErrorInvalidURL', + 'class SDImageCache', + ]); + expect(result.nodes.find((n) => n.name === 'SDImageCacheType')?.docstring).toBe('Where an image came from.'); + }); + + it.each(backends)(`declare enums in C and C++ headers: %s${label}`, (backend) => { + const c = [ + '#include ', + '', + 'typedef CF_ENUM(CFIndex, KSCrashMonitorType) {', + ' KSCrashMonitorTypeNone = 0,', + ' KSCrashMonitorTypeSignal = 1,', + '} CF_SWIFT_NAME(CrashMonitorType);', + '', + ].join('\n'); + expect(symbols(extract(backend, 'src/KSCrashMonitorType.h', eol(c), 'c'))).toEqual([ + 'enum KSCrashMonitorType', + 'enum_member KSCrashMonitorType::KSCrashMonitorTypeNone', + 'enum_member KSCrashMonitorType::KSCrashMonitorTypeSignal', + ]); + const cpp = [ + '#include ', + '', + 'namespace facebook { class Report {}; }', + '', + 'typedef NS_ENUM(NSInteger, RCTSchedulerMode) {', + ' RCTSchedulerModeSync,', + '};', + '', + ].join('\n'); + expect(symbols(extract(backend, 'src/RCTScheduler.h', eol(cpp), 'cpp'))).toEqual([ + 'class facebook::Report', + 'enum RCTSchedulerMode', + 'enum_member RCTSchedulerMode::RCTSchedulerModeSync', + ]); + }); + + it(`are rewritten in place, and the macro's own #define is left alone${label}`, () => { + const source = eol([ + '#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type', + '// typedef NS_ENUM(NSInteger, InAComment) {', + 'typedef NS_ENUM(NSInteger, SDImageCacheType) {', + ' SDImageCacheTypeNone,', + '};', + '', + ].join('\n')); + const rewritten = rewriteFoundationEnumMacros(source); + expect(rewritten.length).toBe(source.length); + expect(rewritten.split(/\r?\n/)).toEqual([ + '#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type', + '// typedef NS_ENUM(NSInteger, InAComment) {', + ' enum SDImageCacheType {', + ' SDImageCacheTypeNone,', + '};', + '', + ]); + }); + } +}); diff --git a/src/extraction/languages/c-cpp.ts b/src/extraction/languages/c-cpp.ts index 377888914..8e75f9454 100644 --- a/src/extraction/languages/c-cpp.ts +++ b/src/extraction/languages/c-cpp.ts @@ -1638,9 +1638,51 @@ export function flattenMidStatementConditionals(source: string): string { return changed ? lines.join('\n') : source; } +// Foundation's enum macros and their CoreFoundation forms, with an optional +// `typedef` before them: `typedef NS_ENUM(NSInteger, SDImageCacheType)`. +const FOUNDATION_ENUM_MACRO_RE = + /\b(?:(typedef)(\s+))?(NS_ENUM|NS_OPTIONS|NS_CLOSED_ENUM|NS_ERROR_ENUM|CF_ENUM|CF_OPTIONS|CF_CLOSED_ENUM)(\s*)\(([^(),]*),(\s*)([A-Za-z_]\w*)(\s*)\)/g; + +/** + * Rewrite a Foundation enum macro as the enum it declares, in place: + * `typedef NS_ENUM(NSInteger, SDImageCacheType) {` becomes + * ` enum SDImageCacheType {`. The parser reads the + * macro as a broken typedef, so the enum and every value in it were missing + * (SDWebImage had no enum at all). The `typedef`, the rest of the macro's + * name, its parentheses and its first argument (the underlying type, or an + * NS_ERROR_ENUM's domain) become spaces, so every offset stays. Matched on + * the code with comments, literals and directives masked: the `#define` of + * the macro itself is left alone. Shared by Objective-C, C and C++, whose + * headers all use the macros. + */ +export function rewriteFoundationEnumMacros(source: string): string { + if (!/\b(?:NS|CF)_[A-Z_]*(?:ENUM|OPTIONS)\s*\(/.test(source)) return source; + let out: string[] | null = null; + for (const m of maskCppCode(source).matchAll(FOUNDATION_ENUM_MACRO_RE)) { + out ??= source.split(''); + const blank = (from: number, to: number): void => { + for (let k = from; k < to; k++) if (out![k] !== '\n' && out![k] !== '\r') out![k] = ' '; + }; + let at = m.index; + if (m[1]) { + blank(at, at + m[1].length); + at += m[1].length + (m[2] as string).length; + } + const macro = m[3] as string; + out.splice(at, 4, 'e', 'n', 'u', 'm'); + blank(at + 4, at + macro.length); + at += macro.length + (m[4] as string).length; + // `(`, the first argument and the `,`. + blank(at, at + 1 + (m[5] as string).length + 1); + at += 1 + (m[5] as string).length + 1 + (m[6] as string).length + (m[7] as string).length + (m[8] as string).length; + blank(at, at + 1); // `)` + } + return out ? out.join('') : source; +} + function preParseCppSource(source: string, filePath?: string): string { const rawStrings = maskCppRawStrings(source); - source = flattenMidStatementConditionals(rewriteCppAccessMacros(rawStrings.source)); + source = flattenMidStatementConditionals(rewriteCppAccessMacros(rewriteFoundationEnumMacros(rawStrings.source))); // blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked // `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape // it matches (the _API token is already spaces by then). The C++ leading @@ -2430,7 +2472,7 @@ export function blankCDesignatedMacroArgs(source: string): string { function preParseCSource(source: string): string { const rawStrings = maskCppRawStrings(source); - source = rawStrings.source; + source = rewriteFoundationEnumMacros(rawStrings.source); const inner = blankCDesignatedMacroArgs(blankCKernelAnnotations(blankCCplusplusGuardBodies(source))); let blanked = blankCLeadingAttrMacros( blankLoneMacroLines( diff --git a/src/extraction/languages/objc.ts b/src/extraction/languages/objc.ts index b8a94489c..88fdc9292 100644 --- a/src/extraction/languages/objc.ts +++ b/src/extraction/languages/objc.ts @@ -1,7 +1,13 @@ import type { Node as SyntaxNode } from 'web-tree-sitter'; import { getChildByField, getNodeText } from '../tree-sitter-helpers'; import type { ExtractorContext, LanguageExtractor } from '../tree-sitter-types'; -import { blankLoneMacroLines, definingDeclaration, OBJC_LONE_MACROS, unnamedTypeVariableName } from './c-cpp'; +import { + blankLoneMacroLines, + definingDeclaration, + OBJC_LONE_MACROS, + rewriteFoundationEnumMacros, + unnamedTypeVariableName, +} from './c-cpp'; function findCompoundStatement(node: SyntaxNode): SyntaxNode | null { for (let i = 0; i < node.namedChildCount; i++) { @@ -100,8 +106,9 @@ export const objcExtractor: LanguageExtractor = { // A macro alone on its line, like `NS_ASSUME_NONNULL_BEGIN` or // `RCT_EXTERN_C_BEGIN`, reads to the parser as the start of a declaration: // it swallowed the `typedef` below it, and an `@interface` below it lost - // its class to error recovery. Blanked as in C, offsets kept. - preParse: (source) => blankLoneMacroLines(source, OBJC_LONE_MACROS), + // its class to error recovery. Blanked as in C, offsets kept. A + // `typedef NS_ENUM(NSInteger, Name) {` is rewritten as `enum Name {`. + preParse: (source) => blankLoneMacroLines(rewriteFoundationEnumMacros(source), OBJC_LONE_MACROS), functionTypes: ['function_definition'], // Only @interface emits a class node; @implementation reuses it via visitNode. classTypes: ['class_interface'],