diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a204592725..1ae94dde61 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: # Native extraction-kernel prebuilds (docs/design/rust-kernel-migration-plan.md). # The kernel is the ONLY parser (no wasm fallback since the kernel-only # extraction plan's Phase 5), so this matrix is REQUIRED: a failed kernel - # build blocks the release, and build-bundle.sh refuses to stage a bundle + # build blocks the release, and the bundle recipe refuses to stage a bundle # without a prebuild. (Formerly: per-file wasm fallback for erroring files and unsupported # platforms — but every published bundle must carry its .node. Note the # vendored-grammar-C languages (kotlin/lua/scala/dart) compile parser.c via diff --git a/BUNDLING.md b/BUNDLING.md index 1960e11da6..4688259af1 100644 --- a/BUNDLING.md +++ b/BUNDLING.md @@ -33,6 +33,10 @@ scripts/build-bundle.sh linux-x64 # -> release/codegraph-linux-x64.ta scripts/build-bundle.sh win32-x64 # -> release/codegraph-win32-x64.zip ``` +The Node runtime is the official nodejs.org release named by the script's +default `NODE_VERSION` (or its second argument). The download is refused +unless it matches the SHA-256 listed in that release's `SHASUMS256.txt`. + Because dropping better-sqlite3 left **zero native addons**, building a bundle is pure file-packaging — **any** target builds on **any** OS (the whole matrix builds on one Linux runner). Cross-compilation isn't a concern; only *run-testing* a diff --git a/CHANGELOG.md b/CHANGELOG.md index faded19993..8fe7c213b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -386,10 +386,45 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In VB.NET, every member of a `Structure` is now indexed, including its fields, properties, methods, constructors and nested enums. Before, only the first member was, so the rest could not be found and their callers looked empty. +- In TypeScript, an awaited call with type arguments like `await fetchApi("/me")` or `await api.get(url)` now links to the function it calls. Before, the call was dropped, so typed fetch wrappers looked like they had no callers. (#2586) +- In JavaScript and TypeScript, a call or `new` in a parameter's default value, like `function renderWithNote(note = buildNote())`, a destructured `{ size = defaultSize() }` or `constructor(private svc = new Service())`, now counts as the function's own call. Before, it was dropped, so `buildNote` was missing that caller and impact on it came back incomplete. Re-index JavaScript and TypeScript projects after upgrading. Thanks @inth3shadows for the report and the pull request. (#2588) +- In React, a component rendered at module scope, in a top-level constant or inside a test's `it(…)` callback, like `createRoot(el).render()` or `render()`, now counts as rendered there, so an app's entry reaches its root component and a component's tests show among its callers. (#2587) +- In JavaScript and TypeScript, react-dom's `root.render()` no longer links to your own class component's `render()`, such as an error boundary's, when `root` is declared outside the function making the call, like a module's `let root: Root | null`, or holds what `createRoot(el)` returned. A call on a variable declared outside the calling function now goes by the type that declaration gives it, so one declared with your own class, like `const store = new Store()` at the top of a file, links to that class's method, and a class component's `render()` and lifecycle methods, which only React calls, are no longer a guess for an untyped value. Re-index JavaScript and TypeScript projects after upgrading. Thanks @inth3shadows for the report. (#2589) +- In JavaScript and TypeScript, a method called on a freshly constructed object, like `new RegExp(p).exec(s)`, `new URL(u).toString()` or `new Runner().run()`, now links only to that class's own or inherited method, on the class the file imports when several share its name, and no longer to an unrelated project method of the same name; a class from outside your project links nothing. Re-index JavaScript and TypeScript projects after upgrading. Thanks @danusha2345. (#1945) +- In Python, `import a.b as X` written inside a function now links its calls like `X.f()`, as the same import at module level already did. Thanks @Egrn for the report. (#2600) +- In Python, `from app import email` no longer links the importing file to a project variable named `app`, such as a FastAPI or Flask `app = FastAPI()`. (#2584) +- In Python, a from-import of a module whose name has no dot, like `from loader import Registry` with `loader.py` at the repository root, or `from mypackage import helper` in a `src/` layout, now links the importing file to that module's file, as `import loader` and `from pkg.loader import Registry` already did. Before, the file's dependency on the module itself was left out, so `codegraph affected` and impact could miss the files that import it. When the file also imported something else under the module's name, like `from django.core import mail` beside `from mail import …`, the module was linked to that instead. Re-index Python projects after upgrading. +- In Python, a call through a module-level instance, like `_svc.delete(obj)` after `_svc = SceneService()`, now links to the method its class has or inherits, even when other classes have a method of that name. (#2582) +- In Python, a call of your own class's method named like a dictionary, list or file method, such as `get`, `update`, `items`, `pop` or `close`, now links to that method when the function says what the variable holds: `reg = SystemPackRegistry()` then `reg.get(…)`, `registry = load_registry()` with `def load_registry() -> SystemPackRegistry`, or a parameter written `reg: SystemPackRegistry`. Before, these calls were dropped as built-in methods unless the variable was named after the class. A variable the function also sets to something else, like `{}` or what `json.load` returns, a class imported from another package, like `queue.Queue`, and a union type still link nothing. Re-index Python projects after upgrading. Thanks @inth3shadows for the report. (#2583) +- In Python, a call on a module imported from outside your project, like `subprocess.run(…)` after `import subprocess`, no longer links to a project method that happens to share its name. Thanks @Egrn for the report. (#2599) +- `codegraph_explore`, the CLI `codegraph explore` and the prompt hook no longer stall for a minute or more on a large project after a full index; the step that finds the project's busiest file no longer slows to a crawl when a few files are huge, like a generated Android `R.java` (#2571). Thanks @yangfengfan2011. +- `codegraph_explore`, the CLI `codegraph explore` and the Claude Code prompt hook now answer a long query, like a pasted log, document or source file, in seconds on a large project instead of minutes. Every word of such a query used to search the whole index several times over, so the prompt hook ran out of time on long prompts and added nothing. Those searches now reuse what they read, and a very long query is searched by its code names and its opening words rather than by every word in it. Answers to ordinary questions and lists of symbol names are unchanged. Thanks @retsamekohC for the report and the first fix. (#2620) +- The Claude Code prompt hook now goes by what you typed. Claude Code's desktop app puts status notes, like which other sessions are linked, in front of your prompt, and the hook read them as part of it, so every turn, even a one-word "yes", could get code context about the note's words, and so could a task notification, a subagent's report or a message from another session that came after one. The notes are now skipped, and a message from another session gets no context, as task notifications and subagent reports already didn't. Thanks @WonderMr for the report and @yang0228 for the first fix. (#2635) +- `codegraph_explore` and `codegraph context` now find your project's busiest file, and favor the code around it, even when twenty or more test or generated files are busier still, such as a package of protobuf stubs. Before, the search looked only at the 20 busiest files and then set the tests and generated ones aside, so it could find no file at all, or judge one far ahead of the next when it was not. `getTopRouteFile()` in the library counts every file's routes the same way. Thanks @inth3shadows for the report. (#2578) +- In JavaScript and TypeScript, a call written as the body of an `if`, `while`, `for` or `do` without braces, like `if (e.key === "Escape") setOpen(false);`, no longer links to a same-named property or method in another file when the name is the file's own, such as a `setOpen` from `useState` (#2585). +- In JavaScript and TypeScript, a method called on an element of a class field, like `this.handlers[type].handle()` or `this.items[0].save()`, now links to the method of the type the field is declared to hold, such as `Item[]` or `Record`, instead of to any project method that shares its name, often another class's, and is left unlinked when that type isn't written down, as in plain JavaScript. Re-index JavaScript and TypeScript projects after upgrading. Thanks @inth3shadows. (#2766) +- `codegraph sync` no longer fails with "FOREIGN KEY constraint failed" in a project whose code lazy-loads a workspace package, like Supabase Studio's `dynamic(() => import('icons').then((mod) => mod.EdgeFunctions))` or vben's `() => import('@vben/layouts').then((m) => m.IFrameView)`. Before, every sync after an edit in such a project rolled back, so the index stopped following your changes until a full re-index. Thanks @bompus for the report and @inth3shadows. (#2565) +- In VB.NET, every member of a `Structure` is now indexed, including its fields, properties, methods, constructors and nested enums. Before, only the first member was, so the rest could not be found and their callers looked empty. +- In Delphi, a routine with an attribute on one of its parameters, like `const [ref] AValue: Integer` or DelphiMVCFramework's `const [MVCFromBody] Person: TPerson`, is now indexed with its calls, and so is every class member declared after it. Before, the routine was lost or its calls were credited to the file, the parameter showed up as a field of the class, and the members after it were missing. Thanks @stalexteam for the report. (#2545) +- In Delphi, a `for … in` loop that declares its variable inline, like `for var lItem in AList do` or `for var lHandle: THandle in lHandles do`, now has its calls indexed, and a call on a typed loop variable reaches that type's method. Before, the whole loop was skipped, so a routine calling a method only from such a loop was missing from that method's callers. Thanks @stalexteam for the report. (#2546) +- In Delphi, a call made on the result of another call, like `MD5FileSized(TMeta.ExtendedPath(APath)).IsSame(AKey)` or `List[IndexOf(Name)].Free`, now has the inner calls indexed too, so `MD5FileSized` and `TMeta.ExtendedPath` list that routine among their callers. A typecast like `TButton(Sender).Caption` or `F := TFoo(Obj)` now counts as a use of the type instead of creating one. Thanks @stalexteam for the report. (#2547) +- In Delphi, a routine called without parentheses, the usual way to call one that takes no arguments, like `Before;`, `if Ready then`, `Result := FProvider.Value + 1` or `QueryTabs.ActiveTab.MemoFilename`, now counts as a call, so it shows up in callers, callees and impact, and a method given to an event, like `OnClick := ButtonClick`, is linked as a reference. A name is linked only to a method or function its code reaches, in its class and the classes it inherits from (`class sealed (TBase)` included), its unit, or the units it uses, never to a field, property or variable of that name. Thanks @OleksiyPenkov for the report, and @drakeo338 and @OleksiyPenkov for their pull requests. (#2514) +- In Delphi, a call to something the compiler provides, like `SizeOf(X)`, `TypeInfo(TFoo)`, `High(A)`, `Inc(N)`, `Assert(…)` or a typecast like `Pointer(P)`, written in any letter case, no longer shows up as a call to a project method, field or variable that happens to share its name, unless the calling code's own class or units declare one, like a list class's own `Delete`. A call through a field, property or variable now counts only when it holds a routine, like an `OnChange` event or a `reference to function` variable, and calls in `{$I}` include files and into units the parser could only partly read now find the routines those units declare. Thanks @OleksiyPenkov for the report. (#2514) +- In Delphi, a routine declared inside another, between its header and its `begin`, is now indexed as a routine of its own under the one it belongs to (`TFoo.Bar`'s `Inner` is `TFoo::Bar::Inner`), with the calls its body makes. Before, its body was skipped, so what it called was missing from callers and impact, and a call to it from its own routine went to any routine of the same name elsewhere. Thanks @OleksiyPenkov for the pull request this follows. (#2550) +- In Delphi, a unit that re-publishes an inherited property, like `property OwnsObjects;` or `property Cursor default crIBeam;`, is now indexed whole. So is one with a conditional right after a routine's header (`{$IFDEF HASINLINE}inline;{$ENDIF}`), an assembler and a Pascal version of one routine, a bare `raise`, a field or variable of an unnamed record type (`fIn: record … end`), a label just before `end`, or two versions of a statement in one `case` branch. Before, the parse broke at such a line, so the routines and class members after it went missing or their calls were credited to the file, which cost SynEdit's main unit most of its code. +- In Delphi, a constant read in a routine's body is no longer credited to the last routine nested inside it. - In VB.NET and C#, what a property's `Get` and `Set` code calls, creates and reads now belongs to that property, as do C#'s `get => …` accessors and `=> …` property bodies. Before, it was dropped, so a method used only from a property looked unused. - In VB.NET, a field or property initializer like `= Compute()` or `As New List(Of Order)` now links what it calls and creates, and so do a `Custom Event`'s `AddHandler`, `RemoveHandler` and `RaiseEvent` blocks. Re-index VB.NET and C# projects after upgrading. +- In C#, a field or property initializer like `private readonly ILogger _log = LogManager.GetLogger(typeof(X));` or `public List Items { get; } = new();` now links what it calls, creates and reads, including inside a lambda, and a target-typed `new()` there counts as creating the declared type. Before, initializers were skipped, so a method or class used only from one — like a converter created in a static list — looked unused, and a method passed as a value there was credited to the whole class instead of the field or property. +- In C# and VB.NET, a constant no longer stands in for a type with the same name: `new Station { … }` links to class `Station` even when another class declares `private const string Station`, `new Version(…)` no longer links to a `const string Version`, and a field like `static readonly Meter Meter` no longer points at itself. Thanks @EvanYu1980 for the report and @drakeo338. (#2337) +- In CFML, `createObject("component", "contenscms.core.queue.batchcontext")`, and the one-argument `createObject("contenscms.core.queue.batchcontext")`, now link to the component they create, found by its path the way `extends` finds one, as `new` already did. A call chained onto the new object, like `createObject(…).init()` or `new contenscms.core.queue.batchcontext().init()`, now links to that component's `init`, or to the one it inherits. In a framework's own source, a path like `wheels.JobWorker`, whose first part is the framework itself, now also finds the component at the project root, for `extends` as well. Before, `createObject` showed up as a call to a function that doesn't exist, and the chained `init` was linked to whichever component's `init` came first, or to none. Thanks @HarryMuc for the report. (#2652) +- In VB.NET, a call on a variable, parameter, field or property now reaches a method of the type it is declared with — one that type inherits, or an extension method written for it — and nothing when that type comes from outside your project, instead of any project method that merely shares the name. +- In VB.NET, a call to one of .NET's own methods such as `Add`, `Contains`, `Clear` or `Dispose` on a value whose type isn't known is no longer linked to a project method of that name, unless the value is named after that method's class. +- In VB.NET, a type name is looked up the way VB.NET does it — through the namespaces around it, the file's and project's `Imports` (aliases included), and the caller's own project — so a class declared in several namespaces or projects no longer draws every call to whichever copy was indexed first, and two candidates nothing tells apart get no link at all. +- In VB.NET, a call with no receiver, or on `Me`, now reaches only a member of the class it is written in, of a class it inherits, or of a Module, and no longer a nearby class's member of the same name. +- VB.NET designer code such as `New System.Drawing.Point(…)` or `Me.Size = New System.Drawing.Size(…)` is no longer linked to a project type or member that only shares the name, and a class nested inside another is matched by its bare name only from inside that class or one that inherits it. - Upgrading CodeGraph while an agent session is open no longer leaves the old version's background server in charge of your project: the first session started from the new install stops it and starts a current one in its place, even while sessions opened before the upgrade are still running. That old server could no longer load the language parsers the upgrade removed, so it saved every file it re-indexed with no symbols, while new sessions could only read the index beside it without keeping it up to date. A background server from a newer install is never stopped, and sessions opened before the upgrade keep the old version until you restart them. Thanks @lipchey for the report. (#2335) - A file is no longer saved with no symbols when its language parser can't be loaded, which is what happened to every file a background server re-indexed after an upgrade removed its install: the file keeps what it had and is indexed again once the parser loads, and files an earlier version emptied this way are re-indexed by the next sync. A background server also exits on its own once its install is upgraded or removed, so the next session starts one from the current install. Thanks @lipchey for the report. (#2335) @@ -412,10 +447,29 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Python global-write inference validates the local binding before scanning other files. The native resolver retains its bounded source and name caches. +- `codegraph index` no longer stops with advice to run `codegraph daemon stop`, a command that doesn't exist, when the project's background-server lock names a process that isn't CodeGraph, such as a lock left by a crash whose process number the system has since given to another program: it now clears that stale lock, leaving the process alone, and rebuilds, and `codegraph unlock` clears it too. When a CodeGraph process that doesn't answer holds the lock, such as an older background server, the message names that process and the command that stops it, and the message about another server already serving a project now points to `codegraph daemon`. Thanks @inth3shadows for the report and @danusha2345 for the first fix. (#2770) +- A program that uses CodeGraph as a library and indexes chosen files with `indexFiles()` now keeps other CodeGraph processes, like a `codegraph sync` run by a git hook, out of the index until those files are stored. Before, it let go of the write lock that keeps two writers apart as soon as indexing started, so another process could write to the index at the same time. +- The shared background server is no longer killed with "Main thread unresponsive" while it brings another project's index up to date, such as one a session names with `projectPath`: only its own project's index counted as progress, so a long update of any other looked like a hang, and the server was stopped and restarted again and again. Every index the server keeps up to date now counts, and so does the one a session updates on its own when the background server is unavailable. Thanks @mradwankhalil for the report and the first fix. (#2404) +- A project a session queries through `projectPath` is now let go once it has gone its own 10 minutes without a query. Before, when its index was still being brought up to date as the session went on to query another project, it was held, locked, until that other project's 10 minutes were up, so its own session and `codegraph index` could wait nearly twice as long to take over. Thanks @bompus. (#2087) +- An index left incomplete by an indexing run that was stopped partway, by a crash, an out-of-memory kill or the server's own hang check, no longer gets the background server killed with "Main thread unresponsive" every time it starts: on a large project, restoring the database indexes the run had set aside took longer than the hang check allows, so each restart died at the same point and the index was never repaired. They are now restored in the background, and the project is served once they are back. Thanks @cbeaulieu-gt for the report and the first fix. (#1887) +- A new agent session no longer waits up to about 12 minutes for its first answer when the background server is stuck: a server whose process still holds its connection open but never responds made the session wait out a long series of retries. The session now gives up within seconds and answers on its own, and switches back to the background server once that has been restarted. +- The bundled Node runtime moves to 24.22.1, Node's October security release, and every release now checks the Node it bundles against the checksums nodejs.org publishes. The new runtime also fixes a crash on Windows as a command like `codegraph sync` exits, after its work is done, printing `Assertion failed: !(handle->flags & UV_HANDLE_CLOSING)`, which made a script or CI run that had succeeded fail. Thanks @Ancienttwo for the report and the first fix, and @IdealBoy709 for the report on the security release. (#2617, #2780) +- Building CodeGraph from source with bun, as in `bun run build`, no longer runs until it is stopped: bun ran one step of the build as the whole build again, so it kept starting itself over. Building with npm is unchanged. Thanks @bompus. +- When CodeGraph can't open a project's database, it now says what went wrong: SQLite's error, the file's path, and what to check, such as another program holding the file open, a folder you can't write to, or a network share. Before, every such failure read "CodeGraph requires node:sqlite (Node.js 22.5+)", and an agent was told no project was loaded. That hint now appears only when Node really can't load its built-in SQLite, a case that used to end in an unhandled error instead. +- `codegraph upgrade` no longer wipes your agent instructions file, such as a global `AGENTS.md`, `CLAUDE.md` or `GEMINI.md`, when it refreshes CodeGraph's section in it: a stray `` line above that section, like one left behind when part of an older section was deleted by hand, made the refresh replace everything in between, often the whole file. CodeGraph now rewrites only its own section, in `codegraph install` and `codegraph uninstall` too, and keeps everything around it exactly as it was, Windows line endings included. Thanks @pzaj2 for the report. (#2506) +- `codegraph install`, `upgrade` and `uninstall` now write through an instructions or config file that is a symbolic link, such as a `~/.codex/AGENTS.md` or `~/.claude.json` linked from your dotfiles, instead of replacing the link with a copy: the shared file gets CodeGraph's settings, uninstall takes them out again, and the link stays in place. A link CodeGraph can't write through, like one to a read-only file or into a folder that doesn't exist, is now reported by name and left alone, and an upgrade still refreshes your other agents. Thanks @jguerra6 for the report and @danusha2345 for the first fix. (#2721) +- `codegraph install` and `upgrade` no longer break Codex's `config.toml` when CodeGraph's `[mcp_servers.codegraph]` entry is written differently from how CodeGraph writes it, such as indented, with spaces or quotes inside the brackets, or with Windows line endings: the entry is now updated in place, where before a second copy was added and Codex could load none of the file. A file already broken that way is repaired on the next install or upgrade, an example of the entry inside a multiline setting such as `developer_instructions` is no longer mistaken for it, uninstall removes the entry however it is written, and an entry you wrote with dotted keys or as an inline table is left as you wrote it. When your `mcp_servers` settings are one inline table, CodeGraph now says it can't add itself instead of breaking the file. Thanks @dencoseca for the report, and @danusha2345 and @rudycelekli for the first fixes. (#2562) +- For Hermes Agent, `codegraph install` and `upgrade` no longer add `mcp-codegraph` to `platform_toolsets.cli`. Hermes up to 0.22.2 reported that name as an unknown toolset on every start, and Hermes 0.22.4 reads it as the only MCP server to load, hiding your others from CLI sessions; an entry an earlier install added is taken out again. A refresh also no longer resets your own `mcp_servers.codegraph` settings, such as an absolute command path, timeouts or `enabled: false`, and a config indented or written differently from Hermes's default no longer gets a duplicate or unreadable entry. Thanks @Huy3ko for the report and @conifercone for confirming it. (#2485) +- `codegraph upgrade` now upgrades a global npm install when run from your home folder, which holds npm's global folder on Windows and with tools like nvm: before, it ran `npm install` without `-g` there, which left the global copy on the old version and put a second copy in your home folder. A copy installed in a project is now upgraded in that project wherever you run the command, with a warning if npm left it on the old version, and a copy installed with pnpm, yarn or bun gets the command that upgrades it instead of npm. Thanks @timlincool for the report and @maxmilian for the first fix. (#2477) +- `codegraph status` no longer says the index is up to date while indexed files are missing their symbols: it now names files the parser couldn't read and files stored without their symbols (which `codegraph sync` repairs), `status --json` counts both, and `codegraph files --json` lists each file's recorded errors. Thanks @lipchey for the report. (#2336) +- In CFML, a `` or `` block the parser can't read no longer marks its whole file as contributing nothing to the graph: `codegraph init` no longer warns about the file, and `codegraph status` no longer lists it among files whose symbols are missing, since its component, functions and calls are all indexed. The file's recorded errors now name the block and its line instead. Thanks @HarryMuc for the report. (#2654) +- In Erlang, a comment inside an argument or parameter list, like the one in `add_to_includes( % give access to .hrl in app's src/` before its two arguments, no longer counts as an argument. The call now links to the function of its real arity, and a function whose parameter list holds a comment keeps its own arity. Before, the comment raised the arity by one, so the call linked to nothing. Re-index Erlang projects after upgrading. Thanks @Dshuishui. (#1619) +- Indexing large Python projects is much faster again and needs less memory: since 1.6.2, resolving Python references re-read source files over and over, so a project the size of CPython took several times as long to index. The graph it builds is unchanged. Thanks @bompus for the report. (#2332) - In VB.NET, reading or setting a `Shared` field or property through its class or module name, like `AppSession.SessionId`, `AppSession.CurrentUser = "demo"` or `AppSession.Items(0)`, now counts as a use of that member and of the class, and so do reading an `Enum` value like `Mode.Fast` and calling a `Shared` function without parentheses. Before, only calls written with parentheses were linked, so `codegraph callers` on such a field, property or class came back empty and impact missed most of the code that depends on it. A local, parameter or field that only shares a class's name is not mistaken for the class. Thanks @serkanince for the report and @ChrisPrapas. (#2305) - In Vue and Nuxt components, calls written in the template (`{{ formatDate(post) }}`, `:to="getAccountRoute(account)"`, `@click="save(item)"`) and calls in a destructuring declaration at the top of `\n\n", + 'pages/push.vue': + '\n' + + '\n', + 'pages/target.vue': '\n', + 'pages/login.vue': '\n', + 'middleware/auth.global.ts': + "export default defineNuxtRouteMiddleware((to) => {\n if (to.path !== '/login') {\n return navigateTo('/login');\n }\n});\n", + 'pages/account.vue': + '\n' + + '\n', + 'composables/useNav.ts': 'export function useNav() {\n return { go: (to: string) => navigateTo(to) };\n}\n', + // Two project functions named navigateTo: a plugin's own, and a helper's. + 'plugins/keys.client.ts': + 'export default defineNuxtPlugin(() => {\n' + + ' const navigateTo = (to: string) => {\n console.log(to);\n };\n' + + " window.addEventListener('keydown', () => navigateTo('/target'));\n" + + '});\n', + 'lib/open.ts': 'export function navigateTo(url: string) {\n return window.open(url);\n}\n', + 'pages/docs.vue': + "\n" + + '\n', +}; + +const roots: string[] = []; +function indexProject(files: Record): Promise { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-nuxt-nav-')); + roots.push(root); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } + return CodeGraph.init(root, { index: true }); +} + +afterAll(() => { + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('Nuxt navigateTo', () => { + let cg: CodeGraph; + beforeAll(async () => { + cg = await indexProject(NUXT_APP); + }); + afterAll(() => cg?.close()); + + const route = (name: string): Node => cg.getNodesByKind('route').find((r) => r.name === name)!; + const sym = (name: string, file: string): Node => cg.getNodesInFile(file).find((n) => n.name === name && n.kind !== 'import')!; + /** The edges a node makes from its `navigateTo` calls. */ + const navigateToEdges = (from: Node): Edge[] => + cg.getOutgoingEdges(from.id).filter((e) => (e.metadata as Record | undefined)?.refName === 'navigateTo'); + const destinations = (from: Node) => + navigateToEdges(from).map((e) => `${e.kind} ${(e.metadata as Record).href} -> ${cg.getNode(e.target)?.name}`); + + it('navigates from a page to the page it names, with no edge from the page to itself', () => { + expect(destinations(sym('index', 'pages/index.vue'))).toEqual(['navigates /target -> /target']); + expect(navigateToEdges(sym('index', 'pages/index.vue'))[0]!.target).toBe(route('/target').id); + // router.push in the same app, unchanged. + expect(cg.getOutgoingEdges(sym('go', 'pages/push.vue').id).filter((e) => e.kind === 'navigates').map((e) => e.target)).toEqual([ + route('/target').id, + ]); + }); + + it('navigates from route middleware', () => { + const file = cg.getNodesInFile('middleware/auth.global.ts').find((n) => n.kind === 'file')!; + expect(destinations(file)).toEqual(['navigates /login -> /login']); + }); + + it('reads an object destination and a call written in the template', () => { + expect(destinations(sym('goLogin', 'pages/account.vue'))).toEqual(['navigates /login -> /login']); + expect(destinations(sym('account', 'pages/account.vue'))).toEqual(['navigates /target -> /target']); + }); + + it('leaves a computed destination, a path no page serves and an external URL unresolved', () => { + for (const [name, file] of [ + ['goNext', 'pages/account.vue'], + ['goMissing', 'pages/account.vue'], + ['goDocs', 'pages/account.vue'], + ['useNav', 'composables/useNav.ts'], + ] as const) { + expect(navigateToEdges(sym(name, file)), name).toEqual([]); + } + // Neither project function named navigateTo is called by any of them, nor + // does any navigateTo call point at its own caller. + const namesakes = cg.getNodesByName('navigateTo').map((n) => n.id); + expect(namesakes).toHaveLength(2); + const intoNamesakes = cg.getIncomingEdgesTo(namesakes, ['calls']); + expect(intoNamesakes.map((e) => `${cg.getNode(e.source)?.name} -> ${cg.getNode(e.target)?.filePath}`)).toEqual([ + 'openDocs -> lib/open.ts', + ]); + }); + + it('calls the navigateTo a file imports, and navigates through the one a file declares', () => { + const docs = cg.getOutgoingEdges(sym('openDocs', 'pages/docs.vue').id).filter((e) => e.kind === 'calls'); + expect(docs.map((e) => cg.getNode(e.target)).map((n) => `${n?.name} ${n?.filePath}`)).toEqual(['navigateTo lib/open.ts']); + // The plugin's wrapper is called with a page's path: where it goes, as before. + const plugin = cg.getNodesInFile('plugins/keys.client.ts').find((n) => n.kind === 'file')!; + expect(destinations(plugin)).toEqual(['navigates /target -> /target']); + }); +}); + +describe('navigateTo outside Nuxt', () => { + it('is the project function a plain Vue app imports', async () => { + const cg = await indexProject({ + 'package.json': JSON.stringify({ name: 'app', dependencies: { vue: '^3.5.0', 'vue-router': '^4.4.0' } }), + 'tsconfig.json': JSON.stringify({ compilerOptions: { baseUrl: '.', paths: { '@/*': ['src/*'] } } }), + 'src/router/index.ts': + "import { createRouter, createWebHistory } from 'vue-router';\n" + + "export default createRouter({ history: createWebHistory(), routes: [{ path: '/login', component: () => import('@/views/Login.vue') }] });\n", + 'src/views/Login.vue': '\n', + 'src/utils/nav.ts': 'export function navigateTo(path: string) {\n window.location.assign(path);\n}\n', + 'src/views/Home.vue': + "\n" + + '\n', + }); + try { + const go = cg.getNodesInFile('src/views/Home.vue').find((n) => n.name === 'go')!; + const calls = cg.getOutgoingEdges(go.id).filter((e) => e.kind === 'calls'); + expect(calls.map((e) => cg.getNode(e.target)?.filePath)).toEqual(['src/utils/nav.ts']); + } finally { + cg.close(); + } + }); +}); diff --git a/__tests__/operator-method-names.test.ts b/__tests__/operator-method-names.test.ts new file mode 100644 index 0000000000..b4ffae8fe4 --- /dev/null +++ b/__tests__/operator-method-names.test.ts @@ -0,0 +1,285 @@ +/** + * A method whose name is not a plain word: C++'s `operator++` and + * `operator()`, Scala's `++`, a Kotlin name written in backticks. The + * resolver writes names into the patterns it reads source with, and these + * went in as pattern syntax: + * + * - `operator++` is "Nothing to repeat". Choosing between a method's + * overloads reads the call's argument list after its name, so one + * `this->operator++();` inside `operator++(int)` stopped a whole index, + * and a sync after an edit, at "Resolving refs" (#2457, #2482, #2636). + * - `operator()` matched the empty `()` of its own name, so a call to the + * two-argument overload read as one with no arguments and stayed linked + * to the overload it was written in. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import type { Node } from '../src/types'; + +const lines = (...l: string[]): string => [...l, ''].join('\n'); + +/** The 1-based line of `text` that contains `needle` (exactly one must). */ +const lineOf = (text: string, needle: string): number => { + const at = text.split('\n').map((l, i) => (l.includes(needle) ? i + 1 : 0)).filter(Boolean); + if (at.length !== 1) throw new Error(`${JSON.stringify(needle)} is on ${at.length} lines`); + return at[0]!; +}; + +const COUNTER_HPP = lines( + '#pragma once', + '#include ', + '#include ', + '', + 'struct Counter {', + ' int value = 0;', + ' Counter operator++(int) {', + ' Counter before = *this;', + ' this->operator++();', + ' return before;', + ' }', + ' Counter& operator++() {', + ' ++value;', + ' return *this;', + ' }', + ' Counter& operator+=(int n) {', + ' value += n;', + ' return *this;', + ' }', + ' Counter& operator+=(const Counter& other) {', + ' value += other.value;', + ' return *this;', + ' }', + ' auto operator<=>(const Counter& other) const = default;', + ' explicit operator bool() const { return value != 0; }', + '};', + '', + 'struct Scale {', + ' int operator()(int x) {', + ' return this->operator()(x, 2);', + ' }', + ' int operator()(int x, int factor) {', + ' return x * factor;', + ' }', + '};', + '', + 'struct Table {', + ' int cells[8] = {};', + ' int& operator[](std::size_t i) { return cells[i]; }', + '};', + '', + 'struct Pointee {', + ' int field = 0;', + ' int read() const { return field; }', + '};', + '', + 'struct Handle {', + ' Pointee* target = nullptr;', + ' Pointee* operator->() const { return target; }', + ' int& operator->*(int Pointee::*m) const { return target->*m; }', + '};', + '', + 'struct Pool {', + ' static void* operator new[](std::size_t n);', + ' static void operator delete(void* p);', + ' static void operator delete[](void* p);', + ' static void* grab(std::size_t n) { return Pool::operator new[](n); }', + ' static void drop(void* p) { Pool::operator delete(p); }', + '};', + '', + 'struct Distance { long double km; };', + 'Distance operator"" _km(long double v);', +); + +const COUNTER_CPP = lines( + '#include "counter.hpp"', + '#include ', + '', + 'void* Pool::operator new[](std::size_t n) { return std::malloc(n); }', + 'void Pool::operator delete(void* p) { std::free(p); }', + 'void Pool::operator delete[](void* p) { Pool::operator delete(p); }', + 'Distance operator"" _km(long double v) { return Distance{v}; }', + '', + 'int useAll() {', + ' Counter c;', + ' c.operator++();', + ' c.operator++(0);', + ' c.operator+=(3);', + ' Counter d;', + ' d.operator+=(c);', + ' bool nonzero = c.operator bool();', + ' auto order = c.operator<=>(d);', + ' (void)order;', + ' Scale s;', + ' int scaled = s.operator()(4);', + ' Table t;', + ' t.operator[](1) = 7;', + ' Pointee pointee;', + ' Handle h{&pointee};', + ' int got = h.operator->()->read() + h.operator->*(&Pointee::field);', + ' void* arr = Pool::operator new[](32);', + ' Pool::operator delete[](arr);', + ' Distance near = operator"" _km(2.0L);', + ' return scaled + got + (nonzero ? 1 : 0) + static_cast(near.km);', + '}', +); + +describe('a method whose name is not a plain word', () => { + const roots: string[] = []; + const graphs: CodeGraph[] = []; + + afterEach(() => { + for (const cg of graphs.splice(0)) cg.close(); + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); + }); + + const project = (files: Record): string => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-operator-names-')); + roots.push(root); + for (const [rel, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), text); + } + return root; + }; + + const index = async (root: string): Promise => { + const cg = await CodeGraph.init(root, { index: true }); + graphs.push(cg); + return cg; + }; + + /** `@:` of each call from the node at `file`:`line`. */ + const callsFrom = (cg: CodeGraph, file: string, line: number): string[] => { + const from = cg.getNodesInFile(file).find((n: Node) => n.startLine === line && (n.kind === 'method' || n.kind === 'function')); + expect(from, `${file}:${line}`).toBeDefined(); + return cg.getCallees(from!.id).filter((r) => r.edge.kind === 'calls') + .map((r) => `${r.node.qualifiedName}@${path.basename(r.node.filePath)}:${r.node.startLine}`) + .sort(); + }; + + const hpp = (needle: string): string => `counter.hpp:${lineOf(COUNTER_HPP, needle)}`; + const cpp = (needle: string): string => `counter.cpp:${lineOf(COUNTER_CPP, needle)}`; + + it('C++: an index of overloaded operators that call each other by name finishes, and each call reaches its overload', async () => { + const cg = await index(project({ 'src/counter.hpp': COUNTER_HPP, 'src/counter.cpp': COUNTER_CPP })); + + // `this->operator++();` inside the postfix `operator++(int)`: the prefix one. + expect(callsFrom(cg, 'src/counter.hpp', lineOf(COUNTER_HPP, 'Counter operator++(int)'))) + .toEqual([`Counter::operator++@${hpp('Counter& operator++()')}`]); + // `this->operator()(x, 2)` inside the one-argument `operator()`: the two-argument one. + expect(callsFrom(cg, 'src/counter.hpp', lineOf(COUNTER_HPP, 'int operator()(int x) {'))) + .toEqual([`Scale::operator()@${hpp('int operator()(int x, int factor)')}`]); + expect(callsFrom(cg, 'src/counter.hpp', lineOf(COUNTER_HPP, 'static void* grab'))) + .toEqual([`Pool::operator new[]@${cpp('void* Pool::operator new[]')}`]); + expect(callsFrom(cg, 'src/counter.hpp', lineOf(COUNTER_HPP, 'static void drop'))) + .toEqual([`Pool::operator delete@${cpp('void Pool::operator delete(void* p)')}`]); + + const useAll = callsFrom(cg, 'src/counter.cpp', lineOf(COUNTER_CPP, 'int useAll()')); + expect(useAll).toEqual(expect.arrayContaining([ + `Counter::operator++@${hpp('Counter& operator++()')}`, + `Counter::operator++@${hpp('Counter operator++(int)')}`, + `Counter::operator+=@${hpp('Counter& operator+=(int n)')}`, + `Counter::operator+=@${hpp('Counter& operator+=(const Counter& other)')}`, + `Counter::operator<=>@${hpp('auto operator<=>')}`, + `Scale::operator()@${hpp('int operator()(int x) {')}`, + `Table::operator[]@${hpp('int& operator[]')}`, + `Handle::operator->@${hpp('Pointee* operator->()')}`, + `Handle::operator->*@${hpp('int& operator->*')}`, + `Pool::operator new[]@${cpp('void* Pool::operator new[]')}`, + `Pool::operator delete[]@${cpp('void Pool::operator delete[]')}`, + `operator"" _km@${cpp('Distance operator"" _km(long double v) {')}`, + ])); + }, 120_000); + + it('C++: a sync that adds such calls to an indexed project finishes, and links them', async () => { + // Indexed first without a single call of `operator++` by its name. + const strip = (text: string): string => text + .replace(' this->operator++();\n', ' ++value;\n') + .replace(' c.operator++();\n', '') + .replace(' c.operator++(0);\n', ''); + const root = project({ 'src/counter.hpp': strip(COUNTER_HPP), 'src/counter.cpp': strip(COUNTER_CPP) }); + const cg = await index(root); + expect(callsFrom(cg, 'src/counter.hpp', lineOf(COUNTER_HPP, 'Counter operator++(int)'))).toEqual([]); + + fs.writeFileSync(path.join(root, 'src/counter.hpp'), COUNTER_HPP); + fs.writeFileSync(path.join(root, 'src/counter.cpp'), COUNTER_CPP); + await cg.sync(); + + expect(callsFrom(cg, 'src/counter.hpp', lineOf(COUNTER_HPP, 'Counter operator++(int)'))) + .toEqual([`Counter::operator++@${hpp('Counter& operator++()')}`]); + expect(callsFrom(cg, 'src/counter.cpp', lineOf(COUNTER_CPP, 'int useAll()'))).toEqual(expect.arrayContaining([ + `Counter::operator++@${hpp('Counter& operator++()')}`, + `Counter::operator++@${hpp('Counter operator++(int)')}`, + ])); + }, 120_000); + + it('C++: the argument list of an `operator()` call is the one after the name, not the name\'s own parentheses', async () => { + // No `operator++` here: nothing threw before, the calls just went to the wrong overloads. + const hppText = lines( + 'struct Scale {', + ' int operator()(int x) { return this->operator()(x, 2); }', + ' int operator()(int x, int factor) { return x * factor; }', + '};', + 'struct Total {', + ' int value = 0;', + ' Total& operator+=(int n) { value += n; return *this; }', + ' Total& operator+=(const Total& other) { value += other.value; return *this; }', + '};', + ); + const cppText = lines( + '#include "scale.hpp"', + 'int addUp() {', + ' Total a;', + ' Total b;', + ' b.operator+=(a);', + ' return b.value;', + '}', + ); + const cg = await index(project({ 'src/scale.hpp': hppText, 'src/scale.cpp': cppText })); + expect(callsFrom(cg, 'src/scale.hpp', lineOf(hppText, 'int operator()(int x) {'))) + .toEqual([`Scale::operator()@scale.hpp:${lineOf(hppText, 'int operator()(int x, int factor)')}`]); + expect(callsFrom(cg, 'src/scale.cpp', lineOf(cppText, 'int addUp()'))) + .toEqual([`Total::operator+=@scale.hpp:${lineOf(hppText, 'Total& operator+=(const Total& other)')}`]); + }, 120_000); + + it('C++: `operator[](0)` still reaches the `size_type` overload, now that its arguments are read', async () => { + // nlohmann/json's `p = &p->operator[](0);`: the index overload, as before, not the C-string key one. + // A container's `size_type` is a number, which a `0` fits better than it fits a pointer. + const hppText = lines( + 'struct Json {', + ' using size_type = unsigned long;', + ' Json& operator[](size_type idx) { return *this; }', + ' template ', + ' JSON_NON_NULL(2)', + ' Json& operator[](T* key) { return *this; }', + '};', + 'Json* walk(Json* p) { return &p->operator[](0); }', + ); + const cg = await index(project({ 'src/json.hpp': hppText })); + expect(callsFrom(cg, 'src/json.hpp', lineOf(hppText, 'Json* walk('))) + .toEqual([`Json::operator[]@json.hpp:${lineOf(hppText, 'operator[](size_type idx)')}`]); + }, 120_000); + + it('Scala and Kotlin: overloads named `++` or in backticks no longer stop the index', async () => { + const scala = lines( + 'class Vec(val x: Int) {', + ' def ++(o: Vec): Vec = new Vec(x + o.x)', + ' def ++(n: Int, m: Int): Vec = this.++(new Vec(n + m))', + ' def twice: Vec = this.++(1, 2)', + '}', + ); + const kotlin = lines( + 'class Weird {', + ' fun `reads (a value`(x: Int): Int = x', + ' fun `reads (a value`(x: Int, y: Int): Int = `reads (a value`(x + y)', + ' fun use(): Int = `reads (a value`(1, 2)', + '}', + ); + const cg = await index(project({ 'src/Vec.scala': scala, 'src/Weird.kt': kotlin })); + expect(callsFrom(cg, 'src/Vec.scala', lineOf(scala, 'def twice'))).toEqual([`Vec::++@Vec.scala:${lineOf(scala, 'def ++(n: Int, m: Int)')}`]); + expect(callsFrom(cg, 'src/Weird.kt', lineOf(kotlin, 'fun use()'))).toEqual([`Weird::\`reads (a value\`@Weird.kt:${lineOf(kotlin, 'x: Int, y: Int')}`]); + }, 120_000); +}); diff --git a/__tests__/out-of-repo-import-shadow.test.ts b/__tests__/out-of-repo-import-shadow.test.ts new file mode 100644 index 0000000000..8e8e431fce --- /dev/null +++ b/__tests__/out-of-repo-import-shadow.test.ts @@ -0,0 +1,99 @@ +/** + * A bare JS/TS call through a name imported from outside the repository + * (`import { resolve } from 'node:path'`) means the package's function, never + * another file's same-named symbol. A declaration of that name in an inner + * scope of the calling file still shadows the import, and the call is that + * local declaration's. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +let tempDir: string; +let cg: CodeGraph | null = null; + +function project(files: Record): void { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-import-shadow-')); + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(tempDir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + } +} + +async function callTargets(caller: string): Promise { + cg = await CodeGraph.init(tempDir, { index: true }); + cg.resolveReferences(); + const from = cg.getNodesByKind('function').find((n) => n.name === caller)!; + expect(from).toBeDefined(); + return cg.getOutgoingEdges(from.id).filter((e) => e.kind === 'calls') + .map((e) => { const t = cg!.getNode(e.target)!; return `${t.filePath}:${t.name}:${t.startLine}`; }); +} + +afterEach(() => { + cg?.close(); + cg = null; + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe('a name imported from outside the repository', () => { + it('is shadowed by a declaration in an inner scope of the calling file', async () => { + project({ + 'plugin.ts': "export function resolve() { return 'plugin'; }", + 'config.ts': + "import { resolve } from 'node:path';\n" + + "export function configure() {\n function resolve() { return 'local'; }\n return resolve();\n}\n", + }); + expect(await callTargets('configure')).toEqual(['config.ts:resolve:3']); + }); + + it('is not shadowed by a declaration nested in a sibling function', async () => { + // `b` calls the package's `resolve`: the one nested in `a` is out of scope there. + project({ + 'plugin.ts': "export function resolve() { return 'plugin'; }", + 'config.ts': + "import { resolve } from 'node:path';\n" + + "export function a() {\n function resolve() { return 'local'; }\n return resolve();\n}\n" + + "export function b() {\n return resolve('src');\n}\n", + }); + expect(await callTargets('b')).toEqual([]); + }); + + it('is not shadowed by a declaration nested a level below the call', async () => { + project({ + 'config.ts': + "import { resolve } from 'node:path';\n" + + "export function outer() {\n function mid() { function resolve() { return 1; } return resolve(); }\n return resolve('x') + mid();\n}\n", + }); + expect((await callTargets('outer')).filter((t) => t.includes(':resolve:'))).toEqual([]); + }); + + it('is shadowed in a function nested inside the declaring one', async () => { + project({ + 'config.ts': + "import { resolve } from 'node:path';\n" + + "export function configure() {\n function resolve() { return 'local'; }\n function inner() { return resolve(); }\n return inner();\n}\n", + }); + expect(await callTargets('inner')).toEqual(['config.ts:resolve:3']); + }); + + it('binds no other file\'s symbol of that name', async () => { + project({ + 'package.json': JSON.stringify({ name: 'app', dependencies: { 'external-resolver': '^1.0.0' } }), + 'plugin.ts': "export function resolve() { return 'plugin'; }", + 'config.ts': "import { resolve } from 'external-resolver';\nexport function configure() { return resolve('src'); }\n", + }); + expect(await callTargets('configure')).toEqual([]); + }); + + it('is not shadowed by its own top-level require binding', async () => { + project({ + 'package.json': JSON.stringify({ name: 'app', devDependencies: { supertest: '^6' } }), + 'app.js': "var request = require('supertest');\nfunction run(app) { return request(app); }\nmodule.exports = run;\n", + }); + expect(await callTargets('run')).toEqual([]); + }); +}); diff --git a/__tests__/override-overload-pairing.test.ts b/__tests__/override-overload-pairing.test.ts new file mode 100644 index 0000000000..168c90782f --- /dev/null +++ b/__tests__/override-overload-pairing.test.ts @@ -0,0 +1,385 @@ +/** + * An overload is overridden by the overload with its parameters (#2663). The + * override edges (`synthesizedBy: interface-impl`) linked every overload of an + * interface or base-class method to every same-named method of the class, so + * `ExampleService.putTairValue(String)` reached + * `ExampleServiceImpl.putTairValue(List, double)` and the other way + * round. Java, Kotlin, C# and Scala make an override repeat the parameter list + * of what it overrides, so an overload now pairs with the methods of its + * parameter count, and of those with the ones whose parameter types are its + * own; where a type can't be compared (a type parameter, a type written in + * another language), the count alone decides. A name declared once on each + * side links as before. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +const FILES: Record = { + // The issue's example. + 'src/main/java/com/example/ExampleService.java': `package com.example; + +import java.util.List; + +interface ExampleService { + Object putTairValue(String tairKey); + + Object putTairValue(List tairKey, double count); +} +`, + 'src/main/java/com/example/ExampleServiceImpl.java': `package com.example; + +import java.util.List; + +class ExampleServiceImpl implements ExampleService { + @Override + public Object putTairValue(String tairKey) { + return null; + } + + @Override + public Object putTairValue(List tairKey, double count) { + return null; + } +} +`, + 'src/main/java/com/example/Shapes.java': `package com.example; + +import java.util.List; + +// One parameter each, told apart by type; \`final\` and annotations aside. +interface Slots { + void put(String key); + void put(int index); +} + +class ArraySlots implements Slots { + public void put(@Deprecated final String key) {} + public void put(int index) {} +} + +// A subclass overriding one overload of its base. +class Base { + void draw(int x) {} + void draw(String label) {} +} + +class Sub extends Base { + @Override + void draw(int x) {} +} + +// An implementation with an overload of its own. +interface Logger { + void log(String msg); +} + +class FileLogger implements Logger { + public void log(String msg) {} + public void log(String msg, int level) {} +} + +// Written another way but the same types: qualified, varargs as an array, +// a C-style array, a raw type. +interface Codec { + void write(java.util.List lines); + void write(String... parts); + void write(int[] data, int length); + void write(List codes, String sep); +} + +class CodecImpl implements Codec { + public void write(List lines) {} + public void write(String[] parts) {} + public void write(int data[], int length) {} + public void write(List codes, String sep) {} +} + +// Generic methods: a type parameter can be any type. +interface Mapper { + T map(Class type); + Object map(String name); +} + +class ReflectMapper implements Mapper { + public U map(Class type) { return null; } + public Object map(String name) { return null; } +} + +// One method on each side: linked as before, whatever its parameters. +interface Single { + void only(int x); +} + +class SingleImpl implements Single { + public void only(long x) {} +} + +// A Java interface a Kotlin class implements: the count decides. +interface Sink { + void accept(int x); + void accept(String s, int n); +} + +// A method's own type parameter stands in for no concrete type of its base: +// the generic appendTo overrides only the generic one. +class Appender { + A appendTo(A appendable, java.util.Iterator parts) { return appendable; } + StringBuilder appendTo(StringBuilder builder, java.util.Iterator parts) { return builder; } +} + +class SkippingAppender extends Appender { + @Override + A appendTo(A appendable, java.util.Iterator parts) { return appendable; } +} +`, + 'src/main/kotlin/com/example/Repos.kt': `package com.example + +interface Repository { + fun save(entity: T) + fun save(entities: List, flush: Boolean) +} + +class UserRepository : Repository { + override fun save(entity: String) {} + fun save(entities: Map) {} + override fun save(entities: List, flush: Boolean) {} +} + +interface Cache { + fun put(key: String): Any? + fun put(keys: List, count: Double = 1.0): Any? + fun put(key: String, value: Int): Any? +} + +class MemoryCache : Cache { + override fun put(key: String): Any? = null + override fun put(keys: List, count: Double): Any? = null + override fun put(key: kotlin.String, value: Int): Any? = null +} + +open class Shape { + open fun draw(x: Int) {} + open fun draw(label: String) {} +} + +class Circle : Shape() { + override fun draw(x: Int) {} +} + +class KotlinSink : Sink { + override fun accept(x: Int) {} + override fun accept(s: String, n: Int) {} +} +`, + 'src/Store.cs': `using System; +using System.Collections.Generic; + +namespace Example +{ + public interface IStore + { + object Put(string key); + object Put(List keys, double count); + object Put(int index); + } + + public class Store : IStore + { + public object Put(string key) { return null; } + public object Put(List keys, double count) { return null; } + public object Put(int index) { return null; } + } + + public class ExplicitStore : IStore + { + object IStore.Put(string key) => null; + object IStore.Put(List keys, double count) => null; + object IStore.Put(int index) => null; + } + + // BCL names for the keywords, a namespace, a default value, an + // attribute and a comment that mention the method. + public class SpelledStore : IStore + { + [Obsolete("Put(int) is the one to use")] + public object Put(String key) => null; + public object Put(System.Collections.Generic.List keys, Double count = 1.0) => null; + // Put(string) and Put(List, double) above. + public object Put( + Int32 index) => null; + } + + public interface IRepo + { + void Save(T entity); + void Save(IList entities, bool flush); + } + + public class UserRepo : IRepo + { + public void Save(string entity) { } + public void Save(IList entities, bool flush) { } + } + + public abstract class Widget + { + public virtual void Draw(int x) { } + public virtual void Draw(string label) { } + public virtual void Move(ref int x) { } + public virtual void Move(int x) { } + } + + public class Button : Widget + { + public override void Draw(int x) { } + public override void Move(ref int x) { } + } +} +`, + 'src/Repo.scala': `package example + +trait Repo { + def find(id: Long): Any + def find(ids: List[Long], limit: Int = 10): Any +} + +class SqlRepo extends Repo { + def find(id: Long): Any = null + def find(ids: List[Long], limit: Int): Any = null +} +`, +}; + +let root = ''; +let graph: CodeGraph; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-override-overloads-')); + for (const [rel, content] of Object.entries(FILES)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + graph = await CodeGraph.init(root, { index: true }); +}, 120_000); + +afterAll(() => { + graph?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +/** `Owner::name:line -> Owner::name:line` for every override edge out of `owner`'s methods. */ +function overrides(owner: string): string[] { + const label = (id: string): string => { + const n = graph.getNode(id)!; + return `${n.qualifiedName.split('::').slice(-2).join('::')}:${n.startLine}`; + }; + return graph + .getNodesByKind('method') + .filter((m) => m.qualifiedName.split('::').slice(-2)[0] === owner) + .flatMap((m) => graph.getOutgoingEdges(m.id) + .filter((e) => e.kind === 'calls' && e.metadata?.synthesizedBy === 'interface-impl') + .map((e) => `${label(e.source)} -> ${label(e.target)}`)) + .sort(); +} + +describe('an overload is overridden by the overload with its parameters (#2663)', () => { + it("Java: the issue's interface and implementation", () => { + expect(overrides('ExampleService')).toEqual([ + 'ExampleService::putTairValue:6 -> ExampleServiceImpl::putTairValue:6', + 'ExampleService::putTairValue:8 -> ExampleServiceImpl::putTairValue:11', + ]); + }); + + it('Java: by parameter types at the same count, past `final` and annotations', () => { + expect(overrides('Slots')).toEqual([ + 'Slots::put:7 -> ArraySlots::put:12', + 'Slots::put:8 -> ArraySlots::put:13', + ]); + }); + + it('Java: a subclass overrides only the overload it repeats', () => { + expect(overrides('Base')).toEqual(['Base::draw:18 -> Sub::draw:23']); + }); + + it("Java: an implementation's own overload is no override", () => { + expect(overrides('Logger')).toEqual(['Logger::log:29 -> FileLogger::log:33']); + }); + + it('Java: the same types written another way', () => { + expect(overrides('Codec')).toEqual([ + 'Codec::write:40 -> CodecImpl::write:47', + 'Codec::write:41 -> CodecImpl::write:48', + 'Codec::write:42 -> CodecImpl::write:49', + 'Codec::write:43 -> CodecImpl::write:50', + ]); + }); + + it('Java: a type parameter matches any type in its place', () => { + expect(overrides('Mapper')).toEqual([ + 'Mapper::map:55 -> ReflectMapper::map:60', + 'Mapper::map:56 -> ReflectMapper::map:61', + ]); + }); + + it("Java: an override's own type parameter stands in for no concrete type", () => { + expect(overrides('Appender')).toEqual(['Appender::appendTo:82 -> SkippingAppender::appendTo:87']); + }); + + it('a name declared once on each side links as before', () => { + expect(overrides('Single')).toEqual(['Single::only:66 -> SingleImpl::only:70']); + }); + + it('Kotlin: by count where a type parameter stands in, by type elsewhere', () => { + expect(overrides('Repository')).toEqual([ + 'Repository::save:4 -> UserRepository::save:10', + 'Repository::save:4 -> UserRepository::save:9', + 'Repository::save:5 -> UserRepository::save:11', + ]); + expect(overrides('Cache')).toEqual([ + 'Cache::put:15 -> MemoryCache::put:21', + 'Cache::put:16 -> MemoryCache::put:22', + 'Cache::put:17 -> MemoryCache::put:23', + ]); + expect(overrides('Shape')).toEqual(['Shape::draw:27 -> Circle::draw:32']); + }); + + it('Kotlin implementing a Java interface: by parameter count', () => { + expect(overrides('Sink')).toEqual([ + 'Sink::accept:75 -> KotlinSink::accept:36', + 'Sink::accept:76 -> KotlinSink::accept:37', + ]); + }); + + it('C#: by parameter types, read from the declarations', () => { + expect(overrides('IStore')).toEqual([ + 'IStore::Put:10 -> ExplicitStore::Put:24', + 'IStore::Put:10 -> SpelledStore::Put:35', + 'IStore::Put:10 -> Store::Put:17', + 'IStore::Put:8 -> ExplicitStore::Put:22', + 'IStore::Put:8 -> SpelledStore::Put:31', + 'IStore::Put:8 -> Store::Put:15', + 'IStore::Put:9 -> ExplicitStore::Put:23', + 'IStore::Put:9 -> SpelledStore::Put:33', + 'IStore::Put:9 -> Store::Put:16', + ]); + expect(overrides('IRepo')).toEqual([ + 'IRepo::Save:41 -> UserRepo::Save:47', + 'IRepo::Save:42 -> UserRepo::Save:48', + ]); + // `Move(ref int)` and `Move(int)` are two overloads. + expect(overrides('Widget')).toEqual([ + 'Widget::Draw:53 -> Button::Draw:61', + 'Widget::Move:55 -> Button::Move:62', + ]); + }); + + it('Scala: by parameter types, past a default value', () => { + expect(overrides('Repo')).toEqual([ + 'Repo::find:4 -> SqlRepo::find:9', + 'Repo::find:5 -> SqlRepo::find:10', + ]); + }); +}); diff --git a/__tests__/param-default-calls.test.ts b/__tests__/param-default-calls.test.ts new file mode 100644 index 0000000000..4fe9221006 --- /dev/null +++ b/__tests__/param-default-calls.test.ts @@ -0,0 +1,269 @@ +/** + * A parameter's default value is code its function runs, each time a call + * leaves that argument out: `function render(note = buildNote())` calls + * buildNote as surely as `const note = buildNote()` in its body would. The + * extractors walked only the body, so a call or `new` in a default was never + * seen (#2588). Each default, and a computed key in a destructured parameter, + * is now walked with the function on the stack — for functions, arrows, + * methods, constructors, class-field arrows, object-literal methods and the + * render functions of React wrappers — in the kernel and the wasm extractor + * alike. The names a parameter binds and its type are not code, and a Python + * default, which runs once when the `def` does, is left as it was. + */ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel'; +import type { ExtractionResult, Language } from '../src/types'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelAvailable = fs.existsSync(KERNEL_PATH) || process.env.CODEGRAPH_KERNEL_EXPECT === '1'; + +const TS_SOURCE = `import { buildNote, fallback, make, pickKey, logError, Options, Service, throttle } from './lib'; +export function plain(note = buildNote()) { return note; } +export const arrow = (note = buildNote()) => note; +export function destr({ n } = buildNote() as any) { return n; } +export function nested({ a = fallback(), b: { c = make() } = {} as any }: any) { return a + c; } +export function computed({ [pickKey()]: v }: any) { return v; } +export function rest(...[first = fallback()]: number[]) { return first; } +export function optional(x?: number, y: number = fallback()) { return (x ?? 0) + y; } +export function typed(x: ReturnType = buildNote()) { return x; } +export function withNew(opts = new Options()) { return opts; } +export function handler(onError = (e: unknown) => logError(e)) { onError(1); } +export async function* stream(note = buildNote()) { yield note; } +export class K { + constructor(private readonly svc = new Service()) {} + m(note = buildNote()) { return note; } + static s(note = buildNote()) { return note; } + field = (note = buildNote()) => note; + wrapped = throttle((note = fallback()) => note); +} +export const api = { load(note = buildNote()) { return note; }, save: (note = make()) => note }; +export function outer() { + function inner(note = buildNote()) { return note; } + return inner(); +} +export function over(x: number): number; +export function over(x: number = fallback()): number { return x; } +export function inBody() { return buildNote(); } +`; + +const TS_EXPECTED = [ + 'api::load calls buildNote', + 'api::save calls make', + 'arrow calls buildNote', + 'computed calls pickKey', + 'K::constructor instantiates Service', + 'destr calls buildNote', + 'K::field calls buildNote', + 'handler calls logError', + 'handler calls onError', + 'inBody calls buildNote', + 'outer::inner calls buildNote', + 'K::m calls buildNote', + 'K::s calls buildNote', + 'nested calls fallback', + 'nested calls make', + 'optional calls fallback', + 'outer calls inner', + 'over calls fallback', + 'plain calls buildNote', + 'rest calls fallback', + 'stream calls buildNote', + // The type annotation's `typeof shape` is a type, not a call. + 'typed calls buildNote', + 'withNew instantiates Options', + 'K::wrapped calls fallback', +]; + +const JS_SOURCE = `import { buildNote, fallback, pickKey, Options } from './lib'; +export function plain(note = buildNote()) { return note; } +export function nested({ a = fallback(), b: { c = buildNote() } = {} }) { return a + c; } +export function arr([x = fallback(), , y = buildNote()] = []) { return x + y; } +export function computed({ [pickKey()]: v }) { return v; } +export const arrow = (note = buildNote()) => note; +export class J { + constructor(o = new Options()) { this.o = o; } + m(note = buildNote()) { return note; } + field = (note = fallback()) => note; +} +export default function (note = buildNote()) { return note; } +export function inBody() { return buildNote(); } +`; + +const JS_EXPECTED = [ + 'J::m calls buildNote', + 'arr calls buildNote', + 'arr calls fallback', + 'arrow calls buildNote', + 'computed calls pickKey', + 'J::constructor instantiates Options', + 'J::field calls fallback', + 'inBody calls buildNote', + 'nested calls buildNote', + 'nested calls fallback', + 'plain calls buildNote', + // An anonymous function's code is its enclosing scope's: here, the file's. + 'src/plain.js calls buildNote', +]; + +const TSX_SOURCE = `import { forwardRef, memo } from 'react'; +import styled from 'styled-components'; +import { buildNote, fallback } from './lib'; +export const Button = forwardRef(function Button({ size = fallback() }: any, ref: any) { return {size}; }); +export const Card = memo(({ note = buildNote() }: any) => {note}); +export const Box = styled.div.attrs((p = buildNote()) => ({ p }))\` + color: red; +\`; +export const Nested = memo(forwardRef(({ title = buildNote() }: any, ref: any) =>

{title}

)); +export function Page({ title = fallback() }: { title?: string }) { return

{title}

; } +setTimeout((n = fallback()) => n, 0); +`; + +const TSX_EXPECTED = [ + 'Box calls buildNote', + 'Button calls fallback', + 'Card calls buildNote', + 'Nested calls buildNote', + 'Page calls fallback', + // An anonymous function's code is its enclosing scope's: here, the file's. + 'src/ui.tsx calls fallback', + 'src/ui.tsx calls setTimeout', +]; + +const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const; + +describe('a call in a parameter default is the function’s call (#2588)', () => { + let savedEnv: Record = {}; + + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['typescript', 'tsx', 'javascript', 'jsx']); + }); + + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + resetKernelForTests(); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + resetKernelForTests(); + }); + + function extract(backend: 'kernel' | 'wasm', file: string, source: string, language: Language): ExtractionResult { + if (backend === 'wasm') { + process.env.CODEGRAPH_KERNEL = '0'; + return extractFromSource(file, source, language); + } + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + const result = tryKernelExtract(file, source, language); + expect(result, `kernel extraction of ${file}`).not.toBeNull(); + return result!; + } + + /** `caller kind callee` for every call and `new` the file records, by its caller's qualified name. */ + function callsOf(result: ExtractionResult): string[] { + const names = new Map(result.nodes.map((n) => [n.id, n.kind === 'file' ? n.filePath : n.qualifiedName])); + return result.unresolvedReferences + .filter((r) => r.referenceKind === 'calls' || r.referenceKind === 'instantiates') + .map((r) => `${names.get(r.fromNodeId)} ${r.referenceKind} ${r.referenceName}`) + .sort(); + } + + const backends = kernelAvailable ? (['kernel', 'wasm'] as const) : (['wasm'] as const); + + for (const crlf of [false, true]) { + const eol = (s: string) => (crlf ? s.replace(/\n/g, '\r\n') : s); + const label = crlf ? ' (CRLF)' : ''; + + it.each(backends)(`in TypeScript functions, arrows, methods, fields and object literals: %s${label}`, (backend) => { + expect(callsOf(extract(backend, 'src/use.ts', eol(TS_SOURCE), 'typescript'))).toEqual([...TS_EXPECTED].sort()); + }); + + it.each(backends)(`in JavaScript, through assignment and destructuring patterns: %s${label}`, (backend) => { + expect(callsOf(extract(backend, 'src/plain.js', eol(JS_SOURCE), 'javascript'))).toEqual([...JS_EXPECTED].sort()); + expect(callsOf(extract(backend, 'src/plain.jsx', eol(JS_SOURCE), 'jsx'))).toEqual( + JS_EXPECTED.map((s) => s.replace('src/plain.js ', 'src/plain.jsx ')).sort() + ); + }); + + it.each(backends)(`in a React wrapper's render function and a styled attrs function: %s${label}`, (backend) => { + expect(callsOf(extract(backend, 'src/ui.tsx', eol(TSX_SOURCE), 'tsx'))).toEqual([...TSX_EXPECTED].sort()); + }); + } + + it.runIf(kernelAvailable)('the kernel and the wasm extractor agree on every node, edge and reference', () => { + const canon = (r: ExtractionResult) => ({ + nodes: r.nodes.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort())).sort(), + edges: r.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(), + refs: r.unresolvedReferences.map((x) => JSON.stringify(x, Object.keys(x).sort())).sort(), + }); + for (const [file, source, language] of [ + ['src/use.ts', TS_SOURCE, 'typescript'], + ['src/plain.js', JS_SOURCE, 'javascript'], + ['src/ui.tsx', TSX_SOURCE, 'tsx'], + ] as const) { + expect(canon(extract('kernel', file, source, language)), file).toEqual(canon(extract('wasm', file, source, language))); + } + }); +}); + +describe('the default value links to what it calls', () => { + let root = ''; + let graph: CodeGraph | null = null; + + afterEach(() => { + graph?.close(); + graph = null; + if (root) fs.rmSync(root, { recursive: true, force: true }); + root = ''; + }); + + it('gives buildNote its callers from defaults, and leaves a Python default alone', async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-param-default-')); + const files: Record = { + 'package.json': JSON.stringify({ name: 'app' }), + // The issue's reproduction, verbatim. + 'src/f.ts': + 'export function buildNote() { return { id: 1 }; }\n' + + 'export function renderWithNote(note = buildNote()) {\n return note.id;\n}\n' + + 'export const arrow = (note = buildNote()) => note.id;\n' + + 'export class K { m(note = buildNote()) { return note.id; } }\n' + + 'export function destr({ n } = buildNote() as any) { return n; }\n' + + 'export function inBody() { const note = buildNote(); return note.id; }\n', + // Python evaluates a default once, when the `def` runs: not the function's call. + 'tool/ctl.py': 'def build():\n return 1\n\n\ndef uses_default(x=build()):\n return x\n\n\ndef in_body():\n return build()\n', + }; + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + graph = await CodeGraph.init(root, { index: true }); + + const callers = (file: string, name: string): string[] => { + const target = graph!.getNodesInFile(file).find((n) => n.name === name)!; + return graph!.getIncomingEdges(target.id) + .filter((e) => e.kind === 'calls') + .map((e) => graph!.getNode(e.source)!.name) + .sort(); + }; + expect(callers('src/f.ts', 'buildNote')).toEqual(['arrow', 'destr', 'inBody', 'm', 'renderWithNote']); + expect(callers('tool/ctl.py', 'build')).toEqual(['in_body']); + }, 120_000); +}); diff --git a/__tests__/pascal-chained-calls.test.ts b/__tests__/pascal-chained-calls.test.ts new file mode 100644 index 0000000000..6a581db249 --- /dev/null +++ b/__tests__/pascal-chained-calls.test.ts @@ -0,0 +1,318 @@ +/** + * The receiver of a chained Delphi call (#2547): in + * `MD5FileSized(TMeta.ExtendedPath(APath)).IsSame(AKey)` only `IsSame` was + * recorded. The receiver call `MD5FileSized` and the `TMeta.ExtendedPath` in + * its arguments were dropped, and so were the paren'd calls deeper in a chain + * (`A(1).B.C`) or in an index (`List[GetIndex(x)].Free`). The extractor now + * reads every paren'd call in a receiver chain. A paren-less link (`.B`) may + * read a field or property, so it is still not a call, and a typecast like + * `TButton(Sender)` creates nothing, so it no longer reads as a construction. + */ +import { describe, it, expect, beforeAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +// The issue's own unit. +const CHAIN_PAS = `unit Chain; + +interface + +type + TKey = record + function IsSame(const AOther: TKey): Boolean; + procedure Reset; + end; + + TMeta = class + class function ExtendedPath(const APath: string): string; static; + end; + +function MD5FileSized(const APath: string): TKey; +function TargetIsTheSame(const APath: string; const AKey: TKey): Boolean; +procedure Stmt(const APath: string); + +implementation + +function TKey.IsSame(const AOther: TKey): Boolean; +begin + Result := True; +end; + +procedure TKey.Reset; +begin +end; + +class function TMeta.ExtendedPath(const APath: string): string; +begin + Result := APath; +end; + +function MD5FileSized(const APath: string): TKey; +begin +end; + +function TargetIsTheSame(const APath: string; const AKey: TKey): Boolean; +begin + Result := MD5FileSized(TMeta.ExtendedPath(APath)).IsSame(AKey); +end; + +procedure Stmt(const APath: string); +begin + MD5FileSized(TMeta.ExtendedPath(APath)).Reset; +end; + +end. +`; + +function refsFrom(code: string, qualifiedName: string, kind = 'calls'): string[] { + const result = extractFromSource('Unit1.pas', code); + const from = result.nodes.find((n) => n.qualifiedName === qualifiedName); + return result.unresolvedReferences + .filter((r) => r.referenceKind === kind && r.fromNodeId === from?.id) + .map((r) => `${r.referenceName}:${r.line}`) + .sort(); +} + +async function withIndex(code: string, check: (cg: CodeGraph) => void): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-chain-')); + try { + fs.writeFileSync(path.join(dir, 'Unit1.pas'), code); + const cg = await CodeGraph.init(dir, { index: true }); + try { + check(cg); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +function edgesFrom(cg: CodeGraph, sourceName: string): string[] { + const source = cg.getNodesByKind('function').concat(cg.getNodesByKind('method')).find((n) => n.name === sourceName); + if (!source) return []; + return cg + .getOutgoingEdges(source.id) + .filter((e) => e.kind !== 'contains') + .map((e) => `${e.kind} ${cg.getNode(e.target)?.qualifiedName ?? '?'}`) + .sort(); +} + +describe('Delphi chained calls (#2547)', () => { + it('records the receiver call and the calls in its arguments', () => { + expect(refsFrom(CHAIN_PAS, 'TargetIsTheSame')).toEqual(['IsSame:41', 'MD5FileSized:41', 'TMeta.ExtendedPath:41']); + expect(refsFrom(CHAIN_PAS, 'Stmt')).toEqual(['MD5FileSized:46', 'Reset:46', 'TMeta.ExtendedPath:46']); + }); + + it('links each routine to all three callees the issue names', async () => { + await withIndex(CHAIN_PAS, (cg) => { + expect(edgesFrom(cg, 'TargetIsTheSame')).toEqual([ + 'calls MD5FileSized', + 'calls TKey::IsSame', + 'calls TMeta::ExtendedPath', + ]); + expect(edgesFrom(cg, 'Stmt')).toEqual(['calls MD5FileSized', 'calls TKey::Reset', 'calls TMeta::ExtendedPath']); + }); + }); + + it('reads paren\'d calls anywhere in a receiver chain, in every context', () => { + const code = `unit Unit1; + +interface + +implementation + +procedure Run; +begin + A(1).B.C(2); + X := A(3).B.C; + if F(4).Ready then Exit; + List[GetIndex(5)].Free; + Y := Items[Find(6)].Value; + GetHandler(7)(8); + Z := Obj.Prop.Method(9); +end; + +end. +`; + // `B`, `C` (line 10), `Ready`, `Value` and `Prop` are paren-less links: reads, not calls. + expect(refsFrom(code, 'Run')).toEqual([ + 'A:10', + 'A:9', + 'C:9', + 'F:11', + 'Find:13', + 'Free:12', + 'GetHandler:14', + 'GetIndex:12', + 'Method:15', + ]); + }); + + it('names a fluent chain\'s inner links only where they can be named precisely', () => { + const code = `unit Unit1; + +interface + +implementation + +procedure Run; +begin + Client.New.BaseURL('x').Accept('y').Get('/'); + RESTClient.HandleRedirects(False).Accept('z').Get('/'); + Builder.WithName(Name).Build; +end; + +end. +`; + // The outer `Get` keeps the name a call has always had. `BaseURL` and + // `Accept` would be bare names on a chain whose type is unknown, which + // match namesakes, so they are left out; `RESTClient.HandleRedirects` + // and `Builder.WithName` name their receiver. + expect(refsFrom(code, 'Run')).toEqual(['Build:11', 'Builder.WithName:11', 'Get:10', 'Get:9', 'RESTClient.HandleRedirects:10']); + }); + + it('gives a factory its caller and still resolves the chain on what it returns', async () => { + const code = `unit Unit1; + +interface + +type + TBar = class + procedure DoIt; + end; + TDecoy = class + procedure DoIt; + end; + TFoo = class + class function GetInstance: TBar; + end; + +procedure Run; + +implementation + +procedure TBar.DoIt; begin end; +procedure TDecoy.DoIt; begin end; +class function TFoo.GetInstance: TBar; begin Result := nil; end; + +procedure Run; +begin + TFoo.GetInstance().DoIt(); +end; + +end. +`; + await withIndex(code, (cg) => { + expect(edgesFrom(cg, 'Run')).toEqual(['calls TBar::DoIt', 'calls TFoo::GetInstance']); + }); + }); + + it('reads a typecast as a reference to its type, never a construction, wherever it is written', async () => { + const code = `unit Unit1; + +interface + +type + TFoo = class + public + procedure DoIt; + function Ready: Boolean; + end; + + TDecoy = class + public + procedure DoIt; + end; + + NullableInt = record + procedure SetNull; + end; + + TPair = record + Lo, Hi: Integer; + end; + PPair = ^TPair; + +function GetObj(N: Integer): TObject; +procedure Run(Sender: TObject; var V; P: Pointer); + +implementation + +procedure TFoo.DoIt; begin end; +function TFoo.Ready: Boolean; begin Result := True; end; +procedure TDecoy.DoIt; begin end; +procedure NullableInt.SetNull; begin end; +function GetObj(N: Integer): TObject; begin Result := nil; end; + +procedure Run(Sender: TObject; var V; P: Pointer); +var + F: TFoo; +begin + TFoo(Sender).DoIt; + TFoo(GetObj(1)).DoIt(); + if TFoo(Sender).Ready then Exit; + F := TFoo(Sender); + NullableInt(V).SetNull; + PPair(P)^.Lo := 1; +end; + +end. +`; + await withIndex(code, (cg) => { + expect(edgesFrom(cg, 'Run')).toEqual([ + 'calls GetObj', + 'calls NullableInt::SetNull', + 'calls TFoo::DoIt', + 'calls TFoo::DoIt', + 'references NullableInt', + 'references PPair', + 'references TFoo', + 'references TFoo', + 'references TFoo', + 'references TFoo', + ]); + }); + }); + + it('captures a routine passed by name to a receiver call, as for any call', async () => { + const code = `unit Unit1; + +interface + +type + TJob = class + procedure Start; + end; + +function Schedule(Proc: TProc): TJob; +procedure Worker; +procedure Run; + +implementation + +procedure TJob.Start; begin end; +function Schedule(Proc: TProc): TJob; begin Result := nil; end; +procedure Worker; begin end; + +procedure Run; +begin + Schedule(Worker).Start; +end; + +end. +`; + await withIndex(code, (cg) => { + expect(edgesFrom(cg, 'Run')).toEqual(['calls Schedule', 'calls TJob::Start', 'references Worker']); + }); + }); +}); diff --git a/__tests__/pascal-delphi-syntax.test.ts b/__tests__/pascal-delphi-syntax.test.ts new file mode 100644 index 0000000000..639e112125 --- /dev/null +++ b/__tests__/pascal-delphi-syntax.test.ts @@ -0,0 +1,521 @@ +/** + * Delphi syntax the Pascal grammar has no rule for, which broke whole units + * of SynEdit, mORMot2, DWScript and HeidiSQL: a property re-published + * without its type (`property OwnsObjects;`, SynEdit.pas from line 303), a + * conditional right after a routine's header that holds no whole body, two + * branches that open the same `begin`, a re-raise, a field or variable of an + * anonymous record type, a label right before `end`, and alternative + * statements where the grammar takes one (`OP_X: {$IFDEF U} … {$ELSE} … + * {$ENDIF}`). The extractor's preParse blanks each, or writes a word the + * grammar knows in its place, offsets kept, so the unit parses whole and the + * routines after it keep their calls. + */ +import { describe, it, expect, beforeAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { getParser, initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { preParsePascal } from '../src/extraction/languages/pascal-preparse'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +/** Whether the unit parses whole: a `unit` node and no error anywhere. */ +function parsesWhole(code: string): boolean { + const tree = getParser('pascal')!.parse(preParsePascal(code))!; + try { + return !tree.rootNode.hasError && tree.rootNode.namedChildren.some((c) => c.type === 'unit'); + } finally { + tree.delete(); + } +} + +/** Every routine's calls, as `Caller->Callee`, and every node's qualified name, from the unit's extraction. */ +function extract(code: string, file: string): { calls: string[]; names: string[] } { + const result = extractFromSource(file, code); + const byId = new Map(result.nodes.map((n) => [n.id, n])); + return { + calls: result.unresolvedReferences + .filter((r) => r.referenceKind === 'calls') + .map((r) => `${byId.get(r.fromNodeId)?.name}->${r.referenceName}`), + names: result.nodes.map((n) => n.qualifiedName), + }; +} + +const OVERRIDES = `unit Marks; + +interface + +type + TMarkList = class(TObjectList) + protected + function GetItem(Index: Integer): TObject; + property OwnsObjects; + public + constructor Create; + property Items[Index: Integer]: TObject read GetItem; default; + end; + + TEditor = class(TCustomControl) + published + property Align; + property Cursor default crIBeam; + class property Tag; + property Lines: TStrings read FLines; + procedure Paint; + end; + +implementation + +procedure TEditor.Paint; +begin + Refresh(1); +end; + +end. +`; + +const HEADERS = `unit Headers; + +interface + +procedure KeyUp; +function Size: Integer; +procedure Generate; + +implementation + +procedure KeyUp; +{$IFDEF LINUX} +var + Code: Byte; +{$ENDIF} +begin + Translate(1); +end; + +function Size: Integer; {$IFDEF HASINLINE}inline;{$ENDIF} +begin + Result := Measure(1); +end; + +procedure Generate; +{$IFDEF LLVM} +{$IFDEF LLVM_EXECUTE} +var + Engine: Pointer; +{$ENDIF} +{$ENDIF} +begin + Emit(1); +end; + +end. +`; + +const BEGINS = `unit Begins; + +interface + +procedure Click; +procedure After; + +implementation + +procedure Click; +var + I: Integer; +{$IFDEF FOLDING} + Index: Integer; +begin + Fold(1); + if I > 0 then + begin + Exit; + end; +{$ELSE} +begin +{$ENDIF} + Paint(1); +end; + +procedure After; +begin + Finish(1); +end; + +end. +`; + +const ASSEMBLER = `unit Assembler; + +interface + +function Log2Of(const V: Double): Double; +function DivMod10(var Dividend: Cardinal): Cardinal; +procedure Fill(P: Pointer); +procedure After; + +implementation + +procedure Fill(P: Pointer); +{$IFDEF FPC}nostackframe; assembler; asm {$ELSE} asm .noframe {$ENDIF} + mov eax, 1 +end; + +function Log2Of(const V: Double): Double; +{$IFDEF WIN32_ASM} +asm + fld1 + fld V + fyl2x +{$ELSE} +begin + Result := Log2(V); +{$ENDIF} +end; + +function DivMod10(var Dividend: Cardinal): Cardinal; +{$IFNDEF WIN32_ASM} +var + Divided: Cardinal; +begin + Divided := Dividend div 10; + Result := Remainder(Dividend, Divided); +{$ELSE} +const + C10: Cardinal = 10; +asm + mov ecx, eax + div C10 +{$ENDIF} +end; + +procedure After; +begin + Finish(1); +end; + +end. +`; + +const RERAISES = `unit Reraises; + +interface + +procedure Load; +procedure Parse; + +implementation + +procedure Load; +begin + try + Read(1); + except + Log(1); + raise; + end +end; + +procedure Parse; +begin + try + Scan(1); + except + on E: EConvertError do + Recover(1); + else raise + end; +end; + +end. +`; + +const RECORDS = `unit Records; + +interface + +type + TClient = class + private + fIn: record + Headers: Pointer; + URL, Method: string; + end; + fRows: array of packed record + Id: Integer; + end; + fCount: Integer; + public + procedure Send; + end; + + TCaches = array[1..4] of record + Size: Integer; + end; + + TState = record + Seed: Integer; + Data: record + case Integer of + 0: (Lc: Pointer) + end + end; + +var + PosixInject: record + GetSmbios: function(Index: Integer): string; + OnIdle: procedure of object; + end; + Ready: Boolean; + +const + Origin: record X, Y: Integer; end = (X: 0; Y: 0); + +implementation + +procedure TClient.Send; +begin + Post(fIn.URL); +end; + +end. +`; + +const LABELS = `unit Labels; + +interface + +function Find(P: PChar): Integer; +procedure After; + +implementation + +function Find(P: PChar): Integer; +label + fnd, quit; +begin + Result := 0; + if P = nil then + goto quit; + if Match(P) then + goto fnd; + goto quit; +fnd: + Result := 1; +quit: +end; + +procedure After; +begin + Finish(1); +end; + +end. +`; + +const ALTERNATIVES = `unit Alternatives; + +interface + +function Repeated(Op: Integer): Integer; +function Pick(Lookup: Pointer): Integer; + +implementation + +function Repeated(Op: Integer): Integer; +begin + Result := 0; + case Op of + 1: + {$IFDEF UNICODE} + begin + CountWide(Result); + end; + {$ELSE} + while Result < 10 do + CountAnsi(Result); + {$ENDIF} + 2: + Result := 2; + end; +end; + +function Pick(Lookup: Pointer): Integer; +begin + if Lookup = nil then + {$IFDEF ASMX86} + Result := PosAsm(0) + {$ELSE} + Result := PosPas(0) + {$ENDIF} + else + Result := PosLookup(Lookup); +end; + +end. +`; + +describe('Delphi syntax the Pascal grammar has no rule for', () => { + it.each([ + ['re-published properties', OVERRIDES], + ['conditionals after routine headers', HEADERS], + ['branches that open the same begin', BEGINS], + ['an assembler and a Pascal version of a body', ASSEMBLER], + ['re-raises', RERAISES], + ['anonymous records', RECORDS], + ['a label that ends a block', LABELS], + ['alternative statements where one goes', ALTERNATIVES], + ])('parses a unit whole past %s', (_name, code) => { + expect(parsesWhole(code)).toBe(true); + }); + + it('keeps a class whole past its re-published properties', () => { + const { calls, names } = extract(OVERRIDES, 'Marks.pas'); + expect(names).toEqual(expect.arrayContaining([ + 'TMarkList::GetItem', 'TMarkList::Create', 'TMarkList::Items', 'TEditor::Lines', 'TEditor::Paint', + ])); + // An override declares nothing new. + expect(names).not.toContain('TMarkList::OwnsObjects'); + expect(names).not.toContain('TEditor::Align'); + expect(calls).toEqual(['Paint->Refresh']); + }); + + it('gives each routine its calls past a conditional after its header', () => { + expect(extract(HEADERS, 'Headers.pas').calls.sort()).toEqual(['Generate->Emit', 'KeyUp->Translate', 'Size->Measure']); + expect(extract(BEGINS, 'Begins.pas').calls.sort()).toEqual(['After->Finish', 'Click->Fold', 'Click->Paint']); + expect(extract(ASSEMBLER, 'Assembler.pas').calls.sort()).toEqual(['After->Finish', 'DivMod10->Remainder', 'Log2Of->Log2']); + }); + + it('gives each routine its calls past a re-raise and a label that ends a block', () => { + expect(extract(RERAISES, 'Reraises.pas').calls.sort()).toEqual(['Load->Log', 'Load->Read', 'Parse->Recover', 'Parse->Scan']); + expect(extract(LABELS, 'Labels.pas').calls.sort()).toEqual(['After->Finish', 'Find->Match']); + }); + + it('keeps a declaration of an anonymous record type, and every declaration after it', () => { + const { calls, names } = extract(RECORDS, 'Records.pas'); + expect(names).toEqual(expect.arrayContaining([ + 'TClient::fIn', 'TClient::fRows', 'TClient::fCount', 'TClient::Send', 'TCaches', 'TState::Seed', 'TState::Data', + 'PosixInject', 'Ready', 'Origin', + ])); + // The record's own fields are no members of the class around it. + expect(names).not.toContain('TClient::URL'); + expect(names).not.toContain('GetSmbios'); + expect(calls).toEqual(['Send->Post']); + }); + + it('reads alternative statements where one goes as one block, keeping the calls of each', () => { + expect(extract(ALTERNATIVES, 'Alternatives.pas').calls.sort()).toEqual([ + 'Pick->PosAsm', 'Pick->PosLookup', 'Pick->PosPas', 'Repeated->CountAnsi', 'Repeated->CountWide', + ]); + }); + + it('keeps every offset and line, CRLF included', () => { + for (const code of [OVERRIDES, HEADERS, BEGINS, ASSEMBLER, RERAISES, RECORDS, LABELS, ALTERNATIVES]) { + const crlf = code.replace(/\n/g, '\r\n'); + const out = preParsePascal(crlf); + expect(out.length).toBe(crlf.length); + const breaks = (s: string) => [...s.matchAll(/\r\n/g)].map((m) => m.index); + expect(breaks(out)).toEqual(breaks(crlf)); + } + }); + + it('writes the words it needs where the grammar reads them', () => { + const blank = (s: string) => s.replace(/[^\r\n]/g, ' '); + expect(preParsePascal(' 1: {$IFDEF U} A(1); {$ELSE} B(2); {$ENDIF}\n')) + .toBe(' 1: begin A(1); ; B(2); end; \n'); + expect(preParsePascal(' Data: record X: Integer; end;\n')).toBe(` Data: string${blank(' X: Integer; end')};\n`); + expect(preParsePascal(' Data: packed record X: Integer; end end;\n')).toBe(` Data: string${blank(' record X: Integer; ')}; end;\n`); + expect(preParsePascal(' else raise\n end;\n')).toBe(' else ; \n end;\n'); + }); + + it('leaves alone what the grammar reads as it is', () => { + const code = `unit Keep; + +interface + +type + TList = class + property Items[Index: Integer]: T read GetItem; default; + property Count: Integer read FCount; + Value: {$IFDEF CPU64} Int64 {$ELSE} Integer {$ENDIF}; + end; + TRec = record + X: Integer; + end; + +function Load: Integer; + +implementation + +function Load: Integer; +{$IFDEF PUREPASCAL} +var + I: Integer; +begin + Result := Read(I); +end; +{$ELSE} +asm + mov eax, 1 +end; +{$ENDIF} + +procedure Run; +begin + with List do + {$IFDEF CPU64} + if A = B then + {$ELSE} + if C = D then + {$ENDIF} + Exit; + if Ready then {$IFDEF X} Go(1); {$ELSE} Stay(1); {$ENDIF} + raise EAbort.Create('stop'); + WriteLn('raise; property X; quit:'); +end; + +end. +`; + expect(preParsePascal(code)).toBe(code); + }); + + it('links a routine past a conditional after its header to what it calls', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-delphi-syntax-')); + try { + fs.writeFileSync(path.join(dir, 'Lib.pas'), `unit Lib; + +interface + +procedure Translate(N: Integer); +procedure Emit(N: Integer); + +implementation + +procedure Translate(N: Integer); begin end; +procedure Emit(N: Integer); begin end; + +end. +`); + fs.writeFileSync(path.join(dir, 'Headers.pas'), HEADERS.replace('implementation\n', 'implementation\n\nuses Lib;\n')); + const cg = await CodeGraph.init(dir, { index: true }); + try { + const callers = (name: string) => { + const target = cg.getNodesByKind('function').find((n) => n.qualifiedName === name && n.filePath === 'Lib.pas'); + return target ? cg.getIncomingEdges(target.id).filter((e) => e.kind === 'calls').map((e) => cg.getNode(e.source)?.name) : []; + }; + expect(callers('Translate')).toEqual(['KeyUp']); + expect(callers('Emit')).toEqual(['Generate']); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/__tests__/pascal-for-in-var.test.ts b/__tests__/pascal-for-in-var.test.ts new file mode 100644 index 0000000000..0d782bdb2f --- /dev/null +++ b/__tests__/pascal-for-in-var.test.ts @@ -0,0 +1,237 @@ +/** + * A Delphi `for … in` loop that declares its variable inline (#2546): + * `for var lItem in AList do`, `for var lHandle: THandle in lHandles do` + * (Delphi 10.3 and later). + * + * The Pascal grammar's for-in rule takes a plain variable, so the whole loop + * was an ERROR node and no call in its body reached the graph. The extractor's + * preParse now blanks the inline `var` and its type, offsets kept, so the loop + * parses the way the grammar knows it. The resolver still finds the type: it + * reads a receiver's declaration from the source as written. + */ +import { describe, it, expect, beforeAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { getParser, initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { preParsePascal } from '../src/extraction/languages/pascal-preparse'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +// The issue's own unit. +const LOOP_PAS = `unit Loop; + +interface + +type + TItem = class + public + function DropIfEmpty: Boolean; + end; + +procedure WithVar(const AList: TArray); +procedure WithDecl(const AList: TArray); + +implementation + +function TItem.DropIfEmpty: Boolean; +begin + Result := True; +end; + +procedure WithVar(const AList: TArray); +begin + for var lItem in AList do + lItem.DropIfEmpty; +end; + +procedure WithDecl(const AList: TArray); +var + lItem: TItem; +begin + for lItem in AList do + lItem.DropIfEmpty; +end; + +end. +`; + +function callsFrom(code: string, qualifiedName: string, file = 'Loop.pas'): string[] { + const result = extractFromSource(file, code); + const from = result.nodes.find((n) => n.qualifiedName === qualifiedName); + return result.unresolvedReferences + .filter((r) => r.referenceKind === 'calls' && r.fromNodeId === from?.id) + .map((r) => `${r.referenceName}:${r.line}`); +} + +async function withIndex(files: Record, check: (cg: CodeGraph) => void): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-for-var-')); + try { + for (const [name, body] of Object.entries(files)) fs.writeFileSync(path.join(dir, name), body); + const cg = await CodeGraph.init(dir, { index: true }); + try { + check(cg); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +function callerNames(cg: CodeGraph, qualifiedName: string): string[] { + const target = cg.getNodesByKind('method').find((n) => n.qualifiedName === qualifiedName); + if (!target) return []; + return cg + .getIncomingEdges(target.id) + .filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.source)?.name ?? '?') + .sort(); +} + +describe('Delphi `for var … in` loops (#2546)', () => { + it('reads the call in the body of a `for var x in` loop, as for a declared loop variable', () => { + expect(callsFrom(LOOP_PAS, 'WithVar')).toEqual(['lItem.DropIfEmpty:24']); + expect(callsFrom(LOOP_PAS, 'WithDecl')).toEqual(['lItem.DropIfEmpty:32']); + }); + + it('links both routines to the method the issue names', async () => { + await withIndex({ 'Loop.pas': LOOP_PAS }, (cg) => { + expect(callerNames(cg, 'TItem::DropIfEmpty')).toEqual(['WithDecl', 'WithVar']); + }); + }); + + it('resolves a typed loop variable on its declared type, not on a namesake', async () => { + const code = `unit Recv; + +interface + +type + THandleBox = class + public + procedure Close; + end; + + TDecoy = class + public + procedure Close; + end; + +procedure UseTyped(const AHandles: TArray); + +implementation + +procedure THandleBox.Close; +begin +end; + +procedure TDecoy.Close; +begin +end; + +procedure UseTyped(const AHandles: TArray); +begin + for var lHandle: THandleBox in AHandles do + lHandle.Close; +end; + +end. +`; + await withIndex({ 'Recv.pas': code }, (cg) => { + expect(callerNames(cg, 'THandleBox::Close')).toEqual(['UseTyped']); + expect(callerNames(cg, 'TDecoy::Close')).toEqual([]); + }); + }); + + it('parses generic and dotted loop types, loops inside anonymous methods and one-line loops', () => { + const code = `unit Forms; + +interface + +implementation + +procedure Run(const D: TDictionary>; const L: TArray); +begin + for var P: TPair> in D do + P.Value.Clear; + for var Q: System.Generics.Collections.TPair in Pairs do Q.Free; + TParallel.For(0, 9, procedure(I: Integer) + begin + for var X in L do X.Reset; + end); + if Assigned(L) then for var Y in L do Y.Done; +end; + +end. +`; + const tree = getParser('pascal')!.parse(preParsePascal(code))!; + try { + expect(tree.rootNode.hasError).toBe(false); + } finally { + tree.delete(); + } + // `P.Value.Clear` keeps the bare method name the extractor gives a chain + // on a variable; the anonymous method's call belongs to `Run`. + expect(callsFrom(code, 'Run', 'Forms.pas')).toEqual([ + 'Clear:10', + 'Q.Free:11', + 'TParallel.For:12', + 'X.Reset:14', + 'Assigned:16', + 'Y.Done:16', + ]); + }); + + it('blanks only the inline `var` and its type, keeping every offset', () => { + const blank = (line: string, ...parts: string[]) => + parts.reduce((s, part) => s.replace(part, ' '.repeat(part.length)), line); + const lines = [ + 'for var lItem in AList do', + 'for var lHandle: THandle in lHandles do', + 'for var P: TPair>', + ' in D do', + ]; + const out = preParsePascal(lines.join('\r\n')); + expect(out.split('\r\n')).toEqual([ + blank(lines[0]!, 'var'), + blank(lines[1]!, 'var', ': THandle'), + blank(lines[2]!, 'var', ': TPair>'), + lines[3], + ]); + }); + + it('leaves every other `for` alone', () => { + const code = `unit Keep; + +interface + +type + TStringHelper = record helper for string + function Upper: string; + end; + +implementation + +procedure Run(const L: TArray); +var + N: Integer; +begin + for var I := 0 to 9 do Use(I); + for var J: Integer := 0 to 9 do Use(J); + for N in L do Use(N); + for N := High(L) downto 0 do Use(N); + // for var Old in L do Use(Old); + { for var Older in L do } + WriteLn('for var Text in L do'); +end; + +end. +`; + expect(preParsePascal(code)).toBe(code); + }); +}); diff --git a/__tests__/pascal-nested-routines.test.ts b/__tests__/pascal-nested-routines.test.ts new file mode 100644 index 0000000000..cc3cf43247 --- /dev/null +++ b/__tests__/pascal-nested-routines.test.ts @@ -0,0 +1,401 @@ +/** + * A Pascal routine may declare routines of its own between its header and + * its `begin`. Their bodies were never walked, so whatever they called was + * missing from the graph, and a call to one from the routine that declares + * it went by name to any routine elsewhere that shares it. Now a nested + * routine is a function qualified under the routine it is nested in + * (`THost::Fire::Inner`), its calls are its own, and its name means it + * wherever it is in scope: in the routine that declares it, in the routines + * nested after it there, and in its own body. Its locals and parameters are + * in scope for its body and for the routines nested inside it. + */ +import { describe, it, expect, beforeAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +async function withIndex(files: Record, check: (cg: CodeGraph) => void): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-nested-')); + try { + for (const [name, code] of Object.entries(files)) fs.writeFileSync(path.join(dir, name), code); + const cg = await CodeGraph.init(dir, { index: true }); + try { + check(cg); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +function routine(cg: CodeGraph, qualifiedName: string) { + const node = cg.getNodesByKind('method').concat(cg.getNodesByKind('function')).find((n) => n.qualifiedName === qualifiedName); + expect(node, qualifiedName).toBeDefined(); + return node!; +} + +/** What a routine calls, as `file:Target`, sorted. */ +function callsFrom(cg: CodeGraph, qualifiedName: string): string[] { + return [...new Set(cg.getOutgoingEdges(routine(cg, qualifiedName).id) + .filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.target)!) + .map((n) => `${path.basename(n.filePath)}:${n.qualifiedName}`))].sort(); +} + +// The routines elsewhere a nested routine's name could be taken for. +const LIB = `unit Lib; + +interface + +procedure Inner(N: Integer); +procedure Helper(N: Integer); +procedure Log(const S: string); + +implementation + +procedure Inner(N: Integer); begin end; +procedure Helper(N: Integer); begin end; +procedure Log(const S: string); begin end; + +end. +`; + +function host(implementation: string): string { + return `unit Host; + +interface + +uses Lib; + +type + THost = class + public + procedure Read; + procedure Fire; + function Count(O: TObject): Integer; + end; + + TOther = class + public + function Count(O: TObject): Integer; + end; + +implementation + +function THost.Count(O: TObject): Integer; begin Result := 0; end; +procedure THost.Read; begin end; +function TOther.Count(O: TObject): Integer; begin Result := 1; end; + +${implementation} + +end. +`; +} + +describe('Pascal nested routines', () => { + it('indexes a nested routine as a function of its own, called by the routine that declares it', async () => { + const main = host(`procedure THost.Fire; +var + Count: Integer; + + procedure Inner(N: Integer); + var + K: Integer; + begin + K := N + Count; + Helper(K); + end; + +begin + Inner(1); +end;`); + await withIndex({ 'Lib.pas': LIB, 'Host.pas': main }, (cg) => { + const inner = routine(cg, 'THost::Fire::Inner'); + expect(inner.kind).toBe('function'); + const contains = cg.getOutgoingEdges(routine(cg, 'THost::Fire').id).filter((e) => e.kind === 'contains'); + expect(contains.map((e) => e.target)).toContain(inner.id); + // Its call is its own, and the routine's call to `Inner` is to it, not to Lib's. + expect(callsFrom(cg, 'THost::Fire::Inner')).toEqual(['Lib.pas:Helper']); + expect(callsFrom(cg, 'THost::Fire')).toEqual(['Host.pas:THost::Fire::Inner']); + }); + }); + + it('follows calls through nesting levels, siblings above and the routine itself', async () => { + const main = host(`procedure THost.Fire; + + procedure First; + begin + Log('first'); + end; + + procedure Again(N: Integer); + begin + if N > 0 then + Again(N - 1); + First; + end; + + procedure Level1; + + procedure Level2; + begin + Again(2); + Level1; + end; + + begin + Level2; + end; + +begin + Level1; +end;`); + await withIndex({ 'Lib.pas': LIB, 'Host.pas': main }, (cg) => { + expect(callsFrom(cg, 'THost::Fire')).toEqual(['Host.pas:THost::Fire::Level1']); + expect(callsFrom(cg, 'THost::Fire::Level1')).toEqual(['Host.pas:THost::Fire::Level1::Level2']); + // A deeper routine reaches a routine nested further out, and the one it is in. + expect(callsFrom(cg, 'THost::Fire::Level1::Level2')).toEqual([ + 'Host.pas:THost::Fire::Again', + 'Host.pas:THost::Fire::Level1', + ]); + // A routine calls itself, and an earlier sibling, with or without parentheses. + expect(callsFrom(cg, 'THost::Fire::Again')).toEqual(['Host.pas:THost::Fire::Again', 'Host.pas:THost::Fire::First']); + expect(callsFrom(cg, 'THost::Fire::First')).toEqual(['Lib.pas:Log']); + }); + }); + + it('keeps a nested routine to the routine it is nested in', async () => { + const main = host(`procedure THost.Fire; + + procedure Outer; + + procedure Hidden(N: Integer); + begin + end; + + begin + Hidden(1); + end; + + procedure Peer; + begin + Hidden(2); + Helper(3); + end; + +begin + Outer; + Peer; +end; + +procedure Tidy; + + procedure Helper(N: Integer); + begin + end; + +begin + Helper(4); +end;`); + await withIndex({ 'Lib.pas': LIB, 'Host.pas': main }, (cg) => { + expect(callsFrom(cg, 'THost::Fire::Outer')).toEqual(['Host.pas:THost::Fire::Outer::Hidden']); + // Neither another routine's nested `Hidden` nor `Tidy`'s nested `Helper` is in scope here. + expect(callsFrom(cg, 'THost::Fire::Peer')).toEqual(['Lib.pas:Helper']); + expect(callsFrom(cg, 'Tidy')).toEqual(['Host.pas:Tidy::Helper']); + }); + }); + + it('keeps a later sibling out of scope, and a nested routine out of another unit', async () => { + const main = host(`procedure THost.Fire; + + procedure Early; + begin + Inner(1); + end; + + procedure Inner(N: Integer); + begin + end; + +begin + Early; + Inner(2); +end;`); + const other = `unit Other; + +interface + +uses Lib, Host; + +procedure Run; + +implementation + +procedure Run; +begin + Inner(3); +end; + +end. +`; + await withIndex({ 'Lib.pas': LIB, 'Host.pas': main, 'Other.pas': other }, (cg) => { + // Declared after it, the sibling isn't one `Early` can call: Lib's `Inner` is. + expect(callsFrom(cg, 'THost::Fire::Early')).toEqual(['Lib.pas:Inner']); + expect(callsFrom(cg, 'THost::Fire')).toEqual(['Host.pas:THost::Fire::Early', 'Host.pas:THost::Fire::Inner']); + expect(callsFrom(cg, 'Run')).toEqual(['Lib.pas:Inner']); + }); + }); + + it('gives a nested routine the class and the locals of the routines around it', async () => { + const main = host(`procedure THost.Fire; +var + Before: TObject; + + procedure Inner; + begin + Read; + Before.Free; + Helper(Count(Before)); + end; + +var + Log: Integer; + + procedure Second(Helper: Integer); + begin + Log := Helper + 1; + end; + +begin + Inner; + Second(1); +end;`); + await withIndex({ 'Lib.pas': LIB, 'Host.pas': main }, (cg) => { + // The class's `Read` and `Count` are in scope there, not `TOther`'s + // `Count`, and `Before` is the routine's local. + expect(callsFrom(cg, 'THost::Fire::Inner')).toEqual(['Host.pas:THost::Count', 'Host.pas:THost::Read', 'Lib.pas:Helper']); + // A parameter and a local of the routine around it hide Lib's routines. + expect(callsFrom(cg, 'THost::Fire::Second')).toEqual([]); + expect(callsFrom(cg, 'THost::Fire')).toEqual(['Host.pas:THost::Fire::Inner', 'Host.pas:THost::Fire::Second']); + }); + }); + + it('takes a routine a broken parse nests, at the start of its line, for one of the unit', async () => { + // The missing `;` after `procedure First` leaves `Second` inside it in the tree. + const script = `procedure First +begin +end; + +function Second : Integer +var + i : Integer; +begin + i := 1; + Result := Helper(i); +end; + +type + TThing = class + method Run : String; + method Stop; + end; + +method TThing.Run : String +begin + Result := ''; +end; + +method TThing.Stop +const + c = 2; +begin +end; +`; + const lib = `unit Lib; + +interface + +function Helper(N: Integer): Integer; + +implementation + +function Helper(N: Integer): Integer; begin Result := N; end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Script.pas': script }, (cg) => { + const names = cg.getNodesByKind('function').filter((n) => n.filePath === 'Script.pas').map((n) => n.qualifiedName); + expect(names).toContain('Second'); + expect(names).not.toContain('First::Second'); + expect(callsFrom(cg, 'Second')).toEqual(['Lib.pas:Helper']); + }); + }); + + it('takes a nested function\'s own name for its result, and a nested name like a System routine\'s for the nested one', async () => { + const main = host(`procedure THost.Fire; + + function Calc: Integer; + begin + Calc := 5; + end; + + procedure Delete(Index: Integer); + begin + end; + +begin + if Calc > 0 then + Delete(1); +end;`); + await withIndex({ 'Lib.pas': LIB, 'Host.pas': main }, (cg) => { + expect(callsFrom(cg, 'THost::Fire::Calc')).toEqual([]); + expect(callsFrom(cg, 'THost::Fire')).toEqual(['Host.pas:THost::Fire::Calc', 'Host.pas:THost::Fire::Delete']); + }); + }); + + it('gives a nested routine the constants it reads, not those its routine\'s body reads after it', async () => { + const code = `unit Values; + +interface + +const + MAX_DEPTH = 10; + OP_EXACTLY = 3; + OP_ANY = 4; + +function Parse(N: Integer): Integer; + +implementation + +function Parse(N: Integer): Integer; + + function Unquote(C: Integer): Integer; + begin + Result := C + OP_ANY; + end; + +begin + if N > MAX_DEPTH then + Result := OP_EXACTLY + else + Result := Unquote(N); +end; + +end. +`; + await withIndex({ 'Values.pas': code }, (cg) => { + const reads = cg.getOutgoingEdges(routine(cg, 'Parse::Unquote').id) + .filter((e) => e.kind === 'references') + .map((e) => cg.getNode(e.target)!.name); + expect(reads).toEqual(['OP_ANY']); + }); + }); +}); diff --git a/__tests__/pascal-param-attributes.test.ts b/__tests__/pascal-param-attributes.test.ts new file mode 100644 index 0000000000..b95ce1165e --- /dev/null +++ b/__tests__/pascal-param-attributes.test.ts @@ -0,0 +1,232 @@ +/** + * An attribute on a Delphi routine parameter (#2545): `const [ref] AValue: + * Integer`, `[ref] const Source: TVarData`, DelphiMVCFramework's + * `const [MVCFromBody] Person: TPerson`. + * + * The Pascal grammar's parameter rule takes no attribute, so the parse broke + * at the `[`: the method was lost, the parameter after the attribute became a + * field of the class, and every member declared after it was missing until + * the next class. The extractor's preParse now blanks such an attribute, + * offsets kept, before the grammar parses the unit. + */ +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { preParsePascal } from '../src/extraction/languages/pascal-preparse'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +// The issue's own unit, with its bodies calling `Self.Before`. +const MIN_PAS = `unit Min; + +interface + +type + TMin = class + public + procedure Before; + procedure WithRef(const [ref] AValue: Integer; const AOther: string); + procedure After; + end; + + TNext = class + public + procedure Other; + end; + +const + CAfterClass = 1; + +implementation + +procedure TMin.Before; +begin +end; + +procedure TMin.WithRef(const [ref] AValue: Integer; const AOther: string); +begin + Self.Before; +end; + +procedure TMin.After; +begin + Self.Before; +end; + +procedure TNext.Other; +begin +end; + +end. +`; + +function members(code: string, file = 'Min.pas') { + const result = extractFromSource(file, code); + const of = (kind: string) => + result.nodes.filter((n) => n.kind === kind).map((n) => `${n.qualifiedName}:${n.startLine}`).sort(); + return { result, methods: of('method'), fields: of('field'), constants: of('constant') }; +} + +describe('Delphi parameter attributes (#2545)', () => { + it('keeps every class member after a `const [ref]` parameter', () => { + const { methods, fields, constants } = members(MIN_PAS); + expect(methods).toEqual([ + 'TMin::After:10', + 'TMin::Before:8', + 'TMin::WithRef:9', + 'TNext::Other:15', + ]); + // The parameter after the attribute is not a field of the class. + expect(fields).toEqual([]); + expect(constants).toEqual(['CAfterClass:19']); + }); + + it('keeps the member after a `[ref]` parameter that ends its list on its own line', () => { + const code = MIN_PAS + .replace('procedure WithRef(const [ref] AValue: Integer; const AOther: string);', 'procedure WithRef(const [ref] AValue: Integer);') + .replace('procedure TMin.WithRef(const [ref] AValue: Integer; const AOther: string);', 'procedure TMin.WithRef(const [ref] AValue: Integer);'); + const { methods } = members(code); + expect(methods).toEqual(['TMin::After:10', 'TMin::Before:8', 'TMin::WithRef:9', 'TNext::Other:15']); + }); + + it('keeps the attribute in the signature, as the source writes it', () => { + const { result } = members(MIN_PAS); + const withRef = result.nodes.find((n) => n.qualifiedName === 'TMin::WithRef'); + expect(withRef?.signature).toBe('(const [ref] AValue: Integer; const AOther: string)'); + }); + + it('reads the calls in the body of a routine whose header has an attribute', () => { + const { result } = members(MIN_PAS); + const withRef = result.nodes.find((n) => n.qualifiedName === 'TMin::WithRef'); + const calls = result.unresolvedReferences + .filter((r) => r.referenceKind === 'calls' && r.fromNodeId === withRef?.id) + .map((r) => `${r.referenceName}:${r.line}`); + expect(calls).toEqual(['Self.Before:29']); + }); + + it('takes attributes before or after the modifier, with arguments, stacked or spanning lines', () => { + const code = `unit People; + +interface + +type + TRec = record + A: Integer; + class operator Implicit(const [ref] Src: TRec): Integer; + procedure CopyFrom([ref] const Source: TRec); + procedure AfterRec; + end; + + TPeopleController = class(TMVCController) + public + [MVCPath('/people')] + [MVCHTTPMethod([httpPOST])] + function CreatePerson([MVCFromBody] const Person: TPerson): IMVCResponse; + function Search(const [MVCFromQueryString('q', '')] Query: string; + [MVCFromHeader('X-Page')] [MVCInject] Page: Integer; + const [MVCFromQueryString('order', + 'name')] Order: string): IMVCResponse; + function Find(const [ref] Key: TKey): T; + procedure Last; + end; + +implementation + +end. +`; + const { result, methods, fields } = members(code, 'People.pas'); + // A method's node starts at the attributes above it (CreatePerson's at 15). + expect(methods).toEqual([ + 'TPeopleController::CreatePerson:15', + 'TPeopleController::Find:22', + 'TPeopleController::Last:23', + 'TPeopleController::Search:18', + 'TRec::AfterRec:10', + 'TRec::CopyFrom:9', + 'TRec::Implicit:8', + ]); + expect(fields).toEqual(['TRec::A:7']); + const signatureOf = (qn: string) => result.nodes.find((n) => n.qualifiedName === qn)?.signature; + expect(signatureOf('TPeopleController::CreatePerson')).toBe('([MVCFromBody] const Person: TPerson): IMVCResponse'); + expect(signatureOf('TRec::CopyFrom')).toBe('([ref] const Source: TRec)'); + }); + + it('blanks only parameter attributes, keeping every offset', () => { + const query = "[MVCFromQueryString('q', '')]"; + const page = "[MVCFromHeader('X-Page')]"; + const code = [`procedure Search(const ${query} Query: string;`, ` ${page}`, ' Page: Integer);'].join('\r\n'); + const out = preParsePascal(code); + expect(out.length).toBe(code.length); + expect(out.split('\r\n')).toEqual([ + `procedure Search(const ${' '.repeat(query.length)} Query: string;`, + ` ${' '.repeat(page.length)}`, + ' Page: Integer);', + ]); + }); + + it('leaves brackets alone everywhere else', () => { + const code = `unit Keep; + +interface + +type + TOwner = class + private + [Weak] FOwner: TObject; + FItems: array[0..1] of Integer; + public + [MVCPath('/x')] + procedure Get(const Options: TOptions = [ooA, ooB]; Count: Integer = 1); + property Items[Index: Integer]: Integer read GetItem; + end; + +implementation + +{ procedure Old(const [ref] A: Integer); } +// function Older([ref] const A: Integer): Integer; + +procedure TOwner.Get(const Options: TOptions = [ooA, ooB]; Count: Integer = 1); +begin + Foo([1, 2]); + Bar(Format('%d', [Count]), [ooA]); + WriteLn('procedure Fake(const [ref] A: Integer)'); + FItems[0] := Ord(ooA in [ooA]); + Count := Operator([1, 2]); // Delphi doesn't reserve \`operator\` +end; + +end. +`; + expect(preParsePascal(code)).toBe(code); + }); + + it('indexes the issue unit with TMin.After resolvable and no field from the parameter', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-attr-')); + try { + fs.writeFileSync(path.join(dir, 'Min.pas'), MIN_PAS); + const cg = await CodeGraph.init(dir, { index: true }); + try { + const after = cg.getNodesByKind('method').find((n) => n.qualifiedName === 'TMin::After'); + expect(after).toBeDefined(); + expect(cg.getNodesByKind('field').map((n) => n.name)).toEqual([]); + const before = cg.getNodesByKind('method').find((n) => n.qualifiedName === 'TMin::Before')!; + const callers = cg + .getIncomingEdges(before.id) + .filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.source)?.name) + .sort(); + expect(callers).toEqual(['After', 'WithRef']); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/__tests__/pascal-parenless-calls.test.ts b/__tests__/pascal-parenless-calls.test.ts new file mode 100644 index 0000000000..04bf175634 --- /dev/null +++ b/__tests__/pascal-parenless-calls.test.ts @@ -0,0 +1,749 @@ +/** + * Delphi calls a parameterless routine without parentheses (#2514): + * `Before;`, `Result := FProvider.Value + 1`, `if Ready then`, and a method + * given to an event, `OnGet := FProvider.Value`. None of these was linked, so + * callers, callees and impact missed most call sites in real Delphi code. + * The same spelling reads a field, a property, a variable or a constant, so a + * name is linked only when Delphi's scope rules reach a method or function of + * that name: the routine's own declarations, its class and ancestors, the + * unit, then the units it uses. Data of that name links nothing. + */ +import { describe, it, expect, beforeAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +async function withIndex(files: Record, check: (cg: CodeGraph) => void): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-parenless-')); + try { + for (const [name, code] of Object.entries(files)) fs.writeFileSync(path.join(dir, name), code); + const cg = await CodeGraph.init(dir, { index: true }); + try { + check(cg); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** The calls and references a routine makes, as `kind Target::Name`. */ +function edgesFrom(cg: CodeGraph, qualifiedName: string): string[] { + const source = cg.getNodesByKind('function').concat(cg.getNodesByKind('method')).find((n) => n.qualifiedName === qualifiedName); + expect(source, qualifiedName).toBeDefined(); + return cg + .getOutgoingEdges(source!.id) + .filter((e) => e.kind === 'calls' || e.kind === 'references') + .map((e) => `${e.kind} ${cg.getNode(e.target)?.qualifiedName ?? '?'}`) + .sort(); +} + +// The issue's own unit. +const UNIT_A = `unit UnitA; + +interface + +type + TGetIntEvent = function: Integer of object; + + TProvider = class + public + function Value: Integer; + function ValueParens(): Integer; + end; + + TConsumer = class + private + FProvider: TProvider; + FOnGet: TGetIntEvent; + public + function UseNoParens: Integer; + function UseParens: Integer; + function UseEvent: Integer; + procedure Wire; + property OnGet: TGetIntEvent read FOnGet write FOnGet; + end; + +implementation + +function TProvider.Value: Integer; +begin + Result := 1; +end; + +function TProvider.ValueParens(): Integer; +begin + Result := 2; +end; + +function TConsumer.UseNoParens: Integer; +begin + Result := FProvider.Value + 1; +end; + +function TConsumer.UseParens: Integer; +begin + Result := FProvider.ValueParens() + 1; +end; + +procedure TConsumer.Wire; +begin + OnGet := FProvider.Value; +end; + +function TConsumer.UseEvent: Integer; +begin + Result := FOnGet; +end; + +end. +`; + +describe('Delphi calls without parentheses (#2514)', () => { + it('links the issue\'s paren-less call as a call and the event\'s method as a reference', async () => { + await withIndex({ 'UnitA.pas': UNIT_A }, (cg) => { + expect(edgesFrom(cg, 'TConsumer::UseNoParens')).toEqual(['calls TProvider::Value']); + expect(edgesFrom(cg, 'TConsumer::UseParens')).toEqual(['calls TProvider::ValueParens']); + expect(edgesFrom(cg, 'TConsumer::Wire')).toEqual(['references TProvider::Value']); + // `FOnGet` is a field: reading it calls whatever was assigned, which the code doesn't say. + expect(edgesFrom(cg, 'TConsumer::UseEvent')).toEqual([]); + }); + }); + + it('links a bare name to the class\'s method or the unit\'s routine, as a statement or for its value', async () => { + const code = `unit Unit1; + +interface + +type + TMin = class + public + procedure Before; + procedure After; + function Value: Integer; + procedure UseValue; + end; + +procedure FreeProc; + +implementation + +procedure FreeProc; +begin +end; + +procedure TMin.Before; +begin +end; + +procedure TMin.After; +begin + Before; + freeproc; +end; + +function TMin.Value: Integer; +begin + Result := 1; +end; + +procedure TMin.UseValue; +var + X: Integer; +begin + X := Value; + if Value > 0 then + Before; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + // Pascal names are case-insensitive: `freeproc` is `FreeProc`. + expect(edgesFrom(cg, 'TMin::After')).toEqual(['calls FreeProc', 'calls TMin::Before']); + expect(edgesFrom(cg, 'TMin::UseValue')).toEqual(['calls TMin::Before', 'calls TMin::Value', 'calls TMin::Value']); + }); + }); + + it('never links a name that reads a field, property, variable, constant or parameter', async () => { + const base = `unit Base; + +interface + +type + TBase = class + protected + FReady: Boolean; + public + property Ready: Boolean read FReady; + end; + +implementation + +end. +`; + const decoys = `unit Decoys; + +interface + +function Ready: Boolean; +function Count: Integer; +function Items: Integer; +function Limit: Integer; +function Total: Integer; +function FItems: Integer; + +implementation + +function Ready: Boolean; begin Result := True; end; +function Count: Integer; begin Result := 0; end; +function Items: Integer; begin Result := 0; end; +function Limit: Integer; begin Result := 0; end; +function Total: Integer; begin Result := 0; end; +function FItems: Integer; begin Result := 0; end; + +end. +`; + const main = `unit Main; + +interface + +uses Decoys, Base; + +type + TList = class(TBase) + private + FItems: Integer; + function GetCount: Integer; + public + property Count: Integer read GetCount; + function Run(Total: Integer): Integer; + end; + +const + Limit = 10; + +var + Items: Integer; + +implementation + +function TList.GetCount: Integer; +begin + Result := 0; +end; + +function TList.Run(Total: Integer): Integer; +begin + Result := Count + Items + Limit + Total + FItems; + if Ready then + Result := 0; +end; + +end. +`; + await withIndex({ 'Base.pas': base, 'Decoys.pas': decoys, 'Main.pas': main }, (cg) => { + // `Ready` is the inherited property, which the unit's own + // `uses Decoys` doesn't reach past. + expect(edgesFrom(cg, 'TList::Run')).toEqual([]); + }); + }); + + it('follows the class\'s ancestors and the units it uses, the last one first', async () => { + const lib = `unit Lib; + +interface + +type + TBase = class + public + procedure Setup; + end; + +procedure Helper; + +implementation + +procedure TBase.Setup; begin end; +procedure Helper; begin end; + +end. +`; + const other = `unit Other; + +interface + +procedure Helper; + +implementation + +procedure Helper; begin end; + +end. +`; + const unused = `unit Unused; + +interface + +procedure Teardown; + +implementation + +procedure Teardown; begin end; + +end. +`; + const main = `unit Main; + +interface + +uses Other, Lib; + +type + TChild = class(TBase) + public + procedure Run; + end; + +implementation + +procedure TChild.Run; +begin + Setup; + Helper; + Teardown; +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Other.pas': other, 'Unused.pas': unused, 'Main.pas': main }, (cg) => { + const helpers = cg.getNodesByKind('function').filter((n) => n.name === 'Helper'); + const libHelper = helpers.find((n) => n.filePath.endsWith('Lib.pas')); + const run = cg.getNodesByKind('method').find((n) => n.qualifiedName === 'TChild::Run')!; + const targets = cg.getOutgoingEdges(run.id).filter((e) => e.kind === 'calls').map((e) => e.target); + // `Lib` is used last, so its `Helper` hides `Other`'s; `Unused` isn't used at all. + expect(targets).toContain(libHelper!.id); + expect(edgesFrom(cg, 'TChild::Run')).toEqual(['calls Helper', 'calls TBase::Setup']); + }); + }); + + it('links a function used as a receiver, through fields, parameters and locals of a declared type', async () => { + // The issue's real project: `FProjectPanel.ActiveModelSeries.Visible`. + const code = `unit Unit1; + +interface + +type + TSeries = class + public + procedure BeginUpdate; + function Visible: Boolean; + end; + + TPanel = class + public + function ActiveSeries: TSeries; + end; + + TForm1 = class + private + FPanel: TPanel; + function CurrentPanel: TPanel; + public + procedure Run; + procedure RunWith(APanel: TPanel); + end; + +implementation + +procedure TSeries.BeginUpdate; begin end; +function TSeries.Visible: Boolean; begin Result := True; end; +function TPanel.ActiveSeries: TSeries; begin Result := nil; end; +function TForm1.CurrentPanel: TPanel; begin Result := FPanel; end; + +procedure TForm1.Run; +begin + FPanel.ActiveSeries.BeginUpdate; + if CurrentPanel.ActiveSeries.Visible then + Exit; +end; + +procedure TForm1.RunWith(APanel: TPanel); +var + S: TSeries; +begin + S := APanel.ActiveSeries; + if S.Visible then + Exit; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + expect(edgesFrom(cg, 'TForm1::Run')).toEqual([ + 'calls TForm1::CurrentPanel', + 'calls TPanel::ActiveSeries', + 'calls TPanel::ActiveSeries', + 'calls TSeries::BeginUpdate', + 'calls TSeries::Visible', + ]); + expect(edgesFrom(cg, 'TForm1::RunWith')).toEqual(['calls TPanel::ActiveSeries', 'calls TSeries::Visible']); + }); + }); + + it('gives a method to an event or an argument as a reference, a method pointer', async () => { + const code = `unit Unit1; + +interface + +type + TNotify = procedure of object; + + TButton = class + public + OnClick: TNotify; + procedure Click; + end; + + TItem = class + public + constructor Create(AOwner: TObject); + end; + + TForm1 = class + private + FButton: TButton; + procedure ButtonClick; + procedure Register(Handler: TNotify); + function Title: string; + function Compare(A, B: Integer): Integer; + procedure Sort(Fn: TCompare); + public + procedure Setup; + procedure SetupWith; + end; + +implementation + +constructor TItem.Create(AOwner: TObject); begin end; +procedure TButton.Click; begin end; +procedure TForm1.ButtonClick; begin end; +procedure TForm1.Register(Handler: TNotify); begin end; +function TForm1.Title: string; begin Result := ''; end; +function TForm1.Compare(A, B: Integer): Integer; begin Result := A - B; end; +procedure TForm1.Sort(Fn: TCompare); begin end; + +procedure TForm1.Setup; +var + Item: TObject; +begin + FButton.OnClick := ButtonClick; + Register(ButtonClick); + // A function that needs arguments, written without them, is its pointer. + Sort(Compare); + // A constructor written without the arguments it needs is an inherited + // one outside the project, TObject.Create. + Item := TItem.Create; +end; + +procedure TForm1.SetupWith; +var + S: string; +begin + with FButton do + begin + Click; + OnClick := ButtonClick; + S := Title; + end; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + expect(edgesFrom(cg, 'TForm1::Setup')).toEqual([ + 'calls TForm1::Register', + 'calls TForm1::Sort', + 'references TForm1::ButtonClick', + 'references TForm1::ButtonClick', + 'references TForm1::Compare', + ]); + // In a `with` body a name may be the `with` object's member: `Click` + // is the button's, and `Title` could be. Only the event's method is + // linked there. + expect(edgesFrom(cg, 'TForm1::SetupWith')).toEqual(['references TForm1::ButtonClick']); + }); + }); + + it('finds the class of a generic class\'s method, whose node spells its constraints', async () => { + const code = `unit Unit1; + +interface + +type + TBox = class + public + procedure Clear; + procedure Reset; + end; + + TOther = class + public + procedure Clear; + procedure Reset; + end; + +implementation + +procedure TOther.Clear; begin end; +procedure TOther.Reset; begin end; +procedure TBox.Clear; begin end; + +procedure TBox.Reset; +begin + Clear; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + const reset = cg.getNodesByKind('method').filter((n) => n.name === 'Reset'); + const targets = reset.flatMap((r) => cg.getOutgoingEdges(r.id).filter((e) => e.kind === 'calls')) + .map((e) => cg.getNode(e.target)?.qualifiedName); + expect(targets).toEqual(['TBox::Clear']); + }); + }); + + it('links a form\'s event handler named after its component, and a C library\'s function', async () => { + const code = `unit Unit1; + +interface + +type + TNotify = procedure(Sender: TObject) of object; + + TButton = class + public + OnClick: TNotify; + end; + + TForm1 = class + private + btnOK: TButton; + public + procedure btnOKClick(Sender: TObject); + procedure Setup; + function Check: Integer; + end; + +function ERR_get_error: Cardinal; cdecl; +function mach_absolute_time: UInt64; cdecl; + +implementation + +function ERR_get_error: Cardinal; cdecl; +begin + Result := 0; +end; + +function mach_absolute_time: UInt64; cdecl; +begin + Result := 0; +end; + +procedure TForm1.btnOKClick(Sender: TObject); +begin +end; + +procedure TForm1.Setup; +begin + btnOK.OnClick := btnOKClick; +end; + +function TForm1.Check: Integer; +begin + Result := ERR_get_error + mach_absolute_time; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + expect(edgesFrom(cg, 'TForm1::Setup')).toEqual(['references TForm1::btnOKClick']); + expect(edgesFrom(cg, 'TForm1::Check')).toEqual(['calls ERR_get_error', 'calls mach_absolute_time']); + }); + }); + + it('reads an ancestor through a class the unit declares ahead (`TBase = class;`)', async () => { + const code = `unit Unit1; + +interface + +type + TChild = class; + + TBase = class + public + function Ready: Boolean; + end; + + TChild = class(TBase) + public + procedure Run; + end; + +implementation + +function TBase.Ready: Boolean; +begin + Result := True; +end; + +procedure TChild.Run; +begin + if Ready then + Exit; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + expect(edgesFrom(cg, 'TChild::Run')).toEqual(['calls TBase::Ready']); + }); + }); + + it('links the calls of a unit\'s initialization and finalization sections', async () => { + const code = `unit Unit1; + +interface + +procedure Setup; +procedure Teardown; + +implementation + +procedure Setup; +begin +end; + +procedure Teardown; +begin +end; + +initialization + Setup; + +finalization + Teardown; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + const file = cg.getNodesByKind('file').find((n) => n.filePath.endsWith('Unit1.pas'))!; + const targets = cg.getOutgoingEdges(file.id).filter((e) => e.kind === 'calls').map((e) => cg.getNode(e.target)?.name).sort(); + expect(targets).toEqual(['Setup', 'Teardown']); + }); + }); + + it('takes a name the class\'s code assigns or passes to Inc for a variable, as Delphi does', async () => { + // `Run` is a field of a base class the project doesn't have (a VCL + // highlighter's, say): the unit's `Run` function is not what it reads. + const helpers = `unit Helpers; + +interface + +function Run: Integer; +function Limit: Integer; + +implementation + +function Run: Integer; begin Result := 0; end; +function Limit: Integer; begin Result := 0; end; + +end. +`; + const main = `unit Main; + +interface + +uses Helpers; + +type + TMyHighlighter = class(TSynCustomHighlighter) + public + procedure Next; + procedure Reset; + function Count: Integer; + end; + +implementation + +procedure TMyHighlighter.Next; +var + C: Char; +begin + Inc(Run); + C := FLine[Run]; + if Limit > 0 then + Exit; +end; + +procedure TMyHighlighter.Reset; +begin + Limit := 0; +end; + +function TMyHighlighter.Count: Integer; +begin + Count := Run; +end; + +end. +`; + await withIndex({ 'Helpers.pas': helpers, 'Main.pas': main }, (cg) => { + expect(edgesFrom(cg, 'TMyHighlighter::Next')).toEqual([]); + expect(edgesFrom(cg, 'TMyHighlighter::Count')).toEqual([]); + }); + }); + + it('leaves out names a routine declares itself, even where a method shares the name', async () => { + const code = `unit Unit1; + +interface + +type + TMin = class + public + function Value: Integer; + function Items: Integer; + procedure Run(Value: Integer); + end; + +implementation + +function TMin.Value: Integer; begin Result := 1; end; +function TMin.Items: Integer; begin Result := 1; end; + +procedure TMin.Run(Value: Integer); +var + X: Integer; +begin + X := Value; + for var Items in [1, 2] do + X := X + Items; +end; + +end. +`; + await withIndex({ 'Unit1.pas': code }, (cg) => { + expect(edgesFrom(cg, 'TMin::Run')).toEqual([]); + }); + }); +}); diff --git a/__tests__/pascal-system-calls.test.ts b/__tests__/pascal-system-calls.test.ts new file mode 100644 index 0000000000..a3741e623d --- /dev/null +++ b/__tests__/pascal-system-calls.test.ts @@ -0,0 +1,783 @@ +/** + * A Pascal call to a name the compiler or the System unit gives meaning to, + * like `SizeOf`, `TypeInfo`, `High`, `Inc`, `Move` or a typecast to a built-in + * type such as `Pointer(P)`, linked to any project member of that name, and + * the check that should have stopped it compared case: Pascal ignores it, so + * `length(S)` and `dec(N)` went to a record's `length` field and a variable + * named `dec`. A call that resolved to a field, a property or a variable + * linked too, though only one holding a routine can run. Now such a call + * links only to a declaration its scope reaches: a member of the class + * (`Delete(I)` in a list class), or a routine of the unit or the units it + * uses, and data only when its declared type is a routine's. That scope + * takes in the include files a unit pulls in and the units whose header the + * parser lost, and stops at the calling routine's own parameters and locals. + */ +import { describe, it, expect, beforeAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; + +beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['pascal']); +}); + +async function withIndex(files: Record, check: (cg: CodeGraph) => void): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-pascal-system-')); + try { + for (const [name, code] of Object.entries(files)) fs.writeFileSync(path.join(dir, name), code); + const cg = await CodeGraph.init(dir, { index: true }); + try { + check(cg); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** The calls a routine makes, as `line: Target::Name`. */ +function callsFrom(cg: CodeGraph, qualifiedName: string): string[] { + const source = cg.getNodesByKind('method').concat(cg.getNodesByKind('function')).find((n) => n.qualifiedName === qualifiedName); + expect(source, qualifiedName).toBeDefined(); + return cg + .getOutgoingEdges(source!.id) + .filter((e) => e.kind === 'calls') + .map((e) => `${e.line}: ${cg.getNode(e.target)?.qualifiedName ?? '?'}`) + .sort(); +} + +// Namesakes of the System unit's names, elsewhere in the project. +const LIB = `unit Lib; + +interface + +type + TRec = record + length: Integer; + high: Integer; + end; + + TOther = class + private + FObj: TObject; + public + function TypeInfo: Pointer; + procedure Move(A, B: Integer); + function SizeOf: Integer; + property Pointer: TObject read FObj; + end; + +var + dec: Integer; + +implementation + +function TOther.TypeInfo: Pointer; begin Result := nil; end; +procedure TOther.Move(A, B: Integer); begin end; +function TOther.SizeOf: Integer; begin Result := 0; end; + +end. +`; + +describe('Pascal calls to the System unit and to data', () => { + it('links an intrinsic, a System routine or a built-in typecast to no project namesake, whatever its case', async () => { + const main = `unit Main; + +interface + +uses Lib; + +type + TList2 = class + public + procedure Run(const S: string; var A: array of Integer); + end; + +implementation + +procedure TList2.Run(const S: string; var A: array of Integer); +var + P: Pointer; + N: Integer; +begin + N := length(S) + High(A) + SIZEOF(N); + dec(N); + P := TypeInfo(TList2); + Move(P, P, 0); + P := Pointer(Self); +end; + +end. +`; + await withIndex({ 'Lib.pas': LIB, 'Main.pas': main }, (cg) => { + expect(callsFrom(cg, 'TList2::Run')).toEqual([]); + }); + }); + + it('takes `system.` for the System unit whatever its case, and a lowercase `data.` for a variable', async () => { + const main = `unit Main; + +interface + +uses Lib; + +type + TStore = class + public + function GetObjectFrom(const Name: string): TObject; + end; + +procedure Run(data: TStore); + +implementation + +function TStore.GetObjectFrom(const Name: string): TObject; begin Result := nil; end; + +procedure Run(data: TStore); +var + P: Pointer; +begin + P := system.TypeInfo(TStore); + data.GetObjectFrom('setting'); +end; + +end. +`; + await withIndex({ 'Lib.pas': LIB, 'Main.pas': main }, (cg) => { + expect(callsFrom(cg, 'Run')).toEqual(['24: TStore::GetObjectFrom']); + }); + }); + + it('links one to the declaration that shadows it in the call\'s scope', async () => { + const main = `unit Main; + +interface + +uses Lib; + +type + TList2 = class + public + procedure Delete(Index: Integer); + procedure Clear; + end; + +function Length(const S: string): Integer; +procedure Tidy; + +implementation + +function Length(const S: string): Integer; begin Result := 0; end; + +procedure TList2.Delete(Index: Integer); begin end; + +procedure TList2.Clear; +begin + Delete(0); +end; + +procedure Tidy; +begin + Length('x'); +end; + +end. +`; + await withIndex({ 'Lib.pas': LIB, 'Main.pas': main }, (cg) => { + // The list's own `Delete`, and the unit's own `Length`. + expect(callsFrom(cg, 'TList2::Clear')).toEqual(['25: TList2::Delete']); + expect(callsFrom(cg, 'Tidy')).toEqual(['30: Length']); + }); + }); + + it('takes a unit\'s stand-in declared under a condition for the System unit\'s own', async () => { + const lib = `unit Lib; + +interface + +type +{$IFNDEF UNICODE} + UnicodeString = WideString; +{$ENDIF} + RawUtf8 = type AnsiString; + +{$IFNDEF COMPILER_3_UP} +procedure Assert(Expr: Boolean); { stub for Delphi 2 } +{$ENDIF} +function UpperCase(const S: RawUtf8): RawUtf8; + +implementation + +{$IFNDEF COMPILER_3_UP} +procedure Assert(Expr: Boolean); begin end; +{$ENDIF} +function UpperCase(const S: RawUtf8): RawUtf8; begin Result := S; end; + +end. +`; + const main = `unit Main; + +interface + +uses SysUtils, Lib; + +procedure Run(P: Pointer; const S: RawUtf8); + +implementation + +procedure Run(P: Pointer; const S: RawUtf8); +begin + Assert(P <> nil); + if UnicodeString(P) <> '' then + UpperCase(S); +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Main.pas': main }, (cg) => { + // The used unit's `UpperCase` hides SysUtils' one; `Assert` and + // `UnicodeString` are the compiler's. + expect(callsFrom(cg, 'Run')).toEqual(['15: UpperCase']); + const run = cg.getNodesByKind('function').find((n) => n.qualifiedName === 'Run')!; + expect(cg.getOutgoingEdges(run.id).filter((e) => e.kind === 'references').map((e) => cg.getNode(e.target)?.name)) + .not.toContain('UnicodeString'); + }); + }); + + it('reaches a class\'s ancestor through a type alias', async () => { + const base = `unit Base; + +interface + +type + TSearchEvent = procedure(Sender: TObject; const Text: string) of object; + + TBaseTree = class + private + FOnSearch: TSearchEvent; + public + function Ready: Boolean; + property OnSearch: TSearchEvent read FOnSearch write FOnSearch; + end; + +implementation + +function TBaseTree.Ready: Boolean; begin Result := True; end; + +end. +`; + const main = `unit Main; + +interface + +uses Base; + +type + TAncestor = TBaseTree; + + TStringTree = class(TAncestor) + public + procedure Search(const Text: string); + end; + +implementation + +procedure TStringTree.Search(const Text: string); +begin + if Ready then + OnSearch(Self, Text); +end; + +end. +`; + await withIndex({ 'Base.pas': base, 'Main.pas': main }, (cg) => { + const search = cg.getNodesByKind('method').find((n) => n.qualifiedName === 'TStringTree::Search')!; + const targets = cg.getOutgoingEdges(search.id).filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.target)?.qualifiedName).sort(); + // `TAncestor = TBaseTree;` names the class the tree descends from: its + // `Ready` runs, and so does what its `OnSearch` event holds. + expect(targets).toEqual(['TBaseTree::OnSearch', 'TBaseTree::Ready']); + }); + }); + + it('links a name its scope doesn\'t declare to the one routine-holding field of that name, unless the routine declares it', async () => { + const lib = `unit Lib; + +interface + +type + THandleAlloc = function(Parent: Pointer; var Handle: Pointer): Integer; + THasher = function(Crc: Cardinal; Buf: Pointer; Len: Cardinal): Cardinal; + + TOciLib = class + public + HandleAlloc: THandleAlloc; + end; + + TDynHasher = class + private + FHasher: THasher; + public + property Hasher: THasher read FHasher; + property Direct: THasher read FHasher; + end; + +implementation + +end. +`; + const main = `unit Main; + +interface + +uses Lib; + +type + TConn = class + private + FOci: TOciLib; + FEnv: Pointer; + public + procedure Connect; + end; + +function Hash(Item: Pointer; Hasher: THasher): Cardinal; +procedure Test; + +implementation + +procedure TConn.Connect; +begin + with FOci do + HandleAlloc(nil, FEnv); +end; + +function Hash(Item: Pointer; Hasher: THasher): Cardinal; +begin + Result := Hasher(0, Item, 4); +end; + +procedure Test; + + procedure Direct(const Url: string); + begin + end; + +begin + Direct('/root'); +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Main.pas': main }, (cg) => { + // The `with` object's member, which the class's scope doesn't hold. + expect(callsFrom(cg, 'TConn::Connect')).toEqual(['24: TOciLib::HandleAlloc']); + // A parameter and a nested routine, not the properties named so. + expect(callsFrom(cg, 'Hash')).toEqual([]); + expect(callsFrom(cg, 'Test')).toEqual(['39: Test::Direct']); + }); + }); + + it('takes a parameter or a local of the calling routine for what its name calls', async () => { + const lib = `unit Lib; + +interface + +type + TLoad = procedure(Data: Pointer); + +var + load: TLoad; + +implementation + +end. +`; + const main = `unit Main; + +interface + +uses Classes, Lib; + +type + TFoo = class + private + FOnChange: TNotifyEvent; + public + procedure Fire(OnChange: TNotifyEvent); + procedure Run(Data: Pointer); + property OnChange: TNotifyEvent read FOnChange; + end; + +implementation + +procedure TFoo.Fire(OnChange: TNotifyEvent); +begin + OnChange(Self); +end; + +procedure TFoo.Run(Data: Pointer); +var + load: TLoad; +begin + load := nil; + load(Data); +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Main.pas': main }, (cg) => { + expect(callsFrom(cg, 'TFoo::Fire')).toEqual([]); + expect(callsFrom(cg, 'TFoo::Run')).toEqual([]); + }); + }); + + it('reads a routine type declared on two lines, generic, or as an alias of another', async () => { + const main = `unit Main; + +interface + +uses SysUtils; + +type + THelperEvent = + procedure(const Value: Variant) of object; + TSetParams = reference to procedure(const Obj: T); + TFormatter = TFunc; + TCounter = Integer; + TCount = TCounter; + + TWriter = class + private + FOnHelp: THelperEvent; + FSetParams: TSetParams; + FFormatter: TFormatter; + FCount: TCount; + public + procedure Write(Obj: T; E: Exception); + end; + +implementation + +procedure TWriter.Write(Obj: T; E: Exception); +begin + FOnHelp(1); + FSetParams(Obj); + FFormatter(E); + FCount(1); +end; + +end. +`; + await withIndex({ 'Main.pas': main }, (cg) => { + expect(callsFrom(cg, 'TWriter::Write')).toEqual([ + '29: TWriter::FOnHelp', + '30: TWriter::FSetParams', + '31: TWriter::FFormatter', + ]); + }); + }); + + it('sees the declarations of a used unit whose header the parser lost', async () => { + // An old-style `begin … end.` initialization costs the parse its `unit` + // node; the unit is still the one its file is named after. + const lib = `unit Lib; + +interface + +type + THasher = function(Crc: Cardinal; Buf: Pointer; Len: Cardinal): Cardinal; + +var + crc32c: THasher; + +function FileSize(const FileName: string): Int64; + +implementation + +function FileSize(const FileName: string): Int64; +begin + Result := 0; +end; + +begin + crc32c := nil; +end. +`; + const main = `unit Main; + +interface + +uses Lib; + +procedure Run(const Name: string); + +implementation + +procedure Run(const Name: string); +begin + if FileSize(Name) > 0 then + crc32c(0, nil, 0); +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Main.pas': main }, (cg) => { + expect(cg.getNodesByKind('module').map((n) => n.name)).toEqual(['Main']); + expect(callsFrom(cg, 'Run')).toEqual(['13: FileSize', '14: crc32c']); + }); + }); + + it('gives an include file\'s code the scope of the unit that includes it', async () => { + const hash = `unit Hash; + +interface + +type + THasher = function(Crc: Cardinal; Buf: Pointer; Len: Cardinal): Cardinal; + +var + DefaultHasher: THasher; + +implementation + +end. +`; + const os = `unit Os; + +interface + +uses Hash; + +procedure Entropy; +procedure Init; + +implementation + +{$I Os.windows.inc} + +procedure Init; +begin + OsLog('init'); +end; + +end. +`; + const extra = `unit Extra; + +interface + +var + ExtraLog: procedure(const S: string); + +implementation + +end. +`; + // An include file may open with a `uses` clause of its own. + const inc = `uses + Extra; + +var + OsLog: procedure(const S: string); + +procedure Entropy; +var + Seed: Cardinal; +begin + Seed := DefaultHasher(0, nil, 0); + ExtraLog('seeded'); +end; +`; + await withIndex({ 'Hash.pas': hash, 'Extra.pas': extra, 'Os.pas': os, 'Os.windows.inc': inc }, (cg) => { + const entropy = cg.getNodesByKind('function').find((n) => n.name === 'Entropy' && n.filePath.endsWith('.inc')); + expect(entropy).toBeDefined(); + expect(cg.getOutgoingEdges(entropy!.id).filter((e) => e.kind === 'calls').map((e) => cg.getNode(e.target)?.qualifiedName).sort()) + .toEqual(['DefaultHasher', 'ExtraLog']); + // And the unit's own code sees what its include file declares. + expect(callsFrom(cg, 'Init')).toEqual(['16: OsLog']); + }); + }); + + it('takes no member of a class such a unit\'s parse lost for a routine of the unit', async () => { + // The anonymous record holding a method, which the parser can't read, + // ends the class early, so `Insert` comes out as a routine of the unit, + // indented as no routine of the unit is. + const lib = `unit Lib; + +interface + +type + TClient = class + Items: record + Name: string; + procedure Clear; + end; + procedure Insert(Index: Integer; const S: string); + end; + +procedure Log(const S: string); + +implementation + +procedure TClient.Insert(Index: Integer; const S: string); begin end; + +procedure Log(const S: string); +begin +end; + +begin +end. +`; + const main = `unit Main; + +interface + +uses Lib; + +procedure Run(var S: string); + +implementation + +procedure Run(var S: string); +begin + Insert('x', S, 1); + Log(S); +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Main.pas': main }, (cg) => { + const insert = cg.getNodesByKind('function').find((n) => n.name === 'Insert'); + expect(insert?.startColumn).toBe(4); + expect(callsFrom(cg, 'Run')).toEqual(['14: Log']); + }); + }); + + it('leaves out such a call in a `with` body, which may be the `with` object\'s', async () => { + const main = `unit Main; + +interface + +type + TItem = class + public + procedure Delete(Index: Integer); + end; + + TItems = class + public + function Add: TItem; + procedure Delete(Index: Integer); + procedure Assign(Source: TObject); + end; + +implementation + +procedure TItem.Delete(Index: Integer); begin end; +function TItems.Add: TItem; begin Result := nil; end; +procedure TItems.Delete(Index: Integer); begin end; + +procedure TItems.Assign(Source: TObject); +begin + with Add do + Delete(0); + Delete(1); +end; + +end. +`; + await withIndex({ 'Main.pas': main }, (cg) => { + // In the `with` body, `Delete` is the item's, not the list's; after it, the list's. + expect(callsFrom(cg, 'TItems::Assign')).toEqual(['28: TItems::Delete']); + }); + }); + + it('links a call to data only when the data holds a routine and the call\'s scope reaches it', async () => { + const lib = `unit Lib; + +interface + +type + TNotifyEvent = procedure(Sender: TObject) of object; + + TStranger = class + private + FOnChange: TNotifyEvent; + end; + +implementation + +end. +`; + const main = `unit Main; + +interface + +uses Lib; + +type + TList2 = class + private + FOnChange: TNotifyEvent; + FCount: Integer; + FProc: procedure of object; + FGetter: reference to function: Integer; + FOnLoad, // a list's type comes at its end + FOnSave: TNotifyEvent; + public + procedure Run; + procedure Other; + property OnChange: TNotifyEvent read FOnChange write FOnChange; + end; + + TOtherList = class + public + procedure Fire; + end; + +var + GLog: procedure(const Msg: string); + GCount: Integer; + +implementation + +procedure TList2.Run; +begin + FOnChange(Self); + OnChange(Self); + FProc(); + FGetter(); + GLog('x'); + FCount(1); + GCount(1); + FOnLoad(Self); +end; + +procedure TList2.Other; begin end; + +procedure TOtherList.Fire; +begin + FOnChange(Self); +end; + +end. +`; + await withIndex({ 'Lib.pas': lib, 'Main.pas': main }, (cg) => { + expect(callsFrom(cg, 'TList2::Run')).toEqual([ + '35: TList2::FOnChange', + '36: TList2::OnChange', + '37: TList2::FProc', + '38: TList2::FGetter', + '39: GLog', + '42: TList2::FOnLoad', + ]); + // No class this method's scope reaches declares `FOnChange`. + expect(callsFrom(cg, 'TOtherList::Fire')).toEqual([]); + }); + }); +}); diff --git a/__tests__/php-namespace-alias-instantiation.test.ts b/__tests__/php-namespace-alias-instantiation.test.ts index 103235c739..009657b604 100644 --- a/__tests__/php-namespace-alias-instantiation.test.ts +++ b/__tests__/php-namespace-alias-instantiation.test.ts @@ -194,4 +194,11 @@ class Personal extends L\\Base `); expect(await edgesFrom('app/Personal.php')).toEqual([]); }); + + it('matches the namespace alias without regard to its case', async () => { + write('app/Blocks/Personal/Fields/FirstName.php', firstName('App\\Blocks\\Personal\\Fields')); + write('app/Blocks/Personal/Personal.php', + consumer('use App\\Blocks\\Personal\\Fields as Field;', 'new field\\FirstName()')); + expect(await instantiated()).toEqual(['App\\Blocks\\Personal\\Fields::FirstName']); + }); }); diff --git a/__tests__/python-from-import-module-file.test.ts b/__tests__/python-from-import-module-file.test.ts new file mode 100644 index 0000000000..a258f7a4cb --- /dev/null +++ b/__tests__/python-from-import-module-file.test.ts @@ -0,0 +1,129 @@ +/** + * The module a Python `from X import y` names is a file of the project when X + * is one, dotted or not: `from loader import X` with `loader.py` at a package + * root links the importing file to it, as `import loader` and `from + * pkg.loader import X` already did. Before, an undotted module of a + * from-import was left unresolved, so the file lost its dependency on the + * module and `codegraph affected` missed it. A module from outside the + * project still links nothing. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +let tempDir: string; +let cg: CodeGraph | null = null; + +function write(files: Record): void { + for (const [rel, source] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(tempDir, rel)), { recursive: true }); + fs.writeFileSync(path.join(tempDir, rel), source); + } +} + +/** `kind:name@file` of every target of an `imports` edge from `fromFile`'s file node, sorted. */ +function importEdges(fromFile: string): string[] { + const file = cg!.getNodesByKind('file').find((n) => n.filePath === fromFile)!; + expect(file).toBeDefined(); + return cg! + .getOutgoingEdges(file.id) + .filter((e) => e.kind === 'imports') + .map((e) => cg!.getNode(e.target)) + .filter((n): n is NonNullable => !!n) + .map((n) => `${n.kind}:${n.name}@${n.filePath}`) + .sort(); +} + +async function indexed(files: Record): Promise { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-from-module-')); + write(files); + cg = await CodeGraph.init(tempDir, { index: true }); +} + +afterEach(() => { + cg?.close(); + cg = null; + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +const LOADER = 'class Registry:\n def get(self, k):\n return k\n\n\ndef load_registry():\n return Registry()\n'; + +describe('the module of a Python from-import links its file', () => { + it('links a top-level module at the project root, as `import loader` does', async () => { + await indexed({ + 'loader.py': LOADER, + 'packs.py': 'from loader import Registry, load_registry\n\n\ndef use():\n return load_registry()\n', + 'plain.py': 'import loader\n\n\ndef use():\n return loader.load_registry()\n', + }); + expect(importEdges('packs.py')).toEqual([ + 'class:Registry@loader.py', + 'file:loader.py@loader.py', + 'function:load_registry@loader.py', + ]); + expect(importEdges('plain.py')).toEqual(['file:loader.py@loader.py']); + }); + + it('links a top-level package to its __init__.py, as a dotted from-import links its package', async () => { + await indexed({ + 'pkg/__init__.py': 'def helper():\n return 1\n', + 'pkg/sub/__init__.py': 'def deep():\n return 2\n', + 'app.py': 'from pkg import helper\nfrom pkg.sub import deep\n', + }); + expect(importEdges('app.py')).toEqual([ + 'file:__init__.py@pkg/__init__.py', + 'file:__init__.py@pkg/sub/__init__.py', + 'function:deep@pkg/sub/__init__.py', + 'function:helper@pkg/__init__.py', + ]); + }); + + it('finds the module from the package roots, as in a src layout', async () => { + await indexed({ + 'pyproject.toml': '[project]\nname = "app"\n', + 'src/app/__init__.py': 'def helper():\n return 1\n', + 'tests/test_app.py': 'from app import helper\n\n\ndef test_helper():\n assert helper() == 1\n', + }); + expect(importEdges('tests/test_app.py')).toEqual([ + 'file:__init__.py@src/app/__init__.py', + 'function:helper@src/app/__init__.py', + ]); + }); + + it('links a module named on a continuation line of the statement', async () => { + await indexed({ 'loader.py': LOADER, 'packs.py': 'from \\\n loader import Registry\n' }); + expect(importEdges('packs.py')).toEqual(['class:Registry@loader.py', 'file:loader.py@loader.py']); + }); + + it('links nothing for a module from outside the project, even one a nested file is named like', async () => { + await indexed({ + 'tests/fixtures/requests.py': 'class Session:\n pass\n', + 'client.py': 'from requests import Session\nimport requests\n', + }); + expect(importEdges('client.py')).toEqual([]); + }); + + it('keeps a submodule the statement imports by name', async () => { + await indexed({ + 'pkg/__init__.py': '', + 'pkg/loader.py': LOADER, + 'app.py': 'from pkg import loader\n', + }); + expect(importEdges('app.py')).toEqual(['file:__init__.py@pkg/__init__.py', 'file:loader.py@pkg/loader.py']); + }); + + it('links the module once it is added, on sync as on a fresh index', async () => { + await indexed({ 'packs.py': 'from loader import Registry\n' }); + expect(importEdges('packs.py')).toEqual([]); + write({ 'loader.py': LOADER }); + await cg!.sync(); + const synced = importEdges('packs.py'); + cg!.close(); + fs.rmSync(path.join(tempDir, '.codegraph'), { recursive: true, force: true }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(synced).toEqual(importEdges('packs.py')); + expect(synced).toEqual(['class:Registry@loader.py', 'file:loader.py@loader.py']); + }); +}); diff --git a/__tests__/python-module-resolution.test.ts b/__tests__/python-module-resolution.test.ts index 8e2b01bd3d..a0ba036e14 100644 --- a/__tests__/python-module-resolution.test.ts +++ b/__tests__/python-module-resolution.test.ts @@ -339,7 +339,8 @@ def fetch(): cg = await CodeGraph.init(tempDir, { index: true }); expect(targets('analysis.py', 'analysis.py', 'imports')).toEqual([]); - expect(targets('report.py', 'report.py', 'imports')).toEqual(['summary@analysis.py']); + // The from-import's module is analysis.py itself; the name is its `summary`. + expect(targets('report.py', 'report.py', 'imports').sort()).toEqual(['analysis.py@analysis.py', 'summary@analysis.py']); expect(cg.findCircularDependencies()).toEqual([]); }); @@ -652,7 +653,8 @@ def main(): 'main.py': 'from pkg import f\n\n\ndef main():\n f()\n', }); cg = await CodeGraph.init(tempDir, { index: true }); - expect(targets('main.py', 'main.py', 'imports')).toEqual(['f@pkg/__init__.py']); + // The package itself is the from-import's module; `f` is its attribute. + expect(targets('main.py', 'main.py', 'imports').sort()).toEqual(['__init__.py@pkg/__init__.py', 'f@pkg/__init__.py']); }); it('binds it through a relative import too', async () => { @@ -676,7 +678,7 @@ def main(): 'main.py': 'from pkg import thing\n\n\ndef main():\n thing()\n', }); cg = await CodeGraph.init(tempDir, { index: true }); - expect(targets('main.py', 'main.py', 'imports')).toEqual(['thing@pkg/impl.py']); + expect(targets('main.py', 'main.py', 'imports').sort()).toEqual(['__init__.py@pkg/__init__.py', 'thing@pkg/impl.py']); }); it('still binds the submodule when the package does not bind the name', async () => { @@ -686,7 +688,7 @@ def main(): 'main.py': 'from pkg import f\n\n\ndef main():\n f.g()\n', }); cg = await CodeGraph.init(tempDir, { index: true }); - expect(targets('main.py', 'main.py', 'imports')).toEqual(['f.py@pkg/f.py']); + expect(targets('main.py', 'main.py', 'imports').sort()).toEqual(['__init__.py@pkg/__init__.py', 'f.py@pkg/f.py']); expect(targets('main', 'main.py', 'calls')).toEqual(['g@pkg/f.py']); }); }); diff --git a/__tests__/python-typed-builtin-method-calls.test.ts b/__tests__/python-typed-builtin-method-calls.test.ts new file mode 100644 index 0000000000..8f826a33a0 --- /dev/null +++ b/__tests__/python-typed-builtin-method-calls.test.ts @@ -0,0 +1,491 @@ +/** + * Python calls of builtin-named methods through a typed local or parameter + * (#2583). + * + * `get`, `update`, `items`, `pop`, `close` and the other dict/list/file method + * names are dropped as builtins before any strategy can type the receiver. + * When the function that binds the receiver says what it holds — a + * constructor or a typed factory as the whole value, or a parameter's + * annotation — the call is that project class's method. The negative cases + * are the shapes where such typing produced a wrong edge in the reporter's + * follow-up: the type's import, a later rebinding, a chained constructor, + * module-level code, a union. Those stay unlinked. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +let dir: string; +let cg: CodeGraph; + +const files: Record = { + // The issue's own files. + 'loader.py': `class SystemPackRegistry: + def get(self, pack_id): + return pack_id + + def update(self): + return None + + def items(self): + return [] + + def pop(self, key): + return key + + def close(self): + return None + + +def load_registry() -> SystemPackRegistry: + return SystemPackRegistry() + + +def make_registry(): + return SystemPackRegistry() + + +async def open_registry() -> "SystemPackRegistry": + return SystemPackRegistry() +`, + 'test_packs.py': `from loader import SystemPackRegistry, load_registry + + +def test_pack(): + registry = load_registry() + return registry.get("x") + + +def test_ctor(): + reg = SystemPackRegistry() + reg.update() + return reg.get("y") +`, + 'more.py': `from typing import Optional +import loader +from loader import SystemPackRegistry, make_registry, open_registry + + +def annotated(reg: SystemPackRegistry): + reg.get("a") + reg.items() + + +def quoted(reg: "SystemPackRegistry"): + reg.pop("b") + + +def optional(reg: Optional[SystemPackRegistry] = None): + reg.close() + + +def ctor_more(): + reg = SystemPackRegistry() + reg.items() + reg.pop("d") + reg.close() + + +def through_module(): + reg = loader.SystemPackRegistry() + reg.items() + + +def untyped_factory(): + reg = make_registry() + reg.update() + + +async def awaited_factory(): + reg = await open_registry() + reg.close() + + +def none_first(flag): + reg = None + if flag: + reg = SystemPackRegistry() + reg.update() + + +def annotated_local(): + reg: SystemPackRegistry = make_registry() + reg.pop("e") + + +def closure(): + reg = SystemPackRegistry() + + def inner(): + return reg.get("f") + return inner +`, + // Two project classes named Handler: the import says which one. + 'a.py': `class Handler: + def get(self): + return "a" +`, + 'b.py': `class Handler: + def get(self): + return "b" +`, + 'handlers.py': `from b import Handler + + +def handle(): + h = Handler() + return h.get() +`, + // A method a declared base in another file has. + 'base.py': `class BaseStore: + def update(self, data): + return data +`, + 'derived.py': `from base import BaseStore + + +class Derived(BaseStore): + pass + + +def refresh(): + d = Derived() + d.update({}) +`, + // The reporter's follow-up: shapes the receiver typing must not link. + 'models.py': `class Queue: + def get(self): + return None + + +class Session: + def get(self, url): + return url + + def close(self): + return None + + +class Store: + def get(self, key): + return key + + def update(self, data): + return data + + def items(self): + return [] + + +class Foo: + def build(self): + return self + + def data(self): + return {} + + +class Counter: + def update(self, other): + return other + + +class StringIO: + def read(self): + return "" +`, + 'traps.py': `import json +from queue import Queue +from requests import Session +from collections import Counter +from io import StringIO +from models import Store, Foo + + +def outside_import(): + q = Queue() + q.get() + s = Session() + s.get("http://x") + c = Counter() + c.update("abc") + buf = StringIO() + buf.read() + + +def rebinding(p): + reg = Store() + reg = {} + reg.get("k") + data = Store() + data = json.load(p) + data.update({}) + + +def branches(flag, rows, path): + a = Store() if flag else {} + a.get("k") + if flag: + b = Store() + else: + b = {} + b.get("k") + c = Store() + for c in rows: + c.items() + d = Store() + with open(path) as d: + d.get("k") + e = Store() + return [e.get("k") for e in rows] + + +def chained(): + x = Foo().data() + x.get("k") + y = Foo().build() + y.get("k") + + +def unions(reg: "Store | dict"): + if isinstance(reg, dict): + reg.get("k") + + +def unannotated(reg): + reg.get("k") + + +def rebound_callee(load): + Store = load() + reg = Store() + reg.get("k") + + +_store = Store() +_store = {} + + +def rebound_global(): + _store.get("k") +`, + 'modlevel.py': `from models import Store + + +def make(): + reg = Store() + return reg + + +reg = {} +reg.get("k") +`, + 'outside_base.py': `from somepkg import BaseStore + + +class Child(BaseStore): + pass + + +def use_child(): + ch = Child() + ch.update({}) +`, +}; + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-2583-')); + for (const [rel, text] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, rel), text); + } + cg = await CodeGraph.init(dir, { index: true }); +}); + +afterAll(() => { + cg?.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +/** The `calls` targets of the named function or method in `file`, as `qualifiedName@file`, sorted. */ +function callsFrom(file: string, qualifiedName: string): string[] { + const src = cg.getNodesInFile(file).find((n) => n.qualifiedName === qualifiedName && + (n.kind === 'function' || n.kind === 'method')); + expect(src, `${file}:${qualifiedName}`).toBeDefined(); + return cg.getOutgoingEdges(src!.id) + .filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.target)!) + .filter((n) => n.kind === 'method') + .map((n) => `${n.qualifiedName}@${n.filePath}`) + .sort(); +} + +/** The `calls` edges into the file-level code of `file`, to methods. */ +function moduleLevelCalls(file: string): string[] { + const fileNode = cg.getNodesInFile(file).find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + return cg.getOutgoingEdges(fileNode!.id) + .filter((e) => e.kind === 'calls') + .map((e) => cg.getNode(e.target)!) + .filter((n) => n.kind === 'method') + .map((n) => `${n.qualifiedName}@${n.filePath}`); +} + +describe('Python builtin-named methods through typed locals and parameters (#2583)', () => { + it('links the issue\'s constructor-typed local', () => { + expect(callsFrom('test_packs.py', 'test_ctor')).toEqual([ + 'SystemPackRegistry::get@loader.py', + 'SystemPackRegistry::update@loader.py', + ]); + }); + + it('links a local typed by its factory\'s return annotation', () => { + expect(callsFrom('test_packs.py', 'test_pack')).toEqual(['SystemPackRegistry::get@loader.py']); + }); + + it('links calls through an annotated parameter, quoted or Optional too', () => { + expect(callsFrom('more.py', 'annotated')).toEqual([ + 'SystemPackRegistry::get@loader.py', + 'SystemPackRegistry::items@loader.py', + ]); + expect(callsFrom('more.py', 'quoted')).toEqual(['SystemPackRegistry::pop@loader.py']); + expect(callsFrom('more.py', 'optional')).toEqual(['SystemPackRegistry::close@loader.py']); + }); + + it('links items, pop and close on a constructed local, and one made through the module', () => { + expect(callsFrom('more.py', 'ctor_more')).toEqual([ + 'SystemPackRegistry::close@loader.py', + 'SystemPackRegistry::items@loader.py', + 'SystemPackRegistry::pop@loader.py', + ]); + expect(callsFrom('more.py', 'through_module')).toEqual(['SystemPackRegistry::items@loader.py']); + }); + + it('links a local typed by what a factory returns, awaited or not', () => { + expect(callsFrom('more.py', 'untyped_factory')).toEqual(['SystemPackRegistry::update@loader.py']); + expect(callsFrom('more.py', 'awaited_factory')).toEqual(['SystemPackRegistry::close@loader.py']); + }); + + it('reads past a None placeholder and trusts a local annotation', () => { + expect(callsFrom('more.py', 'none_first')).toEqual(['SystemPackRegistry::update@loader.py']); + expect(callsFrom('more.py', 'annotated_local')).toEqual(['SystemPackRegistry::pop@loader.py']); + }); + + it('links a call in a nested function through the enclosing function\'s local', () => { + expect(callsFrom('more.py', 'closure::inner')).toEqual(['SystemPackRegistry::get@loader.py']); + }); + + it('records the factory\'s file for a sync to re-open the call', () => { + const src = cg.getNodesInFile('test_packs.py').find((n) => n.qualifiedName === 'test_pack')!; + const edge = cg.getOutgoingEdges(src.id).find((e) => e.kind === 'calls' && cg.getNode(e.target)?.name === 'get'); + expect(edge?.metadata?.typeFrom).toEqual(['loader.py']); + }); + + it('takes the class the import names when two project classes share its name', () => { + expect(callsFrom('handlers.py', 'handle')).toEqual(['Handler::get@b.py']); + }); + + it('links a method a declared base in another file has', () => { + expect(callsFrom('derived.py', 'refresh')).toEqual(['BaseStore::update@base.py']); + }); + + it('links nothing for a class imported from outside the project that a project class is named like', () => { + expect(callsFrom('traps.py', 'outside_import')).toEqual([]); + }); + + it('links nothing for a local rebound to a dict or to what json.load returns', () => { + expect(callsFrom('traps.py', 'rebinding')).toEqual([]); + }); + + it('links nothing for a local bound in a branch, a loop, a with or a comprehension', () => { + expect(callsFrom('traps.py', 'branches')).toEqual([]); + }); + + it('links nothing more for a local holding what a method of a constructed object returns', () => { + expect(callsFrom('traps.py', 'chained')).toEqual(['Foo::build@models.py', 'Foo::data@models.py']); + }); + + it('links nothing for a union, an unannotated parameter, a rebound class name or a rebound global', () => { + expect(callsFrom('traps.py', 'unions')).toEqual([]); + expect(callsFrom('traps.py', 'unannotated')).toEqual([]); + expect(callsFrom('traps.py', 'rebound_callee')).toEqual([]); + expect(callsFrom('traps.py', 'rebound_global')).toEqual([]); + }); + + it('links nothing for module-level code, whatever a function above binds', () => { + expect(moduleLevelCalls('modlevel.py')).toEqual([]); + }); + + it('links nothing for a method a base from outside the project could hold', () => { + expect(callsFrom('outside_base.py', 'use_child')).toEqual([]); + }); +}); + +describe('sync after an edit to the file a typed local reads (#2583)', () => { + let tmp: string; + let graph: CodeGraph | undefined; + + afterEach(() => { + graph?.close(); + graph = undefined; + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + const write = (map: Record) => { + for (const [rel, text] of Object.entries(map)) fs.writeFileSync(path.join(tmp, rel), text); + }; + + const getCalls = (g: CodeGraph, qualifiedName: string): string[] => { + const src = g.getNodesInFile('use.py').find((n) => n.qualifiedName === qualifiedName)!; + return g.getOutgoingEdges(src.id).filter((e) => e.kind === 'calls') + .map((e) => g.getNode(e.target)!).filter((n) => n.kind === 'method') + .map((n) => `${n.qualifiedName}@${n.filePath}`).sort(); + }; + + /** Syncs, then checks the synced graph against a fresh index of the same files. */ + const syncedMatchesFresh = async (qualifiedName: string): Promise => { + await graph!.sync(); + const synced = getCalls(graph!, qualifiedName); + graph!.close(); + fs.rmSync(path.join(tmp, '.codegraph'), { recursive: true, force: true }); + graph = await CodeGraph.init(tmp, { index: true }); + expect(synced).toEqual(getCalls(graph, qualifiedName)); + return synced; + }; + + const USE = 'from loader import Registry, make\n\n\ndef ctor():\n reg = Registry()\n return reg.get("k")\n\n\ndef factory():\n reg = make()\n return reg.get("k")\n'; + + it('re-types the local when the factory changes what it returns', async () => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-2583-sync-')); + const classes = 'class Registry:\n def get(self, k):\n return k\n\n\nclass Other:\n def get(self, k):\n return k\n\n\n'; + write({ 'loader.py': `${classes}def make() -> Registry:\n return Registry()\n`, 'use.py': USE }); + graph = await CodeGraph.init(tmp, { index: true }); + expect(getCalls(graph, 'factory')).toEqual(['Registry::get@loader.py']); + write({ 'loader.py': `${classes}def make() -> Other:\n return Other()\n` }); + expect(await syncedMatchesFresh('factory')).toEqual(['Other::get@loader.py']); + }); + + it('types the local once the factory gains a return annotation', async () => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-2583-sync-')); + const cls = 'class Registry:\n def get(self, k):\n return k\n\n\n'; + write({ 'loader.py': `${cls}def make(kind):\n return kind()\n`, 'use.py': USE }); + graph = await CodeGraph.init(tmp, { index: true }); + expect(getCalls(graph, 'factory')).toEqual([]); + write({ 'loader.py': `${cls}def make(kind) -> Registry:\n return kind()\n` }); + expect(await syncedMatchesFresh('factory')).toEqual(['Registry::get@loader.py']); + }); + + it('links the call once the class gains the method, and drops it when the class loses it', async () => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-2583-sync-')); + const without = 'class Registry:\n def other(self):\n pass\n\n\ndef make():\n return Registry()\n'; + const withGet = 'class Registry:\n def other(self):\n pass\n\n def get(self, k):\n return k\n\n\ndef make():\n return Registry()\n'; + write({ 'loader.py': without, 'use.py': USE }); + graph = await CodeGraph.init(tmp, { index: true }); + expect(getCalls(graph, 'ctor')).toEqual([]); + write({ 'loader.py': withGet }); + expect(await syncedMatchesFresh('ctor')).toEqual(['Registry::get@loader.py']); + write({ 'loader.py': without }); + expect(await syncedMatchesFresh('ctor')).toEqual([]); + }); +}); diff --git a/__tests__/resolution.test.ts b/__tests__/resolution.test.ts index d1d2510906..2bb1eb6a5a 100644 --- a/__tests__/resolution.test.ts +++ b/__tests__/resolution.test.ts @@ -11,8 +11,7 @@ import * as os from 'os'; import { CodeGraph } from '../src'; import { Node, UnresolvedReference } from '../src/types'; import { ReferenceResolver, createResolver, ResolutionContext } from '../src/resolution'; -import { resolveImportPath, importMappingsFromBindings, loadCppIncludeDirs, clearCppIncludeDirCache } from '../src/resolution/import-resolver'; -import { tryKernelBindings, tryKernelExtract } from '../src/extraction/kernel'; +import { resolveImportPath, loadCppIncludeDirs, clearCppIncludeDirCache } from '../src/resolution/import-resolver'; import type { UnresolvedRef } from '../src/resolution/types'; import { detectFrameworks, getAllFrameworkResolvers } from '../src/resolution/frameworks'; import { QueryBuilder } from '../src/db/queries'; @@ -80,6 +79,8 @@ describe('Resolution Module', () => { } }); + + describe('Import Resolver', () => { it('should resolve relative import paths', () => { const context: ResolutionContext = { @@ -125,67 +126,11 @@ describe('Resolution Module', () => { expect(result).toBe('src/helpers.ts'); }); - it('should extract JS/TS import mappings', () => { - const content = ` -import { foo } from './foo'; -import bar from '../bar'; -import * as utils from './utils'; -import { baz, qux } from './baz'; -`; - - // JS/TS mappings come from the binding rows, never from source regexes. - const rows = tryKernelBindings('src/index.ts', content, 'typescript') ?? []; - const mappings = importMappingsFromBindings(rows) ?? []; - - expect(mappings.length).toBeGreaterThan(0); - expect(mappings.some((m) => m.localName === 'foo')).toBe(true); - expect(mappings.some((m) => m.localName === 'bar')).toBe(true); - expect(mappings.find((m) => m.localName === 'utils')).toMatchObject({ isNamespace: true, source: './utils' }); - }); - - it('should extract Python import mappings', () => { - const content = ` -from utils import helper -from .models import User -import os -from ..services import auth_service -`; - - // Python mappings come from the kernel walker's binding rows. - const rows = tryKernelExtract('src/main.py', content, 'python')?.bindings ?? []; - const mappings = importMappingsFromBindings(rows) ?? []; - expect(mappings.length).toBeGreaterThan(0); - expect(mappings.find((m) => m.localName === 'helper')).toMatchObject({ source: 'utils', exportedName: 'helper' }); - expect(mappings.find((m) => m.localName === 'User')).toMatchObject({ source: '.models' }); - expect(mappings.find((m) => m.localName === 'os')).toMatchObject({ isNamespace: true, source: 'os' }); - expect(mappings.find((m) => m.localName === 'auth_service')).toMatchObject({ source: '..services' }); - }); - it('should extract parenthesized Python from-imports', () => { - const content = ` -from store.base import ( - Base, - helper as h, # a comment, with a comma and a call() -) -""" -from docstring import NotAnImport -""" -from store.rows import (Row, Col) -`; - - const rows = tryKernelExtract('src/main.py', content, 'python')?.bindings ?? []; - const mappings = importMappingsFromBindings(rows) ?? []; - - expect(mappings.map((m) => [m.localName, m.exportedName, m.source])).toEqual([ - ['Base', 'Base', 'store.base'], - ['h', 'helper', 'store.base'], - ['Row', 'Row', 'store.rows'], - ['Col', 'Col', 'store.rows'], - ]); - }); }); + describe('Framework Detection', () => { it('should detect React framework', () => { const context: ResolutionContext = { @@ -518,44 +463,6 @@ void initialize() { Packet(); } expect(outgoing.some((e) => e.kind === 'calls' && e.target === packet!.id)).toBe(false); }); - it('keeps calls to a C++ constructor when Type() resolves through an include', async () => { - fs.writeFileSync( - path.join(tempDir, 'proxy.hpp'), - `struct iteration_proxy { - explicit iteration_proxy(int& cont) {} -}; -` - ); - fs.writeFileSync( - path.join(tempDir, 'json.hpp'), - `#include "proxy.hpp" - -struct Json { - int n; - iteration_proxy items() { return iteration_proxy(n); } -}; -` - ); - - cg = await CodeGraph.init(tempDir, { index: true }); - cg.resolveReferences(); - - const items = cg.getNodesByKind('method').find((n) => n.name === 'items'); - const ctor = cg.getNodesByKind('method').find( - (n) => n.name === 'iteration_proxy' && n.filePath.replace(/\\/g, '/').endsWith('proxy.hpp'), - ); - const klass = - cg.getNodesByKind('struct').find((n) => n.name === 'iteration_proxy') ?? - cg.getNodesByKind('class').find((n) => n.name === 'iteration_proxy'); - expect(items).toBeDefined(); - expect(ctor).toBeDefined(); - expect(klass).toBeDefined(); - - const outgoing = cg.getOutgoingEdges(items!.id); - expect(outgoing.some((e) => e.kind === 'calls' && e.target === ctor!.id)).toBe(true); - expect(outgoing.some((e) => e.kind === 'instantiates' && e.target === klass!.id)).toBe(false); - }); - it('resolves a static call through an imported C++ union to its member', async () => { fs.writeFileSync( path.join(tempDir, 'ops.hpp'), @@ -630,42 +537,6 @@ impl Describe for Ctl { fn describe(&self) -> String { "ctl".into() } } ).toBe('interface-impl'); }); - it('bridges TRANSITIVE supertypes — A implements B, B extends C → A overrides C members', async () => { - // A call typed to `Shape` dispatches to `Circle.draw` even though Circle - // only directly `implements Colored` — the interface-impl bridge must - // walk implements/extends chains, not just direct supertypes. - fs.writeFileSync( - path.join(tempDir, 'Shapes.java'), - `public interface Shape { void draw(); } -public interface Colored extends Shape { void fill(); } -public class Circle implements Colored { - public void draw() { } - public void fill() { } -} -` - ); - - cg = await CodeGraph.init(tempDir, { index: true }); - - const methods = cg.getNodesByKind('method'); - const shapeDraw = methods.find((n) => n.qualifiedName === 'Shape::draw'); - const coloredFill = methods.find((n) => n.qualifiedName === 'Colored::fill'); - const circleDraw = methods.find((n) => n.qualifiedName === 'Circle::draw'); - const circleFill = methods.find((n) => n.qualifiedName === 'Circle::fill'); - expect(shapeDraw && coloredFill && circleDraw && circleFill).toBeTruthy(); - - const synth = (id: string) => - cg.getOutgoingEdges(id).filter((e) => e.kind === 'calls' && e.provenance === 'heuristic'); - // Transitive: Shape::draw → Circle::draw (two hops: Circle→Colored→Shape). - expect(synth(shapeDraw!.id).map((e) => e.target)).toEqual([circleDraw!.id]); - // Direct supertype still bridges: Colored::fill → Circle::fill. - expect(synth(coloredFill!.id).map((e) => e.target)).toEqual([circleFill!.id]); - const edge = synth(shapeDraw!.id)[0]!; - expect( - (edge.metadata as { synthesizedBy?: string } | undefined)?.synthesizedBy - ).toBe('interface-impl'); - }); - it('qualifies a generic impl by its type, so trait dispatch reaches it and no edge is invented from its body (#1588)', async () => { // `impl Source for BufSource`: the implementing type parses as a // generic_type, so the old positional receiver scan picked the TRAIT. @@ -1089,9 +960,11 @@ func UseAliased() { `from pkg import module import os + def caller(): return module.func() + def external_caller(): return os.getcwd() ` @@ -1134,6 +1007,7 @@ def external_caller(): path.join(tempDir, 'record.py'), `from . import ledger + def add_outcome(row): if not ledger.append(row): return None @@ -1193,9 +1067,11 @@ def add_outcome(row): `from pkg import module as mod_alias import top_level as tl + def from_import_caller(): return mod_alias.func() + def plain_import_caller(): return tl.top_func() ` @@ -1227,45 +1103,6 @@ def plain_import_caller(): expect(fileImports.map((e) => e.target)).toContain(moduleFile!.id); }); - it('never resolves an absolute Python import into a subpackage of another package', async () => { - // `import json` is the stdlib module: `src/app/json/` is importable as - // `app.json`, never as `json`, because `src/app/` is itself a package. - // A top-level package under `src/` still resolves. - fs.mkdirSync(path.join(tempDir, 'src', 'app', 'json'), { recursive: true }); - fs.writeFileSync(path.join(tempDir, 'src', 'app', '__init__.py'), ''); - fs.writeFileSync(path.join(tempDir, 'src', 'app', 'json', '__init__.py'), 'def load(fp):\n return 1\n'); - fs.writeFileSync(path.join(tempDir, 'src', 'app', 'util.py'), 'def helper():\n return 2\n'); - fs.mkdirSync(path.join(tempDir, 'tests')); - fs.writeFileSync( - path.join(tempDir, 'tests', 'test_config.py'), - `import json -import app.util - -def test_load(): - return json.load(None) - -def test_helper(): - return app.util.helper() -` - ); - - cg = await CodeGraph.init(tempDir, { index: true }); - - const testFile = cg.getNodesByKind('file').find((n) => n.filePath.replace(/\\/g, '/') === 'tests/test_config.py'); - expect(testFile).toBeDefined(); - const importTargets = cg - .getOutgoingEdges(testFile!.id) - .filter((e) => e.kind === 'imports') - .map((e) => cg.getNode(e.target)?.filePath.replace(/\\/g, '/')); - expect(importTargets).not.toContain('src/app/json/__init__.py'); - expect(importTargets).toContain('src/app/util.py'); - - const load = cg.getNodesByKind('function').find((n) => n.name === 'load'); - expect(cg.getIncomingEdges(load!.id).map((e) => cg.getNode(e.source)?.name)).not.toContain('test_load'); - const helper = cg.getNodesByKind('function').find((n) => n.name === 'helper'); - expect(cg.getIncomingEdges(helper!.id).map((e) => cg.getNode(e.source)?.name)).toContain('test_helper'); - }); - it('attaches Go methods to their receiver type across files (#583, cross-file half)', async () => { // In Go a type's methods are commonly declared in a different file from the // `type` declaration (`type Box` in box.go, `func (b *Box) Get()` in @@ -1588,6 +1425,7 @@ func main() { getAllFiles: () => [], }); + it('resolves C++ calls end-to-end to same-named classes in different files (#1079)', async () => { // The exact repro from the issue: two files, each with its own // `Logger::log`. Before the fix both callers pointed at the first def. @@ -1626,6 +1464,7 @@ func main() { expect(callTargets('useB')).toContain(logB.id); }); + it('resolves a static/class-receiver call to the class in the caller\'s file (#1079)', async () => { // `Logger.log()` — the receiver is the class NAME, so this routes through // the class-name-receiver strategy (not the C++ instance path). It was @@ -1743,45 +1582,29 @@ func main() { startLine: 1, endLine: 1, startColumn: 0, endColumn: 0, updatedAt: 0, }); - it('resolveBatchKernelFirst offers a yield checkpoint for every ref', async () => { - fs.writeFileSync( - path.join(tempDir, 'a.ts'), - `export function fnA() { return 1; }\nexport function fnB() { return fnA(); }\nexport function fnC() { return fnB(); }\n`, - ); + + it('the production resolver context memoizes method matches per (language, type, method)', async () => { fs.writeFileSync( - path.join(tempDir, 'b.ts'), - `import { fnA } from './a';\nexport function fnD() { return fnA(); }\n`, + path.join(tempDir, 'svc.ts'), + `class Logger { log() { return 1; } }\nexport function use() { const lg = new Logger(); return lg.log(); }\n`, ); cg = await CodeGraph.init(tempDir, { index: true }); const resolver = (cg as unknown as { resolver: ReferenceResolver }).resolver; + const ctx = (resolver as unknown as { context: ResolutionContext }).context; + + const first = ctx.getMethodMatches!('Logger', 'log', 'typescript'); + const second = ctx.getMethodMatches!('Logger', 'log', 'typescript'); + expect(first.map((n) => n.qualifiedName)).toEqual(['Logger::log']); + // Same array instance = served from the memo, not recomputed. + expect(second).toBe(first); - // `init({ index: true })` already ran resolution, so feed the batch - // directly — resolveBatchKernelFirst takes it as an argument; whether each - // ref resolves is irrelevant to the checkpoint contract. - const refs: UnresolvedReference[] = ['fnA', 'fnB', 'nosuchFn', 'fnA', 'alsoMissing'].map((name, i) => ({ - fromNodeId: `caller-${i}`, - referenceName: name, - referenceKind: 'calls', - line: i + 1, - column: 0, - filePath: 'a.ts', - language: 'typescript', - })); - - let checkpoints = 0; - const countingYield = async () => { checkpoints++; }; - const result = await (resolver as unknown as { - resolveBatchKernelFirst(batch: UnresolvedReference[], maybeYield: () => Promise): Promise<{ stats: { total: number } }>; - closeKernel(): void; - }).resolveBatchKernelFirst(refs, countingYield); - resolver.closeKernel(); - - // One checkpoint per ref: a pocket of pathologically slow refs can never - // run more than ONE ref past the yield budget before the heartbeat gets - // a window — the #1122 kill required 500. - expect(checkpoints).toBe(refs.length); - expect(result.stats.total).toBe(refs.length); + resolver.clearCaches(); + const afterClear = ctx.getMethodMatches!('Logger', 'log', 'typescript'); + expect(afterClear).not.toBe(first); + expect(afterClear.map((n) => n.qualifiedName)).toEqual(['Logger::log']); }); + + }); describe('Local-variable receiver-type inference (#1108)', () => { @@ -1798,6 +1621,9 @@ export class Collision { get(key) { return key; } } export function readKey() { return 'values'; } ${receivers.map((receiver, i) => `export function nested${i}(holder, höldér) { return ${receiver}.get(readKey()); }`).join('\n')} `); + // A dotted chain stays in the index, unresolved, as the call's evidence + // (#1790 dropped these calls outright). + const kept: Record = { 0: 'holder.values.get', 6: 'imported.values.get', 7: 'data.holder.values.get' }; cg = await CodeGraph.init(tempDir, { index: true }); for (let i = 0; i < receivers.length; i++) { const caller = cg.getNodesByName(`nested${i}`).find(n => n.kind === 'function')!; @@ -1806,6 +1632,10 @@ ${receivers.map((receiver, i) => `export function nested${i}(holder, höldér) { // root object nor Collision.get is evidence of the called member. expect(callees.length, receivers[i]).toBeGreaterThan(0); expect(callees.every(({ node }) => node.name === 'readKey'), receivers[i]).toBe(true); + if (kept[i]) { + const refs = cg.getUnresolvedReferencesFrom(caller.id).filter((r) => r.referenceKind === 'calls'); + expect(refs.map((r) => r.referenceName), receivers[i]).toEqual([kept[i]]); + } } }); @@ -2155,6 +1985,7 @@ export function annotated(values: ${type}) { } }); + describe('tsconfig path aliases', () => { it('resolves an aliased import to the alias-mapped file (not a same-named file elsewhere)', async () => { // Two same-named exports in different directories. Without alias @@ -2520,6 +2351,7 @@ export function annotated(values: ${type}) { return join(values) + def report_missing(unresolved): missing_list = ", ".join(sorted(unresolved)) return f"Could not resolve: {missing_list}" @@ -2847,7 +2679,7 @@ export function useGet() { return api.get(); } } }, 30000); - it('does not mistake an imported non-literal receiver for its called member', async () => { + it('leaves a non-literal value receiver on its existing path', async () => { const tmpDir = setup({ 'src/mk.ts': `export function m() { return 'top-level, unrelated to obj'; } export const obj = makeObj(); @@ -2863,14 +2695,14 @@ export function remoteUse() { return obj.m(); } await cg.indexAll(); // `obj` holds a call result, not a literal: the same-named top-level // `m` lies outside its declaration, so containment finds nothing and - // neither call targets `m`. The imported call still goes through the - // receiver constant, so `remoteUse` links to `obj` and nothing else. + // both calls keep today's behavior (unresolved in the defining file; + // the constant edge through the import) rather than guessing. expect(await callersOf(cg, 'm', 'function')).toEqual([]); const obj = (await cg.searchNodes('obj', { limit: 5 })).find((r) => r.node.kind === 'constant'); const remote = (await cg.searchNodes('remoteUse', { limit: 5 })).find((r) => r.node.kind === 'function'); expect( - cg.getOutgoingEdges(remote!.node.id).filter((e) => e.kind === 'calls').map((e) => e.target) - ).toEqual([obj!.node.id]); + cg.getOutgoingEdges(remote!.node.id).some((e) => e.kind === 'calls' && e.target === obj!.node.id) + ).toBe(true); cg.close(); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -3240,26 +3072,6 @@ int run() { expect(result).toBe('mylib/utils.h'); }); - it('should extract C/C++ import mappings from #include directives', () => { - const code = `#include -#include "myheader.h" -#include "utils/helpers.hpp"`; - - // C/C++ mappings come from the kernel walker's include rows. - const rows = tryKernelExtract('main.cpp', code, 'cpp')?.bindings ?? []; - const mappings = importMappingsFromBindings(rows) ?? []; - - expect(mappings.length).toBe(3); - expect(mappings[0]).toEqual({ - localName: 'iostream', - exportedName: '*', - source: 'iostream', - isDefault: false, - isNamespace: true, - }); - expect(mappings[1]!.localName).toBe('myheader'); - expect(mappings[2]).toMatchObject({ localName: 'helpers', source: 'utils/helpers.hpp' }); - }); it('should discover include directories from compile_commands.json', () => { // Create a temp project with compile_commands.json @@ -3351,7 +3163,6 @@ int run() { // feature can't silently regress to a no-op in the indexing flow. it('connects #include to the real header file via include-dir scan (end-to-end)', async () => { const tempProject = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cpp-e2e-')); - let db: ReturnType | undefined; try { fs.mkdirSync(path.join(tempProject, 'include'), { recursive: true }); fs.mkdirSync(path.join(tempProject, 'src'), { recursive: true }); @@ -3374,7 +3185,7 @@ int run() { // The `#include "utils.h"` edge should target the real // `include/utils.h` file node — not a floating `import` node // living inside main.cpp. - db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); + const db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); const rows = db.getDb().prepare(` select dst.kind as dstKind, dst.file_path as dstPath from edges e @@ -3395,13 +3206,13 @@ int run() { ); expect(stdlibFile).toBeUndefined(); } finally { - // Windows will not delete a file that still has an open handle. Both - // holders have to go: the graph's own connection, and the direct one - // this test opens to read edges back. Closing here rather than after - // the query keeps that true when a query or an assertion throws. - // close() is idempotent, so the outer afterEach's destroy() stays safe. - db?.close(); - cg?.close(); + // The graph opened on tempProject has to be closed here: the outer + // afterEach runs after this finally, so on Windows the still-open + // database makes the removal fail with EPERM. + if (cg) { + cg.close(); + cg = undefined as unknown as CodeGraph; + } fs.rmSync(tempProject, { recursive: true, force: true }); } }); @@ -3439,9 +3250,6 @@ class Both : public Base, public Plain {}; // templated + plain in one cla where e.kind = 'extends'` ) .all() as Array<{ fromName: string; toName: string }>; - // Closed before the assertions: an open second connection to the same - // database keeps a handle on tempDir, and afterEach's removal then fails - // with EPERM on Windows. db.close(); const has = (from: string, to: string) => edges.some((r) => r.fromName === from && r.toName === to); @@ -3460,7 +3268,6 @@ class Both : public Base, public Plain {}; // templated + plain in one cla it('resolves require_once to a file→file imports edge (#660)', async () => { const tempProject = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-php-e2e-')); - let db: ReturnType | undefined; try { fs.mkdirSync(path.join(tempProject, 'src'), { recursive: true }); fs.writeFileSync( @@ -3477,7 +3284,7 @@ class Both : public Base, public Plain {}; // templated + plain in one cla // reporter's repro: page.php's `require_once("lib.php")` must resolve // to the real src/lib.php file node — a file→file `imports` edge, so // callers(lib.php) now includes page.php. - db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); + const db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); const rows = db.getDb().prepare(` select dst.kind as dstKind, dst.file_path as dstPath from edges e @@ -3493,15 +3300,19 @@ class Both : public Base, public Plain {}; // templated + plain in one cla ); expect(resolved, 'page.php → src/lib.php imports edge missing').toBeDefined(); } finally { - db?.close(); - cg?.close(); + // The graph opened on tempProject has to be closed here: the outer + // afterEach runs after this finally, so on Windows the still-open + // database makes the removal fail with EPERM. + if (cg) { + cg.close(); + cg = undefined as unknown as CodeGraph; + } fs.rmSync(tempProject, { recursive: true, force: true }); } }); it('resolves a subdirectory include path to the correct file (#660)', async () => { const tempProject = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-php-subdir-')); - let db: ReturnType | undefined; try { fs.mkdirSync(path.join(tempProject, 'inc'), { recursive: true }); fs.writeFileSync( @@ -3515,7 +3326,7 @@ class Both : public Base, public Plain {}; // templated + plain in one cla cg = await CodeGraph.init(tempProject, { index: true }); - db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); + const db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); const rows = db.getDb().prepare(` select dst.kind as dstKind, dst.file_path as dstPath from edges e @@ -3531,15 +3342,19 @@ class Both : public Base, public Plain {}; // templated + plain in one cla 'index.php → inc/db.php imports edge missing' ).toBeDefined(); } finally { - db?.close(); - cg?.close(); + // The graph opened on tempProject has to be closed here: the outer + // afterEach runs after this finally, so on Windows the still-open + // database makes the removal fail with EPERM. + if (cg) { + cg.close(); + cg = undefined as unknown as CodeGraph; + } fs.rmSync(tempProject, { recursive: true, force: true }); } }); it('does not mis-connect an unresolvable include to a same-named file elsewhere (#660)', async () => { const tempProject = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-php-misresolve-')); - let db: ReturnType | undefined; try { // app/page.php's `require "inc/db.php"` resolves relative to app/, where // inc/db.php does NOT exist. A same-named lib/inc/db.php exists elsewhere @@ -3558,7 +3373,7 @@ class Both : public Base, public Plain {}; // templated + plain in one cla cg = await CodeGraph.init(tempProject, { index: true }); - db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); + const db = DatabaseConnection.open(path.join(tempProject, '.codegraph', 'codegraph.db')); const rows = db.getDb().prepare(` select dst.kind as dstKind, dst.file_path as dstPath from edges e @@ -3574,8 +3389,13 @@ class Both : public Base, public Plain {}; // templated + plain in one cla 'app/page.php must NOT mis-connect to unrelated lib/inc/db.php' ).toBeUndefined(); } finally { - db?.close(); - cg?.close(); + // The graph opened on tempProject has to be closed here: the outer + // afterEach runs after this finally, so on Windows the still-open + // database makes the removal fail with EPERM. + if (cg) { + cg.close(); + cg = undefined as unknown as CodeGraph; + } fs.rmSync(tempProject, { recursive: true, force: true }); } }); @@ -5279,21 +5099,6 @@ in }); describe('Bindings in a module that exports nothing (#1719)', () => { - it('does not treat documentation headings as package imports', async () => { - fs.writeFileSync(path.join(tempDir, 'package.json'), JSON.stringify({ workspaces: ['packages/*'] })); - fs.mkdirSync(path.join(tempDir, 'packages/vite'), { recursive: true }); - fs.writeFileSync(path.join(tempDir, 'packages/vite/package.json'), JSON.stringify({ name: 'vite', main: './index.js' })); - fs.writeFileSync(path.join(tempDir, 'packages/vite/index.js'), 'export function defineConfig(x) { return x }'); - fs.writeFileSync(path.join(tempDir, 'guide.md'), '# vite\n'); - fs.writeFileSync(path.join(tempDir, 'consumer.js'), "import { defineConfig } from 'vite'\nexport const config = defineConfig({})\n"); - cg = await CodeGraph.init(tempDir, { index: true }); - cg.resolveReferences(); - const heading = cg.getNodesByKind('module').find((n) => n.language === 'markdown'); - expect(heading).toBeDefined(); - expect(cg.getIncomingEdges(heading!.id).filter((e) => e.kind === 'imports')).toEqual([]); - const definition = cg.getNodesByKind('function').find((n) => n.name === 'defineConfig'); - expect(cg.getIncomingEdges(definition!.id).some((e) => e.kind === 'imports')).toBe(true); - }); it('ignores export examples in strings and comments when checking module visibility', async () => { fs.mkdirSync(path.join(tempDir, 'src')); diff --git a/__tests__/ruby-namespaced-receiver.test.ts b/__tests__/ruby-namespaced-receiver.test.ts new file mode 100644 index 0000000000..e62c0ca03b --- /dev/null +++ b/__tests__/ruby-namespaced-receiver.test.ts @@ -0,0 +1,312 @@ +/** + * A Ruby call on a constant written with its namespace — + * `Billing::ChargeCard.call(…)`, `::Foo.call` — depends on the class it names + * and calls that class's method, as `ChargeCard.call(…)` already did (#2503). + * The constant is the one Ruby's lookup finds from where it is written: the + * nearest namespace around the call that declares it, then the top level; a + * name written from the top level (`::Foo`) is looked up there only. A + * constant the project doesn't declare is a library's (`ActiveRecord::Base`) + * and links nothing, never a project class or method sharing its last name. + * + * Runs on the native kernel (skipped when none is built) and the wasm + * extractor. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { getKernel } from '../src/extraction/kernel/loader'; + +const FILES: Record = { + // The issue's three files. + 'app/services/billing/charge_card.rb': `module Billing + class ChargeCard + def self.call(**kw) = new(**kw).call + def initialize(amount:) = @amount = amount + def call = @amount + end +end +`, + 'app/services/checkout/complete.rb': `module Checkout + class Complete + def call + Billing::ChargeCard.call(amount: 1) + end + end +end +`, + 'app/services/billing/refund.rb': `module Billing + class Refund + def call + ChargeCard.call(amount: 2) + end + end +end +`, + // One class name in two namespaces: the caller names which one it means. + 'app/billing/create.rb': `module Billing + class Create + def self.call = 1 + end +end +`, + 'app/users/create.rb': `module Users + class Create + def self.call = 2 + end +end +`, + 'app/foo.rb': `class Foo + def self.call = 3 +end +`, + 'app/svc/foo.rb': `module Svc + class Foo + def self.call = 4 + end +end +`, + 'app/svc/runner.rb': `module Svc + class Runner + def billing = Billing::Create.call + def users = Users::Create.call + def users_safe = Users::Create&.call + def rooted = ::Billing::Create.call + def rooted_top = ::Foo.call + end +end +`, + // The same relative name under two API versions: the one around the call. + 'app/api/v1/users/create.rb': `module Api + module V1 + module Users + class Create + def self.call = 5 + end + end + end +end +`, + 'app/api/v2/users/create.rb': `module Api + module V2 + module Users + class Create + def self.call = 6 + end + end + end +end +`, + 'app/api/v2/users_controller.rb': `module Api + module V2 + class UsersController + def create + Users::Create.call + end + end + end +end +`, + // A library constant whose last name a project class shares. + 'app/base.rb': `class Base + def self.transaction = 7 +end +`, + 'app/models/order.rb': `class Order + def save! + ActiveRecord::Base.transaction { } + Stripe::Charge.create(amount: 1) + end +end +`, + // A test's stand-in for a library class, which production code never reaches. + 'spec/support/stripe.rb': `module Stripe + class Charge + def self.create(**) = 9 + end +end +`, + 'spec/models/order_spec.rb': `class OrderSpec + def test_charge + Stripe::Charge.create(amount: 1) + end +end +`, + // A module's method, a class method a superclass declares, a class declared + // with a compact name, a method the class doesn't declare, a constant value. + 'app/billing/helpers.rb': `module Billing + module Helpers + def self.fmt(x) = x + end + class Document + def self.build = new + end + class Invoice < Document + end + RATES = { usd: 1 } +end +`, + 'app/billing/statement.rb': `class Billing::Statement + def self.render = 8 +end +`, + 'app/billing/query.rb': `class Query + def where(x) = x +end +`, + 'app/billing/report.rb': `class Report + def run + Billing::Helpers.fmt(1) + Billing::Invoice.build + Billing::Statement.render + Billing::ChargeCard.where(amount: 1) + Billing::RATES.fetch(:usd) + end +end +`, +}; + +let root = ''; +let cg: CodeGraph | undefined; +let savedKernel: string | undefined; + +afterEach(() => { + cg?.close(); + cg = undefined; + if (root) fs.rmSync(root, { recursive: true, force: true }); + root = ''; + if (savedKernel === undefined) delete process.env.CODEGRAPH_KERNEL; + else process.env.CODEGRAPH_KERNEL = savedKernel; +}); + +function write(files: Record): void { + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } +} + +async function index(backend: string, files: Record = FILES): Promise { + savedKernel = process.env.CODEGRAPH_KERNEL; + if (backend === 'wasm') process.env.CODEGRAPH_KERNEL = '0'; + else delete process.env.CODEGRAPH_KERNEL; + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-ruby-nsrecv-')); + write(files); + cg = await CodeGraph.init(root, { index: true }); + return cg; +} + +/** ` :` of every `calls`/`references` edge written on `line` of `file`. */ +function edgesAt(graph: CodeGraph, file: string, line: number): string[] { + const ids = graph.getNodesInFile(file).map((n) => n.id); + return graph + .getOutgoingEdgesFrom(ids) + .filter((e) => (e.kind === 'calls' || e.kind === 'references') && e.line === line) + .map((e) => { + const t = graph.getNode(e.target)!; + return `${e.kind} ${t.qualifiedName}:${t.startLine}`; + }) + .sort(); +} + +/** `:` of every `references` edge into the class or module `qualifiedName`. */ +function referencedFrom(graph: CodeGraph, qualifiedName: string): string[] { + const target = graph.getNodesByQualifiedName(qualifiedName).find((n) => n.kind === 'class' || n.kind === 'module')!; + return graph + .getIncomingEdgesTo([target.id]) + .filter((e) => e.kind === 'references') + .map((e) => `${graph.getNode(e.source)!.filePath}:${e.line}`) + .sort(); +} + +const backends = [ + ['kernel', getKernel() === null], + ['wasm', false], +] as const; + +describe.each(backends)('Ruby call on a namespaced constant (%s)', (backend, skip) => { + it.skipIf(skip)('links the caller to the class, as a bare constant receiver does', async () => { + const graph = await index(backend); + expect(referencedFrom(graph, 'Billing::ChargeCard')).toEqual([ + 'app/billing/report.rb:6', + 'app/services/billing/refund.rb:4', + 'app/services/checkout/complete.rb:4', + ]); + }); + + it.skipIf(skip)("calls the class's method", async () => { + const graph = await index(backend); + expect(edgesAt(graph, 'app/services/checkout/complete.rb', 4)).toEqual([ + 'calls Billing::ChargeCard::call:3', + 'references Billing::ChargeCard:2', + ]); + }); + + it.skipIf(skip)('reaches the class of the namespace the caller names', async () => { + const graph = await index(backend); + const runner = 'app/svc/runner.rb'; + expect(edgesAt(graph, runner, 3)).toEqual(['calls Billing::Create::call:3', 'references Billing::Create:2']); + expect(edgesAt(graph, runner, 4)).toEqual(['calls Users::Create::call:3', 'references Users::Create:2']); + expect(edgesAt(graph, runner, 5)).toEqual(['calls Users::Create::call:3', 'references Users::Create:2']); + }); + + it.skipIf(skip)('looks a ::-rooted name up at the top level only', async () => { + const graph = await index(backend); + const runner = 'app/svc/runner.rb'; + expect(edgesAt(graph, runner, 6)).toEqual(['calls Billing::Create::call:3', 'references Billing::Create:2']); + // `::Foo` inside `module Svc` is the top-level Foo, not Svc::Foo. + expect(edgesAt(graph, runner, 7)).toEqual(['calls Foo::call:2', 'references Foo:1']); + }); + + it.skipIf(skip)('looks a relative name up in the namespaces around the call first', async () => { + const graph = await index(backend); + expect(edgesAt(graph, 'app/api/v2/users_controller.rb', 5)).toEqual([ + 'calls Api::V2::Users::Create::call:5', + 'references Api::V2::Users::Create:4', + ]); + }); + + it.skipIf(skip)('links nothing for a constant the project does not declare', async () => { + const graph = await index(backend); + expect(edgesAt(graph, 'app/models/order.rb', 3)).toEqual([]); + expect(referencedFrom(graph, 'Base')).toEqual([]); + }); + + it.skipIf(skip)('never links production code to a class only a test declares', async () => { + const graph = await index(backend); + expect(edgesAt(graph, 'app/models/order.rb', 4)).toEqual([]); + expect(edgesAt(graph, 'spec/models/order_spec.rb', 3)).toEqual([ + 'calls Stripe::Charge::create:3', + 'references Stripe::Charge:2', + ]); + }); + + it.skipIf(skip)('calls module methods, inherited class methods and compact-named classes, and guesses no method', async () => { + const graph = await index(backend); + const report = 'app/billing/report.rb'; + expect(edgesAt(graph, report, 3)).toEqual(['calls Billing::Helpers::fmt:3', 'references Billing::Helpers:2']); + expect(edgesAt(graph, report, 4)).toEqual(['calls Billing::Document::build:6', 'references Billing::Invoice:8']); + expect(edgesAt(graph, report, 5)).toEqual(['calls Billing::Statement::render:2', 'references Billing::Statement:1']); + // ChargeCard declares no `where`; Query's is not it. + expect(edgesAt(graph, report, 6)).toEqual(['references Billing::ChargeCard:2']); + // A constant holding a value: the call is the value's. + expect(edgesAt(graph, report, 7)).toEqual(['references Billing::RATES:10']); + }); +}); + +describe('Ruby call on a namespaced constant, after a sync', () => { + it('links the calls once a sync adds the class they name', async () => { + const { 'app/services/billing/charge_card.rb': chargeCard, ...rest } = FILES; + const graph = await index('default', rest); + expect(edgesAt(graph, 'app/services/checkout/complete.rb', 4)).toEqual([]); + + write({ 'app/services/billing/charge_card.rb': chargeCard! }); + expect((await graph.sync()).filesAdded).toBe(1); + // complete.rb did not change: only the retry of its parked refs links it. + expect(edgesAt(graph, 'app/services/checkout/complete.rb', 4)).toEqual([ + 'calls Billing::ChargeCard::call:3', + 'references Billing::ChargeCard:2', + ]); + }, 60_000); +}); diff --git a/__tests__/ruby-singleton-methods.test.ts b/__tests__/ruby-singleton-methods.test.ts new file mode 100644 index 0000000000..5307b35a2a --- /dev/null +++ b/__tests__/ruby-singleton-methods.test.ts @@ -0,0 +1,212 @@ +/** + * A Ruby class method — `def self.build`, `def Widget.build`, or a `def` + * inside `class << self` — is the class's own, not one its instances answer + * to. They were indexed as instance methods under the same qualified name as + * an instance method of that name (`Widget::build` twice), so a call picked + * whichever the class declared first: `Widget.build` could land on + * `def build`, and `widget.build` on `def self.build`. Class methods are now + * marked static, and a call reaches the one of the right kind: a call on the + * class (`Widget.build`, `Billing::Payout.call`) its class method, a call on + * an instance (`w = Widget.new; w.build`) its instance method, and a bare + * call the one of the calling method's kind. Only such pairs are split: a + * block may run on another object, so a bare call with no same-named method + * of the other kind keeps the target it had. A receiver known by its name + * alone (`widget.build`) may hold the class itself, so it splits nothing. + * + * Runs on the native kernel (skipped when none is built) and the wasm + * extractor. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { getKernel } from '../src/extraction/kernel/loader'; + +const FILES: Record = { + // The instance method first, so declaration order alone picks it. + 'app/widget.rb': `class Widget + def build + self + end + + def self.build(opts) + new + end + + class << self + def make + new + end + + def build_all + build(1) + end + end + + def run + build + Widget.build(1) + end + + def self.go + build(2) + end +end +`, + 'app/user.rb': `class User + def x + w = Widget.new + w.build + Widget.build(3) + Widget.make + end +end +`, + // #2503's control: an instance of a class whose class method comes first. + 'app/services/billing/charge_card.rb': `module Billing + class ChargeCard + def self.call(**kw) = new(**kw).call + def initialize(amount:) = @amount = amount + def call = @amount + end +end +`, + // A class method's block (a test DSL's) calls what an instance method and a + // top-level function are both named: no pair of one class, so no split. + 'app/spec_like.rb': `class SpecLike + def self.parses + run_case do + mock_app + end + end + + def mock_app = 1 +end +`, + 'app/helpers.rb': `def mock_app = 2 +`, + // A receiver named after a class that holds classes, as rubocop's + // `cop.cop_name` over its registry's cop classes does. + 'app/cop.rb': `class Cop + def self.cop_name = 'Style/Foo' + def cop_name = self.class.cop_name +end +`, + 'app/registry.rb': `class Registry + def names(cops) + cops.map { |cop| cop.cop_name } + end +end +`, + // A namespaced class whose instance method comes first. + 'app/services/billing/payout.rb': `module Billing + class Payout + def call = 1 + def self.call(**kw) = new.call + end +end +`, + 'app/services/checkout/start.rb': `module Checkout + class Start + def call + card = Billing::ChargeCard.new(amount: 3) + card.call + Billing::Payout.call(amount: 4) + payout = Billing::Payout.new + payout.call + end + end +end +`, +}; + +let root = ''; +let cg: CodeGraph | undefined; +let savedKernel: string | undefined; + +afterEach(() => { + cg?.close(); + cg = undefined; + if (root) fs.rmSync(root, { recursive: true, force: true }); + root = ''; + if (savedKernel === undefined) delete process.env.CODEGRAPH_KERNEL; + else process.env.CODEGRAPH_KERNEL = savedKernel; +}); + +async function index(backend: string): Promise { + savedKernel = process.env.CODEGRAPH_KERNEL; + if (backend === 'wasm') process.env.CODEGRAPH_KERNEL = '0'; + else delete process.env.CODEGRAPH_KERNEL; + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-ruby-singleton-')); + for (const [rel, content] of Object.entries(FILES)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + cg = await CodeGraph.init(root, { index: true }); + return cg; +} + +/** `:` of the method each `calls` edge written on `line` of `file` reaches. */ +function callsAt(graph: CodeGraph, file: string, line: number): string[] { + const ids = graph.getNodesInFile(file).map((n) => n.id); + return graph.getOutgoingEdgesFrom(ids) + .filter((e) => e.kind === 'calls' && e.line === line) + .map((e) => { const t = graph.getNode(e.target)!; return `${t.qualifiedName}:${t.startLine}`; }) + .sort(); +} + +const backends = [ + ['kernel', getKernel() === null], + ['wasm', false], +] as const; + +describe.each(backends)('Ruby class methods (%s)', (backend, skip) => { + it.skipIf(skip)('are static, and instance methods are not', async () => { + const graph = await index(backend); + const statics = graph.getNodesInFile('app/widget.rb') + .filter((n) => n.kind === 'method') + .map((n) => `${n.name}:${n.startLine}:${n.isStatic === true ? 'static' : 'instance'}`) + .sort(); + expect(statics).toEqual([ + 'build:2:instance', + 'build:6:static', + 'build_all:15:static', + 'go:25:static', + 'make:11:static', + 'run:20:instance', + ]); + }); + + it.skipIf(skip)('a call on the class reaches its class method', async () => { + const graph = await index(backend); + expect(callsAt(graph, 'app/user.rb', 5)).toEqual(['Widget::build:6']); + expect(callsAt(graph, 'app/user.rb', 6)).toEqual(['Widget::make:11']); + expect(callsAt(graph, 'app/widget.rb', 22)).toEqual(['Widget::build:6']); + expect(callsAt(graph, 'app/services/checkout/start.rb', 6)).toEqual(['Billing::Payout::call:4']); + }); + + it.skipIf(skip)('a call on an instance reaches its instance method', async () => { + const graph = await index(backend); + expect(callsAt(graph, 'app/user.rb', 4)).toEqual(['Widget::build:2']); + expect(callsAt(graph, 'app/services/checkout/start.rb', 5)).toEqual(['Billing::ChargeCard::call:5']); + expect(callsAt(graph, 'app/services/checkout/start.rb', 8)).toEqual(['Billing::Payout::call:3']); + }); + + it.skipIf(skip)('a bare call reaches the method of the calling method kind', async () => { + const graph = await index(backend); + expect(callsAt(graph, 'app/widget.rb', 21)).toEqual(['Widget::build:2']); + expect(callsAt(graph, 'app/widget.rb', 26)).toEqual(['Widget::build:6']); + expect(callsAt(graph, 'app/widget.rb', 16)).toEqual(['Widget::build:6']); + }); + + it.skipIf(skip)('a receiver known by its name alone splits nothing: it may hold the class', async () => { + const graph = await index(backend); + expect(callsAt(graph, 'app/registry.rb', 3)).toEqual(['Cop::cop_name:2']); + }); + + it.skipIf(skip)('a bare call with no method of the other kind by its name keeps its target', async () => { + const graph = await index(backend); + expect(callsAt(graph, 'app/spec_like.rb', 4)).toEqual(['SpecLike::mock_app:8']); + }); +}); diff --git a/__tests__/rust-receiver-chains.test.ts b/__tests__/rust-receiver-chains.test.ts new file mode 100644 index 0000000000..0e213538f0 --- /dev/null +++ b/__tests__/rust-receiver-chains.test.ts @@ -0,0 +1,281 @@ +/** + * A Rust method call keeps the receiver chain it is written on (#2836): a + * chain of fields and method calls on `self` or on a name — + * `self.services.store.put(…)`, `self.store().put(…)`, `vis.map.get(…)` — is + * recorded with that chain, calls written `name()`, as the one-field + * `self.store.put(…)` always was. The extractor used to keep the bare method + * name, so the edge's `refName` said nothing of which value the call goes + * through, and the receiver of a call on a method call's value was never read. + * + * Resolution reads the receiver's type through the whole chain, where the + * call is written: `self.services` is a `Services`, its `store` a + * `Box`, so the call is `Store::put`; `self.store()` is what + * `Svc::store` returns. A later link calling the same name again is a call of + * its own (`self.table.get("a").get(0)` is `Table::get`, then `Row::get`). + */ +import { describe, it, expect, afterAll, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; + +const roots: string[] = []; +const graphs: Record<'issue' | 'chains', CodeGraph> = {} as never; + +function write(root: string, files: Record): void { + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } +} + +async function index(name: keyof typeof graphs, files: Record): Promise { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `cg-rust-chain-${name}-`)); + roots.push(root); + write(root, files); + graphs[name] = await CodeGraph.init(root, { index: true }); +} + +const crate = (name: string, deps = ''): string => + `[package]\nname = "${name}"\nversion = "0.1.0"\nedition = "2021"\n\n[dependencies]\n${deps}`; + +beforeAll(async () => { + // The issue's workspace, with the call on a method call's value it names. + await index('issue', { + 'Cargo.toml': '[workspace]\nmembers = ["api", "app"]\nresolver = "2"\n', + 'api/Cargo.toml': crate('api'), + 'api/src/lib.rs': 'pub trait Store {\n fn put(&self, key: &str);\n}\n', + 'app/Cargo.toml': crate('app', 'api = { path = "../api" }\n'), + 'app/src/lib.rs': [ + 'use api::Store;', + '', + 'pub struct Services { pub store: Box }', + 'pub struct Svc { pub services: Services, pub store: Box }', + '', + 'impl Svc {', + ' pub fn a(&self) { self.services.store.put("a"); }', + ' pub fn b(&self) { let _ = self.services.store.put("b"); }', + ' pub fn c(&self) {', + ' self', + ' .services', + ' .store', + ' .put("c");', + ' }', + ' pub fn d(&self) { self.store.put("d"); }', + ' pub fn e(&self) {', + ' self', + ' .store', + ' .put("e");', + ' }', + ' pub fn m(&self) { self.store().put("m"); }', + ' pub fn store(&self) -> &dyn Store { &*self.store }', + '}', + '', + ].join('\n'), + }); + + // Chains whose links call the same name, through `self`, a parameter and a + // tuple field, and a call on a method call's std value — beside a project + // namesake a guess by the name would take. + await index('chains', { + 'Cargo.toml': crate('chains'), + 'src/lib.rs': [ + 'pub struct Row { cells: Vec }', + '', + 'impl Row {', + ' pub fn get(&self, i: usize) -> u8 { self.cells[i] }', + '}', + '', + 'pub struct Table { rows: Vec, names: Vec }', + '', + 'impl Table {', + ' pub fn get(&self, _k: &str) -> &Row { &self.rows[0] }', + ' pub fn names(&self) -> &Vec { &self.names }', + '}', + '', + 'pub struct Db { pub table: Table }', + 'pub struct Wrap(pub Table);', + '', + '/// A project namesake of std\'s `len`.', + 'pub struct Counter;', + '', + 'impl Counter {', + ' pub fn len(&self) -> usize { 0 }', + '}', + '', + 'impl Db {', + ' pub fn cell(&self) -> u8 { self.table.get("a").get(0) }', + ' pub fn count(&self) -> usize { self.table.names().len() }', + '}', + '', + 'impl Wrap {', + ' pub fn first(&self) -> u8 { self.0.get("c").get(2) }', + '}', + '', + 'pub fn local(db: &Db) -> u8 {', + ' db.table', + ' .get("b")', + ' .get(1)', + '}', + '', + ].join('\n'), + }); +}, 120_000); + +afterAll(() => { + for (const cg of Object.values(graphs)) cg?.close(); + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); +}); + +/** The calls on `line` of `file`, as `qualifiedName resolvedBy refName`, sorted. */ +const callsAt = (name: keyof typeof graphs, file: string, line: number): string[] => { + const cg = graphs[name]; + const out: string[] = []; + for (const node of cg.getNodesInFile(file)) { + for (const edge of cg.getOutgoingEdges(node.id)) { + if (edge.kind !== 'calls' || edge.line !== line || edge.provenance === 'heuristic') continue; + out.push(`${cg.getNode(edge.target)!.qualifiedName} ${String(edge.metadata?.resolvedBy)} ${String(edge.metadata?.refName)}`); + } + } + return [...new Set(out)].sort(); +}; + +describe('the issue\'s workspace (#2836)', () => { + it('a call on two fields keeps its receiver: on one line, after `let _ =`, split by rustfmt', () => { + expect(callsAt('issue', 'app/src/lib.rs', 7)).toEqual(['Store::put instance-method self.services.store.put']); + expect(callsAt('issue', 'app/src/lib.rs', 8)).toEqual(['Store::put instance-method self.services.store.put']); + expect(callsAt('issue', 'app/src/lib.rs', 10)).toEqual(['Store::put instance-method self.services.store.put']); + }); + + it('a call on a method call\'s value keeps the path up to that call, and reaches what it returns', () => { + expect(callsAt('issue', 'app/src/lib.rs', 21)).toEqual([ + 'Store::put instance-method self.store().put', + 'Svc::store qualified-name self.store', + ]); + }); + + it('a call on one field keeps what it had', () => { + expect(callsAt('issue', 'app/src/lib.rs', 15)).toEqual(['Store::put instance-method self.store.put']); + expect(callsAt('issue', 'app/src/lib.rs', 17)).toEqual(['Store::put instance-method self.store.put']); + }); + + it('leaves no call unresolved', () => { + expect(graphs.issue.getUnresolvedReferencesInFile('app/src/lib.rs').filter((r) => r.referenceKind === 'calls')).toEqual([]); + }); +}); + +describe('a Rust call on a receiver chain', () => { + it('is told from a later link calling the same name: through `self`, a tuple field and a parameter', () => { + expect(callsAt('chains', 'src/lib.rs', 25)).toEqual([ + 'Row::get instance-method self.table.get().get', + 'Table::get instance-method self.table.get', + ]); + expect(callsAt('chains', 'src/lib.rs', 30)).toEqual([ + 'Row::get instance-method self.0.get().get', + 'Table::get instance-method self.0.get', + ]); + expect(callsAt('chains', 'src/lib.rs', 34)).toEqual([ + 'Row::get instance-method db.table.get().get', + 'Table::get instance-method db.table.get', + ]); + }); + + it('on a method call\'s std value is no project method', () => { + expect(callsAt('chains', 'src/lib.rs', 26)).toEqual(['Table::names instance-method self.table.names']); + }); +}); + +describe('Rust call references keep their receiver chain', () => { + const ENV_KEYS = ['CODEGRAPH_KERNEL'] as const; + let savedEnv: Record; + + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['rust']); + }); + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + process.env.CODEGRAPH_KERNEL = '0'; + }); + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + }); + + /** The names of the `calls` references extraction records for `body`, a method body's statements, sorted. */ + const callNames = (body: string): string[] => { + const source = `impl Svc {\n fn shapes(&self, x: &X) {\n${body}\n }\n}\n`; + return extractFromSource('shapes.rs', source, 'rust').unresolvedReferences + .filter((r) => r.referenceKind === 'calls') + .map((r) => r.referenceName) + .sort(); + }; + + it('of fields and method calls on `self` or a name, calls written `name()`', () => { + expect(callNames('self.services.store.put("a");')).toEqual(['self.services.store.put']); + expect(callNames('self.store().put("m");')).toEqual(['self.store', 'self.store().put']); + expect(callNames('self.0.len();')).toEqual(['self.0.len']); + expect(callNames('x.map.get("k");')).toEqual(['x.map.get']); + expect(callNames('x.get::(1).put(2);')).toEqual(['x.get', 'x.get().put']); + expect(callNames('self\n .services // the services\n .store\n .put("c");')).toEqual(['self.services.store.put']); + }); + + it('and keep the shapes they had', () => { + expect(callNames('self.store.put("d");')).toEqual(['self.store.put']); + expect(callNames('x.put(1);')).toEqual(['x.put']); + expect(callNames('self.put(1);')).toEqual(['self.put']); + expect(callNames('Foo::new().a().b();')).toEqual(['Foo::new', 'Foo::new().a', 'b']); + }); + + it('but not through parentheses, `?` or `.await`, nor past sixteen links', () => { + expect(callNames('(x).a.b();')).toEqual(['b']); + expect(callNames('self.a?.b();')).toEqual(['b']); + expect(callNames('self.a.b().await.c();')).toEqual(['c', 'self.a.b']); + const links = Array.from({ length: 17 }, (_, i) => `l${i}()`); + const names = callNames(`x.${links.join('.')}.m();`); + expect(names).toHaveLength(18); + expect(names).toContain('x.l0'); + expect(names).toContain(`x.${links.slice(0, 16).join('.')}.l16`); + expect(names).toContain('m'); + }); +}); + +describe('a Rust call on a receiver chain that names no method yet', () => { + let root: string; + let cg: CodeGraph | undefined; + beforeEach(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-rust-chain-sync-')); }); + afterEach(() => { cg?.close(); cg = undefined; fs.rmSync(root, { recursive: true, force: true }); }); + + it('waits under its method\'s name, and resolves once a sync adds the method', async () => { + write(root, { + 'Cargo.toml': crate('later'), + 'src/lib.rs': [ + 'pub mod disk;', + '', + 'pub struct Services { pub disk: disk::Disk }', + 'pub struct Svc { pub services: Services }', + '', + 'impl Svc {', + ' pub fn save(&self) { self.services.disk.flush(); }', + '}', + '', + ].join('\n'), + 'src/disk.rs': 'pub struct Disk;\n', + }); + cg = await CodeGraph.init(root, { index: true }); + const parked = cg.getUnresolvedReferencesInFile('src/lib.rs').filter((r) => r.referenceKind === 'calls'); + expect(parked.map((r) => r.referenceName)).toEqual(['self.services.disk.flush']); + + fs.writeFileSync(path.join(root, 'src/disk.rs'), 'pub struct Disk;\n\nimpl Disk {\n pub fn flush(&self) {}\n}\n'); + await cg.sync(); + const save = cg.getNodesInFile('src/lib.rs').find((n) => n.name === 'save')!; + const calls = cg.getOutgoingEdges(save.id).filter((e) => e.kind === 'calls'); + expect(calls.map((e) => `${cg!.getNode(e.target)!.qualifiedName} ${String(e.metadata?.refName)}`)).toEqual([ + 'Disk::flush self.services.disk.flush', + ]); + }); +}); diff --git a/__tests__/rust-written-receivers.test.ts b/__tests__/rust-written-receivers.test.ts new file mode 100644 index 0000000000..679f7ff400 --- /dev/null +++ b/__tests__/rust-written-receivers.test.ts @@ -0,0 +1,474 @@ +/** + * A Rust call resolves by the type its receiver is written with, where the + * source writes it (#2614): + * + * - a field chain, which the extractor records by the bare method name — + * `vis.map.get("a")` on a `HashMap` field, `self.inner.cfg.port()` — is a + * call on the type the struct declares for the last field; + * - a struct literal, `Xyz { x: 1.0 }.to_val()`, is a call on its own type; + * - a path through a type from std or a dependency — `J::from(i)` under + * `use serde_json::Value as J`, `serde_json::Value::from(i)` — reaches only + * the project's own impls of a trait for that type. + * + * By the name alone, the reporter's `vis.map.get("a")` went to `Ext::get` (an + * associated function, at that), `Xyz { … }.to_val()` to `Stat::to_val` (the + * same trait's other impl) and `J::from(i)` to `impl From for Fail`. + */ +import { describe, it, expect, afterAll, beforeAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +const roots: string[] = []; +const graphs: Record<'issue' | 'variants' | 'keeps', CodeGraph> = {} as never; + +async function index(name: keyof typeof graphs, files: Record): Promise { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `cg-rust-written-${name}-`)); + roots.push(root); + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + graphs[name] = await CodeGraph.init(root, { index: true }); +} + +const manifest = (name: string): string => + `[package]\nname = "${name}"\nversion = "0.1.0"\nedition = "2021"\n\n[dependencies]\nserde_json = "1"\n`; + +beforeAll(async () => { + // The issue's crate, verbatim. + await index('issue', { + 'Cargo.toml': manifest('repro'), + 'src/main.rs': [ + 'use std::collections::HashMap;', + 'use serde_json::Value as J;', + '', + 'pub enum Val {', + ' Null,', + ' Int(i64),', + ' List(Vec),', + '}', + '', + 'pub struct Obj {', + ' keys: Vec,', + '}', + '', + 'impl Obj {', + ' pub fn contains(&self, k: &str) -> bool {', + ' self.keys.iter().any(|x| x == k)', + ' }', + '}', + '', + 'pub struct Ext;', + '', + 'impl Ext {', + ' pub fn get(name: &str) -> Ext {', + ' let _ = name;', + ' Ext', + ' }', + '}', + '', + 'pub struct Fail;', + '', + 'impl From for Fail {', + ' fn from(_: std::io::Error) -> Fail {', + ' Fail', + ' }', + '}', + '', + 'pub struct Xyz {', + ' x: f64,', + '}', + '', + 'pub struct Stat {', + ' n: f64,', + '}', + '', + 'pub trait ToVal {', + ' fn to_val(&self) -> Val;', + '}', + '', + 'impl ToVal for Xyz {', + ' fn to_val(&self) -> Val {', + ' Val::Int(self.x as i64)', + ' }', + '}', + '', + 'impl ToVal for Stat {', + ' fn to_val(&self) -> Val {', + ' Val::Int(self.n as i64)', + ' }', + '}', + '', + '// 1. slice::contains, not Obj::contains', + 'fn slice_has(names: &[&str], k: &str) -> bool {', + ' names.contains(&k)', + '}', + '', + '// 2. HashMap::get on a struct field, not Ext::get', + 'pub struct Vis {', + ' map: HashMap,', + '}', + '', + 'fn map_get(vis: &Vis) -> Option<&i32> {', + ' vis.map.get("a")', + '}', + '', + '// 3. serde_json::Value::from via alias J, not Fail::from', + 'fn to_json(i: i64) -> J {', + ' J::from(i)', + '}', + '', + '// 4. method on struct literal: Xyz::to_val, not Stat::to_val', + 'fn centroid() -> Val {', + ' Xyz { x: 1.0 }.to_val()', + '}', + '', + 'fn main() {', + ' let o = Obj { keys: vec![] };', + ' let _ = o.contains("a");', + ' let _ = Ext::get("x");', + ' let _ = Stat { n: 0.0 }.to_val();', + ' let _ = slice_has(&["a"], "a");', + ' let _ = map_get(&Vis { map: HashMap::new() });', + ' let _ = to_json(1);', + ' let _ = centroid();', + ' let _ = Val::List(vec![Val::Null]);', + '}', + '', + ].join('\n'), + }); + + // Field chains and struct literals of other shapes, beside namesakes of the + // methods they call; and paths through types from outside the project, with + // one `from` in the crate for a guess by the name to take. + await index('variants', { + 'Cargo.toml': manifest('variants'), + 'src/main.rs': 'mod chains;\nmod config;\nmod paths;\n\nfn main() {}\n', + 'src/config.rs': [ + 'pub struct Config {', + ' pub port: u16,', + '}', + '', + 'impl Config {', + ' pub fn load() -> Config {', + ' Config { port: 1 }', + ' }', + '', + ' pub fn port(&self) -> u16 {', + ' self.port', + ' }', + '}', + '', + '/// A project type named like `serde_json::Value`.', + 'pub struct Value;', + '', + 'impl Default for Value {', + ' fn default() -> Value {', + ' Value', + ' }', + '}', + '', + 'pub struct Fail;', + '', + 'impl From for Fail {', + ' fn from(_: std::io::Error) -> Fail {', + ' Fail', + ' }', + '}', + '', + ].join('\n'), + 'src/chains.rs': [ + 'use std::collections::HashMap;', + '', + 'use crate::config::Config;', + '', + 'pub struct Inner {', + ' pub map: HashMap,', + ' pub cfg: Config,', + '}', + '', + 'pub struct Outer {', + ' pub inner: Inner,', + ' pub pair: (Config, u8),', + '}', + '', + 'pub struct Wrap(pub Config);', + '', + '/// Namesakes of the methods the calls below make, beside them.', + 'pub struct Dial;', + '', + 'impl Dial {', + ' pub fn port(&self) -> u16 {', + ' 0', + ' }', + '', + ' pub fn get(&self, _k: &str) -> Option<&i32> {', + ' None', + ' }', + '}', + '', + 'pub enum Shape {', + ' Circle { r: f64 },', + '}', + '', + 'impl Shape {', + ' pub fn area(&self) -> f64 {', + ' 0.0', + ' }', + '}', + '', + 'pub struct Room {', + ' pub w: f64,', + '}', + '', + 'impl Room {', + ' pub fn area(&self) -> f64 {', + ' self.w', + ' }', + '', + ' pub fn fresh(&self) -> f64 {', + ' Self { w: 2.0 }.area()', + ' }', + '}', + '', + 'impl Outer {', + ' pub fn deep_std(&self) -> Option<&i32> {', + ' self.inner.map.get("a")', + ' }', + '', + ' pub fn deep_project(&self) -> u16 {', + ' self.inner.cfg.port()', + ' }', + '', + ' pub fn tuple_field(&self) -> u16 {', + ' self.pair.0.port()', + ' }', + '}', + '', + 'impl Wrap {', + ' pub fn tuple_self(&self) -> u16 {', + ' self.0.port()', + ' }', + '}', + '', + 'pub fn param_chain(o: &Outer) -> u16 {', + ' o.inner.cfg.port()', + '}', + '', + 'pub fn param_chain_std(o: &Outer) -> i32 {', + ' o.inner', + ' .map', + ' .get("b")', + ' .copied()', + ' .unwrap_or(0)', + '}', + '', + 'pub fn local_chain() -> u16 {', + ' let w = Wrap(Config::load());', + ' w.0.port()', + '}', + '', + 'pub fn literal() -> u16 {', + ' Config { port: 2 }.port()', + '}', + '', + 'pub fn literal_in_parens() -> u16 {', + ' (&Config { port: 3 }).port()', + '}', + '', + 'pub fn variant_literal() -> f64 {', + ' Shape::Circle { r: 1.0 }.area()', + '}', + '', + '/// A trait a derive implements (axum-extra\'s `#[derive(TypedPath)]`): the impl is the macro\'s.', + 'pub trait TypedPath {', + ' fn with_query_params(self, q: u8) -> u8', + ' where', + ' Self: Sized,', + ' {', + ' q', + ' }', + '}', + '', + '#[derive(TypedPath, Debug)]', + 'pub struct UsersShow {', + ' pub id: u32,', + '}', + '', + 'pub fn derived() -> u8 {', + ' UsersShow { id: 1 }.with_query_params(2)', + '}', + '', + ].join('\n'), + 'src/paths.rs': [ + 'use serde_json as json;', + 'use serde_json::Value;', + '', + 'pub fn written_outside(i: i64) -> serde_json::Value {', + ' serde_json::Value::from(i)', + '}', + '', + 'pub fn crate_alias(i: i64) -> json::Value {', + ' json::Value::from(i)', + '}', + '', + 'pub fn unrenamed_outside() -> Value {', + ' Value::default()', + '}', + '', + 'pub fn written_default() -> serde_json::Value {', + ' serde_json::Value::default()', + '}', + '', + ].join('\n'), + }); + + // The project's own impls on types from outside it, which those paths do reach. + await index('keeps', { + 'Cargo.toml': manifest('keeps'), + 'src/main.rs': [ + 'use std::io::Write as W;', + 'use std::path::PathBuf;', + '', + 'use serde_json::Value;', + '', + 'pub struct Fail;', + '', + 'impl From for PathBuf {', + ' fn from(_: Fail) -> PathBuf {', + ' PathBuf::new()', + ' }', + '}', + '', + 'pub trait Blank {', + ' fn blank() -> Self;', + '}', + '', + 'impl Blank for serde_json::Value {', + ' fn blank() -> Self {', + ' serde_json::Value::Null', + ' }', + '}', + '', + 'pub struct Sink;', + '', + 'impl std::io::Write for Sink {', + ' fn write(&mut self, buf: &[u8]) -> std::io::Result {', + ' Ok(buf.len())', + ' }', + '', + ' fn flush(&mut self) -> std::io::Result<()> {', + ' Ok(())', + ' }', + '}', + '', + 'fn to_path(f: Fail) -> PathBuf {', + ' PathBuf::from(f)', + '}', + '', + 'fn blank() -> Value {', + ' Value::blank()', + '}', + '', + 'fn drain(s: &mut Sink) {', + ' let _ = W::flush(s);', + '}', + '', + 'fn main() {', + ' let _ = to_path(Fail);', + ' let _ = blank();', + '}', + '', + ].join('\n'), + }); +}, 120_000); + +afterAll(() => { + for (const cg of Object.values(graphs)) cg?.close(); + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); +}); + +/** The calls on `line` of `file`, as `qualifiedName resolvedBy`, sorted. */ +const callsAt = (name: keyof typeof graphs, file: string, line: number): string[] => { + const cg = graphs[name]; + const out: string[] = []; + for (const node of cg.getNodesInFile(file)) { + for (const edge of cg.getOutgoingEdges(node.id)) { + if (edge.kind !== 'calls' || edge.line !== line || edge.provenance === 'heuristic') continue; + out.push(`${cg.getNode(edge.target)!.qualifiedName} ${String(edge.metadata?.resolvedBy)}`); + } + } + return [...new Set(out)].sort(); +}; + +describe('the issue\'s crate (#2614)', () => { + it('a `HashMap` field\'s `get` is no project method', () => { + expect(callsAt('issue', 'src/main.rs', 72)).toEqual([]); + }); + + it('`J::from(i)` under `use serde_json::Value as J` is no project method', () => { + expect(callsAt('issue', 'src/main.rs', 77)).toEqual([]); + }); + + it('a struct literal calls its own type\'s method', () => { + expect(callsAt('issue', 'src/main.rs', 82)).toEqual(['Xyz::to_val instance-method']); + expect(callsAt('issue', 'src/main.rs', 89)).toEqual(['Stat::to_val instance-method']); + }); + + it('keeps what was already right', () => { + expect(callsAt('issue', 'src/main.rs', 63)).toEqual([]); + expect(callsAt('issue', 'src/main.rs', 87)).toEqual(['Obj::contains instance-method']); + expect(callsAt('issue', 'src/main.rs', 88)).toEqual(['Ext::get qualified-name']); + }); +}); + +describe('a Rust call on a field chain', () => { + it('of a std type is no project method', () => { + expect(callsAt('variants', 'src/chains.rs', 56)).toEqual([]); + expect(callsAt('variants', 'src/chains.rs', 79)).toEqual([]); + }); + + it('of a project type is that type\'s method, not a namesake beside the call', () => { + expect(callsAt('variants', 'src/chains.rs', 60)).toEqual(['Config::port instance-method']); + expect(callsAt('variants', 'src/chains.rs', 64)).toEqual(['Config::port instance-method']); + expect(callsAt('variants', 'src/chains.rs', 70)).toEqual(['Config::port instance-method']); + expect(callsAt('variants', 'src/chains.rs', 75)).toEqual(['Config::port instance-method']); + expect(callsAt('variants', 'src/chains.rs', 88)).toEqual(['Config::port instance-method']); + }); +}); + +describe('a Rust call on a struct literal', () => { + it('is its own type\'s method, in parentheses too', () => { + expect(callsAt('variants', 'src/chains.rs', 92)).toEqual(['Config::port instance-method']); + expect(callsAt('variants', 'src/chains.rs', 96)).toEqual(['Config::port instance-method']); + }); + + it('of an enum\'s variant is the enum\'s method, and `Self { … }` the impl type\'s', () => { + expect(callsAt('variants', 'src/chains.rs', 100)).toEqual(['Shape::area instance-method']); + expect(callsAt('variants', 'src/chains.rs', 50)).toEqual(['Room::area instance-method']); + }); + + it('reaches the method a trait it derives provides', () => { + expect(callsAt('variants', 'src/chains.rs', 119)).toEqual(['TypedPath::with_query_params instance-method']); + }); +}); + +describe('a Rust path through a type from outside the project', () => { + it('written out or through a crate\'s alias is no project type\'s function', () => { + expect(callsAt('variants', 'src/paths.rs', 5)).toEqual([]); + expect(callsAt('variants', 'src/paths.rs', 9)).toEqual([]); + }); + + it('a `use` names, or the path writes, is not a project namesake\'s', () => { + expect(callsAt('variants', 'src/paths.rs', 13)).toEqual([]); + expect(callsAt('variants', 'src/paths.rs', 17)).toEqual([]); + }); + + it('still reaches the project\'s own impls for that type, or of that trait', () => { + expect(callsAt('keeps', 'src/main.rs', 37)).toEqual(['PathBuf::from qualified-name']); + expect(callsAt('keeps', 'src/main.rs', 41)).toEqual(['Value::blank qualified-name']); + expect(callsAt('keeps', 'src/main.rs', 45)).toEqual(['Sink::flush instance-method']); + }); +}); diff --git a/__tests__/same-file-match-confidence.test.ts b/__tests__/same-file-match-confidence.test.ts new file mode 100644 index 0000000000..82e02d0413 --- /dev/null +++ b/__tests__/same-file-match-confidence.test.ts @@ -0,0 +1,123 @@ +/** + * Confidence of an exact-name match the call site's own file decided. + * + * With several same-named candidates, the name-matcher ranks the one in the + * call site's file first, then scored the winner by DIRECTORY proximity — the + * number of leading folders its path shares with the caller's. A same-file + * winner shares every folder, but a shallow file has few to share, so + * `scripts/x.py` calling its own `main` got 0.4, the score of a guess across + * unrelated modules. A lone same-file definition now scores like a lone + * candidate; the matcher's pick itself never changes, and a pick the file + * does not settle (two same-file candidates, a sibling class's method, the + * caller itself) keeps the proximity score. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; + +describe('same-file exact-match confidence', () => { + let dir: string; + beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'same-file-conf-')); }); + afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); }); + + const write = (rel: string, body: string) => { + const p = path.join(dir, rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, body); + }; + + type CallEdge = { src: string; tgt: string; tgtQn: string; tgtFile: string; confidence: number }; + + const calls = async (kind = 'calls'): Promise => { + const cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + const db = (cg as any).db.db; + const rows: CallEdge[] = db + .prepare( + `SELECT s.name src, t.name tgt, t.qualified_name tgtQn, t.file_path tgtFile, + json_extract(e.metadata, '$.confidence') confidence + FROM edges e + JOIN nodes s ON s.id = e.source + JOIN nodes t ON t.id = e.target + WHERE e.kind = ?` + ) + .all(kind); + cg.close?.(); + return rows; + }; + + it('scores a shallow-path call to its own file as a strong match', async () => { + write('x.py', 'def helper():\n return 1\n\ndef main():\n return helper()\n'); + write('pkg/other.py', 'def helper():\n return 2\n'); + const edge = (await calls()).find((e) => e.src === 'main' && e.tgt === 'helper'); + expect(edge?.tgtFile).toBe('x.py'); + expect(edge?.confidence).toBe(0.9); + }); + + it('keeps the same-file target when a sibling file defines the name too', async () => { + write('a/b/c/x.py', 'def helper():\n return 1\n\ndef main():\n return helper()\n'); + write('a/b/c/y.py', 'def helper():\n return 2\n'); + const edge = (await calls()).find((e) => e.src === 'main' && e.tgt === 'helper'); + expect(edge?.tgtFile).toBe('a/b/c/x.py'); + expect(edge?.confidence).toBe(0.9); + }); + + it('leaves the proximity score when the file holds several candidates', async () => { + // Both branches of an `#ifdef` define `helper`: line distance picks one. + write( + 'x.c', + '#ifdef FAST\nstatic int helper(void) { return 1; }\n#else\nstatic int helper(void) { return 2; }\n#endif\n' + + 'int main(void) { return helper(); }\n' + ); + write('lib/other.c', 'int helper(void) { return 3; }\n'); + const edge = (await calls()).find((e) => e.src === 'main' && e.tgt === 'helper'); + expect(edge?.tgtFile).toBe('x.c'); + expect(edge?.confidence).toBe(0.4); + }); + + it('leaves the proximity score when the match is the caller itself', async () => { + // Recursion, or a receiver the call lost (`json.dumps` inside `dumps`). + write('x.py', 'def walk(n):\n return walk(n - 1)\n'); + write('pkg/other.py', 'def walk(n):\n return 2\n'); + const edge = (await calls()).find((e) => e.src === 'walk' && e.tgt === 'walk'); + expect(edge?.tgtFile).toBe('x.py'); + expect(edge?.confidence).toBe(0.4); + }); + + it("leaves the proximity score for a sibling class's method", async () => { + // The other `run` is one the file includes: C++ sees no other file's (#2684). + write('x.cpp', '#include "pkg/other.h"\nstruct A {\n int run() { return 1; }\n};\nint go() { return run(); }\n'); + write('pkg/other.h', 'inline int run() { return 2; }\n'); + const edge = (await calls()).find((e) => e.src === 'go' && e.tgt === 'run'); + expect(edge?.tgtQn).toBe('A::run'); + expect(edge?.confidence).toBe(0.4); + }); + + it("outranks a framework's naming guess that sends the name to another file", async () => { + // ASP.NET's naming rule takes `ToastService` for the class in Services/, + // but the component reads its own property named after that class, as + // eShopOnWeb's Blazor ToastComponent does (C#'s "Color Color"). + write('Startup.cs', 'namespace App { public class Startup { } }\n'); + write('Services/ToastService.cs', 'namespace App.Services {\n public class ToastService { public event System.Action OnShow; }\n}\n'); + write( + 'Helpers/ToastComponent.cs', + 'using App.Services;\nnamespace App.Helpers {\n public class ToastComponent {\n ToastService ToastService { get; set; }\n' + + ' void OnInitialized() { ToastService.OnShow += Show; }\n void Show() { }\n }\n}\n' + ); + const edge = (await calls('references')).find((e) => e.src === 'OnInitialized' && e.tgt === 'ToastService'); + expect(edge?.tgtQn).toBe('App.Helpers::ToastComponent::ToastService'); + expect(edge?.confidence).toBe(0.9); + }); + + it("does not raise a sibling type's member found by its bare name", async () => { + // `Dns()` in one class is not the `Dns` entry of the enum beside it + // (okhttp's InterceptorOverridesTest): no more settled by the file than + // a sibling class's method. + write('src/app/Overrides.kt', 'package app\n\nenum class Param { Dns }\n\nclass Override {\n val value = Dns()\n}\n'); + write('src/app/Dns.kt', 'package app\n\nclass Dns\n'); + const raised = (await calls()).filter((e) => e.tgtQn === 'app::Param::Dns' && e.confidence >= 0.9); + expect(raised).toEqual([]); + }); +}); diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index 63c0e31ea1..aff4004303 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -249,6 +249,44 @@ describe('Symlink escape prevention (#527)', () => { expect(validatePathWithinRoot(root, 'escapedir/secret.txt')).toBeNull(); }); + it('rejects a missing leaf below an out-of-root directory symlink', () => { + if (!link(path.join(root, 'escapedir'), path.join(outside, 'pkg'))) return; + expect(validatePathWithinRoot(root, 'escapedir/not-created-yet/file.txt')).toBeNull(); + }); + + // A directory link that needs no privilege on Windows (a junction), so the + // cases below run there without Developer Mode; a symlink elsewhere. Each + // test removes its own link, so nothing recursive ever walks through one. + const dirLink = (linkPath: string, target: string): void => + fs.symlinkSync(target, linkPath, process.platform === 'win32' ? 'junction' : 'dir'); + + it.runIf(process.platform === 'win32')('rejects a missing leaf below an out-of-root directory junction', () => { + const junction = path.join(root, 'escapejunction'); + dirLink(junction, path.join(outside, 'pkg')); + try { + expect(validatePathWithinRoot(root, 'escapejunction/not-created-yet/file.txt')).toBeNull(); + } finally { + fs.unlinkSync(junction); + } + }); + + it('still allows a missing leaf below a directory link that stays WITHIN the root', () => { + fs.mkdirSync(path.join(root, 'src', 'realdir')); + const inLink = path.join(root, 'src', 'linkdir'); + dirLink(inLink, path.join(root, 'src', 'realdir')); + try { + expect(validatePathWithinRoot(root, 'src/linkdir/new/file.ts')).toBe( + path.join(root, 'src', 'linkdir', 'new', 'file.ts') + ); + } finally { + fs.unlinkSync(inLink); + } + }); + + it('rejects every path under a root that does not exist', () => { + expect(validatePathWithinRoot(path.join(root, 'gone'), 'src/in.ts')).toBeNull(); + }); + it('still allows an in-repo symlink that stays WITHIN the root (no over-blocking)', () => { if (!link(path.join(root, 'src', 'inlink.ts'), path.join(root, 'src', 'in.ts'))) return; expect(validatePathWithinRoot(root, 'src/inlink.ts')).not.toBeNull(); @@ -299,6 +337,18 @@ describe('Symlink escape prevention (#527)', () => { expect(validatePathWithinRoot(root, 'escapedir/secret.txt', { allowSymlinkEscape: true })).not.toBeNull(); }); + it('allowSymlinkEscape still allows a missing leaf through an out-of-root DIR link (indexing read)', () => { + const escapeLink = path.join(root, 'escapedir'); + dirLink(escapeLink, path.join(outside, 'pkg')); + try { + expect( + validatePathWithinRoot(root, 'escapedir/not-created-yet/file.txt', { allowSymlinkEscape: true }) + ).not.toBeNull(); + } finally { + fs.unlinkSync(escapeLink); + } + }); + it('allowSymlinkEscape STILL rejects a lexical ../ traversal (guard not waived)', () => { expect( validatePathWithinRoot(root, `../${path.basename(outside)}/pkg/secret.txt`, { allowSymlinkEscape: true }) diff --git a/__tests__/sqlite-open-error.test.ts b/__tests__/sqlite-open-error.test.ts new file mode 100644 index 0000000000..a9aca88e4f --- /dev/null +++ b/__tests__/sqlite-open-error.test.ts @@ -0,0 +1,286 @@ +/** + * A database SQLite cannot open is reported as what went wrong, with the path + * and a short hint, not as "CodeGraph requires node:sqlite (Node.js 22.5+)". + * + * Before, every failure inside the SQLite connection's constructor came out as + * that Node-version hint, so a database another program held open, one in a + * folder the user can't write, or one on a network share sent users after + * their Node version. A failure in the connection's first statements (locked, + * not a database, an I/O error) came out raw, with no path. The hint is now + * for a runtime that cannot load node:sqlite at all; the Windows/WSL shared + * index keeps its own guidance. + */ +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { DatabaseConnection, DatabaseOpenError, NodeSqliteUnavailableError, getDatabasePath } from '../src/db'; +import { toDatabaseOpenError } from '../src/db/open-error'; +import { WslSharedIndexError } from '../src/db/wsl-shared-index'; +import { __setWslWindowsDriveForTests } from '../src/sync/watch-policy'; + +// Through require: vite does not resolve the node:sqlite builtin. +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { DatabaseSync } = require('node:sqlite') as { DatabaseSync: new (path: string) => { exec(sql: string): void; close(): void } }; +const DIST = path.resolve(__dirname, '../dist'); +const BIN = path.join(DIST, 'bin', 'codegraph.js'); +const { MCPEngine } = require(path.join(DIST, 'mcp', 'engine.js')) as typeof import('../src/mcp/engine'); + +beforeAll(() => { + if (!fs.existsSync(BIN)) throw new Error('Build the project first (npm run build).'); +}); + +const dirs: string[] = []; +const cleanups: Array<() => void | Promise> = []; +afterEach(async () => { + for (const fn of cleanups.splice(0).reverse()) await fn(); + __setWslWindowsDriveForTests(null); + for (const dir of dirs.splice(0)) { + const deadline = Date.now() + 10_000; + for (;;) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + break; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code ?? ''; + if (!['EBUSY', 'ENOTEMPTY', 'EPERM'].includes(code) || Date.now() > deadline) throw error; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + } +}); + +function tempDir(): string { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-open-error-'))); + dirs.push(dir); + return dir; +} + +/** An indexed project, closed again. */ +async function indexedProject(): Promise<{ root: string; dbPath: string }> { + const root = tempDir(); + fs.writeFileSync(path.join(root, 'a.ts'), 'export function alpha(): number { return 1; }\n'); + (await CodeGraph.init(root, { index: true })).close(); + return { root, dbPath: getDatabasePath(root) }; +} + +/** A node:sqlite error as SQLite reports it. */ +function sqliteError(errcode: number, message: string): Error { + return Object.assign(new Error(message), { code: 'ERR_SQLITE_ERROR', errcode, errstr: message }); +} + +/** + * Hold `file` open with no sharing, as a backup or antivirus scan can on + * Windows (libuv's UV_FS_O_EXLOCK): SQLite then can't open it at all. + */ +function holdExclusively(file: string): void { + const fd = fs.openSync(file, fs.constants.O_RDWR | 0x10000000); + cleanups.push(() => fs.closeSync(fd)); +} + +/** + * Lock the database from another connection so that no other can read it: + * exclusive locking mode, inside a write transaction. + */ +function lockDatabase(dbPath: string): void { + const holder = new DatabaseSync(dbPath); + holder.exec('PRAGMA locking_mode = EXCLUSIVE'); + holder.exec('BEGIN EXCLUSIVE'); + holder.exec('CREATE TABLE IF NOT EXISTS hold_2795 (x)'); + cleanups.push(() => { holder.exec('ROLLBACK'); holder.close(); }); +} + +function thrownBy(fn: () => unknown): unknown { + try { + fn(); + } catch (err) { + return err; + } + throw new Error('expected a throw'); +} + +describe('naming a failed open', () => { + const dbPath = path.join('/projects', 'app', '.codegraph', 'codegraph.db'); + + it.each([ + [14, 'unable to open database file', 'SQLITE_CANTOPEN', 'Check that the folder exists'], + [526, 'unable to open database file', 'SQLITE_CANTOPEN', 'holds the file open'], + [10, 'disk I/O error', 'SQLITE_IOERR', 'network share'], + [3850, 'disk I/O error', 'SQLITE_IOERR', 'local disk'], + [5, 'database is locked', 'SQLITE_BUSY', 'locked'], + [26, 'file is not a database', 'SQLITE_NOTADB', 'codegraph init'], + [11, 'database disk image is malformed', 'SQLITE_CORRUPT', 'codegraph index'], + [8, 'attempt to write a readonly database', 'SQLITE_READONLY', 'read-only'], + ])('reports SQLite code %i as %s, with the path and a hint', (errcode, message, name, hint) => { + const original = sqliteError(errcode, message); + const err = toDatabaseOpenError(original, dbPath) as DatabaseOpenError; + expect(err).toBeInstanceOf(DatabaseOpenError); + expect(err.message).toContain(dbPath); + expect(err.message).toContain(`${message} (${name}).`); + expect(err.message).toContain(hint); + expect(err.message).not.toContain('node:sqlite'); + expect(err.sqliteCode).toBe(name); + // Callers branching on SQLite's codes see what they saw before. + expect(err.code).toBe('ERR_SQLITE_ERROR'); + expect(err.errcode).toBe(errcode); + expect(err.cause).toBe(original); + }); + + it('names a code it has no hint for, without one', () => { + const err = toDatabaseOpenError(sqliteError(1, 'no such table: nodes'), dbPath) as DatabaseOpenError; + expect(err.message).toBe(`Could not open the CodeGraph database at ${dbPath}: no such table: nodes (SQLITE_ERROR).`); + }); + + it('leaves alone what SQLite did not report', () => { + const plain = new TypeError('The "path" argument must be of type string'); + expect(toDatabaseOpenError(plain, dbPath)).toBe(plain); + const missing = new NodeSqliteUnavailableError(new Error('No such built-in module: node:sqlite')); + expect(toDatabaseOpenError(missing, dbPath)).toBe(missing); + }); + + it('keeps the Windows/WSL guidance for an index the two share', () => { + __setWslWindowsDriveForTests(() => true); + const err = toDatabaseOpenError(sqliteError(10, 'disk I/O error'), '/mnt/c/app/.codegraph/codegraph.db'); + expect(err).toBeInstanceOf(WslSharedIndexError); + expect(toDatabaseOpenError(err, '/mnt/c/app/.codegraph/codegraph.db')).toBe(err); + }); + + it('keeps the node:sqlite explanation for a runtime that cannot load it', () => { + const err = new NodeSqliteUnavailableError(new Error('No such built-in module: node:sqlite')); + expect(err.message).toContain('CodeGraph requires node:sqlite (Node.js 22.5+)'); + expect(err.message).toContain('No such built-in module: node:sqlite'); + }); +}); + +describe('a database SQLite cannot open', () => { + it('reports a path SQLite cannot open as that, not as a Node version problem', () => { + // A directory where the file should be: SQLITE_CANTOPEN in the open itself. + const dbPath = path.join(tempDir(), 'codegraph.db'); + fs.mkdirSync(dbPath); + const err = thrownBy(() => DatabaseConnection.open(dbPath)) as Error; + expect(err).toBeInstanceOf(DatabaseOpenError); + expect(err.message).toContain(`Could not open the CodeGraph database at ${dbPath}: unable to open database file (SQLITE_CANTOPEN).`); + expect(err.message).not.toContain('node:sqlite'); + }); + + it('reports a file that is not a database, with its path', () => { + const dbPath = path.join(tempDir(), 'codegraph.db'); + fs.writeFileSync(dbPath, 'not a database '.repeat(300)); + const err = thrownBy(() => DatabaseConnection.open(dbPath)) as DatabaseOpenError; + expect(err).toBeInstanceOf(DatabaseOpenError); + expect(err.sqliteCode).toBe('SQLITE_NOTADB'); + expect(err.message).toContain(dbPath); + expect(err.message).toContain('codegraph init'); + }); + + it('reports a database another connection keeps locked, and opens it once released', async () => { + const { dbPath } = await indexedProject(); + lockDatabase(dbPath); + const err = thrownBy(() => DatabaseConnection.open(dbPath)) as DatabaseOpenError; + expect(err).toBeInstanceOf(DatabaseOpenError); + expect(err.sqliteCode).toBe('SQLITE_BUSY'); + expect(err.message).toContain(dbPath); + await cleanups.pop()!(); + DatabaseConnection.open(dbPath).close(); + }, 30_000); + + it.runIf(process.platform === 'win32')('reports a database another program holds open on Windows', async () => { + const { root, dbPath } = await indexedProject(); + holdExclusively(dbPath); + let err: unknown; + try { + (await CodeGraph.open(root)).close(); + } catch (e) { + err = e; + } + expect(err).toBeInstanceOf(DatabaseOpenError); + expect((err as Error).message).toContain(`Could not open the CodeGraph database at ${dbPath}: unable to open database file (SQLITE_CANTOPEN).`); + expect((err as Error).message).toContain('holds the file open'); + expect((err as Error).message).not.toContain('node:sqlite'); + }, 30_000); +}); + +/** Run the built CLI; output with colour codes removed. */ +function cli(args: string[], nodeFlags: string[] = []): { status: number | null; output: string } { + const r = spawnSync(process.execPath, [...nodeFlags, '--liftoff-only', '--disable-warning=ExperimentalWarning', BIN, ...args], { + encoding: 'utf8', + env: { ...process.env, CODEGRAPH_NO_DAEMON: '1', CODEGRAPH_TELEMETRY: '0', DO_NOT_TRACK: '1', CODEGRAPH_NO_UPDATE_CHECK: '1', NO_COLOR: '1' }, + timeout: 120_000, + }); + return { status: r.status, output: `${r.stdout}${r.stderr}`.replace(/\x1b\[[0-9;]*m/g, '') }; +} + +describe('the CLI', () => { + it.runIf(process.platform === 'win32')('names the SQLite failure in status, sync and query', async () => { + const { root, dbPath } = await indexedProject(); + holdExclusively(dbPath); + for (const args of [['status', root], ['sync', root], ['query', 'alpha', '--path', root]]) { + const { status, output } = cli(args); + expect(status, output).not.toBe(0); + expect(output).toContain(`Could not open the CodeGraph database at ${dbPath}`); + expect(output).toContain('(SQLITE_CANTOPEN)'); + expect(output).not.toContain('requires node:sqlite'); + } + }, 120_000); + + it('explains a runtime without node:sqlite, without an unhandled rejection', async () => { + const { root } = await indexedProject(); + for (const args of [['status', root], ['sync', root]]) { + const { status, output } = cli(args, ['--no-experimental-sqlite']); + expect(status, output).not.toBe(0); + expect(output).toContain('CodeGraph requires node:sqlite (Node.js 22.5+)'); + expect(output).not.toContain('Unhandled rejection'); + } + }, 120_000); +}); + +describe('the MCP server', () => { + const engines: Array<{ stop(): Promise }> = []; + afterEach(async () => { + for (const engine of engines.splice(0)) await engine.stop(); + }); + + const text = (r: { content: Array<{ text: string }> }): string => r.content.map((c) => c.text).join('\n'); + + it('answers a call on the default project with why its index could not be opened', async () => { + const { root, dbPath } = await indexedProject(); + lockDatabase(dbPath); + const engine = new MCPEngine({ watch: false }); + engines.push(engine); + await engine.ensureInitialized(root); + expect(engine.hasDefaultCodeGraph()).toBe(false); + const result = await engine.getToolHandler().execute('codegraph_search', { query: 'alpha' }); + // A malfunction: isError, with the retry-once note, saying what failed. + expect(result.isError).toBe(true); + expect(text(result)).toContain(`Could not open the CodeGraph database at ${dbPath}`); + expect(text(result)).toContain('(SQLITE_BUSY)'); + expect(text(result)).toContain('Retry the call once'); + expect(text(result)).not.toContain('No CodeGraph project is loaded'); + expect(text(result)).not.toContain('internal codegraph error'); + }, 60_000); + + it('answers a projectPath call with why that index could not be opened', async () => { + const { root, dbPath } = await indexedProject(); + lockDatabase(dbPath); + const engine = new MCPEngine({ watch: false }); + engines.push(engine); + const result = await engine.getToolHandler().execute('codegraph_search', { query: 'alpha', projectPath: root }); + expect(result.isError).toBe(true); + expect(text(result)).toContain(`Could not open the CodeGraph database at ${dbPath}`); + expect(text(result)).toContain('(SQLITE_BUSY)'); + expect(text(result)).toContain('Retry the call once'); + expect(text(result)).not.toContain('node:sqlite'); + }, 60_000); + + it('still answers an unindexed default project with the not-indexed guidance', async () => { + const root = tempDir(); + const engine = new MCPEngine({ watch: false }); + engines.push(engine); + await engine.ensureInitialized(root); + const result = await engine.getToolHandler().execute('codegraph_search', { query: 'alpha' }); + expect(result.isError).toBeFalsy(); + expect(text(result)).toContain('No CodeGraph project is loaded'); + }, 30_000); +}); diff --git a/__tests__/status-index-health.test.ts b/__tests__/status-index-health.test.ts index b94562c8eb..7a5571e103 100644 --- a/__tests__/status-index-health.test.ts +++ b/__tests__/status-index-health.test.ts @@ -5,9 +5,9 @@ * "Up to date" used to mean only "every file's content hash matches the * index". Two kinds of file pass that check with nothing usable in the graph: * - * - a file the parser could not read — `export type * from` is valid - * TypeScript 5.0 the bundled grammar has no rule for, so the file is stored - * with a parse error and no symbols (#2336); + * - a file the parser could not read, like one left with merge-conflict + * markers, which is stored with a parse error and no symbols (#2336; the + * report's file, TypeScript 5.0's `export type * from`, parses since #2474); * - a row stored without its symbols — by an engine whose grammar failed to * load (#2335), or the #1541 wipe — which `codegraph sync` re-indexes. * @@ -96,7 +96,12 @@ describe('status reports files indexed without their symbols (#2336, #2335)', () it('a file the parser could not read keeps status from reporting "up to date"', async () => { fs.writeFileSync( path.join(dir, 'src', 'index.ts'), - 'export * from "./constants/index.js";\nexport type * from "./types/database.js";\n' + 'export * from "./constants/index.js";\n' + + '<<<<<<< HEAD\n' + + 'export type { Row } from "./types/database.js";\n' + + '=======\n' + + 'export type { RowId } from "./types/database.js";\n' + + '>>>>>>> feature\n' ); await index(); diff --git a/__tests__/sync-navigation-retry.test.ts b/__tests__/sync-navigation-retry.test.ts index 359089140d..b1bd911813 100644 --- a/__tests__/sync-navigation-retry.test.ts +++ b/__tests__/sync-navigation-retry.test.ts @@ -315,6 +315,24 @@ const CASES: Case[] = [ from: 'src/views/Home.vue', expected: '/login → /login', }, + { + // Nuxt's auto-imported navigateTo, waiting for its page (#2634). + name: 'Nuxt (navigateTo)', + files: { + 'package.json': JSON.stringify({ name: 'site', dependencies: { nuxt: '4', vue: '3' } }), + 'pages/index.vue': + '\n' + + '\n', + 'pages/login.vue': '\n', + }, + route: { file: 'pages/login.vue', before: null }, + from: 'pages/index.vue', + expected: '/login → /login', + }, { name: 'Angular', files: { diff --git a/__tests__/ts-array-annotated-receivers.test.ts b/__tests__/ts-array-annotated-receivers.test.ts index cd3aabfdb4..91c01f8a00 100644 --- a/__tests__/ts-array-annotated-receivers.test.ts +++ b/__tests__/ts-array-annotated-receivers.test.ts @@ -291,6 +291,24 @@ describe("a call on a TypeScript local annotated as one of a type's properties", expect(calls.filter((c) => c.startsWith('onDone.'))).toEqual(['onDone.fire -> Emitter::fire']); }, 60_000); + it("reads a property typed by a type parameter of the indexed type as no type, whatever class shares the parameter's name", async () => { + // `Item` is each type's own parameter here, not the project class `Item`: + // a generic type alias, and a class whose parameter list holds braces. + // `Stack` has a `flush` too, so the call's name alone picks neither. + const calls = await callsOf({ + ...CLASSES, + 'src/stack.ts': 'export class Stack {\n findLast() { return 1; }\n flush() { return 2; }\n}\n', + 'src/state.ts': + 'export type State = { current: Item };\n' + + 'export class Shaped { current!: Item; }\n', + 'src/use.ts': + "import { State, Shaped } from './state';\n" + + "export function viaAlias(x: State['current']) { return x.flush(); }\n" + + "export function viaShaped(x: Shaped['current']) { return x.flush(); }\n", + }); + expect(calls.filter((c) => c.startsWith('x.'))).toEqual([]); + }, 60_000); + it("is not matched to a method by the call's name when the property's type isn't written in the project", async () => { // vscode's `invocation: Parameters[0]`: // left untyped, the receiver's name alone would pick `Logger.info`. diff --git a/__tests__/ts-export-type-star.test.ts b/__tests__/ts-export-type-star.test.ts new file mode 100644 index 0000000000..4c43299bba --- /dev/null +++ b/__tests__/ts-export-type-star.test.ts @@ -0,0 +1,275 @@ +/** + * TypeScript 5.0's type-only re-export of a whole module parses (#2474): + * + * export { FooClient, type FooClientOptions } from "./client.js"; + * export type * from "./types.js"; + * export type * as Shapes from "./shapes.js"; + * + * The grammar has no rule for `export type *` (upstream's neither), so `type` + * parsed as an error. A barrel of re-exports was then reported as producing no + * symbols, `codegraph status` listed it among the files it could not parse, + * and the native kernel handed every file holding such a line to the wasm + * parser. The TypeScript extractor's preParse now blanks that `type` to spaces + * where a statement starts, offsets kept, so the line reads as the `export *` + * it is to the graph, and both engines parse the same bytes. The import + * resolver reads the statement as written from the file, so a name imported + * through the barrel still reaches the file that declares it. + */ +import { describe, it, expect, beforeAll, beforeEach, afterEach, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { tryKernelExtract, tryKernelExtractRaw, resetKernelForTests } from '../src/extraction/kernel'; +import { blankExportTypeStar, blanksExportTypeStar } from '../src/extraction/languages/typescript'; +import type { ExtractionResult, Language } from '../src/types'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelAvailable = fs.existsSync(KERNEL_PATH) || process.env.CODEGRAPH_KERNEL_EXPECT === '1'; + +/** The issue's barrel, plus the namespace form. */ +const BARREL = [ + 'export { FooClient, FooApiError, type FooClientOptions } from "./client.js";', + 'export { DAYS_OF_WEEK } from "./types.js";', + 'export type * from "./types.js";', + 'export type * as Shapes from "./shapes.js";', + '', +].join('\n'); + +/** Each statement of the barrel, as its import node keeps it. */ +const BARREL_STATEMENTS = BARREL.split('\n').filter(Boolean); + +describe('blankExportTypeStar', () => { + it('blanks the `type` of a type-only star re-export, keeping every offset', () => { + const cases: Array<[string, string]> = [ + ['export type * from "./types";', 'export * from "./types";'], + ['export type * as Types from "./types";', 'export * as Types from "./types";'], + ["export type * as 'my types' from './types';", "export * as 'my types' from './types';"], + ['export type *from"./types";', 'export *from"./types";'], + ['export\ttype\t*\tfrom "./types";', 'export\t \t*\tfrom "./types";'], + // `from` on the next line. + ['export type *\n from "./types";', 'export *\n from "./types";'], + // Indented in an ambient module, and after `{`, `;` or a block comment on the same line. + ['declare module "m" {\n export type * from "./y";\n}', 'declare module "m" {\n export * from "./y";\n}'], + ['declare module "m" { export type * from "./y"; }', 'declare module "m" { export * from "./y"; }'], + ['export * from "./a"; export type * from "./b";', 'export * from "./a"; export * from "./b";'], + ['/* types */ export type * from "./b";', '/* types */ export * from "./b";'], + ['export * from "./a";\r\nexport type * from "./b";\r\n', 'export * from "./a";\r\nexport * from "./b";\r\n'], + ]; + for (const [source, expected] of cases) { + const blanked = blankExportTypeStar(source); + expect(blanked).toBe(expected); + expect(blanked).toHaveLength(source.length); + } + }); + + it('leaves everything else as written', () => { + for (const source of [ + 'export type Star = "*";', + 'export type { A } from "./a";', + 'export { type A } from "./a";', + 'export * from "./a";', + 'export * as ns from "./a";', + // Not where a statement starts: a mention in a comment, a doc or a string. + '// Re-export them with export type * from "./types".', + '/**\n * export type * from "./types";\n */', + 'const hint = "export type * from \'./types\'";', + ]) { + expect(blankExportTypeStar(source)).toBe(source); + } + }); +}); + +const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const; + +describe('a type-only star re-export parses in both engines', () => { + let savedEnv: Record = {}; + + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['typescript', 'tsx']); + }); + + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + resetKernelForTests(); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + resetKernelForTests(); + }); + + function extract(backend: 'kernel' | 'wasm', file: string, source: string, language: Language): ExtractionResult { + if (backend === 'wasm') { + process.env.CODEGRAPH_KERNEL = '0'; + return extractFromSource(file, source, language); + } + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + // Null when the kernel defers the file to the wasm parser, as it does every file with a parse error. + const result = tryKernelExtract(file, source, language); + expect(result, `kernel extraction of ${file}`).not.toBeNull(); + return result!; + } + + const backends = kernelAvailable ? (['kernel', 'wasm'] as const) : (['wasm'] as const); + + for (const crlf of [false, true]) { + const eol = (s: string) => (crlf ? s.replace(/\n/g, '\r\n') : s); + for (const [file, language] of [ + ['src/index.ts', 'typescript'], + ['src/index.tsx', 'tsx'], + ] as const) { + it.each(backends)(`a barrel in ${file}${crlf ? ' (CRLF)' : ''}: %s`, (backend) => { + const result = extract(backend, file, eol(BARREL), language); + expect(result.errors).toEqual([]); + // Each statement imports its module, as `export * from` does. + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => `${n.name}@${n.startLine}`)).toEqual([ + './client.js@1', + './types.js@2', + './types.js@3', + './shapes.js@4', + ]); + expect(result.unresolvedReferences.map((r) => `${r.referenceName}@${r.line}`).sort()).toEqual([ + './client.js@1', + './shapes.js@4', + './types.js@2', + './types.js@3', + 'DAYS_OF_WEEK@2', + 'FooApiError@1', + 'FooClient@1', + 'FooClientOptions@1', + ]); + // Each import node keeps its statement as written, `type` included. + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.signature)).toEqual(BARREL_STATEMENTS); + }); + } + } + + it.runIf(kernelAvailable)('keeps a re-export as written in a file the kernel hands to the wasm parser', () => { + // `export const = ;` is an error, so the kernel defers the whole file. + const source = 'export type * from "./types.js";\nexport const = ;\n'; + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + expect(tryKernelExtract('src/broken.ts', source, 'typescript')).toBeNull(); + const result = extractFromSource('src/broken.ts', source, 'typescript'); + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.signature)).toEqual([ + 'export type * from "./types.js";', + ]); + }); + + it.runIf(kernelAvailable)('leaves the undecoded kernel path to files with nothing to keep as written', () => { + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + // Its import nodes' text is restored on the decoded path, which the parse worker falls back to. + expect(tryKernelExtractRaw('src/index.ts', BARREL, 'typescript')).toBeNull(); + const plain = 'export { FooClient } from "./client.js";\nexport * from "./types.js";\n'; + expect(tryKernelExtractRaw('src/plain.ts', plain, 'typescript')).not.toBeNull(); + expect(blanksExportTypeStar(BARREL)).toBe(true); + expect(blanksExportTypeStar(plain)).toBe(false); + // A mention only in a comment blanks nothing. + expect(blanksExportTypeStar('// prefer export type * from "./types"\nexport * from "./types";\n')).toBe(false); + }); + + it.each(backends)('in a declaration file\'s declare module body: %s', (backend) => { + const source = 'declare module "sdk" {\n export type * from "./types";\n export function connect(): void;\n}\n'; + const result = extract(backend, 'typings/sdk.d.ts', source, 'typescript'); + expect(result.errors).toEqual([]); + expect(result.nodes.filter((n) => n.kind !== 'file').map((n) => `${n.kind} ${n.name}`)).toEqual([ + 'import ./types', + 'function connect', + ]); + }); + + it.runIf(kernelAvailable)('the kernel and the wasm extractor agree', () => { + const canon = (r: ExtractionResult) => [ + ...r.nodes.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort())).sort(), + ...r.unresolvedReferences.map((ref) => JSON.stringify(ref, Object.keys(ref).sort())).sort(), + ]; + for (const [file, language] of [ + ['src/index.ts', 'typescript'], + ['src/index.tsx', 'tsx'], + ] as const) { + expect(canon(extract('kernel', file, BARREL, language))).toEqual(canon(extract('wasm', file, BARREL, language))); + } + }); +}); + +describe('indexing a barrel with `export type *` (#2474)', () => { + let root = ''; + let cg: CodeGraph; + + beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-export-type-star-')); + const files: Record = { + 'packages/sdk/src/client.ts': + 'export interface FooClientOptions { baseUrl: string }\n' + + 'export class FooApiError extends Error {}\n' + + 'export class FooClient {\n' + + ' constructor(private opts: FooClientOptions) {}\n' + + ' get(path: string) { return this.opts.baseUrl + path; }\n' + + '}\n', + 'packages/sdk/src/types.ts': "export const DAYS_OF_WEEK = ['mon', 'tue'];\nexport type Schedule = { day: string };\n", + 'packages/sdk/src/shapes.ts': 'export interface Circle { radius: number }\n', + 'packages/sdk/src/index.ts': BARREL, + // Same names, declared where nothing re-exports them: what a name match could pick. + 'app/shapes.ts': 'export interface FooClientOptions { decoy: true }\nexport type Schedule = { decoy: true };\n', + 'app/main.ts': + 'import { FooClient, type FooClientOptions } from "../packages/sdk/src/index.js";\n' + + 'import type { Schedule } from "../packages/sdk/src/index.js";\n' + + 'export function run(o: FooClientOptions, s: Schedule) {\n' + + ' return new FooClient(o).get(s.day);\n' + + '}\n', + }; + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + cg = await CodeGraph.init(root, { index: true }); + }); + + afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); + }); + + it('stores the barrel without a parse error', () => { + const barrel = cg.getFile('packages/sdk/src/index.ts'); + expect(barrel).not.toBeNull(); + expect(barrel!.errors ?? []).toEqual([]); + }); + + it('stores each re-export as written', () => { + const imports = cg.getNodesInFile('packages/sdk/src/index.ts').filter((n) => n.kind === 'import'); + expect(imports.sort((a, b) => a.startLine - b.startLine).map((n) => n.signature)).toEqual(BARREL_STATEMENTS); + }); + + it('links each name imported through the barrel to the file that declares it', () => { + const ids = cg.getNodesInFile('app/main.ts').map((n) => n.id); + const imported = cg + .getOutgoingEdgesFrom(ids, ['imports']) + .map((e) => cg.getNode(e.target)!) + .filter((t) => t.kind !== 'file') + .map((t) => `${t.name} -> ${t.filePath}`) + .sort(); + expect(imported).toEqual([ + 'FooClient -> packages/sdk/src/client.ts', + 'FooClientOptions -> packages/sdk/src/client.ts', + // Reached only through `export type * from "./types.js"`. + 'Schedule -> packages/sdk/src/types.ts', + ]); + }); +}); diff --git a/__tests__/ts-this-field-element.test.ts b/__tests__/ts-this-field-element.test.ts new file mode 100644 index 0000000000..2a490741a8 --- /dev/null +++ b/__tests__/ts-this-field-element.test.ts @@ -0,0 +1,141 @@ +/** + * A TS/JS method called on an element of a field of the enclosing class — + * `this.handlers[type].handle()`, `this.items[0].save()` — reaches the + * element's type as the field's declaration writes it (`Item[]`, + * `Array`, `Record`), or nothing. It used to keep the + * bare method name, which exact-matched any project method of that name: + * `this.listeners[type].push(h)` reached a stack class's `push` in another + * file. `this['svc'].start()` is the field `svc` itself. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +let dir: string; +let cg: CodeGraph; + +const files: Record = { + 'src/model.ts': + 'export class Item { save() { return 1; } }\n' + + 'export class Handler { handle(t: string) { return t; } start() { return 0; } }\n', + // Same-named methods a bare-name match would reach. + 'src/decoy.ts': + 'export class UndoStack { push(entry: string) { return entry; } }\n' + + 'export class Other { save() { return 2; } handle(t: string) { return t; } }\n', + 'src/holder.ts': + "import { Item, Handler } from './model';\n" + + 'export class Holder {\n' + + ' private items: Item[] = [];\n' + + ' private list: Array = [];\n' + + ' private handlers: Record = {};\n' + + ' private listeners: Record = {};\n' + + ' private names: string[] = [];\n' + + ' private svc: Handler = new Handler();\n' + + ' #store: Item[] = [];\n' + + ' first() { return this.items[0].save(); }\n' + + ' fromList(i: number) { return this.list[i].save(); }\n' + + ' fire(t: string) { return this.handlers[t].handle(t); }\n' + + ' keyed() { return this.handlers["main"].handle("x"); }\n' + + ' cast() { return (this.items[1] as Item).save(); }\n' + + ' asserted() { return this.items![0]!.save(); }\n' + + ' privately() { return this.#store[0].save(); }\n' + + " byKey() { return this['svc'].start(); }\n" + + ' on(type: string, h: string) { this.listeners[type].push(h); }\n' + + ' builtin(i: number) { return this.names[i].trim(); }\n' + + '}\n', + // `Item` here is the class's type parameter, not the project class. + 'src/store.ts': + 'export class Store {\n' + + ' private items: Item[] = [];\n' + + ' private byKey: Record = {};\n' + + ' first() { return this.items[0].save(); }\n' + + ' keyed(k: string) { return this.byKey[k].save(); }\n' + + '}\n' + + // A header over several lines, and a constraint holding braces. + 'export class Wide<\n' + + ' A,\n' + + ' B,\n' + + ' Item\n' + + '> {\n' + + ' private items: Item[] = [];\n' + + ' first() { return this.items[0].save(); }\n' + + '}\n' + + 'export class Shaped {\n' + + ' private items: Item[] = [];\n' + + ' first() { return this.items[0].save(); }\n' + + '}\n', + // Either array's element: no one type to resolve on. + 'src/mixed.ts': + "import { Item } from './model';\n" + + "import { Other } from './decoy';\n" + + 'export class Mixed {\n' + + ' private entries: Item[] | Other[] = [];\n' + + ' private maybe: Item[] | null = null;\n' + + ' either() { return this.entries[0].save(); }\n' + + ' nullable() { return this.maybe![0].save(); }\n' + + ' private below: Array\n' + + ' | Other = [];\n' + + ' private joined: Item[] & Other[] = [];\n' + + ' continued() { return this.below[0].save(); }\n' + + ' both() { return this.joined[0].save(); }\n' + + '}\n', +}; + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-this-element-')); + for (const [rel, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), text); + } + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}); + +afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +const calleesOf = (name: string, file = 'holder.ts') => { + const caller = cg.getNodesByName(name).find((n) => n.filePath.endsWith(file) && n.kind === 'method')!; + return cg.getCallees(caller.id) + .filter(({ edge }) => edge.kind === 'calls') + .map(({ node: n }) => `${path.basename(n.filePath)}:${n.qualifiedName}`) + .sort(); +}; + +describe('a call on an element of a this-field', () => { + it.each([ + ['first', 'model.ts:Item::save'], + ['fromList', 'model.ts:Item::save'], + ['fire', 'model.ts:Handler::handle'], + ['keyed', 'model.ts:Handler::handle'], + ['cast', 'model.ts:Item::save'], + ['asserted', 'model.ts:Item::save'], + ['privately', 'model.ts:Item::save'], + ['byKey', 'model.ts:Handler::start'], + ])('%s reaches the element type', (caller, target) => { + expect(calleesOf(caller)).toEqual([target]); + }); + + it('an element of a builtin type reaches no project method', () => { + expect(calleesOf('on')).toEqual([]); + expect(calleesOf('builtin')).toEqual([]); + }); + + it("a class's type parameter is not the project class of that name", () => { + expect(calleesOf('first', 'store.ts')).toEqual([]); + expect(calleesOf('keyed', 'store.ts')).toEqual([]); + const caller = (cls: string) => cg.getNodesByName('first').find((n) => n.qualifiedName === `${cls}::first`)!; + for (const cls of ['Wide', 'Shaped']) expect(cg.getCallees(caller(cls).id).filter(({ edge }) => edge.kind === 'calls')).toEqual([]); + }); + + it('a union of element types reaches none of them; `| null` keeps the one', () => { + expect(calleesOf('either', 'mixed.ts')).toEqual([]); + expect(calleesOf('nullable', 'mixed.ts')).toEqual(['model.ts:Item::save']); + expect(calleesOf('continued', 'mixed.ts')).toEqual([]); + expect(calleesOf('both', 'mixed.ts')).toEqual([]); + }); +}); diff --git a/__tests__/ui-repeated-keys.test.ts b/__tests__/ui-repeated-keys.test.ts index e848f5348e..491d636939 100644 --- a/__tests__/ui-repeated-keys.test.ts +++ b/__tests__/ui-repeated-keys.test.ts @@ -25,6 +25,16 @@ beforeAll(async () => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-ui-repeated-')); const files: Record = { 'package.json': JSON.stringify({ name: 'app', private: true, dependencies: { express: '^4.0.0', astro: '^4.0.0' } }), + 'dense/app.js': `const express = require('express'); const app = express(); +${Array.from({ length: 24 }, (_, i) => `function task${i}() { return ${i}; }`).join('\n')} +function second(req, res) { res.end(); } +function third(req, res) { res.end(); } +app.get('/dense', (req, res) => { +${Array.from({ length: 24 }, (_, i) => `task${i}();`).join('\n')} +}); +app.get('/second', second); +app.get('/third', third); +`, 'server/app.js': `const express = require('express'); const app = express(); @@ -70,6 +80,62 @@ afterAll(() => { }); describe('the routes list', () => { + it('applies its limit to routes, not to the calls of one inline handler', () => { + const payload = buildRoutes(cg, new URLSearchParams('limit=3')); + expect(payload.entries.map((e) => e.url)).toEqual(['GET /dense', 'GET /second', 'GET /third']); + expect(payload.shown).toBe(3); + expect(payload.truncated).toBe(true); + }); + + it('still calls a project routed when its one inline route has three calls', async () => { + const singleRoot = path.join(root, 'one-route'); + fs.mkdirSync(singleRoot, { recursive: true }); + fs.writeFileSync(path.join(singleRoot, 'package.json'), JSON.stringify({ dependencies: { express: '^4.0.0' } })); + fs.writeFileSync(path.join(singleRoot, 'app.js'), `const app = require('express')(); +function first() {} function second() {} function third() {} +app.get('/one', () => { first(); second(); third(); }); +`); + const single = await CodeGraph.init(singleRoot, { index: true }); + try { + const payload = buildRoutes(single, new URLSearchParams('limit=3')); + expect(payload.routed).toBe(true); + expect(payload.entries.map((e) => e.url)).toEqual(['GET /one']); + } finally { + single.close(); + } + }); + + it('fills the page past routes it drops after reading them, like a test file\'s', async () => { + const testsFirst = path.join(root, 'tests-first'); + const put = (rel: string, text: string) => { + fs.mkdirSync(path.dirname(path.join(testsFirst, rel)), { recursive: true }); + fs.writeFileSync(path.join(testsFirst, rel), text); + }; + put('package.json', JSON.stringify({ dependencies: { express: '^4.0.0' } })); + // Sorts before src/, and every handler is test code, so these rows are + // dropped only once they are read: they must not use up the window. + put('__tests__/routes.test.js', `const app = require('express')(); +${Array.from({ length: 6 }, (_, i) => `function t${i}(req, res) { res.end(); }\napp.get('/t${i}', t${i});`).join('\n')} +`); + put('src/app.js', `const app = require('express')(); +function a(req, res) { res.end(); } +function b(req, res) { res.end(); } +function c(req, res) { res.end(); } +app.get('/a', a); +app.get('/b', b); +app.get('/c', c); +`); + const project = await CodeGraph.init(testsFirst, { index: true }); + try { + const payload = buildRoutes(project, new URLSearchParams('limit=3')); + expect(payload.routed).toBe(true); + expect(payload.entries.map((e) => e.url)).toEqual(['GET /a', 'GET /b', 'GET /c']); + expect(payload.truncated).toBe(false); + } finally { + project.close(); + } + }); + it('has one row per route', () => { const { entries } = buildRoutes(cg, new URLSearchParams()); const ids = entries.map((e) => e.routeId); diff --git a/__tests__/ui-trails.test.ts b/__tests__/ui-trails.test.ts index a669cb1c77..8f6b13e2f9 100644 --- a/__tests__/ui-trails.test.ts +++ b/__tests__/ui-trails.test.ts @@ -18,6 +18,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import * as http from 'http'; +import { execFileSync } from 'child_process'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -26,11 +27,15 @@ import { createGraphApi, startUiServer, type GraphApi, type UiServerHandle } fro import { encodeResolvedRun, isTrailId, + listStoredTrails, parseTrail, slugify, TRAILS_RELATIVE_DIR, + type StoredTrail, type WireTrailHop, } from '../src/ui-server/api'; +import { writeStoredTrail } from '../src/ui-server/api/trail-store'; +import { resetTrailAuthor, trailAuthor } from '../src/ui-server/api/trails'; interface Res { status: number; @@ -115,6 +120,66 @@ function trailsDir(): string { return path.join(projectRoot, TRAILS_RELATIVE_DIR); } +function storedTrail(id: string): StoredTrail { + return { + version: 1, + id, + name: 'Must stay inside', + note: '', + author: 'test', + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), + hops: [ + { + dir: 'start', + name: 'start', + qualifiedName: 'start', + kind: 'function', + file: 'src/start.ts', + line: 1, + id: 'function:start', + }, + ], + }; +} + +/** Remove a link itself, never what it points at, when there is one. */ +function removeLink(link: string): void { + try { + if (fs.lstatSync(link).isSymbolicLink()) fs.unlinkSync(link); + } catch { + // Nothing there. + } +} + +/** + * Whether this machine can create a link of `type`. A Windows junction needs no + * privilege; a symbolic link there needs Developer Mode or admin rights. Where + * one can't be made, its tests are skipped, with the reason in their name. + */ +function canCreateLink(type: 'file' | 'dir' | 'junction'): boolean { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-link-probe-')); + const target = path.join(dir, 'target'); + const link = path.join(dir, 'link'); + try { + if (type === 'file') fs.writeFileSync(target, ''); + else fs.mkdirSync(target); + fs.symlinkSync(target, link, type); + return true; + } catch { + return false; + } finally { + removeLink(link); + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** A `.codegraph/ui` link can be a junction or a directory symlink on Windows. */ +const DIR_LINKS = (process.platform === 'win32' ? ['junction', 'dir'] : ['dir']) as Array< + 'junction' | 'dir' +>; +const NO_LINK = ' (skipped: this link needs Developer Mode or admin rights on Windows)'; + /** Re-index in place, the way a `codegraph sync` would after an edit. */ async function reindex(): Promise { const cg = CodeGraph.openSync(projectRoot); @@ -216,6 +281,153 @@ describe('trail ids', () => { }); }); +describe('trail write containment', () => { + for (const type of DIR_LINKS) { + const canLink = canCreateLink(type); + const kind = type === 'junction' ? 'junction' : 'symlink'; + + it.skipIf(!canLink)( + `refuses a missing trail directory below a ${kind} that leaves the project${canLink ? '' : NO_LINK}`, + () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-trail-symlink-')); + const project = path.join(base, 'project'); + const outside = path.join(base, 'outside'); + const uiLink = path.join(project, '.codegraph', 'ui'); + fs.mkdirSync(path.join(project, '.codegraph'), { recursive: true }); + fs.mkdirSync(outside, { recursive: true }); + fs.symlinkSync(outside, uiLink, type); + + try { + expect(() => writeStoredTrail(project, storedTrail('must-stay-inside'))).toThrow( + /outside the project/ + ); + expect(fs.existsSync(path.join(outside, 'trails'))).toBe(false); + } finally { + removeLink(uiLink); + fs.rmSync(base, { recursive: true, force: true }); + } + } + ); + + it.skipIf(!canLink)( + `allows a project root reached through a ${kind} when the real path stays inside it${canLink ? '' : NO_LINK}`, + () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-trail-root-link-')); + const project = path.join(base, 'project'); + const alias = path.join(base, 'project-link'); + fs.mkdirSync(path.join(project, '.codegraph'), { recursive: true }); + fs.symlinkSync(project, alias, type); + + try { + expect(() => writeStoredTrail(alias, storedTrail('safe-root-link'))).not.toThrow(); + expect( + fs.existsSync(path.join(project, TRAILS_RELATIVE_DIR, 'safe-root-link.json')) + ).toBe(true); + } finally { + removeLink(alias); + fs.rmSync(base, { recursive: true, force: true }); + } + } + ); + } +}); + +describe('trail read containment', () => { + const fileLinks = canCreateLink('file'); + + it.skipIf(!fileLinks)( + `skips a trail file symlinked outside the project instead of reading it${fileLinks ? '' : NO_LINK}`, + () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-trail-read-link-')); + const project = path.join(base, 'project'); + const outside = path.join(base, 'outside'); + const dir = path.join(project, TRAILS_RELATIVE_DIR); + const linked = path.join(dir, 'linked.json'); + fs.mkdirSync(dir, { recursive: true }); + fs.mkdirSync(outside, { recursive: true }); + fs.writeFileSync( + path.join(outside, 'secret.json'), + JSON.stringify({ ...storedTrail('outside'), note: 'must not be read' }) + ); + fs.symlinkSync(path.join(outside, 'secret.json'), linked, 'file'); + + try { + expect(listStoredTrails(project)).toEqual({ trails: [], skipped: 1 }); + } finally { + removeLink(linked); + fs.rmSync(base, { recursive: true, force: true }); + } + } + ); + + it.runIf(process.platform !== 'win32')( + 'skips a FIFO in the trails directory without blocking on it', + () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-trail-read-fifo-')); + const project = path.join(base, 'project'); + const dir = path.join(project, TRAILS_RELATIVE_DIR); + fs.mkdirSync(dir, { recursive: true }); + // A FIFO with no writer: a blocking open would hang here until one showed up. + execFileSync('mkfifo', [path.join(dir, 'blocked.json')]); + + try { + // Run current TS source in a child: a synchronous open blocks Vitest's + // own timeout, but the parent's execFileSync timeout can kill the child. + const script = ` + const fs = require('fs'); + const ts = require('typescript'); + require.extensions['.ts'] = (module, filename) => { + const source = fs.readFileSync(filename, 'utf8'); + const { outputText } = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }, + }); + module._compile(outputText, filename); + }; + const { listStoredTrails } = require(process.argv[1]); + process.stdout.write(JSON.stringify(listStoredTrails(process.argv[2]))); + `; + const result = execFileSync(process.execPath, [ + '-e', script, path.resolve(__dirname, '../src/ui-server/api/trail-store.ts'), project, + ], { + cwd: path.resolve(__dirname, '..'), + encoding: 'utf8', + timeout: 5_000, + killSignal: 'SIGKILL', + }); + expect(JSON.parse(result)).toEqual({ trails: [], skipped: 1 }); + } finally { + fs.rmSync(base, { recursive: true, force: true }); + } + }, + 10_000 + ); +}); + +describe('trail author cache', () => { + it('keeps git user names separate for projects served by one process', () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-trail-authors-')); + const first = path.join(base, 'first'); + const second = path.join(base, 'second'); + + try { + for (const [dir, author] of [ + [first, 'Alice Project'], + [second, 'Bob Project'], + ] as const) { + fs.mkdirSync(dir, { recursive: true }); + execFileSync('git', ['init', '-q'], { cwd: dir }); + execFileSync('git', ['config', 'user.name', author], { cwd: dir }); + } + resetTrailAuthor(); + expect(trailAuthor(first)).toBe('Alice Project'); + expect(trailAuthor(second)).toBe('Bob Project'); + } finally { + resetTrailAuthor(); + fs.rmSync(base, { recursive: true, force: true }); + } + }); +}); + describe('parseTrail', () => { it('rejects a file that is not a trail rather than half-reading it', () => { expect(parseTrail('x', 'not json')).toBeNull(); @@ -225,6 +437,12 @@ describe('parseTrail', () => { expect(parseTrail('x', '{"name":"","hops":[{"qualifiedName":"a"}]}')).toBeNull(); }); + it('rejects a declared format version this build does not understand', () => { + expect( + parseTrail('future', JSON.stringify({ ...storedTrail('future'), version: 2 })) + ).toBeNull(); + }); + it('takes its id from the FILE, not from the field inside it', () => { const trail = parseTrail('on-disk', '{"name":"a","id":"remembered","hops":[{"name":"f"}]}'); expect(trail?.id).toBe('on-disk'); diff --git a/__tests__/upgrade.test.ts b/__tests__/upgrade.test.ts index ad46a17ad4..3c9b698602 100644 --- a/__tests__/upgrade.test.ts +++ b/__tests__/upgrade.test.ts @@ -2,6 +2,8 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; +import { spawnSync } from 'node:child_process'; +import { CODEGRAPH_INSTRUCTIONS_BLOCK } from '../src/installer/instructions-template'; import { detectInstallMethod, deriveInstallDir, @@ -83,8 +85,8 @@ describe('detectInstallMethod', () => { it('detects a local (project) npm install as local', () => { const cwd = '/home/u/project'; const filename = `${cwd}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`; - const m = detectInstallMethod({ filename, platform: 'linux', cwd, exists: () => false }); - expect(m).toEqual({ kind: 'npm', scope: 'local' }); + const m = detectInstallMethod({ filename, platform: 'linux', cwd, exists: bundleExists(new Set([`${cwd}/package.json`])) }); + expect(m).toEqual({ kind: 'npm', scope: 'local', root: cwd }); }); it('detects an npx run from the _npx cache', () => { @@ -115,9 +117,9 @@ describe('detectInstallMethod', () => { const cwd = '/home/u/project'; const root = `${cwd}/node_modules/@colbymchenry/codegraph/node_modules/@colbymchenry/codegraph-darwin-arm64`; const filename = `${root}/lib/dist/bin/codegraph.js`; - const present = new Set([`${root}/node`, `${root}/bin/codegraph`]); + const present = new Set([`${root}/node`, `${root}/bin/codegraph`, `${cwd}/node_modules/.bin/codegraph`]); const m = detectInstallMethod({ filename, platform: 'darwin', cwd, exists: bundleExists(present) }); - expect(m).toEqual({ kind: 'npm', scope: 'local' }); + expect(m).toEqual({ kind: 'npm', scope: 'local', root: cwd }); }); it('still detects an npx run when the cached platform package has the bundle layout', () => { @@ -150,6 +152,99 @@ describe('detectInstallMethod', () => { }); expect(m.kind).toBe('unknown'); }); + + // #2477: the scope came from "is the running file under cwd", so a global + // prefix inside the home folder (Windows %APPDATA%\npm, nvm, ~/.npm-global) + // was taken for a local install whenever upgrade ran from home — npm then + // ran without -g and the global copy stayed old. It is now read from npm's + // layout: a global launcher beside the tree, or a project around it. + describe('npm scope comes from npm\'s layout, never from cwd (#2477)', () => { + const slash = (p: string | undefined) => p?.replace(/\\/g, '/'); + const winHome = 'C:/Users/X'; + const winPrefix = `${winHome}/AppData/Roaming/npm`; + const winThin = `${winPrefix}/node_modules/@colbymchenry/codegraph/node_modules/@colbymchenry/codegraph-win32-x64/lib/dist/bin/codegraph.js`; + + it.each([winHome, 'C:/Users', 'C:/', 'C:/Users/X/projects/app'])('a Windows global install under %%APPDATA%%\\npm is global with cwd %s', (cwd) => { + const present = new Set([`${winPrefix}/codegraph.cmd`, `${winHome}/package.json`]); + for (const filename of [winThin, `${winPrefix}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`]) { + expect(detectInstallMethod({ filename, platform: 'win32', cwd, exists: bundleExists(present) })).toEqual({ kind: 'npm', scope: 'global' }); + } + }); + + it.each(['/home/u', '/', '/home'])('a POSIX global install under nvm in the home folder is global with cwd %s', (cwd) => { + const prefix = '/home/u/.nvm/versions/node/v22.11.0'; + const filename = `${prefix}/lib/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`; + const present = new Set([`${prefix}/bin/codegraph`, '/home/u/package.json']); + expect(detectInstallMethod({ filename, platform: 'linux', cwd, exists: bundleExists(present) })).toEqual({ kind: 'npm', scope: 'global' }); + }); + + it('a global install under a custom prefix is global from home (~/.npm-global, D:\\tools\\npm)', () => { + const posix = '/home/u/.npm-global'; + expect(detectInstallMethod({ + filename: `${posix}/lib/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`, + platform: 'darwin', cwd: '/home/u', exists: bundleExists(new Set([`${posix}/bin/codegraph`])), + })).toEqual({ kind: 'npm', scope: 'global' }); + expect(detectInstallMethod({ + filename: 'D:/tools/npm/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js', + platform: 'win32', cwd: 'D:/', exists: bundleExists(new Set(['D:/tools/npm/codegraph.cmd'])), + })).toEqual({ kind: 'npm', scope: 'global' }); + }); + + it('a local install whose project root is the home folder stays local and belongs to the home folder, from any cwd', () => { + const filename = `${winHome}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`; + const present = new Set([`${winHome}/package.json`, `${winHome}/node_modules/.bin/codegraph.cmd`, `${winHome}/projects/app/package.json`]); + // ~/projects/app is a project of its own: npm run there would install into it. + for (const cwd of [winHome, 'C:/', `${winHome}/projects/app`]) { + const m = detectInstallMethod({ filename, platform: 'win32', cwd, exists: bundleExists(present) }) as Extract; + expect([m.scope, slash(m.root)]).toEqual(['local', winHome]); + } + }); + + it('a local install in a project belongs to that project, from any cwd', () => { + const project = '/home/u/projects/app'; + const filename = `${project}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`; + const present = new Set([`${project}/package.json`, `${project}/node_modules/.bin/codegraph`]); + for (const cwd of [project, `${project}/src`, `${project}/packages/web`, '/home/u', '/', '/home/u/projects/application']) { + expect(detectInstallMethod({ filename, platform: 'linux', cwd, exists: bundleExists(present) })).toEqual({ kind: 'npm', scope: 'local', root: project }); + } + }); + + it('npx and bundled installs under the home folder are unaffected by a home cwd', () => { + expect(detectInstallMethod({ + filename: `${winHome}/AppData/Local/npm-cache/_npx/1a2b3c/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`, + platform: 'win32', cwd: winHome, exists: () => false, + })).toEqual({ kind: 'npx' }); + const winBundle = `${winHome}/AppData/Local/codegraph/current`; + const win = detectInstallMethod({ + filename: `${winBundle}/lib/dist/bin/codegraph.js`, platform: 'win32', cwd: winHome, + exists: bundleExists(new Set([`${winBundle}/node.exe`, `${winBundle}/bin/codegraph.cmd`])), + }); + expect(win.kind).toBe('bundle'); + const unixBundle = '/home/u/.codegraph/versions/v1.6.2'; + expect(detectInstallMethod({ + filename: `${unixBundle}/lib/dist/bin/codegraph.js`, platform: 'linux', cwd: '/home/u', + exists: bundleExists(new Set([`${unixBundle}/node`, `${unixBundle}/bin/codegraph`, '/home/u/.codegraph'])), + })).toEqual({ kind: 'bundle', os: 'unix', bundleRoot: unixBundle, installDir: '/home/u/.codegraph' }); + }); + + it('a pnpm, yarn or bun install is named as such, so npm never runs in its folder', () => { + const pnpm = '/home/u/.local/share/pnpm/global/5'; + expect(detectInstallMethod({ + filename: `${pnpm}/node_modules/.pnpm/@colbymchenry+codegraph@1.6.2/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`, + platform: 'linux', cwd: '/home/u', exists: bundleExists(new Set([`${pnpm}/package.json`])), + })).toEqual({ kind: 'package-manager', manager: 'pnpm', global: true }); + const yarn = '/home/u/.config/yarn/global'; + expect(detectInstallMethod({ + filename: `${yarn}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`, + platform: 'linux', cwd: '/home/u', exists: bundleExists(new Set([`${yarn}/package.json`, `${yarn}/yarn.lock`])), + })).toEqual({ kind: 'package-manager', manager: 'yarn', global: true }); + const project = '/home/u/app'; + expect(detectInstallMethod({ + filename: `${project}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`, + platform: 'linux', cwd: project, exists: bundleExists(new Set([`${project}/package.json`, `${project}/bun.lock`])), + })).toEqual({ kind: 'package-manager', manager: 'bun', global: false }); + }); + }); }); describe('deriveInstallDir', () => { @@ -336,7 +431,7 @@ describe('windows bundle swap script (#2185)', () => { // --------------------------------------------------------------------------- interface Calls { - runs: Array<{ cmd: string; args: string[]; env?: NodeJS.ProcessEnv }>; + runs: Array<{ cmd: string; args: string[]; env?: NodeJS.ProcessEnv; cwd?: string }>; captures: Array<{ cmd: string; args: string[] }>; logs: string[]; errors: string[]; @@ -353,8 +448,8 @@ function makeDeps( currentVersion: overrides.currentVersion, method: overrides.method, resolveLatest: overrides.resolveLatest ?? (async () => 'v0.9.9'), - run: (cmd, args, env) => { - calls.runs.push({ cmd, args, env }); + run: (cmd, args, env, cwd) => { + calls.runs.push({ cmd, args, env, cwd }); return runExit; }, // Default probe: spawn fails → 'inconclusive'. Tests that exercise the @@ -557,6 +652,87 @@ describe('runUpgrade', () => { }); }); +describe('runUpgrade picks npm\'s scope and folder from the install, not from cwd (#2477)', () => { + let tmp: string; + beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-upgrade-2477-')); + }); + afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + const npmRun = (calls: Calls) => calls.runs.find((r) => r.args.join(' ').includes('install')); + + it('a Windows global install run from the home folder upgrades with -g (the report)', async () => { + const home = 'C:/Users/X'; + const prefix = `${home}/AppData/Roaming/npm`; + const method = detectInstallMethod({ + filename: `${prefix}/node_modules/@colbymchenry/codegraph/dist/bin/codegraph.js`, + platform: 'win32', + cwd: home, + exists: (p) => p.replace(/\\/g, '/') === `${prefix}/codegraph.cmd`, + }); + const { deps, calls } = makeDeps({ method, currentVersion: '0.9.8', platform: 'win32' }); + expect(await runUpgrade({}, deps)).toBe(0); + const run = npmRun(calls)!; + expect(run.args).toEqual(['/d', '/s', '/c', `npm install -g ${NPM_PACKAGE}@latest`]); + expect(run.cwd).toBeUndefined(); + }); + + it('a local install is upgraded in its own project, and the version npm left there is checked', async () => { + const project = path.join(tmp, 'app'); + const manifest = path.join(project, 'node_modules', '@colbymchenry', 'codegraph', 'package.json'); + fs.mkdirSync(path.dirname(manifest), { recursive: true }); + fs.writeFileSync(path.join(project, 'package.json'), '{}\n'); + + const method: InstallMethod = { kind: 'npm', scope: 'local', root: project }; + // npm left the project on the old version: say so, don't just claim success. + fs.writeFileSync(manifest, JSON.stringify({ version: '0.9.8' })); + const stale = makeDeps({ method, currentVersion: '0.9.8' }); + expect(await runUpgrade({}, stale.deps)).toBe(0); + expect(npmRun(stale.calls)!).toMatchObject({ cmd: 'npm', args: ['install', `${NPM_PACKAGE}@latest`], cwd: project }); + expect(stale.calls.logs.join('\n')).toContain(`Running: npm install ${NPM_PACKAGE}@latest (in ${project})`); + expect(stale.calls.logs.join('\n')).toContain(`npm finished, but ${project} still has CodeGraph v0.9.8.`); + + fs.writeFileSync(manifest, JSON.stringify({ version: '0.9.9' })); + const done = makeDeps({ method, currentVersion: '0.9.8' }); + await runUpgrade({}, done.deps); + expect(done.calls.logs.join('\n')).toContain(`✓ ${project} now has CodeGraph v0.9.9.`); + }); + + it('a local install in a home folder that is a project is upgraded there, even when run from another project inside it', async () => { + const home = path.join(tmp, 'home'); + const other = path.join(home, 'projects', 'app'); + fs.mkdirSync(path.join(home, 'node_modules', '.bin'), { recursive: true }); + fs.mkdirSync(other, { recursive: true }); + fs.writeFileSync(path.join(home, 'package.json'), '{}\n'); + fs.writeFileSync(path.join(home, 'node_modules', '.bin', 'codegraph'), ''); + fs.writeFileSync(path.join(other, 'package.json'), '{}\n'); + const method = detectInstallMethod({ + filename: path.join(home, 'node_modules', '@colbymchenry', 'codegraph', 'dist', 'bin', 'codegraph.js'), + platform: process.platform, + cwd: other, + }); + const { deps, calls } = makeDeps({ method, currentVersion: '0.9.8' }); + expect(await runUpgrade({}, deps)).toBe(0); + expect(npmRun(calls)!.cwd).toBe(home); + }); + + it('a pnpm, yarn or bun install is not touched with npm: the user gets the command for their manager', async () => { + for (const [manager, global, command] of [ + ['pnpm', true, `pnpm add -g ${NPM_PACKAGE}@latest`], + ['yarn', true, `yarn global add ${NPM_PACKAGE}@latest`], + ['bun', false, `bun add ${NPM_PACKAGE}@latest`], + ] as const) { + const { deps, calls } = makeDeps({ method: { kind: 'package-manager', manager, global }, currentVersion: '0.9.8' }); + expect(await runUpgrade({}, deps)).toBe(1); + expect(calls.runs).toHaveLength(0); + expect(calls.logs.join('\n')).toContain(`installed with ${manager}, so npm can't upgrade it`); + expect(calls.logs.join('\n')).toContain(command); + } + }); +}); + // --------------------------------------------------------------------------- // Beta signup offer — fires ONLY after a real, successful binary update. // (The hook itself gates on TTY + the once-per-machine stored choice; see @@ -824,6 +1000,159 @@ describe('post-upgrade refresh of installed agent surfaces', () => { }); }); +// --------------------------------------------------------------------------- +// The refresh an upgrade runs keeps the user's instructions files (#2506) +// --------------------------------------------------------------------------- + +describe('post-upgrade refresh keeps the user\'s instructions files (#2506)', () => { + const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); + let home: string; + + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-upgrade-2506-')); + }); + + afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + }); + + it('an npm upgrade refreshes the block in the global opencode AGENTS.md and keeps every line around it', async () => { + // opencode configured globally, as a previous `codegraph install` left it. + const dir = path.join(home, '.config', 'opencode'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'opencode.jsonc'), JSON.stringify({ + $schema: 'https://opencode.ai/config.json', + mcp: { servers: { codegraph: { type: 'local', command: ['codegraph', 'serve', '--mcp'], disabled: false, codemode: false } } }, + }, null, 2) + '\n'); + // The user's maintained guidance, opening with what is left of an older + // block whose end marker was deleted, and an old block at the end. + const agentsMd = path.join(dir, 'AGENTS.md'); + const notes = '\n## CodeGraph\n\n# Toolset\n\n- ripgrep, fd\n\n## Commit policy\n\n- conventional commits\n\n'; + const oldBlock = '\n## CodeGraph\n\nPrefer `codegraph_search` over grep.\n\n'; + fs.writeFileSync(agentsMd, notes + oldBlock); + + const work = path.join(home, 'work'); + fs.mkdirSync(work); + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: path.join(home, '.config'), + APPDATA: path.join(home, 'AppData', 'Roaming'), + LOCALAPPDATA: path.join(home, 'AppData', 'Local'), + CODEGRAPH_NO_DAEMON: '1', + CODEGRAPH_TELEMETRY: '0', + CODEGRAPH_NO_UPDATE_CHECK: '1', + DO_NOT_TRACK: '1', + }; + for (const key of ['CODEX_HOME', 'COPILOT_HOME', 'HERMES_HOME', 'CLAUDE_CONFIG_DIR', 'CODEGRAPH_NO_INSTALL_REFRESH']) delete env[key]; + + const { deps, calls } = makeDeps({ + method: { kind: 'npm', scope: 'global' }, + currentVersion: '0.9.8', + hasCommand: (cmd) => cmd === 'codegraph', + }); + // `npm install -g` succeeds without running; the refresh it is followed + // by runs the real CLI against the private home. + deps.run = (cmd, args, runEnv) => { + calls.runs.push({ cmd, args, env: runEnv }); + if (cmd !== 'codegraph') return 0; + const r = spawnSync(process.execPath, [BIN, ...args], { cwd: work, env, encoding: 'utf-8', timeout: 60_000 }); + return r.status ?? -1; + }; + + expect(await runUpgrade({}, deps)).toBe(0); + const refresh = calls.runs[calls.runs.length - 1]; + expect(refresh?.cmd).toBe('codegraph'); + expect(refresh?.args).toEqual(['install', '--refresh']); + expect(fs.readFileSync(agentsMd, 'utf-8')).toBe(notes + CODEGRAPH_INSTRUCTIONS_BLOCK + '\n'); + }, 90_000); +}); + +// --------------------------------------------------------------------------- +// A symbolic link the refresh can't write through (#2721) +// --------------------------------------------------------------------------- + +/** File symlinks need Developer Mode or admin rights on Windows. */ +function canCreateFileSymlink(): boolean { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-symlink-probe-')); + try { + fs.writeFileSync(path.join(dir, 'target'), ''); + fs.symlinkSync(path.join(dir, 'target'), path.join(dir, 'link'), 'file'); + return true; + } catch { + return false; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} +const FILE_SYMLINKS = canCreateFileSymlink(); + +describe.skipIf(!FILE_SYMLINKS)(`post-upgrade refresh with a link it can't write through (#2721)${FILE_SYMLINKS ? '' : ' — skipped: creating a file symlink needs Developer Mode or admin rights on Windows'}`, () => { + const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); + let home: string; + + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-upgrade-2721-')); + }); + + afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + }); + + it('warns, keeps the link, and still refreshes every other agent', async () => { + // Codex and Gemini configured globally. Codex's AGENTS.md is a link into a + // dotfiles folder that isn't there; Gemini's block is out of date. + fs.mkdirSync(path.join(home, '.codex'), { recursive: true }); + fs.writeFileSync(path.join(home, '.codex', 'config.toml'), '[mcp_servers.codegraph]\ncommand = "codegraph"\nargs = ["serve", "--mcp"]\n'); + const agentsMd = path.join(home, '.codex', 'AGENTS.md'); + const missing = path.join(home, 'dotfiles', 'AGENTS.md'); + fs.symlinkSync(missing, agentsMd, 'file'); + fs.mkdirSync(path.join(home, '.gemini'), { recursive: true }); + fs.writeFileSync(path.join(home, '.gemini', 'settings.json'), JSON.stringify({ mcpServers: { codegraph: { type: 'stdio', command: 'codegraph', args: ['serve', '--mcp'] } } }, null, 2) + '\n'); + const geminiMd = path.join(home, '.gemini', 'GEMINI.md'); + fs.writeFileSync(geminiMd, '\n## CodeGraph\n\nPrefer `codegraph_search` over grep.\n\n'); + + const work = path.join(home, 'work'); + fs.mkdirSync(work); + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: path.join(home, '.config'), + APPDATA: path.join(home, 'AppData', 'Roaming'), + LOCALAPPDATA: path.join(home, 'AppData', 'Local'), + CODEGRAPH_NO_DAEMON: '1', + CODEGRAPH_TELEMETRY: '0', + CODEGRAPH_NO_UPDATE_CHECK: '1', + DO_NOT_TRACK: '1', + NO_COLOR: '1', + }; + for (const key of ['CODEX_HOME', 'COPILOT_HOME', 'HERMES_HOME', 'CLAUDE_CONFIG_DIR', 'CODEGRAPH_NO_INSTALL_REFRESH']) delete env[key]; + + const { deps, calls } = makeDeps({ + method: { kind: 'npm', scope: 'global' }, + currentVersion: '0.9.8', + hasCommand: (cmd) => cmd === 'codegraph', + }); + let refreshOutput = ''; + deps.run = (cmd, args, runEnv) => { + calls.runs.push({ cmd, args, env: runEnv }); + if (cmd !== 'codegraph') return 0; + const r = spawnSync(process.execPath, [BIN, ...args], { cwd: work, env, encoding: 'utf-8', timeout: 60_000 }); + refreshOutput = `${r.stdout}\n${r.stderr}`; + return r.status ?? -1; + }; + + expect(await runUpgrade({}, deps)).toBe(0); + expect(calls.logs.join('\n')).toMatch(/Could not refresh the installed agent surfaces/); + expect(refreshOutput).toMatch(/Codex CLI: could not refresh — .*AGENTS\.md is a symbolic link to .*its folder does not exist/); + expect(fs.lstatSync(agentsMd).isSymbolicLink()).toBe(true); + expect(fs.existsSync(path.dirname(missing))).toBe(false); + expect(fs.readFileSync(geminiMd, 'utf-8')).toBe(CODEGRAPH_INSTRUCTIONS_BLOCK + '\n'); + }, 90_000); +}); + // --------------------------------------------------------------------------- // Post-upgrade version probe — does the PATH-resolved `codegraph` serve the // version we just installed, in THIS terminal? diff --git a/__tests__/vbnet-member-access-scope.test.ts b/__tests__/vbnet-member-access-scope.test.ts new file mode 100644 index 0000000000..b912f7ef1b --- /dev/null +++ b/__tests__/vbnet-member-access-scope.test.ts @@ -0,0 +1,483 @@ +/** + * VB.NET member reads that #2355 and #2377 left unlinked (#2305 follow-up): + * + * - A method named without parentheses runs: VB.NET calls the overload that + * takes no arguments (`Me.Load` is `Load(Optional force = False)`, not + * `Load(path)`), and a constructor's `Me.New` / `MyBase.New` is another + * constructor, never itself. + * - A name before the dot can be a namespace: `Billing.Invoice.MaxLines`, + * the project's root namespace in front (`Acme.Billing.Invoice.MaxLines`), + * and a Module's member through its namespace (`Assets.Logo`). The namespace + * is looked up as VB.NET does — in the namespaces around the read, outward to + * the root namespaces, then in the file's `Imports` — so another project's + * namespace of that name is not read. + * - A bare name before the dot that is a field or property of the class around + * the read (or of one it inherits, or a Module's variable) is a read of that + * member too: `Button1.Text` reads the form's own `Button1` — never another + * form's, nor a same-named class's in another project — although `Text` is + * .NET's. A parameter or local of that name hides it; a value named like its + * type (`Property Mode As Mode`) is read only when the member read through it + * is an instance one, and not at all when that type is from outside + * (`Responser As Responser`); a method run through the name is the call's + * alone. + */ +import { describe, it, expect, afterAll, beforeAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +let root = ''; +let cg: CodeGraph; + +const files: Record = { + 'App.vbproj': '\n \n Acme\n \n\n', + 'Plugin/Plugin.vbproj': '\n \n Plugin\n \n\n', + 'AppSession.vb': `Public Class AppSession + Public Shared Property CurrentUser As String +End Class +`, + 'OtherSession.vb': `Public Class OtherSession + Public Shared SessionId As Guid +End Class +`, + // Overloads of a method VB.NET runs without parentheses. + 'Chain.vb': `Public Class Chain + Public Sub New(ByVal name As String) + Me.New + Me.Load + Dim n = Me.Count + End Sub + + Public Sub New() + End Sub + + Public Sub Load(ByVal path As String) + End Sub + + Public Sub Load(Optional ByVal force As Boolean = False) + End Sub + + Public Function Count(ByVal items As Integer()) As Integer + Return 0 + End Function + + Public Function Count(ParamArray items As Integer()) As Integer + Return 0 + End Function +End Class + +Public Class LinkedChain + Inherits Chain + + Public Sub New() + MyBase.New + End Sub +End Class + +Public Class Calc + Public Shared Function GetTargetDAR(ByVal width As Integer) As Double + Return 0 + End Function + + Public Shared Function GetTargetDAR() As Double + Return 0 + End Function +End Class + +Public Class Measurer + Private Tracker As Chain + + Public Function Ratio(ByVal c As Chain) As Double + Dim n = c.Count + Return Calc.GetTargetDAR + End Function + + Public Sub Track() + Tracker.Load + End Sub +End Class +`, + // Types and a Module in namespaces of the Acme project. + 'Billing/Invoice.vb': `Namespace Billing + Public Class Invoice + Public Const MaxLines As Integer = 100 + Public Shared Function Create() As Invoice + Return Nothing + End Function + End Class + + Public Enum Status + Open + Paid + End Enum +End Namespace +`, + 'Assets/Images.vb': `Namespace Assets + Public Module Images + Public Logo As String = "logo.png" + Public Function Pick() As String + Return Logo + End Function + End Module +End Namespace +`, + 'Forms/MainForm.Designer.vb': ` +Partial Class MainForm + Inherits System.Windows.Forms.Form + + Private Sub InitializeComponent() + Me.Button1 = New System.Windows.Forms.Button() + Me.Button1.Size = New System.Drawing.Size(75, 23) + Me.Size = New System.Drawing.Size(300, 200) + End Sub + + Friend WithEvents Button1 As System.Windows.Forms.Button +End Class +`, + 'Forms/MainForm.vb': `Public Class MainForm + Private Clicks As Integer + + Private Sub Button1_Click(sender As Object, e As EventArgs) Handles Button1.Click + Me.Clicks = Me.Clicks + 1 + Button1.Text = AppSession.CurrentUser + With Button1 + .Enabled = False + End With + Me.Text = Console.Out.ToString() + Dim n = Billing.Invoice.MaxLines + Dim m = Acme.Billing.Invoice.MaxLines + Dim s = Billing.Status.Paid + Dim l = Assets.Logo + Dim p = Assets.Pick + Dim i = Billing.Invoice.Create + Dim r = System.Drawing.Color.Red + End Sub +End Class +`, + 'Forms/OtherForm.Designer.vb': `Partial Class OtherForm + Inherits System.Windows.Forms.Form + + Friend WithEvents Button1 As System.Windows.Forms.Button +End Class +`, + 'Forms/OtherForm.vb': `Public Class OtherForm + Private Sub Reload() + Button1.Text = OtherSession.SessionId.ToString() + End Sub +End Class +`, + // The Plugin project sees no \`Billing\` namespace: Acme's is another project's. + 'Plugin/Report.vb': `Public Class Report + Public Sub Run() + Dim n = Billing.Invoice.MaxLines + End Sub +End Class +`, + 'Model.vb': `Public Enum Mode + Fast + Slow +End Enum + +Public Class Settings + Public Property Level As Integer +End Class + +Public Class Widget + Public Property Length As Integer +End Class +`, + 'Globals.vb': `Public Module Globals + Public Current As Settings +End Module +`, + // What the name before the dot binds to: a parameter or local, then a member + // of the class around it (or one it inherits, or a Module's), then a type. + 'Scopes.vb': `Public Class Reporter + Public Sub Report(ByVal Mode As String) + Dim Widget = "x" + Dim n = Mode.Length + Widget.Length + End Sub +End Class + +Public Class SessionView + Private ReadOnly Property AppSession As OtherSession + + Public Sub Show() + Dim id = AppSession.SessionId + End Sub +End Class + +Public Class SettingsUser + Public Property Settings As Settings + Public Property Mode As Mode + + Public Sub Apply() + Dim t = Settings.Level + Dim f = Mode.Fast + End Sub +End Class + +Public Class BaseView + Protected Title As String +End Class + +Public Class DerivedView + Inherits BaseView + + Public Sub Render() + Dim n = Title.Length + End Sub +End Class + +Public Class Viewer + Private Button1 As Settings + + Public Sub Show() + Dim lv = Current.Level + End Sub + + Public Sub Hide() + Dim Button1 As New Settings() + Dim x = Button1.Level + End Sub +End Class + +Public Class Detector + Public Class Data + Public Shared Empty As Integer + End Class + + Public Function Probe() As Integer + Return Data.Empty + End Function +End Class + +Public Class Failure + Inherits System.Exception + + Public Function Describe() As Integer + Return Data.Empty + End Function +End Class +`, + // CRLF line endings, and multibyte text — naming the field — before the read. + 'Weird.vb': [ + 'Public Class Weird', + ' Private Label1 As Settings', + ' Public Sub Go()', + ' Dim s = "Label1 éééé 😀" & Label1.Level & Billing.Invoice.MaxLines', + ' End Sub', + 'End Class', + '', + ].join('\r\n'), + // One class name in two namespaces and in another project: the file's + // \`Imports\` says which \`UserData\` it means. A type in the file's own + // namespace (the root one) comes before an imported one: \`Helper.Name\`. + 'Plugin/UserData.vb': `Public Class UserData + Public Shared ChannelsLabel As String + Private Label As String + + Public Function Size() As Integer + Return Label.Length + End Function +End Class +`, + // Another plugin project declares a \`UserData\` of its own: its members are not Plugin's. + 'Plugin2/Plugin2.vbproj': '\n \n Plugin2\n \n\n', + 'Plugin2/UserData.vb': `Public Class UserData + Private Label As String + + Public Function Size() As Integer + Return Label.Length + End Function +End Class +`, + // A member declared As an outside type of its own name: VB.NET reads that + // type's Shared members and nested types through the type ("Color Color"). + 'Client.vb': `Public Class Client + Protected Responser As Responser + + Public Function Code() As Integer + Dim m = Responser.Modes.Curl + Return Responser.StatusCode + End Function +End Class +`, + 'Helper.vb': `Public Class Helper + Public Shared Name As String +End Class +`, + 'Lib/Helper.vb': `Namespace Lib + Public Class Helper + Public Shared Name As String + End Class +End Namespace +`, + 'Reddit/UserData.vb': `Namespace API.Reddit + Public Class UserData + Public Const ChannelsLabel As String = "Channels" + End Class +End Namespace +`, + 'Twitter/UserData.vb': `Namespace API.Twitter + Public Class UserData + Public Const ChannelsLabel As String = "Lists" + End Class +End Namespace +`, + 'ChannelView.vb': `Imports Acme.API.Reddit +Imports Acme.Lib + +Public Class ChannelView + Public Sub Add() + Dim l = UserData.ChannelsLabel + Dim h = Helper.Name + End Sub +End Class +`, +}; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-vb-scope-')); + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + cg = await CodeGraph.init(root, { index: true }); +}); + +afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +/** The node named `qualifiedName` — at `line`, or in `file`, when the name alone leaves several. */ +function nodeId(qualifiedName: string, at?: number | string): string { + const found = cg.getNodesInFiles(Object.keys(files)).filter((n) => n.qualifiedName === qualifiedName && + (at === undefined || (typeof at === 'number' ? n.startLine === at : n.filePath === at))); + if (found.length !== 1) throw new Error(`${found.length} nodes named ${qualifiedName}${at === undefined ? '' : ` at ${at}`}`); + return found[0]!.id; +} + +/** What links to `qualifiedName` with an edge of `kind`, one per edge: `file:qualifiedName` when `withFiles`. */ +function linkedFrom(qualifiedName: string, kind: 'references' | 'calls', at?: number | string, withFiles = false): string[] { + return cg + .getIncomingEdgesTo([nodeId(qualifiedName, at)], [kind]) + .map((e) => cg.getNode(e.source)!) + .map((n) => (withFiles ? `${n.filePath}:${n.qualifiedName}` : n.qualifiedName)) + .sort(); +} + +describe('VB.NET member reads through a namespace, a member of the class around them, or a method without parentheses (#2305)', () => { + it('calls the overload that takes no arguments, and a constructor never itself', () => { + // Chain.vb: New(name) at 2, New() at 8, Load(path) at 11, Load(Optional) at 14, + // Count(items) at 17, Count(ParamArray) at 21. + expect(linkedFrom('Chain::New', 'calls', 8)).toEqual(['Chain::New', 'LinkedChain::New']); + expect(linkedFrom('Chain::New', 'calls', 2)).toEqual([]); + // `Me.Load`, and `Tracker.Load` through a field, whose read is the call's alone. + expect(linkedFrom('Chain::Load', 'calls', 14)).toEqual(['Chain::New', 'Measurer::Track']); + expect(linkedFrom('Chain::Load', 'calls', 11)).toEqual([]); + expect(linkedFrom('Measurer::Tracker', 'calls')).toEqual([]); + expect(linkedFrom('Chain::Count', 'calls', 21)).toEqual(['Chain::New', 'Measurer::Ratio']); + expect(linkedFrom('Chain::Count', 'calls', 17)).toEqual([]); + // Calc's GetTargetDAR(width) at 35, GetTargetDAR() at 39. + expect(linkedFrom('Calc::GetTargetDAR', 'calls', 39)).toEqual(['Measurer::Ratio']); + expect(linkedFrom('Calc::GetTargetDAR', 'calls', 35)).toEqual([]); + }); + + it('reads a type, a constant, an Enum case and a Module member through their namespace, the root namespace in front or not', () => { + // `Billing.Invoice.MaxLines` and `Acme.Billing.Invoice.MaxLines`. + expect(linkedFrom('Billing::Invoice::MaxLines', 'references')) + .toEqual(['MainForm::Button1_Click', 'MainForm::Button1_Click', 'Weird::Go']); + // The type each of them, and `Billing.Invoice.Create`, names. + expect(linkedFrom('Billing::Invoice', 'references')) + .toEqual(['MainForm::Button1_Click', 'MainForm::Button1_Click', 'MainForm::Button1_Click', 'Weird::Go']); + expect(linkedFrom('Billing::Invoice::Create', 'calls')).toEqual(['MainForm::Button1_Click']); + expect(linkedFrom('Billing::Status::Paid', 'references')).toEqual(['MainForm::Button1_Click']); + expect(linkedFrom('Billing::Status', 'references')).toEqual(['MainForm::Button1_Click']); + // A Module's members are its namespace's; the Module itself is not written. + expect(linkedFrom('Assets::Images::Logo', 'references')).toEqual(['MainForm::Button1_Click']); + expect(linkedFrom('Assets::Images::Pick', 'calls')).toEqual(['MainForm::Button1_Click']); + expect(linkedFrom('Assets::Images', 'references')).toEqual([]); + }); + + it('reads no namespace another project declares', () => { + for (const qn of ['Billing::Invoice::MaxLines', 'Billing::Invoice']) { + expect(linkedFrom(qn, 'references')).not.toContain('Report::Run'); + } + }); + + it('reads a field or property of the class around the read through its bare name', () => { + // `Button1.Text` in Button1_Click and the designer's two `Me.Button1`; the + // `With Button1` block's `.Enabled` is no second read of it. + expect(linkedFrom('MainForm::Button1', 'references')) + .toEqual(['MainForm::Button1_Click', 'MainForm::InitializeComponent', 'MainForm::InitializeComponent']); + expect(linkedFrom('OtherForm::Button1', 'references')).toEqual(['OtherForm::Reload']); + expect(linkedFrom('MainForm::Clicks', 'references')).toEqual(['MainForm::Button1_Click', 'MainForm::Button1_Click']); + // An inherited field, a Module's variable, and a field read after multibyte text. + expect(linkedFrom('BaseView::Title', 'references')).toEqual(['DerivedView::Render']); + expect(linkedFrom('Globals::Current', 'references')).toEqual(['Viewer::Show']); + expect(linkedFrom('Weird::Label1', 'references')).toEqual(['Weird::Go']); + expect(linkedFrom('Settings::Level', 'references')).toEqual(['SettingsUser::Apply', 'Viewer::Hide', 'Viewer::Show', 'Weird::Go']); + }); + + it('binds the name before the dot as VB does: parameter or local, own member, then type', () => { + // `Mode` is a parameter and `Widget` a local of Report; `Button1` a local of Hide. + expect(linkedFrom('Mode', 'references')).toEqual(['SettingsUser::Apply']); + expect(linkedFrom('Widget', 'references')).toEqual([]); + expect(linkedFrom('Widget::Length', 'references')).toEqual([]); + expect(linkedFrom('Viewer::Button1', 'references')).toEqual([]); + // SessionView's own `AppSession` property holds an OtherSession. + expect(linkedFrom('SessionView::AppSession', 'references')).toEqual(['SessionView::Show']); + expect(linkedFrom('OtherSession::SessionId', 'references')).toEqual(['OtherForm::Reload', 'SessionView::Show']); + // "Color Color": `Settings.Level` reads the property, and an instance + // member of its type; `Mode.Fast` reads an Enum case through the type. + expect(linkedFrom('SettingsUser::Settings', 'references')).toEqual(['SettingsUser::Apply']); + expect(linkedFrom('SettingsUser::Mode', 'references')).toEqual([]); + expect(linkedFrom('Mode::Fast', 'references')).toEqual(['SettingsUser::Apply']); + // A type nested in a class is a bare name only inside it: Failure's `Data` is Exception's. + expect(linkedFrom('Detector::Data::Empty', 'references')).toEqual(['Detector::Probe']); + expect(linkedFrom('Detector::Data', 'references')).toEqual(['Detector::Probe']); + // `Responser As Responser`, a type from outside: `Responser.Modes` may name + // the type's nested enum, which nothing here can tell, so no read is linked. + expect(linkedFrom('Client::Responser', 'references')).toEqual([]); + }); + + it("reads a member of the class's own project, not of a same-named class in another", () => { + expect(linkedFrom('UserData::Label', 'references', 'Plugin/UserData.vb', true)).toEqual(['Plugin/UserData.vb:UserData::Size']); + expect(linkedFrom('UserData::Label', 'references', 'Plugin2/UserData.vb', true)).toEqual(['Plugin2/UserData.vb:UserData::Size']); + }); + + it('picks among same-named classes the one VB finds first, for the member and the class', () => { + expect(linkedFrom('API.Reddit::UserData::ChannelsLabel', 'references')).toEqual(['ChannelView::Add']); + expect(linkedFrom('API.Reddit::UserData', 'references')).toEqual(['ChannelView::Add']); + expect(linkedFrom('API.Twitter::UserData::ChannelsLabel', 'references')).toEqual([]); + expect(linkedFrom('UserData::ChannelsLabel', 'references')).toEqual([]); + expect(linkedFrom('Helper::Name', 'references')).toEqual(['ChannelView::Add']); + expect(linkedFrom('Lib::Helper::Name', 'references')).toEqual([]); + }); + + it('does not bind a member of another class that only shares the name', () => { + // `Me.Size` / `Me.Text` on a Form, `Console.Out`, `System.Drawing.Color.Red`. + expect(linkedFrom('MainForm::Button1', 'references')).not.toContain('OtherForm::Reload'); + expect(linkedFrom('OtherForm::Button1', 'references')).not.toContain('MainForm::Button1_Click'); + expect(linkedFrom('Widget::Length', 'references')).toEqual([]); + }); + + it('keeps the edges through an incremental sync', async () => { + for (const rel of ['Forms/MainForm.vb', 'Billing/Invoice.vb', 'Chain.vb', 'Scopes.vb']) { + fs.writeFileSync(path.join(root, rel), `' touched\n${files[rel]}`); + } + await cg.sync(); + expect(linkedFrom('Billing::Invoice::MaxLines', 'references')) + .toEqual(['MainForm::Button1_Click', 'MainForm::Button1_Click', 'Weird::Go']); + expect(linkedFrom('Assets::Images::Logo', 'references')).toEqual(['MainForm::Button1_Click']); + expect(linkedFrom('MainForm::Button1', 'references')) + .toEqual(['MainForm::Button1_Click', 'MainForm::InitializeComponent', 'MainForm::InitializeComponent']); + expect(linkedFrom('SessionView::AppSession', 'references')).toEqual(['SessionView::Show']); + // The touched Chain.vb moved down a line. + expect(linkedFrom('Chain::Load', 'calls', 15)).toEqual(['Chain::New', 'Measurer::Track']); + expect(linkedFrom('Chain::New', 'calls', 9)).toEqual(['Chain::New', 'LinkedChain::New']); + }); +}); diff --git a/__tests__/vue-tsx-script.test.ts b/__tests__/vue-tsx-script.test.ts new file mode 100644 index 0000000000..d20d1ccae0 --- /dev/null +++ b/__tests__/vue-tsx-script.test.ts @@ -0,0 +1,144 @@ +/** + * A Vue ` +`, + // Types and JSX together, which only the TSX grammar reads. + 'src/Card.vue': ` +`, + // Controls: TypeScript stays TypeScript, no lang stays JavaScript. + 'src/Plain.vue': ` + +`, + 'src/Legacy.vue': ` + +`, +}; + +let roots: string[] = []; +let graphs: CodeGraph[] = []; + +afterEach(() => { + for (const graph of graphs) graph.close(); + graphs = []; + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); + roots = []; +}); + +async function indexProject(): Promise { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-vue-tsx-')); + roots.push(root); + for (const [file, text] of Object.entries(PROJECT)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } + const graph = await CodeGraph.init(root, { index: true }); + graphs.push(graph); + return graph; +} + +const named = (nodes: Node[], kind: string, name: string) => nodes.find((n) => n.kind === kind && n.name === name); + +/** + * The file's extraction errors. The file must be stored: one whose grammar + * was not loaded is never parsed, and nothing at all is stored for it. + */ +function errorsOf(graph: CodeGraph, file: string) { + const record = graph.getFile(file); + expect(record, `${file} is stored`).not.toBeNull(); + return record!.errors ?? []; +} + +describe('Vue \n', 'svelte')).toEqual(['javascript']); + }); +}); diff --git a/__tests__/workspace-package-entries.test.ts b/__tests__/workspace-package-entries.test.ts new file mode 100644 index 0000000000..855f36e0b5 --- /dev/null +++ b/__tests__/workspace-package-entries.test.ts @@ -0,0 +1,370 @@ +/** + * Issues #2456 and #2502: a package of the repository is imported by its name, + * and that import has to land on the file the package's manifest names. + * + * - #2502: a pnpm (or npm/yarn/bun) workspace member that declares its entry + * with `main` / `module` / `types` or an `exports` map, like + * `"main": "./src/index.ts"`, had no `index.*` at its root, so + * `import { caseStatus } from '@acme/core'` failed and its calls fell back to + * name matching: a same-named private function nearer the caller took them. + * - #2456: a sibling package an npm `file:` dependency points at, with no root + * `workspaces`, was not known as a package at all. + * + * A declared entry that names build output the index does not hold (`dist/`) + * is read as the source it is built from when the mapping is clear + * (`dist/index.js` → `src/index.ts`), and otherwise links nothing new. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +let roots: string[] = []; +let graphs: CodeGraph[] = []; + +afterEach(() => { + for (const graph of graphs) graph.close(); + graphs = []; + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); + roots = []; +}); + +function write(root: string, files: Record): void { + for (const [rel, content] of Object.entries(files)) { + const file = path.join(root, rel); + if (content === null) { + fs.rmSync(file, { force: true }); + continue; + } + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); + } +} + +async function index(files: Record): Promise { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-ws-entries-')); + roots.push(root); + write(root, files); + const graph = await CodeGraph.init(root, { index: true }); + graphs.push(graph); + return graph; +} + +/** The files `file`'s import statements link it to. */ +function moduleImports(graph: CodeGraph, file: string): string[] { + const out = new Set(); + for (const node of graph.getNodesInFile(file)) { + for (const e of graph.getOutgoingEdges(node.id)) { + if (e.kind !== 'imports') continue; + const target = graph.getNode(e.target)!; + if (target.kind === 'file') out.add(target.filePath); + } + } + return [...out].sort(); +} + +/** `name@file` of every call `file` makes. */ +function callsFrom(graph: CodeGraph, file: string): string[] { + const out: string[] = []; + for (const node of graph.getNodesInFile(file)) { + for (const e of graph.getOutgoingEdges(node.id)) { + if (e.kind !== 'calls') continue; + const target = graph.getNode(e.target)!; + out.push(`${target.name}@${target.filePath}`); + } + } + return out.sort(); +} + +/** A pnpm workspace whose `@acme/core` member has the given manifest. */ +function pnpm(core: Record, extra: Record = {}): Record { + return { + 'package.json': JSON.stringify({ name: 'repo', private: true }), + 'pnpm-workspace.yaml': 'packages:\n - "packages/*"\n - "apps/*"\n', + 'packages/core/package.json': JSON.stringify({ name: '@acme/core', ...core }), + 'packages/core/src/index.ts': 'export function coreFn(): number { return 1; }\n', + 'apps/web/package.json': JSON.stringify({ name: '@acme/web', dependencies: { '@acme/core': 'workspace:*' } }), + 'apps/web/main.ts': "import { coreFn } from '@acme/core';\nexport const run = () => coreFn();\n", + ...extra, + }; +} + +describe('a workspace member imported by name lands on the entry its manifest declares (#2502)', () => { + it.each([ + ['main', { main: './src/index.ts' }], + ['module', { module: './src/index.ts' }], + ['types', { types: './src/index.ts' }], + ['a string `exports`', { exports: './src/index.ts' }], + ['`exports["."]`', { exports: { '.': './src/index.ts' } }], + ['conditions straight under `exports`', { exports: { types: './dist/index.d.ts', import: './src/index.ts' } }], + ])('through %s', async (_label, manifest) => { + const graph = await index(pnpm(manifest)); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + expect(callsFrom(graph, 'apps/web/main.ts')).toContain('coreFn@packages/core/src/index.ts'); + }, 120_000); + + it('through conditions, nested ones included, whichever the index holds', async () => { + // `types` and `require` name build output; the source sits under `import` → `default`. + const exports = { + '.': { + types: './dist/index.d.ts', + import: { types: './dist/index.d.mts', default: './src/index.ts' }, + require: './dist/index.cjs', + }, + }; + const graph = await index(pnpm({ exports })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + }, 120_000); + + it("through the condition its importers resolve by, read as its source, before a custom condition's source", async () => { + // @wdio/logger's layout: `import` names build output, `browserSource` a browser-only source file. + const exports = { + '.': { + browserSource: './src/browser.ts', + browser: './build/browser.js', + types: './build/index.d.ts', + import: './build/index.js', + require: './build/index.cjs', + }, + }; + const graph = await index(pnpm({ exports }, { + 'packages/core/src/browser.ts': 'export function coreFn(): number { return 5; }\n', + })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + }, 120_000); + + it('through the implementation a condition names before a declaration file', async () => { + const graph = await index(pnpm({ exports: { '.': { types: './src/index.d.ts', import: './src/index.ts' } } }, { + 'packages/core/src/index.d.ts': 'export declare function coreFn(): number;\n', + })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + }, 120_000); + + it('never through build output that is on disk but not indexed', async () => { + // A local build left `dist/` behind; `.gitignore` keeps it out of the index. + const graph = await index(pnpm({ main: './dist/index.js' }, { + '.gitignore': 'dist/\n', + 'packages/core/dist/index.js': 'exports.coreFn = function () { return 1; };\n', + })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + }, 120_000); + + it('through a subpath export and a `./*` pattern', async () => { + const graph = await index(pnpm( + { exports: { '.': './src/index.ts', './utils': './src/utils.ts', './schema/*': { import: './src/schema/*.ts' } } }, + { + 'packages/core/src/utils.ts': 'export function trim(s: string): string { return s.trim(); }\n', + 'packages/core/src/schema/case.ts': "export const caseTable = 'case';\n", + 'apps/web/text.ts': "import { trim } from '@acme/core/utils';\nexport const clean = (s: string) => trim(s);\n", + 'apps/web/table.ts': "import { caseTable } from '@acme/core/schema/case';\nexport const tableName = () => caseTable;\n", + }, + )); + expect(moduleImports(graph, 'apps/web/text.ts')).toEqual(['packages/core/src/utils.ts']); + expect(callsFrom(graph, 'apps/web/text.ts')).toContain('trim@packages/core/src/utils.ts'); + expect(moduleImports(graph, 'apps/web/table.ts')).toEqual(['packages/core/src/schema/case.ts']); + }, 120_000); + + it('through build output the index lacks, read as the source it is built from', async () => { + const graph = await index(pnpm({ + main: './dist/cjs/index.cjs', + types: './dist/types/index.d.ts', + exports: { '.': { import: './dist/esm/index.mjs', require: './dist/cjs/index.cjs' }, './utils': './dist/utils.js' }, + }, { + 'packages/core/src/utils.ts': 'export function trim(s: string): string { return s.trim(); }\n', + 'apps/web/text.ts': "import { trim } from '@acme/core/utils';\nexport const clean = (s: string) => trim(s);\n", + })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + expect(moduleImports(graph, 'apps/web/text.ts')).toEqual(['packages/core/src/utils.ts']); + }, 120_000); + + it('through a path into its build output, read as the source it is built from', async () => { + // memfs's layout: `import { … } from '@jsonjoy.com/fs-node-utils/lib/types/misc'` is + // `lib/types/misc.js`, built from `src/types/misc.ts`. A local build on disk changes nothing. + const graph = await index(pnpm({ main: './lib/index.js' }, { + '.gitignore': 'lib/\n', + 'packages/core/lib/strings.js': 'exports.trim = function (s) { return s; };\n', + 'packages/core/src/strings.ts': 'export function trim(s: string): string { return s.trim(); }\n', + 'packages/core/src/types/misc.ts': 'export type Encoding = string;\n', + 'apps/web/text.ts': + "import { trim } from '@acme/core/lib/strings';\nimport type { Encoding } from '@acme/core/lib/types/misc';\nexport const clean = (s: string, e: Encoding) => trim(s) + e;\n", + })); + expect(moduleImports(graph, 'apps/web/text.ts')).toEqual(['packages/core/src/strings.ts', 'packages/core/src/types/misc.ts']); + expect(callsFrom(graph, 'apps/web/text.ts')).toEqual(['trim@packages/core/src/strings.ts']); + }, 120_000); + + it('through the source tree a build folder mirrors before the folder read as a build format', async () => { + // Vite's layout: `dist/node/index.js` is built from `src/node/index.ts`, not `src/index.ts`. + const graph = await index(pnpm({ main: './dist/node/index.js', types: './dist/node/index.d.ts' }, { + 'packages/core/src/node/index.ts': 'export function coreFn(): number { return 3; }\n', + })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/node/index.ts']); + expect(callsFrom(graph, 'apps/web/main.ts')).toContain('coreFn@packages/core/src/node/index.ts'); + }, 120_000); + + it("through `exports` first, then the root's `main`-style fields when `exports` leads to no indexed file", async () => { + // `exports` read as its source beats a root `index.js` that `main` names… + const first = await index(pnpm({ exports: { '.': './dist/index.mjs' }, main: './index.js' }, { + 'packages/core/index.js': "module.exports = require('./dist/index.cjs');\n", + })); + expect(moduleImports(first, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + // …and `source` is read when `exports` names build output with no clear source. + const then = await index(pnpm({ exports: { '.': './dist/acme-core.mjs' }, source: './src/main.ts' }, { + 'packages/core/src/main.ts': 'export function coreFn(): number { return 4; }\n', + })); + expect(moduleImports(then, 'apps/web/main.ts')).toEqual(['packages/core/src/main.ts']); + }, 120_000); + + it('and links nothing new when build output has no clear source', async () => { + // `dist/bundle.js` has no `src/bundle.*`: no guess at another file. + const graph = await index(pnpm({ main: './dist/bundle.js' })); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual([]); + }, 120_000); + + it("so a same-named private function no longer takes the package's callers (the issue's repro)", async () => { + const graph = await index({ + 'package.json': JSON.stringify({ name: 'repro', private: true }), + 'pnpm-workspace.yaml': 'packages:\n - "packages/*"\n - "apps/*"\n', + 'packages/core/package.json': JSON.stringify({ + name: '@acme/core', type: 'module', main: './src/index.ts', exports: { '.': './src/index.ts' }, + }), + 'packages/core/src/index.ts': "export * from './status.js';\n", + 'packages/core/src/status.ts': + "export function caseStatus(input: { open: boolean }): { value: string } {\n return { value: input.open ? 'open' : 'closed' };\n}\n", + 'packages/db/package.json': JSON.stringify({ name: '@acme/db', type: 'module' }), + 'packages/db/src/load.ts': + "import { caseStatus } from '@acme/core';\nexport const loadStatus = (open: boolean) => caseStatus({ open }).value;\n", + 'apps/web/package.json': JSON.stringify({ name: '@acme/web', type: 'module' }), + 'apps/web/lib/list.ts': + "import { caseStatus } from '@acme/core';\nexport const listCases = (cases: { open: boolean }[]) => cases.map((c) => caseStatus(c).value);\n", + 'apps/worker/package.json': JSON.stringify({ name: '@acme/worker', type: 'module' }), + 'apps/worker/src/run.ts': + "import { caseStatus } from '@acme/core';\nexport const run = () => caseStatus({ open: true }).value;\n", + 'apps/worker/src/interpret.ts': + 'function caseStatus(raw: string): string { return raw.trim(); }\nexport const interpret = (raw: string) => caseStatus(raw);\n', + }); + for (const file of ['packages/db/src/load.ts', 'apps/web/lib/list.ts', 'apps/worker/src/run.ts']) { + expect(callsFrom(graph, file)).toContain('caseStatus@packages/core/src/status.ts'); + expect(callsFrom(graph, file)).not.toContain('caseStatus@apps/worker/src/interpret.ts'); + } + }, 120_000); +}); + +describe('a package an npm `file:` dependency points at is imported by its name (#2456)', () => { + /** The issue's layout: no root manifest, `acme-api` depends on `file:../common`. */ + const fileDep = (spec: string, extra: Record = {}): Record => ({ + 'packages/common/package.json': JSON.stringify({ name: 'acme-shared', version: '1.0.0' }), + 'packages/common/utils.ts': + "export function formatLabel(name: string): string { return name.trim(); }\nexport function isAdmin(role: string): boolean { return role === 'admin'; }\n", + 'packages/api/package.json': JSON.stringify({ name: 'acme-api', dependencies: { 'acme-shared': spec } }), + 'packages/api/src/handler.ts': + "import { formatLabel, isAdmin } from 'acme-shared/utils';\nexport function handleRequest(name: string, role: string): string {\n return isAdmin(role) ? formatLabel(name) : 'forbidden';\n}\n", + ...extra, + }); + + it.each(['file:../common', 'link:../common'])('with %s and no root `workspaces`', async (spec) => { + const graph = await index(fileDep(spec, { + // A same-named function nearer the caller, which name matching would pick. + 'packages/api/src/format.ts': "export function formatLabel(name: string): string { return '!' + name; }\n", + })); + expect(moduleImports(graph, 'packages/api/src/handler.ts')).toEqual(['packages/common/utils.ts']); + expect(callsFrom(graph, 'packages/api/src/handler.ts')).toEqual([ + 'formatLabel@packages/common/utils.ts', 'isAdmin@packages/common/utils.ts', + ]); + }, 120_000); + + it('for every file when the root manifest declares it', async () => { + // An Expo app's local native module, as `link:` from the root package.json. + const graph = await index({ + 'package.json': JSON.stringify({ name: 'app', dependencies: { '@acme/native': 'link:./modules/native' } }), + 'modules/native/package.json': JSON.stringify({ name: '@acme/native', main: './src/index.ts' }), + 'modules/native/src/index.ts': 'export function share(text: string): string { return text; }\n', + 'src/screens/Share.ts': "import { share } from '@acme/native';\nexport const onShare = () => share('hi');\n", + }); + expect(moduleImports(graph, 'src/screens/Share.ts')).toEqual(['modules/native/src/index.ts']); + expect(callsFrom(graph, 'src/screens/Share.ts')).toEqual(['share@modules/native/src/index.ts']); + }, 120_000); + + it("only for the files under the manifest that declares it", async () => { + // A test fixture's `left-pad` stub is the fixture's; the app's `left-pad` stays the npm package. + const graph = await index({ + 'package.json': JSON.stringify({ name: 'app', dependencies: { 'left-pad': '^1.3.0' } }), + 'src/pad.ts': "import leftPad from 'left-pad';\nexport const pad = (s: string) => leftPad(s, 4);\n", + 'test/fixture/package.json': JSON.stringify({ name: 'fixture', dependencies: { 'left-pad': 'file:../stub-left-pad' } }), + 'test/fixture/use.ts': "import leftPad from 'left-pad';\nexport const use = () => leftPad('x', 2);\n", + 'test/stub-left-pad/package.json': JSON.stringify({ name: 'left-pad', main: 'index.ts' }), + 'test/stub-left-pad/index.ts': 'export default function leftPad(s: string, n: number): string { return s.padStart(n); }\n', + }); + expect(moduleImports(graph, 'test/fixture/use.ts')).toEqual(['test/stub-left-pad/index.ts']); + expect(moduleImports(graph, 'src/pad.ts')).toEqual([]); + expect(callsFrom(graph, 'src/pad.ts')).toEqual([]); + }, 120_000); + + it("through the target's own `exports`", async () => { + const graph = await index({ + 'packages/common/package.json': JSON.stringify({ name: 'acme-shared', exports: { './utils': './src/utils.ts' } }), + 'packages/common/src/utils.ts': 'export function formatLabel(name: string): string { return name; }\n', + 'packages/api/package.json': JSON.stringify({ name: 'acme-api', dependencies: { 'acme-shared': 'file:../common' } }), + 'packages/api/handler.ts': "import { formatLabel } from 'acme-shared/utils';\nexport const h = (n: string) => formatLabel(n);\n", + }); + expect(moduleImports(graph, 'packages/api/handler.ts')).toEqual(['packages/common/src/utils.ts']); + }, 120_000); +}); + +describe('sync follows a manifest edit (#2456, #2502)', () => { + it("re-links importers when a member's `main` or `exports` changes, as a fresh index would", async () => { + const files = pnpm({ main: './dist/bundle.js' }, { + 'packages/core/src/next.ts': 'export function coreFn(): number { return 2; }\n', + }); + const graph = await index(files); + const root = roots[roots.length - 1]!; + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual([]); + + write(root, { 'packages/core/package.json': JSON.stringify({ name: '@acme/core', main: './src/index.ts' }) }); + await graph.sync(); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + + write(root, { 'packages/core/package.json': JSON.stringify({ name: '@acme/core', exports: { '.': { import: './src/next.ts' } } }) }); + await graph.sync(); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/next.ts']); + expect(callsFrom(graph, 'apps/web/main.ts')).toContain('coreFn@packages/core/src/next.ts'); + }, 120_000); + + it('re-links importers when a `file:` dependency appears', async () => { + const graph = await index({ + 'packages/common/package.json': JSON.stringify({ name: 'acme-shared' }), + 'packages/common/utils.ts': 'export function formatLabel(name: string): string { return name; }\n', + 'packages/api/package.json': JSON.stringify({ name: 'acme-api' }), + 'packages/api/handler.ts': "import { formatLabel } from 'acme-shared/utils';\nexport const h = (n: string) => formatLabel(n);\n", + }); + const root = roots[roots.length - 1]!; + expect(moduleImports(graph, 'packages/api/handler.ts')).toEqual([]); + write(root, { 'packages/api/package.json': JSON.stringify({ name: 'acme-api', dependencies: { 'acme-shared': 'file:../common' } }) }); + await graph.sync(); + expect(moduleImports(graph, 'packages/api/handler.ts')).toEqual(['packages/common/utils.ts']); + }, 120_000); +}); + +describe('sync follows the entry a manifest declares when its file appears (#2502)', () => { + it.each([ + ['an `exports` entry', { exports: { '.': './src/index.ts' } }], + ['the source of a `main` in `dist/`', { main: './dist/index.js' }], + ])("re-links importers that fell back to the package's root index, through %s", async (_label, manifest) => { + const files = pnpm(manifest, { + 'packages/core/index.ts': "export function caseStatus(open: boolean): string { return 'root'; }\n", + 'packages/core/src/status.ts': "export function caseStatus(open: boolean): string { return open ? 'open' : 'closed'; }\n", + 'apps/web/main.ts': "import { caseStatus } from '@acme/core';\nexport const run = () => caseStatus(true);\n", + }); + delete files['packages/core/src/index.ts']; + const graph = await index(files); + const root = roots[roots.length - 1]!; + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/index.ts']); + expect(callsFrom(graph, 'apps/web/main.ts')).toEqual(['caseStatus@packages/core/index.ts']); + + // The barrel names no `caseStatus` of its own: only the import leads the call there. + write(root, { 'packages/core/src/index.ts': "export * from './status';\n" }); + await graph.sync(); + expect(moduleImports(graph, 'apps/web/main.ts')).toEqual(['packages/core/src/index.ts']); + expect(callsFrom(graph, 'apps/web/main.ts')).toEqual(['caseStatus@packages/core/src/status.ts']); + }, 120_000); +}); diff --git a/__tests__/writer-lock.test.ts b/__tests__/writer-lock.test.ts index 60a89f241a..57bc3e642c 100644 --- a/__tests__/writer-lock.test.ts +++ b/__tests__/writer-lock.test.ts @@ -135,7 +135,10 @@ describe('writer lock (#1740)', () => { const msg = writerLockHeldMessage(r.existing, r.pidPath); expect(msg).toMatch(/writer lock held/i); expect(msg).toMatch(/CODEGRAPH_NO_DAEMON/); - expect(msg).toMatch(/daemon stop/); + // `codegraph daemon` is the picker that stops the shared daemon; the CLI + // rejects `codegraph daemon stop` (#2770). + expect(msg).toContain('pick it in `codegraph daemon`'); + expect(msg).not.toMatch(/daemon stop/); } }); diff --git a/__tests__/wsl-shared-index.test.ts b/__tests__/wsl-shared-index.test.ts index a23a1a6859..fc57e8114a 100644 --- a/__tests__/wsl-shared-index.test.ts +++ b/__tests__/wsl-shared-index.test.ts @@ -25,7 +25,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import CodeGraph from '../src/index'; -import { DatabaseConnection } from '../src/db'; +import { DatabaseConnection, DatabaseOpenError } from '../src/db'; import { WslSharedIndexError, isSqliteIoError, toWslSharedIndexError } from '../src/db/wsl-shared-index'; import { codeGraphDirNameFor, findNearestCodeGraphRoot, getCodeGraphDir, isInitialized } from '../src/directory'; import { __setWslWindowsDriveForTests, isWslWindowsDrive } from '../src/sync/watch-policy'; @@ -252,7 +252,7 @@ describe.runIf(process.platform !== 'win32')('a real SQLite I/O error on open', expect((thrown as Error).message).toContain('CODEGRAPH_DIR=.codegraph-wsl'); }); - it('stays the raw SQLite error everywhere else', () => { + it('stays the SQLite error everywhere else, named and placed', () => { __setWslWindowsDriveForTests(() => false); let thrown: unknown; try { @@ -260,9 +260,11 @@ describe.runIf(process.platform !== 'win32')('a real SQLite I/O error on open', } catch (err) { thrown = err; } - expect(thrown).toBeInstanceOf(Error); + expect(thrown).toBeInstanceOf(DatabaseOpenError); expect(thrown).not.toBeInstanceOf(WslSharedIndexError); - expect((thrown as Error).message).toBe('disk I/O error'); + // SQLite's own failure with the path, not the WSL guidance. + expect((thrown as Error).message).toContain(`${dbPath}: disk I/O error (SQLITE_IOERR).`); + expect((thrown as Error).message).not.toContain('CODEGRAPH_DIR'); }); describe('through the MCP engine', () => { diff --git a/codegraph-kernel/src/dart/mod.rs b/codegraph-kernel/src/dart/mod.rs index 61c1466266..3c31ad43ea 100644 --- a/codegraph-kernel/src/dart/mod.rs +++ b/codegraph-kernel/src/dart/mod.rs @@ -105,6 +105,22 @@ fn next_named<'t>(node: Node<'t>) -> Option> { next } +/// dartTypeAliasName (dart.ts) — the name a typedef declares: the last type +/// name before the `=`, the type parameters or the parameter list. The +/// legacy form writes its return type first (`typedef Report LegacyMaker(int +/// n);`), so the first type name is not always it. +fn type_alias_name<'t>(node: Node<'t>) -> Option> { + let mut name = None; + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + match child.kind() { + "=" | "type_parameters" | "formal_parameter_list" => break, + "type_identifier" => name = Some(child), + _ => {} + } + } + name +} #[derive(Default)] struct Extra { @@ -461,6 +477,14 @@ impl<'t> Walker<'t> { /// name field → the method_signature inner unwrap → identifier-ish /// child → `` (operators land here). fn extract_name(&self, node: Node<'t>) -> String { + // resolveName hook (dart.ts): a typedef has no `name` field, and a + // legacy one writes its return type before its name (`typedef Report + // LegacyMaker(int n);`). + if node.kind() == "type_alias" { + if let Some(name) = type_alias_name(node) { + return self.text(name).to_string(); + } + } // resolveName hook (dart.ts:244-260): named ctor/factory → ctor name. if let Some((class_name, ctor_name)) = self.ctor_info(node) { if ctor_name != class_name { @@ -830,13 +854,77 @@ impl<'t> Walker<'t> { return false; } let docstring = self.docstring_of(node); - // `value` field is null (type_alias has no fields) → no refs from - // the aliased type; returns false → children re-visited. - self.create_node("type_alias", &name, node, Extra { docstring, ..Default::default() }); + // type_alias has no fields (no `value`): the dart branch reads the + // aliased type from the rest of the declaration (#2327). Returns + // false → children re-visited (they match nothing type-related). + let row = self.create_node("type_alias", &name, node, Extra { docstring, ..Default::default() }); + if let Some(row) = row { + self.type_alias_refs(node, row); + } false } - // --- extractImport (:3170; hook dart.ts:261-304) ---------------------- + /// pushDartTypeAliasRefs (dart.ts) — a `references` ref from the typedef + /// for every type its aliased type names; its annotations are not its + /// type, and a type parameter is no type of the project's. + fn type_alias_refs(&mut self, node: Node<'t>, from_row: u32) { + let name = type_alias_name(node); + let scope = self.type_parameter_scope(node, &[]).unwrap_or_default(); + let mut cursor = node.walk(); + let kids: Vec> = node.named_children(&mut cursor).collect(); + for child in kids { + if child.kind() == "annotation" || Some(child) == name { + continue; + } + self.aliased_type_refs(child, from_row, &scope); + } + } + + /// dartTypeParameterScope (dart.ts) — `scope` and the type parameters + /// `node` declares (`Mapper`, `T Function(T a)`), or None when it + /// declares none. + fn type_parameter_scope(&self, node: Node<'t>, scope: &[&'t str]) -> Option> { + let mut cursor = node.walk(); + let parameters = node.named_children(&mut cursor).find(|c| c.kind() == "type_parameters")?; + let mut inner = scope.to_vec(); + let mut pc = parameters.walk(); + for parameter in parameters.named_children(&mut pc) { + if parameter.kind() != "type_parameter" { + continue; + } + let mut ic = parameter.walk(); + let name = parameter.named_children(&mut ic).find(|c| c.kind() == "type_identifier"); + if let Some(name) = name { + inner.push(self.text(name)); + } + } + Some(inner) + } + + /// pushDartAliasedTypeRefs (dart.ts) — type_refs_dart for a typedef's + /// aliased type, where a type parameter in `scope` is no type of its own + /// (but `p.T` is the type an import names). + fn aliased_type_refs(&mut self, node: Node<'t>, from_row: u32, scope: &[&'t str]) { + stack_guard!(); + if node.kind() == "type_identifier" { + let text = self.text(node); + let parameter = + scope.contains(&text) && !node.prev_sibling().map(|s| s.kind() == ".").unwrap_or(false); + if !parameter && self.is_type_name_dart(node) { + self.push_ref_at(from_row, text, crate::buffers::EDGE_REFERENCES, node); + } + return; + } + let inner = self.type_parameter_scope(node, scope); + let scope = inner.as_deref().unwrap_or(scope); + let mut cursor = node.walk(); + let kids: Vec> = node.named_children(&mut cursor).collect(); + for child in kids { + self.aliased_type_refs(child, from_row, scope); + } + } + + // --- extractImport (:3170; hook dart.ts extractImport) ----------------- fn extract_import(&mut self, node: Node<'t>) { let find_child = |parent: Node<'t>, kind: &str| -> Option> { diff --git a/codegraph-kernel/src/go/mod.rs b/codegraph-kernel/src/go/mod.rs index 570a3aba62..49a2e26e78 100644 --- a/codegraph-kernel/src/go/mod.rs +++ b/codegraph-kernel/src/go/mod.rs @@ -16,10 +16,10 @@ mod bindings; mod refs; use crate::buffers::{ node_kind_index, Arena, BoolFlags, EmitOut, NodeRow, - RefRow, Tables, BINDING_DECL, BINDING_IMPORT, BINDING_LOCAL, BINDING_PARAM, FLAG_IS_EXPORTED, NONE, NONE_STR, + RefRow, Tables, BINDING_DECL, BINDING_IMPORT, BINDING_LOCAL, BINDING_PARAM, FLAG_IS_EXPORTED, FUNCTION_REF_CODE, NONE, NONE_STR, }; use crate::walker::named_kids; -use crate::walker::{Scope, ValueScope, Cand, scope_qualified_name}; +use crate::walker::{Scope, ValueScope, scope_qualified_name}; use crate::textutil::{is_builtin_type, is_literal_receiver}; use crate::docstring::preceding_go_doc_comment; use crate::ids; @@ -66,6 +66,44 @@ struct Extra { } +struct Cand<'t> { + from: u32, + name: String, + line: u32, + column_byte: usize, + row: usize, + /// The value container the candidate came from: the flush reads its + /// scope from there when the function shadows the name (#2596). + node: Node<'t>, +} + +impl<'t> Cand<'t> { + /// Cand::at (walker.rs) plus the value container (#2596). + fn at(from: u32, name: impl Into, node: Node<'t>, container: Node<'t>) -> Option> { + let name = name.into(); + if name.is_empty() || crate::textutil::is_stoplisted(&name) { + return None; + } + let p = node.start_position(); + Some(Cand { from, name, line: p.row as u32 + 1, column_byte: node.start_byte(), row: p.row, node: container }) + } + + /// Cand::member (walker.rs) plus the value container (#2596): a member + /// value that keeps its receiver path, anchored at the member name. + fn member(from: u32, path: &str, member: Node<'t>, container: Node<'t>) -> Option> { + let path = path.split('.').map(str::trim).collect::>().join("."); + let named = path.contains('.') + && path.split('.').all(|part| { + !part.is_empty() + && part.bytes().enumerate().all(|(i, c)| c == b'_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())) + }); + if !named { + return None; + } + let p = member.start_position(); + Some(Cand { from, name: path, line: p.row as u32 + 1, column_byte: member.start_byte(), row: p.row, node: container }) + } +} /// Per-node metadata for the receiver-method owner lookup (mirrors the TS /// side's scan over `this.nodes` — FIRST match wins, earlier-in-file only). @@ -90,7 +128,7 @@ pub struct Walker<'t> { /// Go binds a name to one import, so an unaliased import whose last path /// element is such a name (`"github.com/cheggaaa/pb/v3"`) does not bind it. explicit_import_names: HashSet, - fn_ref_cands: Vec, + fn_ref_cands: Vec>, fs_values: HashMap, fs_value_counts: HashMap, value_scopes: Vec>, @@ -418,7 +456,9 @@ impl<'t> Walker<'t> { signature: self.signature_of(node), return_type: self.return_type_of(node), qualified_name: receiver_type.as_ref().map(|r| format!("{r}::{name}")), - ..Extra::default() // extractMethod passes no isExported + // methodsAreTopLevel: a Go method is a top-level declaration, so + // extractMethod gives it the function's exportedness (name case). + is_exported: Some(self.is_exported(node)), }; let Some(row) = self.create_node("method", &name, node, extra) else { return }; @@ -895,7 +935,7 @@ impl<'t> Walker<'t> { /// embedding names — `T`, `*T` (the `*` is a sibling token), `pkg.T` and /// `T[X]` all embed `T`. A qualified type yields its name; resolution reads /// the package back from the source at that position. None for any other - /// type and for a predeclared one. + /// type and for a predeclared one but `error`, an interface with a method. fn embedded_type_name(&self, ty: Option>) -> Option> { let mut ty = ty?; if ty.kind() == "generic_type" { @@ -903,7 +943,10 @@ impl<'t> Walker<'t> { } match ty.kind() { "qualified_type" => ty.child_by_field_name("name"), - "type_identifier" if !is_go_predeclared_type(self.text(ty)) => Some(ty), + "type_identifier" => { + let name = self.text(ty); + (name == "error" || !is_go_predeclared_type(name)).then_some(ty) + } _ => None, } } @@ -1062,7 +1105,45 @@ impl<'t> Walker<'t> { // --- fn refs (GO_SPEC, with the literal_element/expression_list layers) -------- - flush_fn_ref_candidates_impl!(); + fn flush_fn_ref_candidates(&mut self) { + let cands = std::mem::take(&mut self.fn_ref_cands); + if cands.is_empty() || util::is_generated_file(self.file_path) { + return; + } + let survivors: Vec> = cands + .into_iter() + .filter(|c| { + c.name.starts_with("this.") + || c.name.contains('.') + || c.name.contains("::") + || self.defined_fn_names.contains(&c.name) + || self.imported_names.contains(&c.name) + }) + .collect(); + // One reference per function and name, at its first use; one per + // scope where the function binds the name itself (#2596). Mirrors + // flushFnRefCandidates / goScopedFnRefKeys. + let scopes = go_scoped_fn_ref_scopes(&survivors, &self.node_ids, self.src); + let mut seen: HashSet<(String, String, Option)> = HashSet::new(); + for (i, c) in survivors.into_iter().enumerate() { + let scope = scopes.get(&i).copied(); + if !seen.insert((self.node_ids[c.from as usize].clone(), c.name.clone(), scope)) { + continue; + } + let column = self.cols.col(self.src, c.row, c.column_byte); + let name_ref = self.arena.put(&c.name); + self.tables.push_ref(&RefRow { + from_idx: c.from, + kind: FUNCTION_REF_CODE, + line: c.line, + column, + reference_name: name_ref, + candidates: NONE_STR, + from_id_str: NONE_STR, + }); + } + } + // --- value refs ------------------------------------------------------------------- @@ -1081,3 +1162,155 @@ fn is_go_predeclared_type(name: &str) -> bool { | "uint8" | "uint16" | "uint32" | "uint64" | "uintptr" ) } + +// --- a local that shadows the function a value names (#2596) --------------------- + +/// Go nodes that open a scope a local can be declared in: a block, the +/// implicit block of an `if` / `for` / `switch` / `select` statement, and each +/// clause of a `switch` or `select`. Mirrors GO_SCOPE_TYPES (function-ref.ts). +const GO_SCOPE_TYPES: [&str; 10] = [ + "block", + "if_statement", + "for_statement", + "expression_switch_statement", + "type_switch_statement", + "select_statement", + "expression_case", + "default_case", + "type_case", + "communication_case", +]; + +/// Where a function-value candidate sits: the innermost scope node's id +/// (usize::MAX when there is none) and the unit it is written in: the +/// function or method declaration, or for a func literal at file scope the +/// outermost literal, else None. Mirrors goFnRefSite. +fn go_fn_ref_site<'t>(node: Node<'t>) -> (usize, Option>) { + let mut scope = usize::MAX; + let mut literal: Option> = None; + let mut cur = Some(node); + while let Some(n) = cur { + if scope == usize::MAX && GO_SCOPE_TYPES.contains(&n.kind()) { + scope = n.id(); + } + if matches!(n.kind(), "function_declaration" | "method_declaration") { + return (scope, Some(n)); + } + if n.kind() == "func_literal" { + literal = Some(n); + } + cur = n.parent(); + } + (scope, literal) +} + +/// The names among `names` that a function binds itself, as (unit id, name): +/// its func literals' parameters, `:=`, `var` and `const`, range and +/// type-switch variables, and a select case's received value. A name the +/// function's own receiver, parameters or results bind is left out: those +/// cover its whole body. Mirrors goLocalBindings. +fn go_local_bindings<'t>(units: &[Node<'t>], names: &HashSet<&str>, src: &str) -> HashSet<(usize, String)> { + let mut out: HashSet<(usize, String)> = HashSet::new(); + for unit in units { + let mut bind = |id: Option>| { + if let Some(id) = id { + if id.kind() == "identifier" { + let name = &src[id.start_byte()..id.end_byte()]; + if names.contains(name) { + out.insert((unit.id(), name.to_string())); + } + } + } + }; + let declares = |n: Node<'t>| { + (0..n.child_count()).any(|i| n.child(i).is_some_and(|c| !c.is_named() && c.kind() == ":=")) + }; + let mut stack: Vec> = vec![*unit]; + while let Some(n) = stack.pop() { + let list = match n.kind() { + "short_var_declaration" => n.child_by_field_name("left"), + "range_clause" | "receive_statement" if declares(n) => n.child_by_field_name("left"), + "type_switch_statement" => n.child_by_field_name("alias"), + _ => None, + }; + if let Some(list) = list { + for i in 0..list.named_child_count() { + bind(list.named_child(i)); + } + } + if matches!(n.kind(), "var_spec" | "const_spec" | "parameter_declaration" | "variadic_parameter_declaration") { + let mut cursor = n.walk(); + for c in n.children_by_field_name("name", &mut cursor) { + bind(Some(c)); + } + } + for i in (0..n.named_child_count()).rev() { + if let Some(c) = n.named_child(i) { + stack.push(c); + } + } + } + for field in ["receiver", "parameters", "result"] { + let Some(list) = unit.child_by_field_name(field) else { continue }; + if list.kind() != "parameter_list" { + continue; + } + for i in 0..list.named_child_count() { + let Some(p) = list.named_child(i) else { continue }; + if !matches!(p.kind(), "parameter_declaration" | "variadic_parameter_declaration") { + continue; + } + let mut cursor = p.walk(); + for c in p.children_by_field_name("name", &mut cursor) { + if c.kind() == "identifier" { + out.remove(&(unit.id(), src[c.start_byte()..c.end_byte()].to_string())); + } + } + } + } + } + out +} + +/// The candidates, by index, that get one reference per scope instead of +/// one per function, each with the id of the scope it sits in: names a +/// function uses at least twice and binds itself. Only those candidates' +/// positions are read from the tree. Mirrors goScopedFnRefScopes. +fn go_scoped_fn_ref_scopes<'t>(cands: &[Cand<'t>], node_ids: &[String], src: &str) -> HashMap { + let mut groups: HashMap<(&str, &str), Vec> = HashMap::new(); + for (i, c) in cands.iter().enumerate() { + if c.name.contains('.') { + continue; + } + groups.entry((node_ids[c.from as usize].as_str(), c.name.as_str())).or_default().push(i); + } + let mut sites: Vec<(usize, usize, Node<'t>)> = Vec::new(); + let mut units: Vec> = Vec::new(); + let mut unit_ids: HashSet = HashSet::new(); + let mut names: HashSet<&str> = HashSet::new(); + for members in groups.values() { + if members.len() < 2 { + continue; + } + for &i in members { + let (scope, unit) = go_fn_ref_site(cands[i].node); + let Some(unit) = unit else { continue }; + sites.push((i, scope, unit)); + if unit_ids.insert(unit.id()) { + units.push(unit); + } + names.insert(cands[i].name.as_str()); + } + } + let mut out: HashMap = HashMap::new(); + if units.is_empty() { + return out; + } + let bound = go_local_bindings(&units, &names, src); + for (i, scope, unit) in sites { + if bound.contains(&(unit.id(), cands[i].name.clone())) { + out.insert(i, scope); + } + } + out +} diff --git a/codegraph-kernel/src/go/refs.rs b/codegraph-kernel/src/go/refs.rs index 7bf1732535..8d6431ad03 100644 --- a/codegraph-kernel/src/go/refs.rs +++ b/codegraph-kernel/src/go/refs.rs @@ -14,6 +14,9 @@ impl<'t> Walker<'t> { "literal_value" => ("list", ""), _ => return, }; + if self.stack.is_empty() { + return; + } let from = self.top_row(); let mut values: Vec = Vec::new(); @@ -62,13 +65,13 @@ impl<'t> Walker<'t> { } for v in values { - self.normalize_fn_ref_value(v, from, 0); + self.normalize_fn_ref_value(v, from, 0, node); } } /// normalizeValue with GO_SPEC's transparent layers (literal_element, /// expression_list — both fan out to named children). - pub(super) fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + pub(super) fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32, container: Node<'t>) { stack_guard!(); if depth > 4 { return; @@ -76,18 +79,18 @@ impl<'t> Walker<'t> { match v.kind() { "identifier" => { let name = self.text(v).to_string(); - self.fn_ref_cands.extend(Cand::at(from, name, v)); + self.fn_ref_cands.extend(Cand::at(from, name, v, container)); } // A method value `c.store.Fetch` keeps its receiver (#1820). "selector_expression" => { if let Some(field) = v.child_by_field_name("field") { let path = self.selector_path(v); - self.fn_ref_cands.extend(Cand::member(from, &path, field)); + self.fn_ref_cands.extend(Cand::member(from, &path, field, container)); } } "literal_element" | "expression_list" => { for c in named_kids(v) { - self.normalize_fn_ref_value(c, from, depth + 1); + self.normalize_fn_ref_value(c, from, depth + 1, container); } } _ => {} diff --git a/codegraph-kernel/src/java/mod.rs b/codegraph-kernel/src/java/mod.rs index 44d57a6efe..dfa8e78e67 100644 --- a/codegraph-kernel/src/java/mod.rs +++ b/codegraph-kernel/src/java/mod.rs @@ -8,6 +8,7 @@ //! are walked like any other (tree-sitter's recovery is canonical). mod lombok; +mod records; mod bindings; mod refs; use crate::buffers::{ @@ -27,7 +28,11 @@ use tree_sitter::Node; fn is_method_type(kind: &str) -> bool { - matches!(kind, "method_declaration" | "constructor_declaration") + matches!(kind, "method_declaration" | "constructor_declaration" | "compact_constructor_declaration") +} +/// classTypes (languages/java.ts): a `record` is a class. +fn is_class_type(kind: &str) -> bool { + matches!(kind, "class_declaration" | "record_declaration") } fn is_interface_type(kind: &str) -> bool { matches!(kind, "interface_declaration" | "annotation_type_declaration") @@ -343,7 +348,7 @@ impl<'t> Walker<'t> { let md_owner = self.top_row(); self.markdown_refs_from_string(node, md_owner); - if kind == "class_declaration" { + if is_class_type(kind) { self.extract_class(node); skip_children = true; } else if is_method_type(kind) { @@ -415,7 +420,7 @@ impl<'t> Walker<'t> { // Static-member / value-read (`Type.CONST`) — self-gates on field_access. self.extract_static_member_ref(node); - if kind == "class_declaration" { + if is_class_type(kind) { self.extract_class(node); return; } @@ -452,7 +457,11 @@ impl<'t> Walker<'t> { for c in named_kids(body) { self.visit_node(c); } - // Lombok member synthesis (#912) — class still on the stack. + // synthesizeJavaMembers — class still on the stack: record + // components first, then Lombok (#912). + if node.kind() == "record_declaration" { + self.synthesize_record_components(node, row); + } self.synthesize_lombok_members(node, row); self.stack.pop(); } @@ -855,6 +864,20 @@ impl<'t> Walker<'t> { } } + /// supertypeName — a parameterized supertype (`generic_type`) by the name + /// before its type arguments: `Store` for `implements Store`. + fn supertype_name(&self, node: Node<'t>) -> String { + if node.kind() == "generic_type" { + let named = (0..node.named_child_count()) + .filter_map(|j| node.named_child(j)) + .find(|c| c.kind() != "type_arguments"); + if let Some(named) = named { + return self.text(named).to_string(); + } + } + self.text(node).to_string() + } + /// extractInheritance — the Java clauses (type_list-aware). fn extract_inheritance(&mut self, node: Node<'t>, class_row: u32) { let extends_kind = crate::buffers::EDGE_EXTENDS; @@ -870,7 +893,7 @@ impl<'t> Walker<'t> { None => child.named_child(0).into_iter().collect(), }; for target in targets { - let name = self.text(target).to_string(); + let name = self.supertype_name(target); self.push_ref_at(class_row, &name, extends_kind, target); } } @@ -882,7 +905,7 @@ impl<'t> Walker<'t> { None => named_kids(child).collect(), }; for iface in targets { - let name = self.text(iface).to_string(); + let name = self.supertype_name(iface); self.push_ref_at(class_row, &name, implements_kind, iface); } } diff --git a/codegraph-kernel/src/java/records.rs b/codegraph-kernel/src/java/records.rs new file mode 100644 index 0000000000..76e5c96a26 --- /dev/null +++ b/codegraph-kernel/src/java/records.rs @@ -0,0 +1,82 @@ +//! Record component synthesis (languages/java.ts synthesizeRecordComponents, #2639): +//! a `record` declares a private field and a public accessor per component. + +use super::*; + +impl<'t> Walker<'t> { + /// Members already declared directly in a class (exact `classQN::name` + /// matches), split into the method and field namespaces. + fn declared_member_names(&self, class_row: u32) -> (HashSet, HashSet) { + let class_qn = self.nodes_meta[class_row as usize].qualified_name.clone(); + let mut taken_methods: HashSet = HashSet::new(); + let mut taken_fields: HashSet = HashSet::new(); + for m in &self.nodes_meta { + if m.qualified_name == format!("{class_qn}::{}", m.name) { + match m.kind { + "method" | "function" => { + taken_methods.insert(m.name.clone()); + } + "field" | "variable" | "constant" | "property" => { + taken_fields.insert(m.name.clone()); + } + _ => {} + } + } + } + (taken_methods, taken_fields) + } + + /// synthesizeRecordComponents (languages/java.ts): per record component a + /// private field and a public accessor, anchored on the component's name. + pub(super) fn synthesize_record_components(&mut self, record_node: Node<'t>, record_row: u32) { + let Some(params) = record_node.child_by_field_name("parameters") else { return }; + let (mut taken_methods, mut taken_fields) = self.declared_member_names(record_row); + for i in 0..params.named_child_count() { + let Some(param) = params.named_child(i) else { continue }; + if param.kind() != "formal_parameter" { + continue; + } + let (Some(name_node), Some(type_node)) = + (param.child_by_field_name("name"), param.child_by_field_name("type")) + else { + continue; + }; + let name = self.text(name_node).trim().to_string(); + let type_text = self.text(type_node).trim().to_string(); + if name.is_empty() { + continue; + } + if !taken_fields.contains(&name) { + taken_fields.insert(name.clone()); + self.create_node( + "field", + &name, + name_node, + Extra { + visibility: Some(2), + is_static: Some(false), + signature: Some(format!("{type_text} {name}")), + ..Extra::default() + }, + ); + } + if !taken_methods.contains(&name) { + taken_methods.insert(name.clone()); + let return_type = self.normalize_java_type(Some(type_node)); + self.create_node( + "method", + &name, + name_node, + Extra { + visibility: Some(1), + is_static: Some(false), + signature: Some(format!("{type_text} {name}()")), + docstring: Some("Implicit record component accessor".to_string()), + return_type, + ..Extra::default() + }, + ); + } + } + } +} diff --git a/codegraph-kernel/src/resolve/bound.rs b/codegraph-kernel/src/resolve/bound.rs index cb8216a6eb..e6f05b674a 100644 --- a/codegraph-kernel/src/resolve/bound.rs +++ b/codegraph-kernel/src/resolve/bound.rs @@ -737,6 +737,13 @@ if site.language == "dart" && site.reference_kind == "calls" { // Of a member template's explicit specializations, the one the call's template arguments // name (cpp_specialization_pool). let members = self.cpp_specialization_pool(members, site); + // A receiver typed by `x = Widget.new` is an instance: `def call` + // before `def self.call` when both share `Widget::call` (#2834). + let members = if site.language == "ruby" { + super::ruby_paths::ruby_methods_of_kind_first(members, "instance") + } else { + members + }; if members.len() > 1 && site.language == "cpp" && site.reference_kind == "calls" && !site.reference_name.contains("().") { // A C++ call on a typed receiver takes the overload its arguments fit, not the first // one declared (a chained call `inner(…).name(…)` keeps the first: the arguments at @@ -1318,6 +1325,13 @@ Ok(self.dart_member_of(type_name, method, r, false)? let node = self.cpp_overload_for(ordered, method, r)?; return Ok(Some(KCand { node, confidence, resolved_by })); } + // A receiver typed by `x = Widget.new` is an instance: `def m` before + // `def self.m` when both share `Owner::m` (#2834). + let matches = if r.language == "ruby" { + super::ruby_paths::ruby_methods_of_kind_first(matches, "instance") + } else { + matches + }; Ok(Self::pick_method_match(matches, r, confidence, resolved_by, preferred_fqn)) } diff --git a/codegraph-kernel/src/resolve/call_shape.rs b/codegraph-kernel/src/resolve/call_shape.rs index cb46c1e07c..cda904aea8 100644 --- a/codegraph-kernel/src/resolve/call_shape.rs +++ b/codegraph-kernel/src/resolve/call_shape.rs @@ -457,10 +457,33 @@ impl KernelResolver { links=field_links; site=r.clone().at(&call.node); rust_return_owner(ret,owner(&call.node).unwrap_or("")) + } else if links.clone().next().is_some() { + // `self..<…>` — the bound-member walk can't see rust + // struct fields (they are not its member nodes), so resolve + // the receiver as a standalone call: `self.table.get` is + // `Table::get`, whose return type owns the outer method. + let Some(inner)=self.match_method_call_free(&r.clone().naming(receiver,"calls"))? else {return Ok(None)}; + let Some(ret)=inner.node.return_type.as_deref() else {return Ok(None)}; + site=r.clone().at(&inner.node); + for _ in links.by_ref() {} + rust_return_owner(ret,owner(&inner.node).unwrap_or("")) } else { self.node_by_id(&r.from_node_id)?.and_then(|n|n.qualified_name.rsplit_once("::").map(|(owner,_)|owner.to_string())).unwrap_or_default() } - } else if let Some(ty)=self.infer_local_receiver_type(root,r,true)? {ty} + } else if let Some(ty)=self.infer_local_receiver_type(root,r,true)? { + // `..<…>` — the bound-member walk below can't see + // rust struct fields, so a receiver with deeper links resolves + // as the standalone call it already is: `db.table.get` is + // `Table::get`, whose return type owns the outer method. + if links.clone().nth(1).is_some() { + if let Some(inner)=self.match_method_call_free(&r.clone().naming(receiver,"calls"))? { + let Some(ret)=inner.node.return_type.as_deref() else {return Ok(None)}; + site=r.clone().at(&inner.node); + for _ in links.by_ref() {} + rust_return_owner(ret,owner(&inner.node).unwrap_or("")) + } else {ty} + } else {ty} + } else { let reference=r.clone().naming(root,"calls"); // A local function shadows an imported factory in its lexical scope. diff --git a/codegraph-kernel/src/resolve/fields.rs b/codegraph-kernel/src/resolve/fields.rs index 3625aaefeb..519b53f21a 100644 --- a/codegraph-kernel/src/resolve/fields.rs +++ b/codegraph-kernel/src/resolve/fields.rs @@ -989,6 +989,23 @@ impl KernelResolver { }; let start = (s.start_line - 1).max(0) as usize; let end = (s.end_line as usize).min(source.len()); + if field.bytes().all(|b| b.is_ascii_digit()) { + // `self.N` — the Nth positional field of a tuple struct + // (`struct Wrap(pub Table)`), which the `name:` scan below + // cannot see. + let idx: usize = field.parse().unwrap_or(usize::MAX); + let head = source.get(start..end).unwrap_or_default().join("\n"); + let Some(m) = re!(r"\bstruct\s+[A-Za-z_]\w*\s*(?:<[^>]*>)?\s*\(([^)]*)\)").captures(&head) else { + continue; + }; + let types: Vec<&str> = m[1].split(',').collect(); + let Some(decl) = types.get(idx) else { return Ok(None) }; + let decl = decl.trim().trim_start_matches("pub").trim().to_string(); + let Some(field_type) = rust_field_type_name(&decl) else { + return Ok(None); + }; + return self.rust_field_method(&decl, &field_type, method, &s, r); + } for raw in source.get(start..end).unwrap_or_default() { let line = strip_line_comments(raw); let Some(declared) = RUST_FIELD_TYPE.capture(&line, field) else { diff --git a/codegraph-kernel/src/resolve/imports.rs b/codegraph-kernel/src/resolve/imports.rs index 31e1bc6bf8..cb1f7f23aa 100644 --- a/codegraph-kernel/src/resolve/imports.rs +++ b/codegraph-kernel/src/resolve/imports.rs @@ -1230,10 +1230,15 @@ impl KernelResolver { } let class_name = segments[segments.len() - 1]; let dir_segments = &segments[..segments.len() - 1]; - let mut best: Option> = None; - let mut best_score = 0usize; - let mut tie = false; + // Production code never names a component in a test suite; a test can + // name one, by the path to it. + let from_test = super::resolver_upstream::test_suite_path(&r.file_path); + let mut best_score = 0f64; + let mut tied: Vec> = Vec::new(); for cand in candidates(self, class_name)? { + if !from_test && super::resolver_upstream::is_test_suite_path(&cand.file_path) { + continue; + } let path = cand.file_path.replace('\\', "/"); let mut dirs: Vec<&str> = path.split('/').collect(); dirs.pop(); @@ -1244,18 +1249,154 @@ impl KernelResolver { { score += 1; } + // The one exception to corroboration: a one-folder path naming a + // component at the project root, `wheels` the mapping onto it + // (#2652); the file must be named like the component. + let mut score = score as f64; + if score == 0.0 && dirs.is_empty() && dir_segments.len() == 1 { + let stem = cand.file_path.replace('\\', "/").to_lowercase(); + if stem.strip_suffix(".cfc").is_some_and(|s| s == class_name.to_lowercase()) { + score = 0.5; + } + } if score > best_score { - best = Some(cand); best_score = score; - tie = false; - } else if score == best_score && score > 0 { - tie = true; + tied = vec![cand]; + } else if score == best_score && score > 0.0 { + tied.push(cand); + } + } + // A test's tie between a component and a suite mirroring its path + // names the component, as it does from production code (#2650). + if tied.len() > 1 { + tied.retain(|c| !super::resolver_upstream::is_test_suite_path(&c.file_path)); + } + if best_score == 0.0 || tied.len() != 1 { + return Ok(None); + } + Ok(tied + .into_iter() + .next() + .map(|node| KCand { node, confidence: if best_score >= 1.0 { 0.9 } else { 0.8 }, resolved_by: "qualified-name" })) + } + + /// resolveCfmlConstructedCall: a call chained onto a construction + /// (#2652) — `createObject("component", "a.b.Batch").init()` or `new + /// a.b.Batch().init()`, encoded `a.b.Batch().init`. The component is the + /// one its path names, by resolveCfmlComponentPath's rules, or for a bare + /// name the one beside the calling file, else the only one of that name. + /// The method is the component's own or one it inherits. + pub(super) fn resolve_cfml_constructed_call(&mut self, r: &ResolveRefIn) -> Res> { + let Some(m) = re!(r"^([A-Za-z_$][\w$.-]*)\(\)\.([A-Za-z_$][\w$]*)$").captures(&r.reference_name) else { + return Ok(None); + }; + let component = m[1].to_string(); + let cls = if component.contains('.') { + let as_instantiates = r.clone().naming(&component, "instantiates"); + match self.resolve_cfml_component_path(&as_instantiates)? { + Some(c) => Some(c.node), + None => None, + } + } else { + self.cfml_component_by_name(&component, &r.file_path)? + }; + let Some(cls) = cls else { return Ok(None) }; + let method = self.cfml_method_of(&cls, &m[2].to_lowercase(), &mut HashSet::new())?; + Ok(method.map(|node| KCand { node, confidence: 0.9, resolved_by: "instance-method" })) + } + + /// cfmlComponentByName: the CFML component a bare name reaches from + /// `from_file` — the one in its directory, else the only one so named. + fn cfml_component_by_name(&mut self, name: &str, from_file: &str) -> Res>> { + let components: Vec> = self + .nodes_by_lower_name(&name.to_lowercase())? + .iter() + .filter(|n| (n.kind == "class" || n.kind == "interface") && (n.language == "cfml" || n.language == "cfscript")) + .cloned() + .collect(); + fn dir_of(p: &str) -> &str { + p.rfind('/').map(|i| &p[..=i]).unwrap_or("") + } + let beside: Vec<&Arc> = components.iter().filter(|n| dir_of(&n.file_path) == dir_of(from_file)).collect(); + if beside.len() == 1 { + return Ok(Some(beside[0].clone())); + } + Ok((components.len() == 1).then(|| components[0].clone())) + } + + /// cfmlMethodOf: a component's method by its lowercased name (CFML names + /// ignore case) — its own, else one a supertype has. + fn cfml_method_of(&mut self, cls: &KNode, lower_name: &str, seen: &mut HashSet) -> Res>> { + if !seen.insert(cls.id.clone()) || seen.len() > 8 { + return Ok(None); + } + let own = self.nodes_in_file(&cls.file_path)?.iter().find(|n| { + n.kind == "method" + && n.name.to_lowercase() == lower_name + && n.qualified_name.to_lowercase() == format!("{}::{}", cls.name, n.name).to_lowercase() + && n.start_line >= cls.start_line + && n.end_line <= cls.end_line + }).cloned(); + if own.is_some() { + return Ok(own); + } + for supertype in self.supertype_nodes(&cls.id)? { + if let Some(inherited) = self.cfml_method_of(&supertype, lower_name, seen)? { + return Ok(Some(inherited)); + } + } + Ok(None) + } + + /// resolveCfmlIncludePath: the file a CFML include names (#2651) — + /// `include "x.cfm";`, ``, + /// `cfinclude(template="x.cfm")`. Relative to the including file's + /// directory, or to the web (project) root when the path starts with + /// `/`. One case-insensitive match counts when the exact case misses. + pub(super) fn resolve_cfml_include_path(&mut self, r: &ResolveRefIn) -> Res> { + let written = r.reference_name.trim().replace('\\', "/"); + let from_root = written.starts_with('/'); + let mut parts: Vec = if from_root { + Vec::new() + } else { + let mut v: Vec = r.file_path.replace('\\', "/").split('/').map(String::from).collect(); + v.pop(); + v + }; + for seg in written.split('/') { + match seg { + "" | "." => {} + ".." => { + if parts.pop().is_none() { + return Ok(None); + } + } + s => parts.push(s.to_string()), } } - if best_score == 0 || tie { + let Some(file_name) = parts.last().filter(|s| !s.is_empty()).cloned() else { return Ok(None); + }; + let wanted = parts.join("/"); + let files: Vec> = self + .nodes_by_lower_name(&file_name.to_lowercase())? + .iter() + .filter(|n| n.kind == "file") + .cloned() + .collect(); + if let Some(exact) = files.iter().find(|n| n.file_path == wanted) { + return Ok(Some(KCand { + node: exact.clone(), + confidence: if from_root { 0.85 } else { 0.95 }, + resolved_by: "file-path", + })); } - Ok(best.map(|node| KCand { node, confidence: 0.9, resolved_by: "qualified-name" })) + let folded: Vec<&Arc> = files.iter().filter(|n| n.file_path.eq_ignore_ascii_case(&wanted)).collect(); + Ok((folded.len() == 1).then(|| KCand { + node: folded[0].clone(), + confidence: if from_root { 0.8 } else { 0.9 }, + resolved_by: "file-path", + })) } /// The resolved import path's file node @0.9, or nothing. @@ -1540,7 +1681,9 @@ impl KernelResolver { &mut self, r: &ResolveRefIn, ) -> Res> { - if r.reference_kind != "imports" || !r.reference_name.contains('.') { + if r.reference_kind != "imports" + || (!r.reference_name.contains('.') && !self.is_python_from_import_module(r.clone())) + { return Ok(None); } Ok(self @@ -1597,6 +1740,13 @@ fn is_declaration_file(path: &str) -> bool { [".d.ts", ".d.mts", ".d.cts"].iter().any(|ext| path.ends_with(ext)) } +/// isCfmlTemplatePath: the name a CFML include carries — a path with a +/// separator, or one ending in a template extension. +pub(super) fn is_cfml_template_path(name: &str) -> bool { + !name.contains(':') + && (name.contains('/') || name.contains('\\') || re!(r"(?i)\.(?:cfml?|cfc|cfs)$").is_match(name.trim())) +} + /// Whether the `import` statement on `line` names the module `source` as a /// plain dotted name (`import a.b, c`, not `import a.b as c`). `None` when /// the line is not a plain one-line `import` statement, so the caller reads diff --git a/codegraph-kernel/src/resolve/js_objects_upstream.rs b/codegraph-kernel/src/resolve/js_objects_upstream.rs index bec6322e45..0f9f809454 100644 --- a/codegraph-kernel/src/resolve/js_objects_upstream.rs +++ b/codegraph-kernel/src/resolve/js_objects_upstream.rs @@ -344,3 +344,193 @@ impl KernelResolver { Ok(false) } } + +const JS_SUPER_LANGS: &[&str] = &["typescript", "tsx", "javascript", "jsx"]; + +/// `this.m()` written inside a class member names the member on `this`'s +/// class (its own first, then the nearest `extends` ancestor) — the +/// extractors strip `this.` so the ref arrives bare and the claim gate's +/// `this.` exclusion would otherwise leave it to name guesses. +impl KernelResolver { + pub(super) fn match_js_this_call(&mut self, r: &ResolveRefIn) -> Res> { + if r.reference_kind != "calls" || !JS_SUPER_LANGS.contains(&r.language.as_str()) + || !re!(r"^[A-Za-z_$][\w$]*$").is_match(&r.reference_name) { return Ok(None); } + let Some(lines) = self.read_file(&r.file_path) else { return Ok(None) }; + let Some(line) = lines.get((r.line - 1).max(0) as usize) else { return Ok(None) }; + let Some(slice) = line.get(r.column.max(0) as usize..) else { return Ok(None) }; + if !this_dot_name(slice, &r.reference_name) { return Ok(None); } + let Some(caller) = self.node_by_id(&r.from_node_id)? else { return Ok(None) }; + let Some((member, cls)) = self.js_class_member_around(&caller)? else { return Ok(None) }; + if let Some(own) = self.js_own_method(&cls, &r.reference_name, member.is_static)? { + return Ok(Some(ResolveOutcome::resolved(&own, 0.9, "exact-match", true, None))); + } + match self.js_inherited_method(&cls, &r.reference_name, member.is_static)? { + Some(node) => Ok(Some(ResolveOutcome::resolved(&node, 0.9, "exact-match", true, None))), + None => Ok(None), + } + } +} + +/// matchJsSuperCall (js-super-calls.ts): a TS/JS call written `super.name(…)` +/// runs the method of the class's parent, or the nearest ancestor of that +/// one that has the method, static matching the caller's kind (#2497). +/// `Ok(None)` leaves the ref to the other strategies; a site whose +/// ancestors don't carry the method is terminal — never a guess by name. +impl KernelResolver { + pub(super) fn match_js_super_call(&mut self, r: &ResolveRefIn) -> Res> { + if !self.is_js_super_call_site(r)? { return Ok(None); } + let Some(caller) = self.node_by_id(&r.from_node_id)? else { return Ok(None) }; + let Some((member, cls)) = self.js_class_member_around(&caller)? else { return Ok(None) }; + if self.js_writes_class_before(&member, r)? { return Ok(None); } + match self.js_inherited_method(&cls, &r.reference_name, member.is_static)? { + // The `extends` walk and static match prove the ancestor; the + // name-guess post guards in `finish` would veto it (`run` is a + // local method name), so the verdict is built directly. + Some(node) => Ok(Some(ResolveOutcome::resolved(&node, 0.9, "instance-method", true, None))), + None => Ok(Some(ResolveOutcome::unresolved())), + } + } + + /// isJsSuperCallSite: the call is written `super.name(…)` where it + /// starts (the member on the next line or two included). + fn is_js_super_call_site(&mut self, r: &ResolveRefIn) -> Res { + if r.reference_kind != "calls" || !JS_SUPER_LANGS.contains(&r.language.as_str()) + || !re!(r"^[A-Za-z_$][\w$]*$").is_match(&r.reference_name) { return Ok(false); } + let Some(lines) = self.read_file(&r.file_path) else { return Ok(false) }; + let Some(line) = lines.get((r.line - 1).max(0) as usize) else { return Ok(false) }; + let column = r.column.max(0) as usize; + let Some(slice) = line.get(column..) else { return Ok(false) }; + if !slice.starts_with("super") { return Ok(false); } + if super_dot_name(slice, &r.reference_name) { return Ok(true); } + let at = (r.line - 1).max(0) as usize; + let window = lines.get(at..(at + 3).min(lines.len())).unwrap_or_default().join("\n"); + Ok(window.get(column..).is_some_and(|w| super_dot_name(w, &r.reference_name))) + } + + /// classMemberAround: the class member `caller` belongs to, and its + /// class — following arrow functions nested in it up to 4 deep. + fn js_class_member_around(&mut self, caller: &Arc) -> Res, Arc)>> { + let mut member = caller.clone(); + for _ in 0..4 { + if member.kind != "function" { break; } + let Some(outer) = self.js_owner_of(&member, &["function", "method", "property"])? else { return Ok(None) }; + member = outer; + } + if member.kind != "method" && member.kind != "property" { return Ok(None); } + let Some(cls) = self.js_owner_of(&member, &["class"])? else { return Ok(None) }; + Ok(Some((member, cls))) + } + + /// ownerOf: the node `node` is declared in, by its qualified name, in + /// its file and around it. + fn js_owner_of(&mut self, node: &Arc, kinds: &[&str]) -> Res>> { + let Some(cut) = node.qualified_name.rfind("::") else { return Ok(None) }; + if cut == 0 { return Ok(None); } + Ok(self.nodes_by_qualified_name(&node.qualified_name[..cut])?.iter() + .find(|n| kinds.contains(&n.kind.as_str()) && n.file_path == node.file_path + && n.start_line <= node.start_line && n.end_line >= node.end_line) + .cloned()) + } + + /// writesClassBefore: a `class` is written between where `member` + /// starts and the call (a class expression's `super` is that class's). + fn js_writes_class_before(&mut self, member: &KNode, r: &ResolveRefIn) -> Res { + let Some(lines) = self.read_file(&r.file_path) else { return Ok(false) }; + if member.start_line > r.line { return Ok(false); } + let from = (member.start_line - 1).max(0) as usize; + let to = (r.line as usize).min(lines.len()); + if from >= to { return Ok(false); } + let mut span: Vec = lines[from..to].to_vec(); + let last = span.len() - 1; + let col = (r.column.max(0) as usize).min(span[last].len()); + if !span[last].is_char_boundary(col) { return Ok(false); } + span[last] = span[last][..col].to_string(); + let first_col = if member.start_line == r.line { + (member.start_column as usize).min(col) + } else { + (member.start_column as usize).min(span[0].len()) + }; + if !span[0].is_char_boundary(first_col) { return Ok(false); } + span[0] = span[0][first_col..].to_string(); + let text = span.join("\n"); + if !has_class_keyword(&text) { return Ok(false); } + Ok(has_class_keyword(&awaited::blank_string_contents(&awaited::strip_ts_comments(&text)))) + } + + /// inheritedMethod: the nearest method named `name` of `cls`'s + /// ancestors, static or instance as asked (BFS over `extends`). + fn js_inherited_method(&mut self, cls: &Arc, name: &str, is_static: bool) -> Res>> { + let mut seen: HashSet = HashSet::from([cls.id.clone()]); + let mut level = self.js_superclasses_of(cls)?; + for _ in 0..8 { + if level.is_empty() { break; } + let mut next = Vec::new(); + for parent in level { + if !seen.insert(parent.id.clone()) { continue; } + if let Some(own) = self.js_own_method(&parent, name, is_static)? { return Ok(Some(own)); } + next.extend(self.js_superclasses_of(&parent)?); + } + level = next; + } + Ok(None) + } + + fn js_superclasses_of(&mut self, cls: &KNode) -> Res>> { + let ids = self.outgoing_edge_targets(&cls.id, &["extends"])?; + let mut out = Vec::new(); + for id in ids { + if let Some(n) = self.node_by_id(&id)? { + if n.kind == "class" && JS_SUPER_LANGS.contains(&n.language.as_str()) { out.push(n); } + } + } + Ok(out) + } + + /// ownMethod: a method `cls` itself declares under `name`; of + /// TypeScript's overload signatures and the body, the body (last one). + fn js_own_method(&mut self, cls: &KNode, name: &str, is_static: bool) -> Res>> { + let mut own: Vec> = self.nodes_by_qualified_name(&format!("{}::{}", cls.qualified_name, name))? + .iter() + .filter(|m| m.kind == "method" && m.file_path == cls.file_path + && m.start_line >= cls.start_line && m.end_line <= cls.end_line + && m.is_static == is_static) + .cloned() + .collect(); + own.sort_by_key(|m| (m.start_line, m.start_column)); + Ok(own.pop()) + } +} + +/// `class` written as a keyword — not an identifier tail or a member +/// (upstream's `(? bool { + let bytes = text.as_bytes(); + for m in re!(r"class").find_iter(text) { + let before = m.start().checked_sub(1).map(|i| bytes[i]); + let after = bytes.get(m.end()).copied(); + let word = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b'$'; + let before_ok = before.is_none_or(|b| !word(b) && b != b'.'); + let after_ok = after.is_none_or(|b| !word(b)); + if before_ok && after_ok { return true; } + } + false +} + +/// The text starts `super.` with `name` not an identifier's head +/// (upstream's `^super\s*\.\s*NAME(?![\w$])`, written out by hand). +fn super_dot_name(slice: &str, name: &str) -> bool { + let rest = slice.strip_prefix("super").unwrap_or(""); + if rest.is_empty() { return false; } + let rest = rest.trim_start(); + let Some(rest) = rest.strip_prefix('.') else { return false }; + let rest = rest.trim_start(); + if !rest.starts_with(name) { return false; } + rest[name.len()..].chars().next().is_none_or(|c| !(c.is_alphanumeric() || c == '_' || c == '$')) +} + +/// The text starts `this.` with `name` not an identifier's head. +fn this_dot_name(slice: &str, name: &str) -> bool { + let Some(rest) = slice.strip_prefix("this.") else { return false }; + if !rest.starts_with(name) { return false; } + rest[name.len()..].chars().next().is_none_or(|c| !(c.is_alphanumeric() || c == '_' || c == '$')) +} diff --git a/codegraph-kernel/src/resolve/lang_scope.rs b/codegraph-kernel/src/resolve/lang_scope.rs index b18f67a777..ff43137acd 100644 --- a/codegraph-kernel/src/resolve/lang_scope.rs +++ b/codegraph-kernel/src/resolve/lang_scope.rs @@ -157,6 +157,7 @@ impl KernelResolver { kept.push(n); } if kotlin_call { kept = self.lexical_kotlin_members(kept, r)?; } + if ruby_bare { kept = self.ruby_methods_of_caller_kind(kept, r)?; } Ok((kept, shrank)) } @@ -561,7 +562,7 @@ impl KernelResolver { /// rubyAncestry: a class's qualified name, its superclasses', and those of /// every module mixed into any of them (at most 60). - fn ruby_ancestry(&mut self, qn: &str) -> Res>> { + pub(super) fn ruby_ancestry(&mut self, qn: &str) -> Res>> { if let Some(hit) = self.ruby_ancestry_memo.get(qn) { return Ok(hit.clone()); } diff --git a/codegraph-kernel/src/resolve/method_call.rs b/codegraph-kernel/src/resolve/method_call.rs index 93e5ece17e..65e4e03511 100644 --- a/codegraph-kernel/src/resolve/method_call.rs +++ b/codegraph-kernel/src/resolve/method_call.rs @@ -1411,6 +1411,15 @@ if inferred.is_none() && r.language == "dart" { let own = found.qualified_name == format!("{}::{method}", c.qualified_name); return Ok(Some(KCand { node: found, confidence: if own { 0.85 } else { 0.8 }, resolved_by })); } + // A call on a Ruby class or module constant takes the class + // method (`def self.m`) when it and `def m` share the qualified + // name, then the owner's Ruby ancestry (#2834). + if c.language == "ruby" { + if let Some((found, own)) = self.ruby_method_of(&c.qualified_name, method, r)? { + return Ok(Some(KCand { node: found, confidence: if own { confidence } else { 0.8 }, resolved_by })); + } + continue; + } let in_file = self.nodes_in_file(&c.file_path)?; if let Some(mn) = in_file .iter() @@ -1821,3 +1830,153 @@ impl KernelResolver { Ok(TsTypedLocal::Pass(file)) } } + +/// matchNewReceiverCall (name-matcher.ts): `new Runner(args).run()`, encoded +/// by the extractor as `new Runner().run`. The receiver is an instance of the +/// class written at the call, so this shape resolves only through that class. +fn new_receiver_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"^new ([\w$.]+)\(\)\.([\w$]+)$").unwrap()) +} + +impl KernelResolver { + /// matchNewReceiverCall — the class a `new C().m` ref constructs, as the + /// name its declaration carries: `ns.Runner` → `Runner`, an aliased named + /// import `R` → `Runner`. A JS/TS built-in (`RegExp`, `Map`, `Date`…) + /// counts only when the file imports or declares a class of that name — + /// otherwise a project class that merely shares the name would take + /// every `new Map().get()` in the repo. + pub(super) fn match_new_receiver_call(&mut self, r: &ResolveRefIn) -> Res> { + let Some(m) = new_receiver_re().captures(&r.reference_name) else { return Ok(None) }; + let written = m.get(1).unwrap().as_str(); + let method = m.get(2).unwrap().as_str(); + let last = written.rsplit('.').next().unwrap_or(written); + let mut cls = last.to_string(); + if !written.contains('.') { + let imported = self + .import_mappings(&r.file_path)? + .iter() + .find(|i| i.local_name == written) + .cloned(); + if let Some(imp) = imported { + if !imp.is_default + && !imp.is_namespace + && crate::textutil::ascii_ident_re().is_match(&imp.exported_name) + { + cls = imp.exported_name; + } + } else if JS_BUILT_INS.contains(last) + && !self.nodes_in_file(&r.file_path)?.iter().any(|n| n.kind == "class" && n.name == last) + { + return Ok(None); + } + } + if r.language == "java" || r.language == "csharp" { + return self.match_java_csharp_new_receiver_call(&cls, written, method, r); + } + let file = self.ts_imported_type_file(written, r)?; + if let Some(hit) = self.resolve_method_on_type(&cls, method, r, 0.9, "instance-method", file.as_deref())? { + return Ok(Some(hit)); + } + // An ES5 constructor function (`function Params() {}`) keeps its + // methods as functions assigned onto its prototype in its own file. + let ctors: Vec> = self + .nodes_by_name(&cls)? + .iter() + .filter(|n| n.kind == "function" && same_language_family(&n.language, &r.language)) + .cloned() + .collect(); + let [ctor] = ctors.as_slice() else { return Ok(None) }; + let lines = self.read_file(&ctor.file_path); + let on_proto: Vec> = self + .nodes_in_file(&ctor.file_path)? + .iter() + .filter(|n| { + n.name == method + && (n.kind == "function" || n.kind == "method") + && lines + .as_ref() + .and_then(|l| l.get((n.start_line - 1).max(0) as usize)) + .is_some_and(|line| line.contains("prototype")) + }) + .cloned() + .collect(); + Ok(match on_proto.as_slice() { + [only] => Some(KCand { node: only.clone(), confidence: 0.8, resolved_by: "instance-method" }), + _ => None, + }) + } + + /// matchJavaCsharpNewReceiverCall — a Java or C# `new Foo().bar()`. With + /// one project class of the name, its method or an inherited one, as for + /// any typed call. With several, the class the language picks, or none: + /// Java's written package (`new a.b.Foo()`), the file's single-type + /// import, else the caller's own package; C#'s one type the file's + /// namespaces and usings can see (isCsharpTypeVisible). Then only that + /// class's own method. + fn match_java_csharp_new_receiver_call( + &mut self, + cls: &str, + written: &str, + method: &str, + r: &ResolveRefIn, + ) -> Res> { + let twins: Vec> = self + .nodes_by_name(cls)? + .iter() + .filter(|n| (n.kind == "class" || n.kind == "struct") && same_language_family(&n.language, &r.language)) + .cloned() + .collect(); + if twins.len() <= 1 { + return self.resolve_method_on_type(cls, method, r, 0.9, "instance-method", None); + } + let dir_of = |file: &str| file.rsplit('/').skip(1).collect::>().into_iter().rev().collect::>().join("/"); + let mut chosen: Option> = None; + if r.language == "csharp" { + // isCsharpTypeVisible (name-matcher.ts): `ns::Type` is visible + // from a file whose own namespaces cover `ns` or whose usings + // name it. A `new X().m` ref never matches a using alias, so that + // arm does not apply. + let own = self.csharp_namespaces_at(r).unwrap_or_default(); + let usings = self.csharp_namespace_usings_at(r); + let mut visible = Vec::new(); + for n in &twins { + let ns = n.qualified_name.split("::").next().unwrap_or(""); + if own.iter().any(|o| o == ns || o.starts_with(&format!("{ns}."))) + || usings.contains(ns) + { + visible.push(n.clone()); + } + } + if visible.len() == 1 { + chosen = Some(visible.remove(0)); + } + } else { + let fqn = if written.contains('.') { + Some(written.to_string()) + } else { + self.imported_fqn_of(cls, r)? + }; + if let Some(fqn) = fqn { + let path = fqn.replace('.', "/"); + chosen = twins + .iter() + .find(|n| n.file_path.strip_suffix(".java").unwrap_or(&n.file_path).ends_with(&format!("/{path}"))) + .cloned(); + } else { + let own_dir = dir_of(&r.file_path); + let own: Vec<&Arc> = twins.iter().filter(|n| dir_of(&n.file_path) == own_dir).collect(); + if own.len() == 1 { + chosen = Some(own[0].clone()); + } + } + } + let Some(chosen) = chosen else { return Ok(None) }; + let want_qn = format!("{}::{}", chosen.qualified_name, method); + Ok(self + .nodes_in_file(&chosen.file_path)? + .iter() + .find(|n| n.kind == "method" && n.name == method && n.qualified_name == want_qn) + .map(|node| KCand { node: node.clone(), confidence: 0.9, resolved_by: "instance-method" })) + } +} diff --git a/codegraph-kernel/src/resolve/mod.rs b/codegraph-kernel/src/resolve/mod.rs index ccbff475f0..f00a04b897 100644 --- a/codegraph-kernel/src/resolve/mod.rs +++ b/codegraph-kernel/src/resolve/mod.rs @@ -577,6 +577,8 @@ mod dart_local; mod dart_calls; mod lua_alias; mod member_scope; +mod pascal_scope; +mod ruby_paths; mod vbnet; mod receivers; mod bound; @@ -680,6 +682,7 @@ pub struct KernelResolver { php_supers_memo: HashMap>>, /// dartSupertypesOf, by type name. vb: vbnet::VbMemo, + pascal: pascal_scope::PasMemo, js_objects: js_objects_upstream::JsObjects, dart_libraries: dart_libraries::DartLibraries, dart_locals: dart_local::DartLocals, @@ -915,6 +918,7 @@ impl KernelResolver { java_supers_memo: HashMap::new(), php_supers_memo: HashMap::new(), vb: vbnet::VbMemo::default(), + pascal: pascal_scope::PasMemo::default(), js_objects: Default::default(), dart_libraries: Default::default(), dart_locals: Default::default(), diff --git a/codegraph-kernel/src/resolve/pascal_scope.rs b/codegraph-kernel/src/resolve/pascal_scope.rs new file mode 100644 index 0000000000..e7c9ec5049 --- /dev/null +++ b/codegraph-kernel/src/resolve/pascal_scope.rs @@ -0,0 +1,2029 @@ +//! Pascal scope resolution — port of upstream's `resolution/pascal-scope.ts`. +//! +//! Covers three upstream features sharing one scope model: +//! +//! - Names written without parentheses (#2514): a parameterless routine is +//! called that way in Delphi — `Before;`, `Result := FProvider.Value + 1`, +//! `if Ready then` — and the same spelling reads a field, a property, a +//! variable or a constant. The extractor records such a name as a +//! `references` ref named `{mode}.Name` or `{mode}.Receiver.Name`: +//! - `{call}` — a statement on its own, which only a routine can be; +//! - `{read}` — a name in an expression or an argument, where a function +//! (or a constructor) is called for its value and a procedure, which +//! has none, can only be a method pointer; +//! - `{ref}` — the value given to an event (`OnClick := Handler`), a +//! method pointer; +//! - `{recv}` — a receiver (`GetList` in `GetList.Count`, `B` in +//! `A.B.C`) or an indexed value, which only a function can be. +//! +//! Written without its arguments, a routine that needs some is a method +//! pointer too, in any of them. The routine's class follows an `@` +//! (`{call@TConsumer}.Before`), because the reference's own node is the +//! file when an implementation matched no declaration. A receiver the +//! routine declares itself is written as its declared type after a `:` +//! (`{read}.:TProvider.Value`). +//! +//! - Calls to the names Delphi's compiler and System unit give meaning to +//! (`SizeOf`, `TypeInfo`, `length(S)`, `Pointer(P)`), whatever their case: +//! only a declaration in the call's scope shadows them (#2803). +//! +//! - Bare calls to routines nested where the call is made (`Inner(1)` in the +//! routine that declares `procedure Inner(N: Integer);`), and calls made +//! inside a nested routine, which reach the rest of that routine's scope +//! the same way (#2810). +//! +//! Every lookup follows Delphi's scope rules — the routine's class and its +//! ancestors, then the unit, then the units it uses, last first. A unit is +//! its own file plus the include files it pulls in (`{$I name.inc}`). The +//! first declaration of the name in that order decides: when it is data (a +//! field, property, variable, constant, enum value or type), nothing is +//! linked unless the data's declared type is a routine's. + +use super::*; +use tables::{PASCAL_SYSTEM_NAMES, RTL_PROCEDURAL_TYPES}; + +const PARENLESS: &str = r"^\{(call|read|ref|recv)(?:@([^}]+))?\}\.(.+)$"; +const CLASS_KINDS: &[&str] = &["class", "struct", "interface"]; +/// What a unit declares by name at its top level. +const UNIT_KINDS: &[&str] = &[ + "function", "variable", "constant", "class", "struct", "interface", + "enum", "type_alias", "enum_member", +]; +/// What a class declares as its members. +const MEMBER_KINDS: &[&str] = + &["method", "field", "property", "constant", "variable"]; +/// Declarations that hold a value rather than run code. +const DATA_KINDS: &[&str] = + &["field", "property", "variable", "constant", "enum_member"]; +/// What a Pascal typecast's name resolves to (`TFoo(x)` casts, it doesn't +/// call). +pub(super) const CAST_TARGET_KINDS: &[&str] = + &["class", "struct", "interface", "enum", "type_alias"]; +const PASCAL_SOURCE: &str = r"(?i)\.(?:pas|dpr|dpk|lpr|inc)$"; +/// A routine type written out: `procedure(...) of object`, `function: T`, +/// `reference to procedure`. +const PROCEDURAL_TYPE: &str = r"(?i)^(?:reference\s+to\s+)?(?:procedure|function)\b"; +const MAX_HIERARCHY_DEPTH: usize = 8; + +#[derive(Clone)] +struct PasFound { + node: Arc, + resolved_by: &'static str, + confidence: f64, +} + +/// What a dotted name's first link is: a value of a known class (a local's +/// declared type, written `:TFoo`), a class itself, or a unit. +struct PasHead { + ty: Option>, + unit_file: Option, +} + +/// A routine header or scope boundary found while scanning a file once +/// (headerClassAt's per-file table). +struct PasHeader { + line: i64, + class_qn: Option, + routine: bool, +} + +/// What scope lookups need of a file's nodes, read once: its classes, its +/// `uses` in order, how far its own routines are indented at most. +struct PasOutline { + classes: Vec>, + imports: Vec>, + /// In a unit whose `unit` header the parser lost, the members of a class + /// it couldn't read come out as routines of the unit, indented deeper + /// than the unit's own; i64::MAX elsewhere (any column). + routine_column: i64, +} + +/// The project's files by lowercased name, and its Pascal sources. +struct PasProjectFiles { + by_name: HashMap>, + pascal: Vec, +} + +/// pascal-scope.ts's per-context memos, held for the resolver's run. +#[derive(Default)] +pub(super) struct PasMemo { + in_scope: HashMap>, + class_by_qn: HashMap>>, + ancestors: HashMap>>>, + used_files: HashMap>, + class_by_name: HashMap>>, + units_by_name: Option>>>, + generic_types: Option>>>>, + project_files: Option>, + includes: HashMap>>, + includers: HashMap>, + own_files: HashMap>>, + unit_file_sets: HashMap>>, + unit_files: HashMap>, + hierarchy: HashMap>>, + member_types: HashMap>>, + keywords: HashMap>, + procedural: HashMap, + header_classes: HashMap>>, + outlines: HashMap>, + condition_depths: HashMap>>, + nested_in_scope: HashMap>, + routine_nodes: HashMap>>, + routine_qns: HashMap, + supers: HashMap>>, +} + +impl KernelResolver { + // ------------------------------------------------------------------ + // Ref matchers + // ------------------------------------------------------------------ + + /// isPascalParenlessRef — a pascal `references` ref named `{mode}.path`. + pub(super) fn is_pascal_parenless_ref(&self, r: &ResolveRefIn) -> bool { + r.language == "pascal" && r.reference_kind == "references" + && re!(PARENLESS).is_match(&r.reference_name) + } + + /// matchPascalParenlessRef — the routine a paren-less Pascal name + /// reaches in its scope, or None (see the file comment). + pub(super) fn resolve_pascal_parenless( + &mut self, + r: &ResolveRefIn, + ) -> Res> { + if !self.is_pascal_parenless_ref(r) { + return Ok(None); + } + let m = re!(PARENLESS).captures(&r.reference_name).unwrap(); + let mode = m[1].to_string(); + let path: Vec = + m[3].split('.').map(|s| s.to_string()).collect(); + // Most such names read data, often the framework's: a name no + // Pascal routine has is settled here, before any lookup in scope. + let last = path[path.len() - 1].to_lowercase(); + let routine_exists = self.nodes_by_lower_name(&last)?.iter().any(|n| { + n.language == "pascal" + && matches!(n.kind.as_str(), "method" | "function") + }); + if !routine_exists { + return Ok(Some(ResolveOutcome::unresolved())); + } + let cls = if let Some(qn) = m.get(2) { + self.pascal_class_by_qn(qn.as_str(), &r.file_path)? + } else { + match self.node_by_id(&r.from_node_id)? { + Some(routine) => self.pascal_class_of_routine(&routine)?, + None => None, + } + }; + let found = if path.len() == 1 { + self.pascal_name_in_scope(&path[0], cls.as_deref(), r)? + } else { + self.pascal_member_through_path(&path, cls.as_deref(), r)? + }; + let Some(found) = found else { + return Ok(Some(ResolveOutcome::unresolved())); + }; + // Written without arguments, a routine that needs some can only be + // a method pointer — save a constructor, whose name then means an + // inherited one outside the project (`TObject.Create`) — and so is + // a procedure read for a value, and an event's method; any other + // runs. + let needs_args = pascal_needs_arguments(&found.node); + if needs_args && self.pascal_routine_keyword(&found.node)? == Some("constructor".into()) { + return Ok(Some(ResolveOutcome::unresolved())); + } + let pointer = mode == "ref" || needs_args + || (mode != "call" && !self.pascal_yields_value(&found.node)?); + // A statement runs its routine, and a receiver is a routine's value. + if pointer && (mode == "call" || mode == "recv") { + return Ok(Some(ResolveOutcome::unresolved())); + } + let edge_kind = + if pointer { "references".to_string() } else { "calls".to_string() }; + let mut out = self.finish_pre_framework( + r, + KCand { + node: found.node, + confidence: found.confidence, + resolved_by: found.resolved_by, + }, + )?; + if out.status == "resolved" { + out.edge_kind = Some(edge_kind); + } + Ok(Some(out)) + } + + /// isPascalSystemCall — a bare Pascal call to one of the System unit's + /// names (see PASCAL_SYSTEM_NAMES), whatever its case. + pub(super) fn is_pascal_system_call(&mut self, r: &ResolveRefIn) -> Res { + Ok(r.language == "pascal" + && r.reference_kind == "calls" + && PASCAL_SYSTEM_NAMES.contains(r.reference_name.to_lowercase().as_str()) + && self.pascal_is_bare_call(r)?) + } + + /// matchPascalSystemCall — a call to one of the System unit's names is + /// the System unit's, which the project doesn't hold, unless a + /// declaration in the call's scope shadows it: a member of the class + /// (`Delete(I)` in a list class), a routine of the unit or the units it + /// uses. A project type of that name is a typecast. Data of that name + /// runs only when it holds a routine (see isProceduralData). A unit that + /// declares one under a condition stands in for compilers without it + /// (`{$IFNDEF UNICODE} UnicodeString = WideString; {$ENDIF}`): the code + /// is most likely built with one that has it. + pub(super) fn match_pascal_system_call( + &mut self, + r: &ResolveRefIn, + ) -> Res { + let cls = self.pascal_class_of_call(r)?; + let decl = self.pascal_declaration_in_scope( + &r.reference_name.to_lowercase(), + cls.as_deref(), + r, + )?; + let Some(decl) = decl else { + return Ok(ResolveOutcome::unresolved()); + }; + if DATA_KINDS.contains(&decl.node.kind.as_str()) + && !self.pascal_is_procedural_data(&decl.node)? + { + return Ok(ResolveOutcome::unresolved()); + } + if !decl.node.qualified_name.contains("::") + && self.pascal_declared_under_condition(&decl.node)? + { + return Ok(ResolveOutcome::unresolved()); + } + self.finish_pre_framework( + r, + KCand { + node: decl.node, + confidence: decl.confidence, + resolved_by: decl.resolved_by, + }, + ) + } + + /// matchPascalNestedCall — a bare Pascal call (`Inner(1)`) to a routine + /// nested where the call is made (see nestedRoutineInScope). Made inside + /// a nested routine, it reaches the rest of that routine's scope the + /// same way: the class of the method it is nested in, its unit, the + /// units it uses. None when that scope holds no routine of the name, + /// for the other strategies to resolve. + pub(super) fn resolve_pascal_nested_call( + &mut self, + r: &ResolveRefIn, + ) -> Res> { + if r.language != "pascal" + || r.reference_kind != "calls" + || !self.pascal_is_bare_call(r)? + { + return Ok(None); + } + let lower = r.reference_name.to_lowercase(); + let mut found = self.pascal_nested_routine_in_scope(&lower, r)?; + if found.is_none() { + let Some(caller) = self.pascal_routine_node(&r.from_node_id)? else { + return Ok(None); + }; + if !self.pascal_is_nested_routine(&caller)? { + return Ok(None); + } + let cls = self.pascal_class_of_call(r)?; + let Some(decl) = + self.pascal_declaration_in_scope(&lower, cls.as_deref(), r)? + else { + return Ok(None); + }; + if !pascal_is_routine(&decl.node) + && !(DATA_KINDS.contains(&decl.node.kind.as_str()) + && self.pascal_is_procedural_data(&decl.node)?) + { + return Ok(None); + } + found = Some(decl); + } + let found = found.unwrap(); + self.finish_pre_framework( + r, + KCand { + node: found.node, + confidence: found.confidence, + resolved_by: found.resolved_by, + }, + ) + .map(Some) + } + + /// gatePascalCall — what a Pascal call another strategy resolved stands + /// for, when it landed on data or on a routine nested in another. Runs + /// on every resolved pascal `calls` verdict, after resolve_ref_inner. + /// + /// A call resolved to data — a field, a property, a variable, a + /// constant — runs what the data holds, so it stands only for data + /// whose declared type is a routine's. A bare name is what its scope + /// declares (`FOnChange(Self)` in the class that declares its own + /// `FOnChange`): the routine or the routine-holding data there, + /// whichever namesake the strategy chose. Any other is a namesake: + /// `length(S)` was a record's `length` field, `Pointer(P)` a property + /// named `Pointer`. A name the calling routine declares itself, as a + /// parameter, a local or a nested routine, is that declaration, which + /// the graph doesn't hold. A name its scope doesn't declare at all is a + /// `with` object's member (`with OCI do HandleAlloc(...)`) or a + /// declaration the parse put in the wrong place: it stands for the one + /// routine-holding field or variable the project declares by that name. + pub(super) fn gate_pascal_call( + &mut self, + out: ResolveOutcome, + r: &ResolveRefIn, + ) -> Res { + let Some(tid) = &out.target_node_id else { + return Ok(out); + }; + let Some(target) = self.node_by_id(tid)? else { + return Ok(out); + }; + if DATA_KINDS.contains(&target.kind.as_str()) { + return self.gate_pascal_data_call(&target, out, r); + } + if self.pascal_is_nested_routine(&target)? { + return self.gate_pascal_nested_target(out, r); + } + Ok(out) + } + + /// See gate_pascal_call: a call resolved to data. + fn gate_pascal_data_call( + &mut self, + target: &KNode, + out: ResolveOutcome, + r: &ResolveRefIn, + ) -> Res { + if !self.pascal_is_bare_call(r)? { + return Ok( + if target.language == "pascal" && self.pascal_is_procedural_data(target)? { + out + } else { + ResolveOutcome::unresolved() + }, + ); + } + let lower = r.reference_name.to_lowercase(); + if self.pascal_declared_in_routine(&lower, r)? { + return Ok(ResolveOutcome::unresolved()); + } + let cls = self.pascal_class_of_call(r)?; + let decl = self.pascal_declaration_in_scope(&lower, cls.as_deref(), r)?; + let Some(decl) = decl else { + let only = target.language == "pascal" + && self + .pascal_named(&lower)? + .iter() + .all(|n| n.id == target.id || n.kind == "import"); + return Ok( + if only && self.pascal_is_procedural_data(target)? { + out + } else { + ResolveOutcome::unresolved() + }, + ); + }; + if !pascal_is_routine(&decl.node) + && !(DATA_KINDS.contains(&decl.node.kind.as_str()) + && self.pascal_is_procedural_data(&decl.node)?) + { + return Ok(ResolveOutcome::unresolved()); + } + if Some(decl.node.id.as_str()) == out.target_node_id.as_deref() { + return Ok(out); + } + Ok(retarget(out, &decl)) + } + + /// A Pascal call another strategy resolved to a routine nested in some + /// other routine, which nothing outside that routine can call: a bare + /// name is then what the call's scope declares, a routine or data + /// holding one (mORMot's `StrLen` variable), and any other call links + /// nothing. + fn gate_pascal_nested_target( + &mut self, + out: ResolveOutcome, + r: &ResolveRefIn, + ) -> Res { + if !self.pascal_is_bare_call(r)? { + return Ok(ResolveOutcome::unresolved()); + } + let lower = r.reference_name.to_lowercase(); + let cls = self.pascal_class_of_call(r)?; + let Some(decl) = self.pascal_declaration_in_scope(&lower, cls.as_deref(), r)? + else { + return Ok(ResolveOutcome::unresolved()); + }; + if !pascal_is_routine(&decl.node) + && !(DATA_KINDS.contains(&decl.node.kind.as_str()) + && self.pascal_is_procedural_data(&decl.node)?) + { + return Ok(ResolveOutcome::unresolved()); + } + if Some(decl.node.id.as_str()) == out.target_node_id.as_deref() { + return Ok(out); + } + Ok(retarget(out, &decl)) + } + + /// isBareCall — whether a Pascal call named by a bare name is written + /// bare: its expression starts with the name (`Reset(F)`). The last + /// call of a chain is named bare too (`Reset` in `GetKey(P).Reset`), + /// and its scope is its receiver's type. + fn pascal_is_bare_call(&mut self, r: &ResolveRefIn) -> Res { + if !re!(r"^[A-Za-z_]\w*$").is_match(&r.reference_name) { + return Ok(false); + } + let Some(source) = self.read_file(&r.file_path) else { + return Ok(false); + }; + let line = source + .get((r.line - 1).max(0) as usize) + .cloned() + .unwrap_or_default(); + let tail = line.get(r.column.max(0) as usize..).unwrap_or(""); + Ok(re!(r"^[A-Za-z_]\w*") + .find(tail) + .is_some_and(|m| m.as_str().eq_ignore_ascii_case(&r.reference_name))) + } + + // ------------------------------------------------------------------ + // Scope lookup + // ------------------------------------------------------------------ + + /// A bare name, looked up the way Delphi looks it up from a routine of + /// class `cls` (None: a free routine): a routine, or None. + fn pascal_name_in_scope( + &mut self, + name: &str, + cls: Option<&KNode>, + r: &ResolveRefIn, + ) -> Res> { + let decl = + self.pascal_declaration_in_scope(&name.to_lowercase(), cls, r)?; + Ok(decl.filter(|f| pascal_is_routine(&f.node))) + } + + /// declarationInScope — the first declaration a bare name reaches from + /// a routine of class `cls` (None: a free routine), whatever it + /// declares: a routine nested in the calling routine or one around it, + /// a member of the class or its ancestors, then a declaration of the + /// unit, then of the units it uses, last first. None when the project + /// declares none of them, as for the System unit's names. + fn pascal_declaration_in_scope( + &mut self, + lower: &str, + cls: Option<&KNode>, + r: &ResolveRefIn, + ) -> Res> { + if let Some(nested) = self.pascal_nested_routine_in_scope(lower, r)? { + return Ok(Some(nested)); + } + let key = format!( + "{}|{}|{}", + cls.map(|c| c.id.as_str()).unwrap_or(""), + r.file_path, + lower + ); + if let Some(hit) = self.pascal.in_scope.get(&key) { + return Ok(hit.clone()); + } + let mut found: Option = None; + let member = match cls { + Some(cls) => self.pascal_member_in_hierarchy(cls, lower)?, + None => None, + }; + if let Some(member) = member { + found = Some(PasFound { + node: member, + resolved_by: "instance-method", + confidence: 0.9, + }); + } else if let Some(own) = + self.pascal_own_unit_declaration(&r.file_path, lower)? + { + found = Some(PasFound { + node: own, + resolved_by: "exact-match", + confidence: 0.9, + }); + } else { + for file in + self.pascal_used_unit_files(&r.file_path)?.into_iter().rev() + { + if let Some(decl) = + self.pascal_unit_declaration(&file, lower, &r.file_path)? + { + found = Some(PasFound { + node: decl, + resolved_by: "import", + confidence: 0.85, + }); + break; + } + } + } + self.pascal.in_scope.insert(key, found.clone()); + Ok(found) + } + + /// nestedRoutineInScope — the routine nested in the calling routine, or + /// in one around it, that a bare name there names (`procedure Inner;` + /// between `procedure TFoo.Bar;` and its `begin`, + /// `TFoo::Bar::Inner`): the innermost one declared above the call, the + /// calling routine itself among them. It comes before anything of the + /// class or the unit. None when none is. + fn pascal_nested_routine_in_scope( + &mut self, + lower: &str, + r: &ResolveRefIn, + ) -> Res> { + // Most names no nested routine has: settled before any other lookup. + let named = self.pascal_named(lower)?; + if !named + .iter() + .any(|n| n.kind == "function" && n.qualified_name.contains("::")) + { + return Ok(None); + } + let key = format!("{}|{}", r.from_node_id, lower); + if let Some(hit) = self.pascal.nested_in_scope.get(&key) { + return Ok(hit.clone()); + } + let mut found: Option = None; + if let Some(caller) = self.pascal_routine_node(&r.from_node_id)? { + if caller.language == "pascal" { + let chain = caller.qualified_name.to_lowercase(); + let mut best: Option> = None; + let mut depth = -1i64; + for n in &named { + if n.file_path != caller.file_path + || n.start_line > r.line + || !self.pascal_is_nested_routine(n)? + { + continue; + } + let outer = pascal_owner_of(n); + // Nested in the caller, or in a routine the caller is + // nested in. + if chain != outer + && !chain.starts_with(&format!("{outer}::")) + { + continue; + } + if outer.len() as i64 > depth + || (outer.len() as i64 == depth + && n.start_line + < best.as_ref().map(|b| b.start_line).unwrap_or(i64::MAX)) + { + best = Some(n.clone()); + depth = outer.len() as i64; + } + } + if let Some(best) = best { + found = Some(PasFound { + node: best, + resolved_by: "exact-match", + confidence: 0.95, + }); + } + } + } + self.pascal.nested_in_scope.insert(key, found.clone()); + Ok(found) + } + + /// isNestedRoutine — whether a node is a routine nested in another: a + /// Pascal function qualified under a routine, which no code outside the + /// one it is nested in can call. + fn pascal_is_nested_routine(&mut self, node: &KNode) -> Res { + if node.kind != "function" || node.language != "pascal" { + return Ok(false); + } + let Some(cut) = node.qualified_name.rfind("::") else { + return Ok(false); + }; + let outer = node.qualified_name[..cut].to_string(); + if let Some(hit) = self.pascal.routine_qns.get(&outer) { + return Ok(*hit); + } + let routine = self + .nodes_by_qualified_name(&outer)? + .iter() + .any(|n| n.language == "pascal" && pascal_is_routine(n)); + self.pascal.routine_qns.insert(outer, routine); + Ok(routine) + } + + /// routineNode — the routine a reference comes from, or None when it + /// comes from a file or anything else. + fn pascal_routine_node(&mut self, id: &str) -> Res>> { + if let Some(hit) = self.pascal.routine_nodes.get(id) { + return Ok(hit.clone()); + } + let node = self.node_by_id(id)?; + let routine = node.filter(|n| pascal_is_routine(n)); + self.pascal + .routine_nodes + .insert(id.to_string(), routine.clone()); + Ok(routine) + } + + /// declaredInRoutine — whether the routine making a call declares the + /// called name itself, as a parameter, a local or a nested routine + /// (`function Hash(Item: Pointer; Hasher: THasher)` calling + /// `Hasher(0, Item, 4)`): from the routine's header down to the call. + fn pascal_declared_in_routine( + &mut self, + lower: &str, + r: &ResolveRefIn, + ) -> Res { + let Some(source) = self.read_file(&r.file_path) else { + return Ok(false); + }; + // The header in the text: a method's node may start at its + // declaration in the class, above the fields and the unit's + // variables. + let from = self.pascal_routine_header_at(&r.file_path, r.line)?; + let mentions = + regex::Regex::new(&format!(r"(?i)\b{}\b", regex::escape(lower))) + .unwrap(); + let declares_routine = regex::Regex::new(&format!( + r"(?i)\b(?:procedure|function)\s+{}\b", + regex::escape(lower) + )) + .unwrap(); + let declares_var = regex::Regex::new(&format!( + r"(?i)\b{}\b\s*(?:,\s*[A-Za-z_]\w*\s*)*:", + regex::escape(lower) + )) + .unwrap(); + let strip = re!(r"//.*$|\{[^}]*\}|'[^']*'"); + for i in from.max(1)..=r.line.min(source.len() as i64) { + let text = &source[(i - 1) as usize]; + if !mentions.is_match(text) { + continue; + } + let stripped = strip.replace_all(text, " "); + // `name:` declares a variable or parameter; `name :=` assigns. + if declares_routine.is_match(&stripped) + || declares_var.find(&stripped).is_some_and(|m| { + !stripped[m.end()..].starts_with('=') + }) + { + return Ok(true); + } + } + Ok(false) + } + + /// The line of the nearest routine header at column 0 at or above a + /// line, else the line itself. + fn pascal_routine_header_at( + &mut self, + file_path: &str, + line: i64, + ) -> Res { + let headers = self.pascal_header_classes(file_path)?; + let mut found = line; + for h in headers.iter() { + if h.line > line { + break; + } + if h.routine { + found = h.line; + } + } + Ok(found) + } + + /// The class whose method makes a call: the one the nearest method + /// header above the call names in the source text + /// (`procedure TFoo.Bar;`), which holds where the tree had errors and + /// gave the code to the file, or to a routine that swallowed the ones + /// after it; else the reference's method's class. + fn pascal_class_of_call( + &mut self, + r: &ResolveRefIn, + ) -> Res>> { + if let Some(qn) = self.pascal_header_class_at(&r.file_path, r.line)? { + if let Some(cls) = self.pascal_class_by_qn(&qn, &r.file_path)? { + return Ok(Some(cls)); + } + } + match self.node_by_id(&r.from_node_id)? { + Some(routine) if routine.kind == "method" => { + self.pascal_class_of_routine(&routine) + } + _ => Ok(None), + } + } + + /// headerClassAt — the class the nearest routine header at or above a + /// line names (`procedure TFoo.Bar;` → `TFoo`), or None when it is a + /// free routine's, or the unit's code resumes there: after an `end;` + /// that starts its line, or at `initialization`. + fn pascal_header_class_at( + &mut self, + file_path: &str, + line: i64, + ) -> Res> { + let headers = self.pascal_header_classes(file_path)?; + let mut hit: Option = None; + for h in headers.iter() { + if h.line <= line { + hit = h.class_qn.clone(); + } else { + break; + } + } + Ok(hit) + } + + /// The per-file table of routine headers and scope boundaries. + fn pascal_header_classes( + &mut self, + file_path: &str, + ) -> Res>> { + if let Some(hit) = self.pascal.header_classes.get(file_path) { + return Ok(hit.clone()); + } + let mut headers = Vec::new(); + if let Some(source) = self.read_file(file_path) { + let header = re!( + r"(?i)^(?:class\s+)?(?:procedure|function|constructor|destructor|operator)\s+([A-Za-z_][\w.<>, ]*?)\s*[(:;]" + ); + for (i, text) in source.iter().enumerate() { + // Each starts its line with one of c, d, e, f, i, o, p + // (either case). + let first = text.as_bytes().first().map(|b| b | 0x20); + if !matches!( + first, + Some(0x63 | 0x64 | 0x65 | 0x66 | 0x69 | 0x6f | 0x70) + ) { + continue; + } + if re!(r"(?i)^(?:end;|(?:initialization|finalization)\b)") + .is_match(text) + { + headers.push(PasHeader { + line: i as i64 + 1, + class_qn: None, + routine: false, + }); + continue; + } + let Some(m) = header.captures(text) else { + continue; + }; + let cleaned = re!(r"<[^>]*>").replace_all(&m[1], ""); + let cleaned = re!(r"\s+").replace_all(&cleaned, ""); + let parts: Vec<&str> = cleaned.split('.').collect(); + let class_qn = if parts.len() > 1 { + Some(parts[..parts.len() - 1].join("::")) + } else { + None + }; + headers.push(PasHeader { + line: i as i64 + 1, + class_qn, + routine: true, + }); + } + } + let rc = Rc::new(headers); + self.pascal + .header_classes + .insert(file_path.to_string(), rc.clone()); + Ok(rc) + } + + /// declaredUnderCondition — whether a declaration is inside an + /// `{$IFDEF}`, `{$IFNDEF}`, `{$IF}` or `{$IFOPT}` block, or its + /// `{$ELSE}`. + fn pascal_declared_under_condition(&mut self, node: &KNode) -> Res { + let depths = self.pascal_condition_depths(&node.file_path)?; + let base = depths + .get((node.start_line - 1).max(0) as usize) + .copied() + .unwrap_or(0); + let line = self + .read_file(&node.file_path) + .and_then(|s| { + s.get((node.start_line - 1).max(0) as usize).cloned() + }) + .unwrap_or_default(); + let head = line + .get(..node.start_column.max(0) as usize) + .unwrap_or(&line); + Ok(pascal_condition_depth_after(head, base) > 0) + } + + /// The depth of conditional blocks at each line of a file, once. + fn pascal_condition_depths( + &mut self, + file_path: &str, + ) -> Res>> { + if let Some(hit) = self.pascal.condition_depths.get(file_path) { + return Ok(hit.clone()); + } + let mut depths = Vec::new(); + if let Some(source) = self.read_file(file_path) { + let mut depth = 0i64; + for text in source.iter() { + depths.push(depth); + if text.contains("{$") { + depth = pascal_condition_depth_after(text, depth); + } + } + } + let rc = Rc::new(depths); + self.pascal + .condition_depths + .insert(file_path.to_string(), rc.clone()); + Ok(rc) + } + + /// A dotted name: the method its receiver's declared type has. + fn pascal_member_through_path( + &mut self, + path: &[String], + cls: Option<&KNode>, + r: &ResolveRefIn, + ) -> Res> { + let Some(head) = self.pascal_head_of(&path[0], cls, r)? else { + return Ok(None); + }; + let mut ty: Option>; + let mut rest: &[String] = &path[1..]; + if let Some(unit_file) = head.unit_file { + // `UnitName.Routine`, `UnitName.GlobalVar.Method`. + let Some(decl) = self.pascal_unit_declaration( + &unit_file, + &rest[0].to_lowercase(), + &r.file_path, + )? + else { + return Ok(None); + }; + if rest.len() == 1 { + return Ok(if pascal_is_routine(&decl) { + Some(PasFound { + node: decl, + resolved_by: "import", + confidence: 0.9, + }) + } else { + None + }); + } + ty = self.pascal_type_of_member(&decl)?; + rest = &rest[1..]; + } else { + ty = head.ty; + } + for link in &rest[..rest.len() - 1] { + let member = match ty.as_deref() { + Some(t) => { + self.pascal_member_in_hierarchy(t, &link.to_lowercase())? + } + None => None, + }; + let Some(member) = member else { return Ok(None) }; + ty = self.pascal_type_of_member(&member)?; + } + let member = match ty.as_deref() { + Some(t) => self + .pascal_member_in_hierarchy(t, &rest[rest.len() - 1].to_lowercase())?, + None => None, + }; + Ok(member.and_then(|m| { + if pascal_is_routine(&m) { + Some(PasFound { + node: m, + resolved_by: "instance-method", + confidence: 0.9, + }) + } else { + None + } + })) + } + + /// headOf — what a dotted name's first link is: a value of a known + /// class (a local's declared type, written `:TFoo`), a class itself, + /// or a unit. + fn pascal_head_of( + &mut self, + name: &str, + cls: Option<&KNode>, + r: &ResolveRefIn, + ) -> Res> { + if let Some(ty_name) = name.strip_prefix(':') { + return Ok(Some(PasHead { + ty: self.pascal_class_named(ty_name, &r.file_path)?, + unit_file: None, + })); + } + let lower = name.to_lowercase(); + if lower == "self" { + return Ok(cls.map(|cls| PasHead { + ty: Some(Arc::new(cls.clone())), + unit_file: None, + })); + } + if let Some(cls) = cls { + if let Some(member) = self.pascal_member_in_hierarchy(cls, &lower)? { + return Ok(Some(PasHead { + ty: self.pascal_type_of_member(&member)?, + unit_file: None, + })); + } + } + let mut decl = self.pascal_own_unit_declaration(&r.file_path, &lower)?; + if decl.is_none() { + for file in + self.pascal_used_unit_files(&r.file_path)?.into_iter().rev() + { + decl = self.pascal_unit_declaration(&file, &lower, &r.file_path)?; + if decl.is_some() { + break; + } + } + } + if let Some(decl) = decl { + return Ok(Some(PasHead { + ty: if CLASS_KINDS.contains(&decl.kind.as_str()) { + Some(decl) + } else { + self.pascal_type_of_member(&decl)? + }, + unit_file: None, + })); + } + match self.pascal_unit_file_named(&lower, &r.file_path)? { + Some(file) => Ok(Some(PasHead { + ty: None, + unit_file: Some(file), + })), + None => Ok(None), + } + } + + // ------------------------------------------------------------------ + // Classes, units and declarations + // ------------------------------------------------------------------ + + /// The Pascal nodes a name names, whatever their case. + fn pascal_named(&mut self, lower: &str) -> Res>> { + Ok(self + .nodes_by_lower_name(lower)? + .iter() + .filter(|n| n.language == "pascal") + .cloned() + .collect()) + } + + /// classByQualifiedName — the class (or record) an implementation + /// header names (`TOuter::TInner`, generic parameters dropped): the + /// one `file_path`'s unit declares, whatever its case or parameters, + /// else one the file sees. + fn pascal_class_by_qn( + &mut self, + qn: &str, + file_path: &str, + ) -> Res>> { + let key = format!("{file_path}|{qn}"); + if let Some(hit) = self.pascal.class_by_qn.get(&key) { + return Ok(hit.clone()); + } + let exact: Vec> = self + .nodes_by_qualified_name(qn)? + .iter() + .filter(|n| { + n.language == "pascal" && CLASS_KINDS.contains(&n.kind.as_str()) + }) + .cloned() + .collect(); + let lower = qn.to_lowercase(); + let mut cls: Option> = None; + for file in self.pascal_own_unit_files(file_path)?.iter().cloned().collect::>() { + cls = pascal_declaration_of( + exact.iter().filter(|n| n.file_path == file).cloned().collect(), + ); + if cls.is_none() { + let outline = self.pascal_outline(&file)?; + cls = pascal_declaration_of( + outline + .classes + .iter() + .filter(|n| { + strip_generic_params(&n.qualified_name) + .to_lowercase() + == lower + }) + .cloned() + .collect(), + ); + } + if cls.is_some() { + break; + } + } + if cls.is_none() { + cls = pascal_declaration_of(exact); + if cls.is_none() { + cls = self.pascal_class_named( + qn.rsplit("::").next().unwrap_or(qn), + file_path, + )?; + } + } + self.pascal.class_by_qn.insert(key, cls.clone()); + Ok(cls) + } + + /// classOfRoutine — the class a method belongs to: its qualified + /// name's owner, declared in the method's own file. + fn pascal_class_of_routine( + &mut self, + routine: &KNode, + ) -> Res>> { + if routine.kind != "method" { + return Ok(None); + } + let owner = routine + .qualified_name + .rsplit_once("::") + .map(|(o, _)| o.to_string()) + .unwrap_or_default(); + let candidates: Vec> = if owner.is_empty() { + Vec::new() + } else { + self.nodes_by_qualified_name(&owner)? + .iter() + .filter(|n| { + n.language == "pascal" + && CLASS_KINDS.contains(&n.kind.as_str()) + }) + .cloned() + .collect() + }; + Ok(pascal_declaration_of( + candidates + .iter() + .filter(|n| n.file_path == routine.file_path) + .cloned() + .collect(), + ) + .or_else(|| pascal_declaration_of(candidates))) + } + + /// ancestorsOf — a class and its ancestors the project declares, + /// nearest first. + fn pascal_ancestors(&mut self, cls: &KNode) -> Res>> { + if let Some(hit) = self.pascal.ancestors.get(&cls.id) { + return Ok(hit.as_ref().clone()); + } + let mut order: Vec> = Vec::new(); + let mut seen: HashSet = HashSet::new(); + let mut frontier = vec![Arc::new(cls.clone())]; + for _ in 0..MAX_HIERARCHY_DEPTH { + if frontier.is_empty() { + break; + } + let mut next: Vec> = Vec::new(); + for c in frontier { + if !seen.insert(c.id.clone()) { + continue; + } + order.push(c.clone()); + for name in self.pascal_declared_supertypes(&c)? { + if let Some(sup) = + self.pascal_class_named(&name, &c.file_path)? + { + next.push(sup); + } + } + } + frontier = next; + } + self.pascal + .ancestors + .insert(cls.id.clone(), Rc::new(order.clone())); + Ok(order) + } + + /// declaredSupertypes — the supertypes a Pascal class's own + /// declaration names (`TFoo = class(TBar)`), memoized. + fn pascal_declared_supertypes(&mut self, cls: &KNode) -> Res> { + if let Some(hit) = self.pascal.supers.get(&cls.id) { + return Ok(hit.as_ref().clone()); + } + let mut supers = Vec::new(); + if let Some(source) = self.read_file(&cls.file_path) { + let lines: &[String] = &source[..]; + let from = (cls.start_line - 1).max(0) as usize; + let head = lines + .get(from..(from + 3).min(lines.len())) + .map(|s| s.join(" ")) + .unwrap_or_default(); + if let Some(m) = re!( + r"(?i)=\s*class\s*(?:sealed|abstract)?\s*\(([^)]*)\)" + ) + .captures(&head) + { + supers = re!(r"[A-Za-z_][\w.:\\]*") + .find_iter(&m[1]) + .map(|w| { + w.as_str() + .rsplit([':', '.', '\\']) + .next() + .unwrap_or(w.as_str()) + .to_string() + }) + .filter(|w| { + !matches!( + w.to_lowercase().as_str(), + "metaclass" | "object" + ) + }) + .collect(); + } + } + self.pascal + .supers + .insert(cls.id.clone(), Rc::new(supers.clone())); + Ok(supers) + } + + /// memberInHierarchy — the member a name reaches in a class or, + /// failing that, in its ancestors, nearest first; a parameterless + /// overload before the others. None when no class of the hierarchy + /// declares it. + fn pascal_member_in_hierarchy( + &mut self, + cls: &KNode, + lower: &str, + ) -> Res>> { + let key = format!("{}|{}", cls.id, lower); + if let Some(hit) = self.pascal.hierarchy.get(&key) { + return Ok(hit.clone()); + } + let mut found: Option> = None; + let candidates: Vec> = self + .pascal_named(lower)? + .into_iter() + .filter(|n| MEMBER_KINDS.contains(&n.kind.as_str())) + .collect(); + if !candidates.is_empty() { + for c in self.pascal_ancestors(cls)? { + let owner = c.qualified_name.to_lowercase(); + let members: Vec> = candidates + .iter() + .filter(|n| { + n.file_path == c.file_path + && pascal_owner_of(n) == owner + }) + .cloned() + .collect(); + if !members.is_empty() { + found = Some( + members + .iter() + .find(|n| !pascal_needs_arguments(n)) + .cloned() + .unwrap_or_else(|| members[0].clone()), + ); + break; + } + } + } + self.pascal.hierarchy.insert(key, found.clone()); + Ok(found) + } + + /// unitDeclaration — a unit's top-level declaration of a name (its + /// enum values included), as code in `from_file` sees it: another + /// unit's member that its parse lost (see PasOutline::routine_column) + /// is not one, while in its own unit it is most likely the member the + /// code calls. + fn pascal_unit_declaration( + &mut self, + file_path: &str, + lower: &str, + from_file: &str, + ) -> Res>> { + let routine_column = if from_file != file_path { + self.pascal_outline(file_path)?.routine_column + } else { + i64::MAX + }; + let decls: Vec> = self + .pascal_named(lower)? + .into_iter() + .filter(|n| { + n.file_path == file_path + && UNIT_KINDS.contains(&n.kind.as_str()) + && (n.kind == "enum_member" + || !n.qualified_name.contains("::")) + && !(from_file != file_path + && n.kind == "function" + && n.start_column > routine_column) + }) + .collect(); + // A routine declared in the interface and defined in the + // implementation is one name; of overloads, one called without + // arguments first. + Ok(decls + .iter() + .find(|n| !pascal_is_routine(n)) + .cloned() + .or_else(|| { + decls + .iter() + .find(|n| !pascal_needs_arguments(n)) + .cloned() + }) + .or_else(|| decls.first().cloned())) + } + + /// usedUnitFiles — the files of the units a file uses, in the order + /// its `uses` clauses name them. The unit's `uses` clauses, then those + /// of the files it includes: mORMot's `.inc` files open the unit's + /// implementation with one. + fn pascal_used_unit_files(&mut self, from_file: &str) -> Res> { + if let Some(hit) = self.pascal.used_files.get(from_file) { + return Ok(hit.clone()); + } + let unit = self.pascal_unit_files_of(from_file)?; + let mut files: Vec = Vec::new(); + for own in unit.clone() { + let imports = self.pascal_outline(&own)?.imports.clone(); + for imp in imports { + if let Some(file) = + self.pascal_unit_file_named(&imp.name.to_lowercase(), &own)? + { + if !unit.contains(&file) && !files.contains(&file) { + files.push(file); + } + } + } + } + self.pascal + .used_files + .insert(from_file.to_string(), files.clone()); + Ok(files) + } + + /// ownUnitDeclaration — the unit's own declaration of a name, seen + /// from one of its files (see pascal_own_unit_files). + fn pascal_own_unit_declaration( + &mut self, + from_file: &str, + lower: &str, + ) -> Res>> { + for file in self.pascal_own_unit_files(from_file)?.iter().cloned().collect::>() { + if let Some(decl) = self.pascal_unit_declaration(&file, lower, &file)? { + return Ok(Some(decl)); + } + } + Ok(None) + } + + /// ownUnitFiles — the files whose declarations are a file's unit's + /// own, the file first: the unit is its own file and the include + /// files it pulls in, so code in `mormot.core.os.pas` sees a variable + /// `mormot.core.os.posix.inc` declares, and that file's code sees the + /// unit's (see pascal_unit_files_of). + fn pascal_own_unit_files(&mut self, file_path: &str) -> Res>> { + if let Some(hit) = self.pascal.own_files.get(file_path) { + return Ok(hit.clone()); + } + let unit = self.pascal_unit_files_of(file_path)?; + let own: Vec = if unit.first().map(String::as_str) == Some(file_path) { + unit + } else { + let mut v = vec![file_path.to_string()]; + v.extend(unit.into_iter().filter(|f| f != file_path)); + v + }; + let rc = Rc::new(own); + self.pascal + .own_files + .insert(file_path.to_string(), rc.clone()); + Ok(rc) + } + + /// unitFilesOf — the files one unit is made of: its own file, then + /// the include files it pulls in (`{$I name.inc}`, `{$INCLUDE name}`), + /// nested ones too. For an include file, the unit that includes it, + /// nearest it when several do. + fn pascal_unit_files_of(&mut self, file_path: &str) -> Res> { + if let Some(hit) = self.pascal.unit_file_sets.get(file_path) { + return Ok(hit.as_ref().clone()); + } + let mut root = file_path.to_string(); + for _ in 0..3 { + if !re!(r"(?i)\.inc$").is_match(&root) { + break; + } + match self.pascal_includer_of(&root)? { + Some(up) if up != file_path => root = up, + _ => break, + } + } + let mut files = vec![root]; + let mut i = 0usize; + while i < files.len() && files.len() < 32 { + let incs = self.pascal_includes_of(&files[i].clone())?; + for inc in incs.iter() { + if !files.contains(inc) { + files.push(inc.clone()); + } + } + i += 1; + } + if !files.iter().any(|f| f == file_path) { + files.push(file_path.to_string()); + } + let rc = Rc::new(files); + self.pascal + .unit_file_sets + .insert(file_path.to_string(), rc.clone()); + Ok(rc.as_ref().clone()) + } + + /// includesOf — the include files a Pascal file pulls in + /// (`{$I name.inc}`, `{$INCLUDE name}`), each the one of that name + /// nearest it. + fn pascal_includes_of(&mut self, file_path: &str) -> Res>> { + if let Some(hit) = self.pascal.includes.get(file_path) { + return Ok(hit.clone()); + } + let mut includes: Vec = Vec::new(); + if re!(PASCAL_SOURCE).is_match(file_path) { + if let Some(source) = self.read_file(file_path) { + for text in source.iter() { + if !text.contains("{$") + || !re!(r"(?i)\{\$i(?:nclude)?\s").is_match(text) + { + continue; + } + let project = self.pascal_project_files()?; + for m in + re!(r#"(?i)\{\$(?:I|INCLUDE)\s+['"]?([^'"}\s*]+)"#) + .captures_iter(text) + { + let name = m[1].replace('\\', "/"); + let mut name = name + .rsplit('/') + .next() + .unwrap_or(&name) + .to_lowercase(); + if !name.contains('.') { + name += ".inc"; + } + let candidates = project + .by_name + .get(&name) + .cloned() + .unwrap_or_default(); + if let Some(inc) = pascal_nearest(candidates, file_path) { + if inc != file_path && !includes.contains(&inc) { + includes.push(inc); + } + } + } + } + } + } + let rc = Rc::new(includes); + self.pascal + .includes + .insert(file_path.to_string(), rc.clone()); + Ok(rc) + } + + /// includerOf — the Pascal file that includes an include file: of the + /// forty nearest it, the nearest whose text does. The files of its + /// directory first, those named most like it first of them + /// (`mormot.core.os.pas` for `mormot.core.os.posix.inc`): how many + /// characters of its path a file shares, the directory counting + /// first. + fn pascal_includer_of(&mut self, inc_file: &str) -> Res> { + if let Some(hit) = self.pascal.includers.get(inc_file) { + return Ok(hit.clone()); + } + let pascal = self.pascal_project_files()?.pascal.clone(); + let ib = inc_file.as_bytes(); + let shared = |f: &str| -> i64 { + let fb = f.as_bytes(); + let mut n = 0usize; + while n < fb.len() && n < ib.len() && fb[n] == ib[n] { + n += 1; + } + let slash = f[..n.min(f.len())].rfind('/').map(|i| i as i64).unwrap_or(-1); + slash * 1000 + n as i64 + }; + let mut candidates: Vec<(i64, String)> = pascal + .iter() + .filter(|f| f.as_str() != inc_file) + .map(|f| (shared(f), f.clone())) + .collect(); + candidates.sort_by_key(|a| std::cmp::Reverse(a.0)); + candidates.truncate(40); + let mut includer: Option = None; + for (_, f) in candidates { + if self.pascal_includes_of(&f)?.iter().any(|i| i == inc_file) { + includer = Some(f); + break; + } + } + self.pascal + .includers + .insert(inc_file.to_string(), includer.clone()); + Ok(includer) + } + + /// projectFiles — the project's files by lowercased name, and its + /// Pascal sources. + fn pascal_project_files(&mut self) -> Res> { + if let Some(hit) = &self.pascal.project_files { + return Ok(hit.clone()); + } + let files: Vec = match self.sorted_files() { + Some(f) => f.as_ref().clone(), + None => { + let mut f: Vec = + self.table()?.files.iter().cloned().collect(); + f.sort(); + f + } + }; + let mut by_name: HashMap> = HashMap::new(); + let mut pascal: Vec = Vec::new(); + for file in &files { + let base = file.rsplit('/').next().unwrap_or(file).to_lowercase(); + by_name.entry(base).or_default().push(file.clone()); + if re!(PASCAL_SOURCE).is_match(file) { + pascal.push(file.clone()); + } + } + let rc = Rc::new(PasProjectFiles { by_name, pascal }); + self.pascal.project_files = Some(rc.clone()); + Ok(rc) + } + + /// unitFileNamed — the file of the unit a name names, nearest to + /// `from_file` when several units share it. + fn pascal_unit_file_named( + &mut self, + lower: &str, + from_file: &str, + ) -> Res> { + let key = format!("{from_file}|{lower}"); + if let Some(hit) = self.pascal.unit_files.get(&key) { + return Ok(hit.clone()); + } + if self.pascal.units_by_name.is_none() { + let mut units: HashMap> = HashMap::new(); + let mut headed: HashSet = HashSet::new(); + for file in self.pascal_project_files()?.pascal.clone() { + for n in self.nodes_in_file(&file)?.iter() { + if n.kind == "module" && n.language == "pascal" { + units + .entry(n.name.to_lowercase()) + .or_default() + .push(n.file_path.clone()); + headed.insert(n.file_path.clone()); + } + } + } + // A unit's file is named after it, which Delphi enforces: one + // whose `unit` header the parser lost to an error is the unit + // its file names. + for file in self.pascal_project_files()?.pascal.clone() { + if headed.contains(&file) + || !file.to_lowercase().ends_with(".pas") + { + continue; + } + let base = file.rsplit('/').next().unwrap_or(&file); + if let Some(stem) = base.get(..base.len().saturating_sub(4)) { + units + .entry(stem.to_lowercase()) + .or_default() + .push(file.clone()); + } + } + self.pascal.units_by_name = Some(Rc::new(units)); + } + let units = self.pascal.units_by_name.as_ref().unwrap().clone(); + let file = pascal_nearest( + units.get(lower).cloned().unwrap_or_default(), + from_file, + ); + self.pascal.unit_files.insert(key, file.clone()); + Ok(file) + } + + /// What scope lookups need of a file's nodes, read once: its classes, + /// its `uses` in order, its routine indentation limit. + fn pascal_outline(&mut self, file_path: &str) -> Res> { + if let Some(hit) = self.pascal.outlines.get(file_path) { + return Ok(hit.clone()); + } + let nodes = self.nodes_in_file(file_path)?; + let mut routine_column = i64::MAX; + if !nodes.iter().any(|n| n.kind == "module") { + for n in nodes.iter() { + if n.kind == "function" && !n.qualified_name.contains("::") { + routine_column = routine_column.min(n.start_column); + } + } + } + let classes: Vec> = nodes + .iter() + .filter(|n| CLASS_KINDS.contains(&n.kind.as_str())) + .cloned() + .collect(); + let mut imports: Vec> = nodes + .iter() + .filter(|n| n.kind == "import" && n.language == "pascal") + .cloned() + .collect(); + imports.sort_by_key(|n| (n.start_line, n.start_column)); + let rc = Rc::new(PasOutline { + classes, + imports, + routine_column, + }); + self.pascal + .outlines + .insert(file_path.to_string(), rc.clone()); + Ok(rc) + } + + /// classNamed — a class (or record or interface) a type name names, + /// seen from `from_file`; a type alias names the class it stands for + /// (`TVTAncestor = TVTAncestorVcl;`). + fn pascal_class_named( + &mut self, + type_name: &str, + from_file: &str, + ) -> Res>> { + self.pascal_class_named_depth(type_name, from_file, 0) + } + + fn pascal_class_named_depth( + &mut self, + type_name: &str, + from_file: &str, + depth: usize, + ) -> Res>> { + let simple = type_name + .rsplit('.') + .next() + .unwrap_or(type_name) + .to_lowercase(); + let key = format!("{from_file}|{simple}"); + if let Some(hit) = self.pascal.class_by_name.get(&key) { + return Ok(hit.clone()); + } + let named = self.pascal_named(&simple)?; + let mut cls = + pascal_declaration_of(self.pascal_visible_named(&named, CLASS_KINDS, from_file)?); + if cls.is_none() && depth < 4 { + for alias in + self.pascal_visible_named(&named, &["type_alias"], from_file)? + { + let line = self + .read_file(&alias.file_path) + .and_then(|s| { + s.get((alias.start_line - 1).max(0) as usize).cloned() + }) + .unwrap_or_default(); + let stripped = re!(r"//.*$|\{[^}]*\}") + .replace_all(&line, "") + .into_owned(); + if let Some(m) = + re!(r"=\s*([A-Za-z_][\w.]*)\s*;").captures(&stripped) + { + if let Some(c) = self.pascal_class_named_depth( + &m[1], + &alias.file_path, + depth + 1, + )? { + cls = Some(c); + break; + } + } + } + } + self.pascal.class_by_name.insert(key, cls.clone()); + Ok(cls) + } + + /// The candidates of `kinds` Delphi sees from `from_file`: in the + /// unit's own files, then in the unit used last, which declares the + /// name Delphi sees; else the one unit that does. + fn pascal_visible_named( + &mut self, + named: &[Arc], + kinds: &[&str], + from_file: &str, + ) -> Res>> { + let candidates: Vec> = named + .iter() + .filter(|n| kinds.contains(&n.kind.as_str())) + .cloned() + .collect(); + if candidates.is_empty() { + return Ok(candidates); + } + for file in self.pascal_own_unit_files(from_file)?.iter() { + let own: Vec> = candidates + .iter() + .filter(|n| n.file_path == *file) + .cloned() + .collect(); + if !own.is_empty() { + return Ok(own); + } + } + for file in self.pascal_used_unit_files(from_file)?.into_iter().rev() { + let there: Vec> = candidates + .iter() + .filter(|n| n.file_path == file) + .cloned() + .collect(); + if !there.is_empty() { + return Ok(there); + } + } + if candidates + .iter() + .all(|n| n.file_path == candidates[0].file_path) + { + return Ok(candidates); + } + Ok(Vec::new()) + } + + // ------------------------------------------------------------------ + // Declared types + // ------------------------------------------------------------------ + + /// typeOfMember — the class a member's value has: a field's or + /// variable's declared type, a property's, a function's result. + fn pascal_type_of_member( + &mut self, + member: &KNode, + ) -> Res>> { + if let Some(hit) = self.pascal.member_types.get(&member.id) { + return Ok(hit.clone()); + } + let type_name: Option = if matches!( + member.kind.as_str(), + "method" | "function" + ) { + member.return_type.clone().or_else(|| { + re!(r":\s*([A-Za-z_][\w.]*)\s*;?\s*$") + .captures(member.signature.as_deref().unwrap_or("")) + .map(|m| m[1].to_string()) + }) + } else { + let text = self.pascal_declared_type_text(member)?; + re!(r"^([A-Za-z_][\w.]*)") + .captures(&text) + .map(|m| m[1].to_string()) + }; + let ty = match type_name { + Some(t) => self.pascal_class_named(&t, &member.file_path)?, + None => None, + }; + self.pascal.member_types.insert(member.id.clone(), ty.clone()); + Ok(ty) + } + + /// declaredTypeText — the text of the type a field, variable, + /// constant or property declares, from its declaration on: `TFoo; + /// // the foo` for `FFoo: TFoo; // the foo`. A name in a comma list + /// (`FOnLoad,` above `FOnSave: TSaveEvent;`) takes the type the list + /// ends with, a few lines down. Empty when none is written, as for a + /// published `property OnClick;`. + fn pascal_declared_type_text(&mut self, node: &KNode) -> Res { + let Some(source) = self.read_file(&node.file_path) else { + return Ok(String::new()); + }; + let name = regex::escape(&node.name); + // `:` that does not start `:=` — Rust regex has no lookahead, so + // the colon is found first and the next byte checked by hand. + let rx = if node.kind == "property" { + regex::Regex::new(&format!( + r"(?i)\bproperty\s+{name}\s*(?:\[[^\]]*\])?\s*:" + )) + .unwrap() + } else { + regex::Regex::new(&format!( + r"(?i)\b{name}\b(?:\s*,\s*[A-Za-z_]\w*)*\s*,?\s*:" + )) + .unwrap() + }; + let strip = re!(r"//.*$|\{[^}]*\}|\(\*.*?\*\)"); + let mut text = String::new(); + let start = (node.start_line - 1).max(0) as usize; + for i in start..(start + 6).min(source.len()) { + text.push(' '); + text.push_str(&strip.replace_all(&source[i], " ")); + if let Some(m) = rx.find(&text) { + let after = &text[m.end()..]; + if !after.starts_with('=') { + return Ok(after.trim().to_string()); + } + } + // A list goes on only after a comma. + if node.kind == "property" || !re!(r",\s*$").is_match(&text) { + break; + } + } + Ok(String::new()) + } + + /// isProceduralData — whether data holds a routine, by its declared + /// type: one written in place (`FProc: procedure of object`, + /// `F: reference to function: Integer`), a project type declared as + /// one (`TOnLog = procedure(...) of object;`), or a type the project + /// doesn't declare that the RTL's names say is one (`TNotifyEvent`, + /// `TWndMethod`, `TProc`). + fn pascal_is_procedural_data(&mut self, node: &KNode) -> Res { + if let Some(hit) = self.pascal.procedural.get(&node.id) { + return Ok(*hit); + } + let text = self.pascal_declared_type_text(node)?; + let procedural = self.pascal_is_procedural_type_text(&text, 0)?; + self.pascal.procedural.insert(node.id.clone(), procedural); + Ok(procedural) + } + + /// isProceduralTypeText — whether a written type is a routine's: + /// written out (`reference to function: Integer`), or named, with the + /// name's declaration saying so, through an alias to another such + /// name (`TFormatter = TFunc;`). A name the + /// project doesn't declare is a routine type when the RTL's names + /// say so: `TProc`, `TFunc`, and the names Delphi gives them + /// (`TNotifyEvent`, `TWndMethod`). + fn pascal_is_procedural_type_text( + &mut self, + text: &str, + depth: usize, + ) -> Res { + if re!(PROCEDURAL_TYPE).is_match(text) { + return Ok(true); + } + let type_name = re!(r"(?i)^(?:type\s+)?([A-Za-z_][\w.]*)") + .captures(text) + .map(|m| { + m[1].rsplit('.').next().unwrap_or(&m[1]).to_lowercase() + }); + let Some(type_name) = type_name else { + return Ok(false); + }; + if depth > 3 { + return Ok(false); + } + let types = self.pascal_types_named(&type_name)?; + if types.is_empty() { + return Ok(RTL_PROCEDURAL_TYPES.contains(type_name.as_str()) + || re!(r"(?i)^t\w*(?:event|proc|method|callback)$") + .is_match(&type_name)); + } + for t in types { + if t.kind == "type_alias" { + let aliased = self.pascal_aliased_type_text(&t)?; + if self.pascal_is_procedural_type_text(&aliased, depth + 1)? { + return Ok(true); + } + } + } + Ok(false) + } + + /// aliasedTypeText — what a type declaration says after its `=`, + /// which may start on the next line: `procedure(...) of object` for + /// `TOnHelper =` above it. + fn pascal_aliased_type_text(&mut self, alias: &KNode) -> Res { + let Some(source) = self.read_file(&alias.file_path) else { + return Ok(String::new()); + }; + let strip = re!(r"//.*$|\{[^}]*\}|\(\*.*?\*\)"); + let mut text = String::new(); + let start = (alias.start_line - 1).max(0) as usize; + for i in start..(start + 3).min(source.len()) { + text.push(' '); + text.push_str(&strip.replace_all(&source[i], " ")); + if let Some(eq) = text.find('=') { + let rest = text[eq + 1..].trim(); + if !rest.is_empty() { + return Ok(rest.to_string()); + } + } + } + Ok(String::new()) + } + + /// typesNamed — the project's Pascal types a name names, generic + /// ones by the name before their parameters: `TSetParams` + /// for `TSetParams`. + fn pascal_types_named(&mut self, lower: &str) -> Res>> { + let generic = self.pascal_generic_types()?; + let mut out: Vec> = self + .pascal_named(lower)? + .into_iter() + .filter(|t| { + t.kind == "type_alias" || CLASS_KINDS.contains(&t.kind.as_str()) + }) + .collect(); + out.extend(generic.get(lower).cloned().unwrap_or_default()); + Ok(out) + } + + /// The generic Pascal types by the name before their parameters. + fn pascal_generic_types( + &mut self, + ) -> Res>>>> { + if let Some(hit) = &self.pascal.generic_types { + return Ok(hit.clone()); + } + let mut generic: HashMap>> = HashMap::new(); + for file in self.pascal_project_files()?.pascal.clone() { + for n in self.nodes_in_file(&file)?.iter() { + if n.language != "pascal" { + continue; + } + let Some(open) = n.name.find('<') else { + continue; + }; + if open == 0 + || !(n.kind == "type_alias" + || CLASS_KINDS.contains(&n.kind.as_str())) + { + continue; + } + let base = n.name[..open].trim().to_lowercase(); + generic.entry(base).or_default().push(n.clone()); + } + } + let rc = Rc::new(generic); + self.pascal.generic_types = Some(rc.clone()); + Ok(rc) + } + + /// yieldsValue — whether a routine gives a value — a function or a + /// constructor — rather than a procedure. + fn pascal_yields_value(&mut self, routine: &KNode) -> Res { + if routine.return_type.is_some() { + return Ok(true); + } + Ok(matches!( + self.pascal_routine_keyword(routine)?.as_deref(), + Some("function") | Some("constructor") + )) + } + + /// routineKeyword — the keyword that declares a routine + /// (`function`, `procedure`, `constructor`, …), lowercased. + fn pascal_routine_keyword( + &mut self, + routine: &KNode, + ) -> Res> { + if let Some(hit) = self.pascal.keywords.get(&routine.id) { + return Ok(hit.clone()); + } + let keyword = self + .read_file(&routine.file_path) + .and_then(|s| { + s.get((routine.start_line - 1).max(0) as usize).cloned() + }) + .and_then(|line| { + re!(r"(?i)\b(function|procedure|constructor|destructor|operator)\s") + .captures(&line) + .map(|m| m[1].to_lowercase()) + }); + self.pascal + .keywords + .insert(routine.id.clone(), keyword.clone()); + Ok(keyword) + } +} + +/// `{...result, targetNodeId, confidence, resolvedBy}` — the gate's +/// retarget keeps the rest of the verdict. +fn retarget(mut out: ResolveOutcome, decl: &PasFound) -> ResolveOutcome { + out.target_node_id = Some(decl.node.id.clone()); + out.confidence = Some(decl.confidence); + out.resolved_by = Some(decl.resolved_by.to_string()); + out +} + +fn pascal_is_routine(node: &KNode) -> bool { + matches!(node.kind.as_str(), "method" | "function") +} + +/// The qualified name of what declares a node (`TFoo::Bar` → `tfoo`), +/// lowercased. +fn pascal_owner_of(node: &KNode) -> String { + node.qualified_name + .rsplit_once("::") + .map(|(o, _)| o.to_lowercase()) + .unwrap_or_default() +} + +/// The class declaration among same-named nodes, not a forward +/// declaration (`TFoo = class;`, which names no members or ancestors): +/// the longest one. +fn pascal_declaration_of(classes: Vec>) -> Option> { + classes + .into_iter() + .max_by_key(|c| c.end_line - c.start_line) +} + +/// conditionDepthAfter — the depth of conditional blocks after a line's +/// directives (`{$IFDEF X}` opens one, `{$ENDIF}` closes it). +fn pascal_condition_depth_after(text: &str, mut depth: i64) -> i64 { + for m in re!(r"(?i)\{\$(ifdef|ifndef|ifopt|ifend|endif|if)\b") + .captures_iter(text) + { + let directive = m[1].to_lowercase(); + depth = if directive == "endif" || directive == "ifend" { + (depth - 1).max(0) + } else { + depth + 1 + }; + } + depth +} + +/// The file whose path shares the longest directory prefix with `from`. +fn pascal_nearest(files: Vec, from: &str) -> Option { + if files.len() <= 1 { + return files.into_iter().next(); + } + let dirs: Vec<&str> = from.split('/').collect(); + let mut best: Option = None; + let mut best_score = -1i64; + for f in files { + let parts: Vec<&str> = f.split('/').collect(); + let mut score = 0usize; + while score < parts.len() && score < dirs.len() + && parts[score] == dirs[score] + { + score += 1; + } + if score as i64 > best_score { + best = Some(f); + best_score = score as i64; + } + } + best +} + +/// `TList` → `TList` — qualified names carry no generic parameters. +fn strip_generic_params(qn: &str) -> String { + re!(r"<[^>]*>").replace_all(qn, "").into_owned() +} + +/// Whether a routine has a parameter without a default value, which a +/// call must pass. +fn pascal_needs_arguments(routine: &KNode) -> bool { + let Some(signature) = routine.signature.as_deref() else { + return false; + }; + let signature = signature.trim(); + if !signature.starts_with('(') { + return false; + } + let mut depth = 0i32; + let mut end = signature.len(); + for (i, ch) in signature.char_indices() { + if ch == '(' { + depth += 1; + } else if ch == ')' { + depth -= 1; + if depth == 0 { + end = i; + break; + } + } + } + signature[1..end] + .split(';') + .any(|param| !param.trim().is_empty() && !param.contains('=')) +} diff --git a/codegraph-kernel/src/resolve/php_scope.rs b/codegraph-kernel/src/resolve/php_scope.rs index 4c317e9608..cf29c1c6bc 100644 --- a/codegraph-kernel/src/resolve/php_scope.rs +++ b/codegraph-kernel/src/resolve/php_scope.rs @@ -11,8 +11,8 @@ const PHP_ANCESTRY_CAP: usize = 80; /// alias → fully qualified name (phpFileScope). #[derive(Default)] pub(crate) struct PhpFileScope { - namespace: String, - uses: HashMap, + pub(crate) namespace: String, + pub(crate) uses: HashMap, } pub(super) fn collect_php_file_scope(text: &str) -> PhpFileScope { @@ -45,7 +45,9 @@ pub(super) fn collect_php_file_scope(text: &str) -> PhpFileScope { .get(2) .map(|a| a.as_str().to_string()) .unwrap_or_else(|| fqn.rsplit('\\').next().unwrap_or("").to_string()); - uses.insert(alias, fqn); + // PHP names resolve case-insensitively (`new field\X()` finds the + // `Field` alias), so the map keys the alias lowercased. + uses.insert(alias.to_lowercase(), fqn); } } PhpFileScope { namespace, uses } @@ -109,7 +111,7 @@ impl KernelResolver { let scope = source.as_ref().map(|s| s.php_file_scope()); let Some(scope) = scope else { return Ok(true); }; let fqn = n.qualified_name.replace("::", "\\"); - let expected = scope.uses.get(&r.reference_name).cloned().unwrap_or_else(|| if scope.namespace.is_empty() { r.reference_name.clone() } else { format!("{}\\{}",scope.namespace,r.reference_name) }); + let expected = scope.uses.get(&r.reference_name.to_lowercase()).cloned().unwrap_or_else(|| if scope.namespace.is_empty() { r.reference_name.clone() } else { format!("{}\\{}",scope.namespace,r.reference_name) }); Ok(fqn.eq_ignore_ascii_case(&expected)) } @@ -287,7 +289,7 @@ impl KernelResolver { Some(at) => (&name[..at], &name[at..]), None => (name, ""), }; - if let Some(imported) = scope.and_then(|s| s.uses.get(head)) { + if let Some(imported) = scope.and_then(|s| s.uses.get(&head.to_lowercase())) { format!("{imported}{rest}") } else if let Some(s) = scope.filter(|s| !s.namespace.is_empty()) { format!("{}\\{name}", s.namespace) diff --git a/codegraph-kernel/src/resolve/pipeline.rs b/codegraph-kernel/src/resolve/pipeline.rs index 442bae50da..b4237a37d1 100644 --- a/codegraph-kernel/src/resolve/pipeline.rs +++ b/codegraph-kernel/src/resolve/pipeline.rs @@ -182,7 +182,14 @@ impl KernelResolver { .import_mappings(&r.file_path)? .iter() .find(|i| i.local_name == head) - .map(|i| i.source.trim_start_matches('\\').to_string()); + .map(|i| i.source.trim_start_matches('\\').to_string()) + // PHP aliases resolve case-insensitively (`new field\X()` + // finds `use … as Field`), which the file scope's lowercased + // keys capture where the exact-name mapping misses. + .or_else(|| { + self.read_file(&r.file_path) + .and_then(|s| s.php_file_scope().uses.get(&head.to_lowercase()).cloned()) + }); match imported { Some(source) => format!("{source}{}", &name[separator..]), None => { @@ -275,6 +282,53 @@ if super::lang_scope::is_dart_member_read(r) { None => Ok(ResolveOutcome::unresolved()), }; } + if r.language == "cfml" || r.language == "cfscript" { + // The component a `createObject("component", "a.b.Batch")` makes, + // named by its component path as `extends` names one (#2652), + // and a call chained onto a construction — + // `createObject(…).init()`, `new a.b.Batch().init()` — a method + // of that component or of nothing: no name match guesses past it. + if r.reference_kind == "instantiates" && r.reference_name.contains('.') { + return match self.resolve_cfml_component_path(r)? { + Some(c) => match self.gate_target_kind(c, r)? { + Some(winner) => self.finish_pre_framework(r, winner), + None => Ok(ResolveOutcome::unresolved()), + }, + None => Ok(ResolveOutcome::unresolved()), + }; + } + if r.reference_kind == "calls" + && re!(r"^([A-Za-z_$][\w$.-]*)\(\)\.([A-Za-z_$][\w$]*)$").is_match(&r.reference_name) + { + return match self.resolve_cfml_constructed_call(r)? { + Some(c) => match self.gate_target_kind(c, r)? { + Some(winner) => self.finish_pre_framework(r, winner), + None => Ok(ResolveOutcome::unresolved()), + }, + None => Ok(ResolveOutcome::unresolved()), + }; + } + // A CFML include names a template by its path from the including + // file's directory, or from the web root (#2651). A path no file + // matches falls to file-name matching — a file only, never an + // import statement, never the including file itself. + if r.reference_kind == "imports" && imports::is_cfml_template_path(&r.reference_name) { + let included = match self.resolve_cfml_include_path(r)? { + Some(c) => Some(c), + None => self.match_by_file_path(r)?, + }; + if included.as_ref().is_some_and(|c| c.node.file_path == r.file_path) { + return Ok(ResolveOutcome::unresolved()); + } + return match self.gate_language(included, r) { + Some(c) => match self.gate_target_kind(c, r)? { + Some(winner) => self.finish_pre_framework(r, winner), + None => Ok(ResolveOutcome::unresolved()), + }, + None => Ok(ResolveOutcome::unresolved()), + }; + } + } // A PHP class written with a namespace in it answers ahead of the // prefilter, which would drop most of them (#2256). if let Some(outcome) = self.resolve_php_qualified_class(r)? { @@ -294,6 +348,9 @@ if super::lang_scope::is_dart_member_read(r) { let pre_pass = probe!(r, "pre-pass", is_nix_path_import_ref(r) || is_js_path_import_ref(r) || self.is_js_bare_module_ref(r)? + // `new C().m` names no node by its full text; the newReceiver arm + // decides it downstream. + || r.reference_name.starts_with("new ") || self.has_any_possible_match_in(existence, &r.language) || self.matches_any_import(r)? || self.framework_claims(&r.reference_name)); @@ -400,7 +457,9 @@ if super::lang_scope::is_dart_member_read(r) { } } - // matchBoundReceiverCall — claimed refs are terminal either way. + // A rust `inner().method` chain types the inner call's return before + // the bound-receiver claim can bind the whole text to its first link. +// matchBoundReceiverCall — claimed refs are terminal either way. let namespace_chain = is_unresolved_js_member_call(r) && self.import_mappings(&r.file_path)?.iter().any(|m| m.is_namespace && m.local_name == r.reference_name.split('.').next().unwrap_or("")); @@ -459,7 +518,7 @@ if super::lang_scope::is_dart_member_read(r) { // method's, so it skips the import arm and matchReference answers alone. if r.reference_kind == "calls" && chain_shape_re().is_match(&r.reference_name) - && is_store_chain_language(&r.language) + && (is_store_chain_language(&r.language) || r.reference_name.starts_with("new ")) { return self.resolve_call_chain(r); } @@ -816,6 +875,28 @@ if super::lang_scope::is_dart_member_read(r) { } else { None }; let answer = attribute.as_ref().map(|a| (a.claimed, a.target_node_id.clone())); let mut outcome = self.resolve_ref_inner(r, answer)?; + // A Pascal call to data runs only a routine the data holds, in + // scope, and one to a routine nested in another comes only from + // inside that one (gatePascalCall, resolveOne). + if r.language == "pascal" && r.reference_kind == "calls" && outcome.status == "resolved" { + outcome = self.gate_pascal_call(outcome, r)?; + } + // A Pascal call written with a bare name that resolves to a type is + // a typecast: `TButton(Sender).Caption`, `PInt64Rec(P)^.Lo`, `F := + // TFoo(Obj)`. Delphi builds an object only through a constructor + // (`TFoo.Create`), so the code refers to the type and creates + // nothing (createEdges). + if r.language == "pascal" && r.reference_kind == "calls" + && outcome.status == "resolved" && !r.reference_name.contains('.') + { + if let Some(tid) = outcome.target_node_id.clone() { + if self.node_by_id(&tid)?.is_some_and(|t| { + pascal_scope::CAST_TARGET_KINDS.contains(&t.kind.as_str()) + }) { + outcome.edge_kind = Some("references".to_string()); + } + } + } if let Some(attribute) = attribute { if !attribute.type_from.is_empty() { outcome.type_from = Some(attribute.type_from); } } @@ -881,6 +962,30 @@ if super::lang_scope::is_dart_member_read(r) { if self.python_receiver_uncertain(r)? { return Ok(ResolveOutcome::unresolved()); } if let Some(outcome) = self.resolve_dart_written(r)? { return Ok(outcome); } if let Some(outcome) = self.resolve_vb_explicit(r)? { return Ok(outcome); } + // A Pascal name written without parentheses (#2514) links only to + // a routine its scope reaches, or through a receiver to a method + // of the receiver's declared type; data of that name links nothing. + if let Some(outcome) = self.resolve_pascal_parenless(r)? { return Ok(outcome); } + // So does a Pascal call to a name the compiler or the System unit + // gives meaning to (`SizeOf`, `TypeInfo`, `length(S)`, `Pointer(P)`), + // whatever its case: only a declaration in the call's scope shadows + // it (#2803). + if self.is_pascal_system_call(r)? { + return self.match_pascal_system_call(r); + } + // A bare Pascal call to a routine nested where it is made + // (`Inner(1)` in the routine that declares `procedure Inner(N: + // Integer);`) is that one, and one made in a nested routine is what + // its scope declares first (#2810). Undefined when that scope holds + // no routine of the name, for the other strategies to resolve. + if let Some(outcome) = self.resolve_pascal_nested_call(r)? { return Ok(outcome); } + // A Ruby namespaced constant path (`Billing::ChargeCard`, + // `::Foo.call`) resolves only to a constant the project declares, + // looked up from the lexical namespace outward; a call on one + // resolves a class method of that class or module through its Ruby + // ancestry. An unknown path or method resolves to nothing rather + // than falling back to a same-name guess (#2827, fixing #2503). + if let Some(outcome) = self.resolve_ruby_constant_path_ref(r)? { return Ok(outcome); } // Rust pure-`::` path refs (`crate::m::Item`, `a::b::c`): TS // resolves them through resolveViaImport's module-file arm, which // needs no bindings rows — run it ahead of the eligibility gate. @@ -920,6 +1025,16 @@ if super::lang_scope::is_dart_member_read(r) { }; } if is_js_family(&r.language) && r.reference_kind == "calls" { + // matchJsSuperCall: `super.m()` binds to the nearest `extends` + // ancestor's method before any name strategy can guess (#2497). + if let Some(outcome) = self.match_js_super_call(r)? { + return Ok(outcome); + } + // `this.m()` inside a class member — the extractors strip the + // receiver so only a source-form check sees it. + if let Some(outcome) = self.match_js_this_call(r)? { + return Ok(outcome); + } if let Some(caller) = self.node_by_id(&r.from_node_id)? { let name = r.reference_name.rsplit(['.', ':']).next().unwrap_or(&r.reference_name); if caller.name == name && caller.kind == "method" @@ -1184,9 +1299,12 @@ if r.language == "dart" && r.reference_kind == "references" } if cand.is_none() { // Python's `super()` is the one inner call whose result the - // class around the call names. + // class around the call names; a `new C().m` receiver names the + // class it constructs (name-matcher's newReceiver arm). cand = if r.language == "python" && r.reference_name.starts_with("super().") { self.python_super_method(r)? + } else if r.reference_name.starts_with("new ") { + self.match_new_receiver_call(r)? } else { self.match_store_accessor_chain(r)? }; diff --git a/codegraph-kernel/src/resolve/prefilter.rs b/codegraph-kernel/src/resolve/prefilter.rs index a0607a2e53..8ca8f89f89 100644 --- a/codegraph-kernel/src/resolve/prefilter.rs +++ b/codegraph-kernel/src/resolve/prefilter.rs @@ -61,11 +61,18 @@ impl KernelResolver { return true; } } - // Pascal/Delphi built-ins and standard library units. - if r.language == "pascal" - && (PASCAL_UNIT_PREFIXES.iter().any(|p| name.starts_with(p)) || PASCAL_BUILT_INS.contains(name)) - { - return true; + // Pascal/Delphi built-ins and standard library units. Pascal names + // ignore case: `writeln`, `WRITELN`, `system.TypeInfo` are the same + // built-in — the check is lowercased on both sides (upstream's + // `lower.startsWith('system.')` covers only the System unit). + if r.language == "pascal" { + let lower = name.to_lowercase(); + if PASCAL_UNIT_PREFIXES.iter().any(|p| name.starts_with(p)) + || lower.starts_with("system.") + || PASCAL_BUILT_INS.contains(lower.as_str()) + { + return true; + } } if r.language == "c" || r.language == "cpp" { let name = name.strip_prefix("::").unwrap_or(name); diff --git a/codegraph-kernel/src/resolve/resolver_upstream.rs b/codegraph-kernel/src/resolve/resolver_upstream.rs index a428a0a940..3f12b26bc3 100644 --- a/codegraph-kernel/src/resolve/resolver_upstream.rs +++ b/codegraph-kernel/src/resolve/resolver_upstream.rs @@ -13,6 +13,24 @@ pub(super) fn test_suite_path(path: &str) -> bool { || re!(r"(?:^|/)[A-Za-z0-9]*(?:Test|Tests|Spec)/").is_match(path) } +/// isTestSuitePath (name-matcher.ts): a test path whose name or directory is +/// shaped like a test suite — narrower than `test_suite_path`, which also +/// names hooks and helpers (`useTests.ts`, `IndexSpecs`). +pub(super) fn is_test_suite_path(path: &str) -> bool { + if !test_suite_path(path) { + return false; + } + let lower = path.to_ascii_lowercase(); + let name = lower.rsplit('/').next().unwrap_or(&lower); + let original = path.rsplit('/').next().unwrap_or(path); + name.starts_with("test_") + || name == "conftest.py" + || re!(r"[._-](?:test|tests|unittest|unittests)\.[a-z0-9]+$|[._](?:spec|specs)\.[a-z0-9]+$").is_match(name) + || re!(r"(?:Test|Tests|TestCase)\.(?:java|kt|kts|swift|cs|scala|groovy|m|mm|vb|fs)$").is_match(original) + || re!(r"(?:^|/)(?:tests?|__tests__|specs?|e2e|unittests|[\w.]+[-_]unittests?)/").is_match(&lower) + || re!(r"(?:^|/)[A-Za-z0-9]*(?:Test|Tests|Spec)/").is_match(path) +} + /// The binding patterns `python_binds_name` tests for a name. #[derive(Clone, Copy)] enum PyBinding { diff --git a/codegraph-kernel/src/resolve/ruby_paths.rs b/codegraph-kernel/src/resolve/ruby_paths.rs new file mode 100644 index 0000000000..a1f1ce3218 --- /dev/null +++ b/codegraph-kernel/src/resolve/ruby_paths.rs @@ -0,0 +1,206 @@ +//! matchRubyConstantPathRef (upstream src/resolution/ruby-constant-paths.ts) +//! and the Ruby class/instance method split (upstream #2834). +//! +//! A namespaced Ruby constant — `Billing::ChargeCard`, `::Foo`, +//! `::Billing::ChargeCard` — resolves only to a constant the project itself +//! declares, looked up from the lexical namespace around the reference +//! outward and then at top level (an absolute `::` path searches top level +//! only). A call on that path resolves a method of the class or module the +//! constant names, through its Ruby ancestry. A path the project does not +//! declare, or a method it does not have, resolves to nothing rather than +//! falling back to a same-name guess (#2827, fixing #2503). +//! +//! `def self.m` and `def m` share `Owner::m` in the index; `is_static` +//! distinguishes them. A call on a class constant prefers the class +//! method, one on a receiver `x = Widget.new` prefers the instance method, +//! and a bare call prefers the kind of the method it is written inside — +//! but only when both kinds really exist under the same qualified name, +//! never when the receiver is just uncertain (#2834). + +use super::*; + +impl KernelResolver { + /// matchRubyConstantPathRef: `references` names that are constant paths + /// resolve to the constant; `calls` names `Path.method` resolve to the + /// owner's method. Returns `Some` (resolved or unresolved) whenever the + /// ref spells a constant path — the claim is exclusive — and `None` for + /// every other ref. + pub(super) fn resolve_ruby_constant_path_ref(&mut self, r: &ResolveRefIn) -> Res> { + if r.language != "ruby" { + return Ok(None); + } + // A bare `Foo` or `Foo.call` is left to the name strategies; only a + // path written with its namespace (`Foo::Bar`, `::Foo`) is claimed. + let namespaced = |c: ®ex::Captures| c.get(1).is_some() || c[2].contains("::"); + if r.reference_kind == "references" { + let Some(c) = re!(r"^(::)?([A-Z]\w*(?:::[A-Z]\w*)*)$").captures(r.reference_name.trim_start_matches("()")) else { + return Ok(None); + }; + if !namespaced(&c) { + return Ok(None); + } + return match self.ruby_constants_named(&c[2], r, c.get(1).is_some())?.into_iter().next() { + Some(target) => Ok(Some(ResolveOutcome::resolved(&target, 0.9, "qualified-name", true, None))), + None => Ok(Some(ResolveOutcome::unresolved())), + }; + } + if r.reference_kind == "calls" { + let Some(c) = re!(r"^(::)?([A-Z]\w*(?:::[A-Z]\w*)*)\.([A-Za-z_]\w*[?!=]?)$").captures(r.reference_name.trim_start_matches("()")) else { + return Ok(None); + }; + if !namespaced(&c) { + return Ok(None); + } + let owner = self + .ruby_constants_named(&c[2], r, c.get(1).is_some())? + .into_iter() + .find(|n| matches!(n.kind.as_str(), "class" | "module")); + let Some(owner) = owner else { + return Ok(Some(ResolveOutcome::unresolved())); + }; + return match self.ruby_method_of(&owner.qualified_name, &c[3], r)? { + Some((method, own)) => Ok(Some(ResolveOutcome::resolved( + &method, + if own { 0.85 } else { 0.8 }, + "qualified-name", + true, + None, + ))), + None => Ok(Some(ResolveOutcome::unresolved())), + }; + } + Ok(None) + } + + /// constantsNamed: the declarations of the first of `::`, + /// `::`, … `` the project declares — the scope being + /// the class or module around the reference, `` alone for a + /// `::`-rooted name. Production code never names a constant only a test + /// declares. + fn ruby_constants_named(&mut self, path: &str, r: &ResolveRefIn, absolute: bool) -> Res>> { + let production = !super::resolver_upstream::test_suite_path(&r.file_path); + let mut prefix = if absolute { String::new() } else { self.ruby_lexical_scope(r)? }; + loop { + let qn = if prefix.is_empty() { path.to_string() } else { format!("{prefix}::{path}") }; + let found: Vec> = self + .nodes_by_qualified_name(&qn)? + .iter() + .filter(|n| { + n.language == "ruby" + && matches!(n.kind.as_str(), "class" | "module" | "variable" | "constant") + && !(production && super::resolver_upstream::test_suite_path(&n.file_path)) + }) + .cloned() + .collect(); + if !found.is_empty() { + return Ok(ruby_ordered(found, r)); + } + if prefix.is_empty() { + return Ok(Vec::new()); + } + prefix = prefix.rfind("::").map_or(String::new(), |i| prefix[..i].to_string()); + } + } + + /// lexicalScope: the qualified name of the innermost class or module + /// around `r`, or empty at the top level of its file. + fn ruby_lexical_scope(&mut self, r: &ResolveRefIn) -> Res { + let mut inner: Option> = None; + for n in self.nodes_in_file(&r.file_path)?.iter() { + if n.language != "ruby" || !matches!(n.kind.as_str(), "class" | "module") || n.start_line > r.line || n.end_line < r.line { + continue; + } + if inner.as_ref().is_none_or(|i| n.start_line > i.start_line || (n.start_line == i.start_line && n.start_column > i.start_column)) { + inner = Some(n.clone()); + } + } + Ok(inner.map(|i| i.qualified_name.clone()).unwrap_or_default()) + } + + /// methodOf: `name` declared on `owner_qn` itself (kind `class` first), + /// else on the first ancestor in its Ruby ancestry that declares it. + /// Returns the method and whether it is the owner's own. `r` is the call + /// site: production code never reaches a method only a test declares. + pub(super) fn ruby_method_of(&mut self, owner_qn: &str, name: &str, r: &ResolveRefIn) -> Res, bool)>> { + let production = !super::resolver_upstream::test_suite_path(&r.file_path); + let own: Vec> = self + .nodes_by_qualified_name(&format!("{owner_qn}::{name}"))? + .iter() + .filter(|n| n.language == "ruby" && n.kind == "method" && !(production && super::resolver_upstream::test_suite_path(&n.file_path))) + .cloned() + .collect(); + if let Some(m) = ruby_methods_of_kind_first(ruby_ordered(own, r), "class").into_iter().next() { + return Ok(Some((m, true))); + } + let ancestry = self.ruby_ancestry(owner_qn)?; + let mut ancestors: Vec<&String> = ancestry.iter().filter(|q| q.as_str() != owner_qn).collect(); + ancestors.sort(); + for qn in ancestors { + let found: Vec> = self + .nodes_by_qualified_name(&format!("{qn}::{name}"))? + .iter() + .filter(|n| n.language == "ruby" && n.kind == "method" && !(production && super::resolver_upstream::test_suite_path(&n.file_path))) + .cloned() + .collect(); + if let Some(m) = ruby_methods_of_kind_first(ruby_ordered(found, r), "class").into_iter().next() { + return Ok(Some((m, false))); + } + } + Ok(None) + } + + /// rubyMethodsOfCallerKind: of the methods `build` could mean, the ones + /// whose `is_static` matches the enclosing method's stay. Applies only + /// when a qualified name holds both kinds and the caller is itself a + /// Ruby method — a bare call from a block or the file's top level keeps + /// every candidate. + pub(super) fn ruby_methods_of_caller_kind(&mut self, methods: Vec>, r: &ResolveRefIn) -> Res>> { + let mut static_qns = HashSet::new(); + let mut instance_qns = HashSet::new(); + for m in methods.iter().filter(|m| m.language == "ruby" && m.kind == "method") { + if m.is_static { + static_qns.insert(m.qualified_name.clone()); + } else { + instance_qns.insert(m.qualified_name.clone()); + } + } + let twins: HashSet = static_qns.intersection(&instance_qns).cloned().collect(); + if twins.is_empty() { + return Ok(methods); + } + let caller_static = match self.node_by_id(&r.from_node_id)? { + Some(c) if c.language == "ruby" && c.kind == "method" => c.is_static, + _ => return Ok(methods), + }; + Ok(methods + .into_iter() + .filter(|m| !(m.language == "ruby" && m.kind == "method" && twins.contains(&m.qualified_name) && m.is_static != caller_static)) + .collect()) + } +} + +/// `ordered`: anything before a value, then the reference's own file, then +/// production code; the index's order within each. +fn ruby_ordered(mut nodes: Vec>, r: &ResolveRefIn) -> Vec> { + if nodes.len() < 2 { + return nodes; + } + let rank = |n: &KNode| { + matches!(n.kind.as_str(), "variable" | "constant") as u8 * 4 + + (n.file_path != r.file_path) as u8 * 2 + + super::resolver_upstream::test_suite_path(&n.file_path) as u8 + }; + nodes.sort_by_key(|n| rank(n)); + nodes +} + +/// rubyMethodsOfKindFirst: methods whose `is_static` matches the wanted +/// kind, then the rest in their original order. `kind` is `"class"` for +/// calls on a class or module constant and `"instance"` for calls on a +/// receiver bound to an instance. +pub(super) fn ruby_methods_of_kind_first(methods: Vec>, kind: &str) -> Vec> { + let want = kind == "class"; + let (mut same, other): (Vec>, Vec>) = methods.into_iter().partition(|m| m.is_static == want); + same.extend(other); + same +} diff --git a/codegraph-kernel/src/resolve/rust_types.rs b/codegraph-kernel/src/resolve/rust_types.rs index ebfad90403..0a811164d3 100644 --- a/codegraph-kernel/src/resolve/rust_types.rs +++ b/codegraph-kernel/src/resolve/rust_types.rs @@ -995,6 +995,33 @@ impl KernelResolver { let Some(body) = node.child_by_field_name("body") else { continue; }; + // `self.N` — a tuple struct's fields are unnamed; the Nth + // positional member's type is the field's type. + if let Ok(index) = field.parse::() { + if body.kind() == "ordered_field_declaration_list" { + let members: Vec<_> = rust_children(body) + .into_iter() + .filter(|n| !matches!(n.kind(), "visibility_modifier" | "attribute_item")) + .collect(); + if let Some(ty) = members.get(index).map(|n| { + n.child_by_field_name("type").unwrap_or(*n) + }) { + let field_type = rust_text(ty, source.text()).into(); + return self.rust_instantiate_type( + Written { + text: field_type, + file: decl.file_path.clone(), + at: node.start_byte(), + }, + Some(&w), + node, + source.text(), + r, + depth, + ); + } + } + } let field_node = rust_children(body).into_iter().find(|n| { n.child_by_field_name("name") .is_some_and(|n| rust_text(n, source.text()) == field) @@ -1697,6 +1724,14 @@ impl KernelResolver { if r.language != "rust" || r.reference_kind != "calls" { return Ok(None); } + // `inner().method` — the ref's position anchors on the INNER call, + // so the source anchor below would answer for the wrong link; the + // chain arm types each link's return by name instead. + if call_chain_re().is_match(&r.reference_name) { + if let Some(hit) = self.match_call_chain(r)? { + return Ok(Some(Some(hit))); + } + } let Some((source, tree)) = self.rust_tree_for(&r.file_path) else { return Ok(None); }; diff --git a/codegraph-kernel/src/resolve/tables.rs b/codegraph-kernel/src/resolve/tables.rs index 4aedd9e378..e34a7ce77f 100644 --- a/codegraph-kernel/src/resolve/tables.rs +++ b/codegraph-kernel/src/resolve/tables.rs @@ -1033,9 +1033,57 @@ pub(super) const PASCAL_UNIT_PREFIXES: [&str; 15] = [ "IdHTTP", "IdTCP", "IdSSL", ]; -/// PASCAL_BUILT_INS (index.ts): standard units, intrinsics and RTL classes. -pub(super) static PASCAL_BUILT_INS: LazyLock> = LazyLock::new(|| { +/// PASCAL_SYSTEM_NAMES (extraction/languages/pascal-system.ts): the names +/// Delphi's compiler and System unit give meaning to — intrinsics, the +/// System/SysUtils routines everyone calls, and the built-in types a +/// typecast names. All lowercase; Pascal names ignore case. +pub(super) static PASCAL_SYSTEM_NAMES: LazyLock> = LazyLock::new(|| { + [ + // Intrinsics and System routines + "abs", "addr", "arctan", "assert", "assigned", "assignfile", "atomiccmpexchange", + "atomicdecrement", "atomicexchange", "atomicincrement", "blockread", "blockwrite", + "break", "chdir", "chr", "closefile", "concat", "continue", "copy", "cos", "dec", + "default", "delete", "dispose", "eof", "eoln", "exclude", "exit", "exp", "fail", + "filepos", "filesize", "fillchar", "finalize", "frac", "freemem", "getdir", + "getmem", "gettypekind", "halt", "hasweakref", "hi", "high", "inc", "include", + "initialize", "insert", "int", "ioresult", "isconstvalue", "ismanagedtype", + "length", "ln", "lo", "low", "mkdir", "move", "new", "odd", "ord", "paramcount", + "paramstr", "pi", "pred", "ptr", "random", "randomize", "read", "readln", + "reallocmem", "returnaddress", "rmdir", "round", "runerror", "setlength", + "setstring", "sin", "sizeof", "slice", "sqr", "sqrt", "str", "succ", "swap", + "trunc", "typehandle", "typeinfo", "typeof", "uniquestring", "upcase", "val", + "vararrayredim", "write", "writeln", + // SysUtils routines the resolver has always taken for the RTL's + "abort", "date", "datetostr", "floattostr", "format", "freeandnil", "inttostr", + "lowercase", "now", "pos", "strtodate", "strtofloat", "strtoint", "time", + "trim", "uppercase", + // Built-in types, called as typecasts + "ansichar", "ansistring", "boolean", "byte", "bytebool", "cardinal", "char", + "comp", "currency", "double", "extended", "int16", "int32", "int64", "int8", + "integer", "longbool", "longint", "longword", "nativeint", "nativeuint", + "olevariant", "pansichar", "pchar", "pointer", "pwidechar", "rawbytestring", + "real", "shortint", "shortstring", "single", "smallint", "string", "uint16", + "uint32", "uint64", "uint8", "unicodestring", "utf8string", "variant", + "widechar", "widestring", "word", "wordbool", + ] + .into_iter() + .collect() +}); + +/// RTL routine-type names (`TProc`, `TFunc`) and the names Delphi gives +/// them — used by pascal_scope's procedural-data check. +pub(super) static RTL_PROCEDURAL_TYPES: LazyLock> = LazyLock::new(|| { [ + "tproc", "tfunc", "tpredicate", "tcomparison", "tthreadprocedure", + "tlistsortcompare", "tstringlistsortcompare", + ] + .into_iter() + .collect() +}); + +/// PASCAL_BUILT_INS (index.ts): standard units, intrinsics and RTL classes. +pub(super) static PASCAL_BUILT_INS: LazyLock> = LazyLock::new(|| { + let names: HashSet = [ "System", "SysUtils", "Classes", "Types", "Variants", "StrUtils", "Math", "DateUtils", "IOUtils", "Generics.Collections", "Generics.Defaults", "Rtti", "TypInfo", "SyncObjs", "RegularExpressions", @@ -1055,7 +1103,9 @@ pub(super) static PASCAL_BUILT_INS: LazyLock> = LazyLock:: "IInterface", "IUnknown", ] .into_iter() - .collect() + .map(|s: &str| s.to_lowercase()) + .collect(); + names }); pub(super) static GO_BUILT_INS: LazyLock> = LazyLock::new(|| { diff --git a/codegraph-kernel/src/resolve/ts_known.rs b/codegraph-kernel/src/resolve/ts_known.rs index 1db65e0bce..a6cd234f06 100644 --- a/codegraph-kernel/src/resolve/ts_known.rs +++ b/codegraph-kernel/src/resolve/ts_known.rs @@ -403,7 +403,9 @@ impl KernelResolver { let Some(source) = self.read_file(&cls.file_path) else { return false }; let from = (cls.start_line - 1).max(0) as usize; let head = source.get(from..(from + 3).min(source.len())).unwrap_or_default().join(" "); - let head = head.split('{').next().unwrap_or(""); + // No `{`-cut: `class Shaped` carries a + // brace inside its parameter list; the `>` scan below is depth-aware. + let head = head.as_str(); let Ok(open) = Self::cached_regex(&format!(r"(?-u:\b){}\s*<", regex::escape(&cls.name))) else { return false }; let Some(open) = open.find(head) else { return false }; // The parameters between the `<` after the name and its own `>`, split diff --git a/codegraph-kernel/src/ruby.rs b/codegraph-kernel/src/ruby.rs index 4eaf8497a0..bb239b341c 100644 --- a/codegraph-kernel/src/ruby.rs +++ b/codegraph-kernel/src/ruby.rs @@ -17,7 +17,7 @@ use crate::buffers::{ node_kind_index, Arena, BoolFlags, EmitOut, NodeRow, - RefRow, Tables, NONE_STR, + RefRow, Tables, FLAG_IS_STATIC, NONE_STR, REF_FLAG_FILE_PATH, }; use crate::walker::named_kids; @@ -85,6 +85,9 @@ struct Extra { docstring: Option, signature: Option, visibility: Option, + /// The isStatic hook (languages/ruby.ts): set on every def, true for a + /// class method (is_class_method). + is_static: Option, } @@ -169,10 +172,15 @@ impl<'t> Walker<'t> { let id_ref = self.arena.put(&id); let doc_ref = self.arena.put_opt(extra.docstring.as_deref()); let sig_ref = self.arena.put_opt(extra.signature.as_deref()); + // No isExported/isAsync hooks; isStatic only for defs. + let mut flags = BoolFlags::default(); + if let Some(v) = extra.is_static { + flags.set(FLAG_IS_STATIC, v); + } let row = self.tables.push_node(&NodeRow { kind: node_kind_index(kind).unwrap(), visibility: extra.visibility.unwrap_or(0), - flags: BoolFlags::default(), // no isExported/isAsync/isStatic hooks + flags, start_line, end_line, start_column: self.col_of(node), @@ -472,6 +480,7 @@ impl<'t> Walker<'t> { docstring: preceding_docstring(node, self.src), signature: None, // no getSignature hook visibility: Some(self.visibility_of(node)), + is_static: Some(is_class_method(node)), }; let Some(row) = self.create_node("function", &name, node, extra) else { return }; // (ruby ∉ TYPE_ANNOTATION_LANGUAGES; decorators are a structural no-op) @@ -490,6 +499,7 @@ impl<'t> Walker<'t> { docstring: preceding_docstring(node, self.src), signature: None, visibility: Some(self.visibility_of(node)), + is_static: Some(is_class_method(node)), }; let Some(row) = self.create_node("method", &name, node, extra) else { return }; self.stack.push(Scope { row, kind: "method", name }); @@ -506,6 +516,7 @@ impl<'t> Walker<'t> { docstring: preceding_docstring(node, self.src), signature: None, visibility: Some(self.visibility_of(node)), + ..Extra::default() }; let Some(row) = self.create_node("class", &name, node, extra) else { return }; @@ -548,7 +559,7 @@ impl<'t> Walker<'t> { let name = self.text(left).to_string(); let signature = right.map(|r| util::init_signature(self.text(r))); let row = - self.create_node("variable", &name, node, Extra { docstring, signature, visibility: None }); + self.create_node("variable", &name, node, Extra { docstring, signature, visibility: None, ..Extra::default() }); // visit_node skips an assignment's children, so the right-hand side's // string literals are reached here, owned by the assigned name. if let (Some(row), Some(r)) = (row, right) { @@ -674,10 +685,12 @@ impl<'t> Walker<'t> { }; self.push_ref(caller, &callee, calls_kind, line, col); - // Capitalized constant receiver (`Klass.static_call`, `RETRY_MAX.times`) - // → an ADDITIONAL references ref at the RECEIVER's position. - // (scope_resolution receivers get none — type ≠ constant.) - if !skip && receiver.kind() == "constant" { + // Capitalized constant receiver (`Klass.static_call`, `RETRY_MAX.times`), + // or one written with its namespace (`Billing::ChargeCard.call`, + // `::Foo.call` — #2503) → an ADDITIONAL references ref at the + // RECEIVER's position, under the receiver as written. A path through + // a value (`obj::Thing`, `self.class::LIMIT`) names no constant. + if !skip && is_constant_path(receiver) { self.push_ref_at( caller, receiver_name, @@ -874,6 +887,35 @@ impl<'t> Walker<'t> { } } +/// A constant, or a `scope_resolution` path of constants (`A::B::C`, a +/// top-level `::A` / `::A::B`): what Ruby reads as a constant (isRubyConstantPath). +fn is_constant_path(node: Node) -> bool { + match node.kind() { + "constant" => true, + "scope_resolution" => { + node.child_by_field_name("name").is_some_and(|n| n.kind() == "constant") + && node.child_by_field_name("scope").is_none_or(is_constant_path) + } + _ => false, + } +} + +/// isRubyClassMethod (languages/ruby.ts): `def self.x` / `def Widget.x` (a +/// singleton_method), or a `def` inside `class << self`. +fn is_class_method(node: Node) -> bool { + match node.kind() { + "singleton_method" => true, + "method" => { + let mut parent = node.parent(); + if parent.is_some_and(|p| p.kind() == "body_statement") { + parent = parent.and_then(|p| p.parent()); + } + parent.is_some_and(|p| p.kind() == "singleton_class") + } + _ => false, + } +} + /// The Ruby `call` node whose argument_list (or keyword pair) contains `node` /// — nearest `call` ancestor within 4 parent hops (function-ref.ts:837). fn ruby_enclosing_call(node: Node) -> Option { diff --git a/codegraph-kernel/src/rustlang/mod.rs b/codegraph-kernel/src/rustlang/mod.rs index dfc767d5f3..bbf8230c3c 100644 --- a/codegraph-kernel/src/rustlang/mod.rs +++ b/codegraph-kernel/src/rustlang/mod.rs @@ -24,10 +24,13 @@ //! - Chained-call re-encode is scoped_identifier-gated (`Foo::new().bar()` → //! `Foo::new().bar`); a call through a field of the enclosing type keeps //! the owner-field shape (`self.inner.run()` → `self.inner.run`, #1585); -//! instance chains, parens, `.await`, deeper/non-self field chains, and -//! bare `self` receivers all collapse to the bare method name (`self` is -//! node kind `self`, not `identifier`, so it dodges SKIP_RECEIVERS by -//! falling through). Turbofish callees keep the raw `helper::` text. +//! a longer chain of fields and method calls on `self` or a name keeps +//! that chain, calls as `name()` (`self.services.store.put()` → +//! `self.services.store.put`, `self.store().put()` → `self.store().put`, +//! #2836); other instance chains, parens, `?`, `.await`, and longer +//! path-rooted chains collapse to the bare method name. A bare `self` +//! receiver keeps `self.` (#1861). Turbofish callees drop their type +//! arguments (`helper::` → `helper`). //! - `use` emits an import node named by the ROOT module (`crate`/`self`/…), //! one root `imports` ref, then one FULL-path `imports` ref per binding; //! `use x::*` (use_wildcard) emits nothing at all. @@ -662,10 +665,13 @@ impl<'t> Walker<'t> { } } "call_expression" => { - // Chained-call re-encode: ONLY an associated- + // Chained-call re-encode: an associated- // function chain (`Foo::new().bar()`, inner - // callee a scoped_identifier). Instance chains - // keep the bare method name. + // callee a scoped_identifier). A chain of + // fields and method calls on `self` or a name + // (`self.store().put()`, #2836) keeps that + // chain; any other instance chain the bare + // method name. let inner_fn = r.child_by_field_name("function").map(unwrap_generic_function); let reencode = inner_fn.map(|f| f.kind() == "scoped_identifier").unwrap_or(false); @@ -678,6 +684,8 @@ impl<'t> Walker<'t> { .collect(); let inner = strip_turbofish(&inner); callee_name = format!("{inner}().{method_name}"); + } else if let Some(chain) = receiver_chain(r, self.src) { + callee_name = format!("{chain}.{method_name}"); } else { callee_name = method_name.to_string(); } @@ -687,9 +695,11 @@ impl<'t> Walker<'t> { // field of the enclosing type (#1585): keep the // `self.` prefix so the resolver can type the // field from the owner struct's declaration - // (or leave it unresolved). Any other - // field_expression receiver — a deeper chain, - // a non-self base — keeps the bare name. + // (or leave it unresolved). A longer chain of + // fields and method calls on `self` or a name + // (`self.services.store.put()`, `vis.map.get()`, + // #2836) keeps that chain; any other + // field_expression receiver the bare name. let base = r.child_by_field_name("value"); let field = r.child_by_field_name("field"); match (base, field) { @@ -699,7 +709,12 @@ impl<'t> Walker<'t> { let field_name = self.text(f); callee_name = format!("self.{field_name}.{method_name}"); } - _ => callee_name = method_name.to_string(), + _ => { + callee_name = match receiver_chain(r, self.src) { + Some(chain) => format!("{chain}.{method_name}"), + None => method_name.to_string(), + } + } } } // `self.method()` — keep the `self.` prefix so the @@ -1037,6 +1052,64 @@ fn unwrap_generic_function(func: Node<'_>) -> Node<'_> { } } +/// The most links a receiver chain is kept with (receiver_chain). +const RECEIVER_CHAIN_MAX_LINKS: usize = 16; + +fn is_rust_ident(s: &str) -> bool { + let mut chars = s.chars(); + matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// The receiver chain a method call is written on, when it is fields and +/// method calls on `self` or on a name: `self.services.store`, +/// `self.store()`, `vis.map`, `self.0`, `x.get::(1)` — each field by its +/// name or index, each call as `name()`, its arguments and turbofish +/// dropped. None for any other receiver (a path's call, a literal, +/// parentheses, `?`, `.await`, an index, a macro), and past +/// RECEIVER_CHAIN_MAX_LINKS links. Mirrors rustReceiverChain in +/// TreeSitterExtractor (#2836): both emit the same bytes. +fn receiver_chain(node: Node<'_>, src: &str) -> Option { + let mut links: Vec = Vec::new(); + let mut cur = node; + while links.len() <= RECEIVER_CHAIN_MAX_LINKS { + if cur.kind() == "self" || cur.kind() == "identifier" { + let root = &src[cur.byte_range()]; + if links.is_empty() || !is_rust_ident(root) { + return None; + } + links.reverse(); + let mut out = root.to_string(); + for link in &links { + out.push('.'); + out.push_str(link); + } + return Some(out); + } + let call = cur.kind() == "call_expression"; + let field_of = if call { + unwrap_generic_function(cur.child_by_field_name("function")?) + } else { + cur + }; + if field_of.kind() != "field_expression" { + return None; + } + let name = field_of.child_by_field_name("field").map(|f| &src[f.byte_range()]).unwrap_or(""); + let ok = if call { + is_rust_ident(name) + } else { + is_rust_ident(name) || (!name.is_empty() && name.chars().all(|c| c.is_ascii_digit())) + }; + if !ok { + return None; + } + links.push(if call { format!("{name}()") } else { name.to_string() }); + cur = field_of.child_by_field_name("value")?; + } + None +} + /// A callee path without its turbofish type arguments: `Vec::::new` → /// `Vec::new`. The brackets nest (`::>`), and the `>` of a `->` is /// no closing one (`:: u8>`). Mirrors stripRustTurbofish in diff --git a/codegraph-kernel/src/tsjs/calls.rs b/codegraph-kernel/src/tsjs/calls.rs index 1c970d5420..eac0849a10 100644 --- a/codegraph-kernel/src/tsjs/calls.rs +++ b/codegraph-kernel/src/tsjs/calls.rs @@ -132,6 +132,12 @@ impl<'t> Walker<'t> { // resolver can read its declared type (#1496). Mirrors // TreeSitterExtractor.extractCall. callee_name = format!("this.{field}.{method_name}"); + } else if let Some(element) = receiver.and_then(|r| self.this_subscript_of(r)) { + // `this.[k].()` — an element of the + // field, typed by the resolver from the field's + // declaration; `this['svc']` is the field itself. + // Mirrors TreeSitterExtractor.extractCall. + callee_name = format!("{element}.{method_name}"); } else if let Some(r) = receiver.filter(|r| r.kind() == "call_expression") { // Call receiver — `make().run()` (#1683): keep the inner // callee as `().`, or emit nothing when it @@ -139,6 +145,27 @@ impl<'t> Walker<'t> { // TreeSitterExtractor.extractCall. let Some(inner) = self.plain_inner_callee(r) else { return }; callee_name = format!("{inner}().{method_name}"); + } else if let Some(r) = receiver.filter(|r| r.kind() == "new_expression") { + // Constructor receiver — `new Runner().run()`: keep the + // class as `new ().` so the resolver + // resolves the method on it or nowhere; emit nothing for + // a constructor that is not a plain name / member chain. + // Mirrors TreeSitterExtractor.extractCall. + let Some(ctor) = r + .child_by_field_name("constructor") + .and_then(|c| Self::plain_member_name(self.text(c))) + else { + return; + }; + // `new this.constructor().copy(this)`, a static + // `new this().init()`: an instance of the enclosing + // class keeps the bare method name, as a call on + // `this` does. + callee_name = if ctor == "this" || ctor == "this.constructor" { + method_name.to_string() + } else { + format!("new {ctor}().{method_name}") + }; } else if receiver.is_some_and(|r| !keeps_bare_receiver(r, self.src)) { // An expression receiver with no static type // (`(a ?? b).map()`, `f().list.map()`): emit nothing @@ -174,6 +201,36 @@ impl<'t> Walker<'t> { } } + /// A receiver subscripting the enclosing class's object: `this.[k]` + /// → `this.[]`, `this['svc']` → `this.svc`; None for any other + /// subscript. Mirrors tsThisSubscriptReceiver. + pub(super) fn this_subscript_of(&self, receiver: Node<'t>) -> Option { + if receiver.kind() != "subscript_expression" { + return None; + } + let object = peel_receiver(receiver.child_by_field_name("object")?); + if object.kind() == "this" { + let index = receiver.child_by_field_name("index")?; + if index.kind() != "string" { + return None; + } + let text = self.text(index); + let quoted = text.len() >= 2 + && (text.starts_with('\'') || text.starts_with('"')) + && text.ends_with(&text[..1]); + let key = if quoted { &text[1..text.len() - 1] } else { return None }; + let mut chars = key.chars(); + let first = chars.next()?; + if !(first.is_ascii_alphabetic() || first == '_' || first == '$') + || !chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '$') + { + return None; + } + return Some(format!("this.{key}")); + } + self.this_field_of(object).map(|field| format!("this.{field}[]")) + } + /// `this.` as a member_expression receiver → Some(field) (#1496, #1987). pub(super) fn this_field_of(&self, receiver: Node<'t>) -> Option { if receiver.kind() != "member_expression" { @@ -211,6 +268,9 @@ impl<'t> Walker<'t> { } pub(super) fn extract_instantiation(&mut self, node: Node<'t>) { + if self.stack.is_empty() { + return; + } let ctor = node .child_by_field_name("constructor") .or_else(|| node.child_by_field_name("type")) @@ -306,13 +366,13 @@ impl<'t> Walker<'t> { // TS `extends_clause` (the other spellings are other grammars'). "extends_clause" | "superclass" | "base_clause" | "extends_interfaces" => { if let Some(target) = child.named_child(0) { - let name = self.text(target).to_string(); + let name = self.supertype_name(target); self.push_ref(class_row, &name, extends_kind, target); } } "implements_clause" | "class_interface_clause" | "super_interfaces" | "interfaces" => { for iface in named_kids(child) { - let name = self.text(iface).to_string(); + let name = self.supertype_name(iface); self.push_ref(class_row, &name, implements_kind, iface); } } @@ -347,6 +407,20 @@ impl<'t> Walker<'t> { } } + /// supertypeName — a parameterized supertype (`generic_type`) by the name + /// before its type arguments: `Store` for `implements Store`. + fn supertype_name(&self, node: Node<'t>) -> String { + if node.kind() == "generic_type" { + let named = (0..node.named_child_count()) + .filter_map(|j| node.named_child(j)) + .find(|c| c.kind() != "type_arguments"); + if let Some(named) = named { + return self.text(named).to_string(); + } + } + self.text(node).to_string() + } + pub(super) fn extract_type_annotations(&mut self, node: Node<'t>, from_row: u32) { if !self.variant.is_ts() { return; @@ -427,12 +501,15 @@ fn peel_receiver(node: Node<'_>) -> Node<'_> { cur } -/// `this` / `super`, a member chain rooted at either or at `window`, or -/// `new C()` still collapse to the bare method name (mirrors -/// keepsBareTsJsReceiver). +/// `this` / `super`, or a member chain rooted at either or at `window`, still +/// collapse to the bare method name; a subscript anywhere in the chain does +/// not (mirrors keepsBareTsJsReceiver). fn keeps_bare_receiver(node: Node<'_>, src: &str) -> bool { let mut cur = node; while matches!(cur.kind(), "member_expression" | "subscript_expression") { + if cur.kind() == "subscript_expression" { + return false; + } match cur.child_by_field_name("object") { Some(object) => cur = peel_receiver(object), None => return false, @@ -441,7 +518,6 @@ fn keeps_bare_receiver(node: Node<'_>, src: &str) -> bool { match cur.kind() { "this" | "super" => true, "identifier" => &src[cur.byte_range()] == "window", - "new_expression" => cur.id() == node.id(), _ => false, } } diff --git a/codegraph-kernel/src/tsjs/extractors.rs b/codegraph-kernel/src/tsjs/extractors.rs index 33385e062b..cacc2aea3e 100644 --- a/codegraph-kernel/src/tsjs/extractors.rs +++ b/codegraph-kernel/src/tsjs/extractors.rs @@ -46,8 +46,10 @@ impl<'t> Walker<'t> { if name == "" { // Still walk the body: module wrappers hold named inner functions // and calls that would otherwise be lost (#528). - if let Some(body) = body_of(node) { - self.visit_for_calls_and_structure(body); + let body = body_of(node); + self.visit_parameter_defaults(node, body); + if let Some(body) = body { + self.visit_function_body(body); } return; } @@ -69,12 +71,73 @@ impl<'t> Walker<'t> { self.extract_decorators_for(node, row); self.stack.push(Scope { row, kind: "function", name }); - if let Some(body) = body_of(node) { - self.visit_for_calls_and_structure(body); + let body = body_of(node); + self.visit_parameter_defaults(node, body); + if let Some(body) = body { + self.visit_function_body(body); } self.stack.pop(); } + /// A parameter's default value is code the function runs, each time a + /// call leaves that argument out (#2588). Each default, and a computed key + /// in a destructured parameter, is walked as the body is, with the same + /// scope on the stack; the names a parameter binds and its type annotation + /// are not. A function with no body never runs. Mirrors + /// TreeSitterExtractor.visitParameterDefaults. + fn visit_parameter_defaults(&mut self, func: Node<'t>, body: Option>) { + let Some(body) = body else { return }; + // A class field holds its function: `onPick = (item = first()) => {…}`. + let owner = if matches!(func.kind(), "public_field_definition" | "field_definition") { + body.parent() + } else { + Some(func) + }; + if let Some(params) = owner.and_then(|o| o.child_by_field_name("parameters")) { + self.visit_pattern_defaults(params); + } + } + + /// The defaults in a parameter list or a destructuring pattern, in source + /// order. Mirrors TreeSitterExtractor.visitPatternDefaults. + fn visit_pattern_defaults(&mut self, node: Node<'t>) { + stack_guard!(); + match node.kind() { + "formal_parameters" | "object_pattern" | "array_pattern" | "rest_pattern" => { + for i in 0..node.named_child_count() { + if let Some(child) = node.named_child(i) { + self.visit_pattern_defaults(child); + } + } + } + "required_parameter" | "optional_parameter" => { + if let Some(pattern) = node.child_by_field_name("pattern") { + self.visit_pattern_defaults(pattern); + } + if let Some(value) = node.child_by_field_name("value") { + self.visit_function_body(value); + } + } + "assignment_pattern" | "object_assignment_pattern" => { + if let Some(left) = node.child_by_field_name("left") { + self.visit_pattern_defaults(left); + } + if let Some(right) = node.child_by_field_name("right") { + self.visit_function_body(right); + } + } + "pair_pattern" => { + if let Some(key) = node.child_by_field_name("key").filter(|k| k.kind() == "computed_property_name") { + self.visit_function_body(key); + } + if let Some(value) = node.child_by_field_name("value") { + self.visit_pattern_defaults(value); + } + } + _ => {} + } + } + /// The property a CommonJS export assignment binds a function to — /// `exports.NAME = ` / `module.exports.NAME = ` — or None for /// any other assignment. The node must be the assignment's whole @@ -173,8 +236,10 @@ impl<'t> Walker<'t> { self.stack.push(Scope { row, kind: "component", name: name.to_string() }); match inner_fn { Some(inner) => { - if let Some(body) = body_of(inner) { - self.visit_for_calls_and_structure(body); + let body = body_of(inner); + self.visit_parameter_defaults(inner, body); + if let Some(body) = body { + self.visit_function_body(body); } } // Without a render function, the initializer is walked instead. @@ -245,8 +310,10 @@ impl<'t> Walker<'t> { } else if matches!(arg.kind(), "arrow_function" | "function_expression") { // A style or `.attrs()` function runs as the component's // code: its body is walked, the function never named. - if let Some(body) = body_of(arg) { - self.visit_for_calls_and_structure(body); + let body = body_of(arg); + self.visit_parameter_defaults(arg, body); + if let Some(body) = body { + self.visit_function_body(body); } } else { self.visit_for_calls_and_structure(arg); @@ -301,10 +368,15 @@ impl<'t> Walker<'t> { stack_guard!(); let resolved_body = body_of(node); // skipBodilessClass unset for TS/JS let name = self.extract_name(node); + // A class expression's name is bound inside the expression alone: + // never exported by it, and its head is its signature, which a + // declaration lacks (#2499). Mirrors TreeSitterExtractor.extractClass. + let expression = node.kind() == "class"; let extra = Extra { docstring: crate::docstring::preceding_docstring(node, self.src), visibility: self.visibility_of(node), - is_exported: Some(self.is_exported(node)), + is_exported: Some(!expression && self.is_exported(node)), + signature: if expression { resolved_body.map(|body| class_expression_head(self.src, node, body)) } else { None }, ..Extra::default() }; let Some(row) = self.create_node("class", &name, node, extra) else { @@ -330,8 +402,10 @@ impl<'t> Walker<'t> { // Object-literal methods are ephemeral: walk the body only. if let Some(parent) = node.parent() { if matches!(parent.kind(), "object" | "object_expression") { - if let Some(body) = body_of(node) { - self.visit_for_calls_and_structure(body); + let body = body_of(node); + self.visit_parameter_defaults(node, body); + if let Some(body) = body { + self.visit_function_body(body); } return; } @@ -357,8 +431,10 @@ impl<'t> Walker<'t> { self.extract_decorators_for(node, row); self.stack.push(Scope { row, kind: "method", name }); - if let Some(body) = body_of(node) { - self.visit_for_calls_and_structure(body); + let body = body_of(node); + self.visit_parameter_defaults(node, body); + if let Some(body) = body { + self.visit_function_body(body); } self.stack.pop(); } @@ -999,6 +1075,15 @@ impl<'t> Walker<'t> { } +/// A class expression's head, `class Widget extends Base`: its text up to the +/// body, trailing spaces and line breaks dropped. Mirrors classExpressionHead +/// (tree-sitter.ts). +fn class_expression_head(src: &str, node: Node<'_>, body: Node<'_>) -> String { + src[node.start_byte()..body.start_byte()] + .trim_end_matches(|c: char| [' ', '\t', '\r', '\n'].contains(&c)) + .to_string() +} + /// `.replace(/\s+/g, ' ')` for the tuple-contract signature. pub(super) fn collapse_ws(s: &str) -> String { let mut out = String::with_capacity(s.len()); diff --git a/codegraph-kernel/src/tsjs/mod.rs b/codegraph-kernel/src/tsjs/mod.rs index c49a9cdab5..764e2ee7bc 100644 --- a/codegraph-kernel/src/tsjs/mod.rs +++ b/codegraph-kernel/src/tsjs/mod.rs @@ -53,7 +53,7 @@ impl Variant { } } /// TS-family (typescript/tsx): type annotations, interfaces, enums, - /// aliases, visibility, isStatic. The JS family lacks all of those hooks. + /// aliases, visibility. The JS family lacks all of those hooks. fn is_ts(self) -> bool { matches!(self, Variant::Typescript | Variant::Tsx) } @@ -108,7 +108,16 @@ pub(super) fn is_function_type(kind: &str) -> bool { } fn is_class_type(v: Variant, kind: &str) -> bool { - kind == "class_declaration" || (v.is_ts() && kind == "abstract_class_declaration") + kind == "class_declaration" || kind == "class" || (v.is_ts() && kind == "abstract_class_declaration") +} + +/// Whether `node` is a class to extract: a class declaration, or a class +/// expression (`class`) with a name, `register(class Widget extends Base +/// {…})` (#2499); an anonymous one is passed over. Mirrors +/// TreeSitterExtractor.isClassTypeNode. +fn is_class_node(v: Variant, node: Node) -> bool { + let kind = node.kind(); + is_class_type(v, kind) && (kind != "class" || node.child_by_field_name("name").is_some()) } fn is_variable_type(kind: &str) -> bool { @@ -443,7 +452,7 @@ impl<'t> Walker<'t> { // `declare function`, a function in a `declare module` body or a `.d.ts`. self.extract_function(node, None); skip_children = true; - } else if is_class_type(self.variant, kind) { + } else if is_class_node(self.variant, node) { self.extract_class(node); skip_children = true; } else if is_method_type(self.variant, kind) @@ -529,6 +538,9 @@ impl<'t> Walker<'t> { // --- visitFunctionBody ------------------------------------------------------ + fn visit_function_body(&mut self, body: Node<'t>) { + self.visit_for_calls_and_structure(body); + } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { stack_guard!(); @@ -590,7 +602,7 @@ impl<'t> Walker<'t> { } } - if is_class_type(self.variant, kind) { + if is_class_node(self.variant, node) { self.extract_class(node); return; } @@ -861,13 +873,10 @@ impl<'t> Walker<'t> { self.has_keyword_child(node, "async") } - /// TS has an isStatic hook; JS does not (None = field absent). + /// A member's own `static` keyword: the isStatic hook TS and JS share + /// (hasStaticModifier, #2761). fn is_static(&self, node: Node) -> Option { - if self.variant.is_ts() { - Some(self.has_keyword_child(node, "static")) - } else { - None - } + Some(self.has_keyword_child(node, "static")) } fn is_const_decl(&self, node: Node) -> bool { diff --git a/docs/design/dart-kernel-port-checklist.md b/docs/design/dart-kernel-port-checklist.md index b978620257..f732180f6b 100644 --- a/docs/design/dart-kernel-port-checklist.md +++ b/docs/design/dart-kernel-port-checklist.md @@ -237,7 +237,12 @@ Grammar-shape facts the config leans on (all probed, `mini-cst.txt` / keyword node, identifier, on-types as bare type_identifiers, class_body). `enum_declaration`: `name:` + `body:` (enum_body); `enum_constant.name` ✓. `type_alias` has NO fields (both modern `typedef X = T` and legacy - `typedef void X(int)` are node `type_alias`). + `typedef void X(int)` are node `type_alias`). Its name is the last direct + `type_identifier` before the `=`, `type_parameters` or + `formal_parameter_list` child: a legacy return type comes first (`typedef + Report X(int n)`), and until 2026-10-21 the first-identifier fallback + named such a typedef after its return type (`dartTypeAliasName` / + `type_alias_name` now, via resolveName). `initialized_variable_definition`: `name:` + one-or-more `value:` fields (the RHS postfix chain is FLATTENED into multiple value-fielded siblings). - **In-class member shapes:** methods/getters/setters/operators/factories @@ -483,7 +488,7 @@ Registration: `EXTRACTORS.dart` (languages/index.ts:56), `FN_REF_SPECS.dart` | `mixin_declaration`, `extension_declaration` | extraClassNodeTypes:1022 | extractClass(node) → kind **`class`** | | `method_signature`, `constructor_signature` | methodTypes:1027 | **NOT class-gated at the ladder** — extractMethod:1737 runs anywhere; its own gate :1747 (not class-like, no methodsAreTopLevel, no receiver) falls back: parent `object`/`object_expression`? (never in dart) else **extractFunction** — this is how extension-TYPE members and any stray non-class method_signature become plain `function` nodes (pinned: `extension type MetersT` → function `km`, bare QN) | | `enum_declaration` | enumTypes:1064 → extractEnum:1914 | §Class family | -| `type_alias` | typeAliasTypes:1071 → extractTypeAlias:2890 | plain type_alias node; `getChildByField(node,'value')` → **null** (no fields) → **NO refs from the aliased type** (`typedef MapAlias = Map` emits nothing); returns false → children re-visited (function_type/formal_parameter_list children match nothing) | +| `type_alias` | typeAliasTypes:1071 → extractTypeAlias:2890 | type_alias node named by the resolveName hook (§Extractor config); `getChildByField(node,'value')` → **null** (no fields), so the dart branch — `pushDartTypeAliasRefs` / `type_alias_refs`, since 2026-10-21 (#2327; nothing before) — emits a `references` ref from the node for each type named in every named child but the name and `annotation`s, through the isDartTypeName gate (`typedef MapAlias = Map` → `Map`, `WidgetT`). A type parameter in scope is skipped unless written `p.T`: the typedef's own, and those of any node that declares `type_parameters` (a generic `function_type`, an old-style generic function-typed `formal_parameter`) within it. Returns false → children re-visited (they match nothing type-related; an annotated typedef's arguments are fn-ref capture points) | | `import_or_export`, `part_directive` | importTypes:1209 → extractImport:3170 | §Extractor config. No skipChildren: the directive's children are still recursed, so an annotated directive's arguments are fn-ref capture points (`@Tag(f) import 'a.dart';` captures `f`). The kernel arm recurses too — since 2026-10-06; before, its `import_or_export` arm returned and missed those (TortureDirectives.dart pins both) | | `new_expression` | INSTANTIATION_KINDS:1255 (`new_expression` ∈ :354-361) | extractInstantiation:4610 → ctor field lookups null → namedChild(0) = type_identifier → `instantiates` ref from stack top; `<`-strip + last-`.`-segment apply (`new p.Foo()` → `Foo`). findAnonymousClassBody → always null for dart. Children still recursed | | `function_body` (sibling of a consumed signature) | **no branch** | recursed → THE DOUBLE-WALK (§below) | @@ -1310,8 +1315,9 @@ NO unwrap, NO ungatedModes, NO addressOfOnly. Pins: sealed/base modifiers (plain classes); enum (simple + enhanced: ctor'd constants at constant positions, args unwalked, `with` GAP vs `implements` ✓, members after `;`); typedef ×3 (modern/legacy/generic — - zero refs); extension type (container invisible, **members leak as bare - `function`s**); the FULL call matrix (§Calls rows verbatim — incl. + zero refs until 2026-10-21, the aliased types since; every form is in + TortureTypedefs.dart); extension type (container invisible, **members + leak as bare `function`s**); the FULL call matrix (§Calls rows verbatim — incl. cascade invisibility, `?.`, const-object args, chain re-encodes, interpolation calls, `$x` vs `${x}`); static reads (`X.member`, `X.method()` double-emission, 3-segment nothing, case-pattern GAP); diff --git a/docs/design/framework-coverage.md b/docs/design/framework-coverage.md index d1f4dd2ec4..e399c7ed1c 100644 --- a/docs/design/framework-coverage.md +++ b/docs/design/framework-coverage.md @@ -1,6 +1,6 @@ # Framework & language coverage — what is done, what is left -**Last verified: 2026-08-29** (Angular row: 2026-10-06; React Router row: 2026-10-07) against the build at that date. Re-verify with the +**Last verified: 2026-08-29** (Angular row: 2026-10-06; React Router row: 2026-10-19; Vue Router / Nuxt row: 2026-10-22) against the build at that date. Re-verify with the queries in [Checking this file is still true](#checking-this-file-is-still-true) before trusting a row; this is a snapshot, not a live view. @@ -40,7 +40,7 @@ arms disagree unresolved rather than guessed. | Next.js | `frameworks/nextjs.ts` | `next-router-synthesizer.ts` | `nextjs.test.ts` | next-saas-starter | | React Router / Remix | `frameworks/react-router.ts` | `react-router-synthesizer.ts` | `react-router.test.ts`, `react-router-framework.test.ts`, `remix-routes.test.ts`, `react-router-returned-tables.test.ts` | proshop (44 edges), proshop-v2 (28), react-redux-realworld (22), react-boilerplate (`styled(Link)`), takenote (v5 ``); pinned official framework config and flat filenames; bulletproof-react: nested children, lazy routes and paths.x.path constants through per-app aliases; paths.x.getHref() links and navigate calls read literal or template destinations; upstream route-table and JSX index/layout regressions cover imported route tables, nested layouts and index pages; fork recheck: 11 navigates on bulletproof-react `9506629` (2026-10-01), all destinations checked against config and route registrations. Lazy loaders that pick their page (`lazyRouteReference` in `frameworks/react.ts`, trap 18): milo/ui's three tabs (`const { TestTab } = await import('…/tabs')`), 20 apps from a GitHub survey of `createBrowserRouter` + `lazy` (636 picked-page edges, each checked against the target file's own export: 467 by name, 169 as its `export default`; 42 routes written with `async lazy()` now indexed, among them React Router's own `lazy-loading-router-provider` example), an AST oracle on the TypeScript compiler agreeing with the text reader on all 2,538 loaders of 876 surveyed router files, a late-file sync equal to a fresh index on 8 of those apps, and 26 control repos byte-identical, among them ghostfolio, whose 30 Angular lazy routes stay Angular's with React detected in the same project Barrels that pass on a page they import (`import { UserLayout } from './UserLayout'; export { UserLayout }`, `import Login from './Login'; export default Login`): `exportedComponent` follows the import, reading the export from binding rows; upstream: Arshia76/DigiShop.Frontend (0 → 27 route and layout edges), 49 other repos byte-identical; `export default withAuth(Page)` is followed nowhere. A page in a folder below the barrel (`pages/Home/index.tsx` behind `@/pages`) whose route failed waits under the barrel's module key: a sync that adds the page, or gives it a definition, re-opens it (`reopenReferencesThroughBarrels` in `resolution/module-importers.ts`; upstream: a late-file sync on DigiShop equals a fresh index). A name the route file declares itself (trap 19) binds to what that declaration renders in routes, layouts and JSX alike; upstream: outline (+31 edges), mastodon (+20), SigNoz (+7), excalidraw (+3), bulletproof-react (+6), with matx, mantis, proshop, next-saas-starter and berry byte-identical. Upstream's 2026-10-19 measurements, taken on its own resolver and not re-measured on the fork's: a table a function returns handed over through a variable (Medusa's admin dashboard, trap 13), `` written in a route object (trap 14), and the `baseUrl` / `"*"` imports of matx, mantis and berry (trap 21). A table behind `useMemo`, and every `return` of a route-building function rather than the first (trap 13), are upstream's 2026-10-19 change too (`tableRoutes` in `frameworks/react.ts`, ported to the fork's top-level table reader; upstream: tg-tt-bot-stage, bewild-clone, SpaceR, BANKFORM1; not re-measured here). | | TanStack Router / Start | `frameworks/tanstack-router.ts` | `tanstack-router-synthesizer.ts` | `tanstack-router.test.ts`, `tanstack-start.test.ts` | TanStack examples, fastapi-template frontend; pinned Start server-handler syntax | -| Vue Router / Nuxt | `frameworks/vue-router.ts` (Nuxt file routes: `nuxtResolver` in `frameworks/vue.ts`) | `vue-router-synthesizer.ts` | `vue-router.test.ts` | vue-realworld (23 edges); vue-element-admin (62 routes), vue-admin-template (14), vben (192), halo console (34) — named tables, module files, `children` + layouts; Nuxt: mealie, elk, nuxt/movies | +| Vue Router / Nuxt | `frameworks/vue-router.ts` (Nuxt file routes: `nuxtResolver` in `frameworks/vue.ts`) | `vue-router-synthesizer.ts` | `vue-router.test.ts`, `nuxt-navigate-to.test.ts` | vue-realworld (23 edges); vue-element-admin (62 routes), vue-admin-template (14), vben (192), halo console (34) — named tables, module files, `children` + layouts; Nuxt: mealie, elk, nuxt/movies. Nuxt's `navigateTo` (upstream's 2026-10-22 measurements, taken on its own resolver and not re-measured on the fork's): mealie 13 of its 14 calls, nuxt.com 17 of 24, elk 9 of 10 outside the plugin that declares its own, atidone 2 of 2, pages and route middleware alike; the rest are computed, an API endpoint (`{ external: true }`), or a page of a Nuxt layer (`extends: ['./layers/nuxi']`), whose pages sit in a route table of their own. nuxt/ui's docs: 0 of 4, two computed and two whose only page is a catch-all (`[...slug].vue`, named `*slug`) or an optional parameter (`[[tag]].vue`, `:tag?`), which `matchRoute` doesn't read yet, for markup links either | | SvelteKit | `frameworks/sveltekit-router.ts` | `sveltekit-synthesizer.ts` | `sveltekit-router.test.ts`, `sveltekit-route-names.test.ts` | sveltekit-realworld (31 edges); shadcn-svelte and skeleton (`(group)` layouts: 13 and 23 edges), svelte.dev (74), kit's test apps (47) | | Angular | `frameworks/angular-router.ts` | `angular-template-synthesizer.ts` | `angular-router.test.ts`, `angular-routes.test.ts` | angular-realworld (31 edges, 18 renders, 32 bindings), Ghostfolio (189 edges, 170 renders, 215 bindings), ngx-admin (routes and renders; its menus are config; 13 bindings), CleanArchitecture's `src/Web/ClientApp` (signals; 12 bindings), angular-spotify (Nx libs behind barrels: 14 routes), jira-clone (class-constant paths, mount-only redirects), jhipster (60), ionic-conference (18), Angular-JumpStart (18) | | Astro | `frameworks/astro.ts` | — | `astro-routes.test.ts` | pinned endpoint fixture; exact page components and source navigation | @@ -583,6 +583,20 @@ Each of these cost real debugging time; they are not hypothetical. project keeps for it (`"*": ["./typings/*"]`, as tsc reads it). A route resolves its lazy module as TypeScript whatever the file is written in, which is why the TypeScript extension list probes `/index.jsx` last. +21. **A navigation call named like a function must not fall back to one.** + Nuxt auto-imports `navigateTo`, so a call to it is written bare, with no + import to say where it comes from. When Vue Router can't read where it goes + — a computed destination, a path no page serves, an external URL — the + reference would reach the name matcher, which links any project function + of that name: elk declares a `navigateTo` of its own inside a plugin. The + Vue resolver used to sink every `navigateTo` into an edge to its caller, + which hid the navigation too, and which sync never revisits (trap 16). + `FrameworkResolver.ownsReference` names the references a framework answers + alone: `nuxtResolver` owns a bare `navigateTo` that no declaration or + import in the file binds, and one no router links stays unresolved, so a + sync binds it when its page appears. An absolute URL is no path of the app: + Vue Router's call reader keeps only destinations that start with `/`, as + its markup reader does. --- diff --git a/docs/design/ruby-kernel-port-checklist.md b/docs/design/ruby-kernel-port-checklist.md index 3a02bac069..26f1993e6c 100644 --- a/docs/design/ruby-kernel-port-checklist.md +++ b/docs/design/ruby-kernel-port-checklist.md @@ -253,14 +253,18 @@ Hooks PRESENT (port each exactly): (see extractImport below). Hooks ABSENT (the walker must NOT do these): `preParse`, `resolveName`, -`recoverMangledName`, `isMisparsedFunction`, `isConst`, `isStatic`, +`recoverMangledName`, `isMisparsedFunction`, `isConst`, `isExported`, `isAsync`, `getSignature`, `getReturnType`, `getReceiverType`, `resolveBody`, `classifyClassNode`, `classifyMethodNode`, `extractPropertyName`, `interfaceKind`, `extraClassNodeTypes`, `packageTypes`/`extractPackage`, `extractModifiers`, `synthesizeMembers`, `skipBodilessClass`, `methodsAreTopLevel`. Consequences: every ruby -function/method node has signature/isAsync/isStatic/returnType/isExported -**undefined** (file node isExported:false); **no isConst means every +function/method node has signature/isAsync/returnType/isExported +**undefined** (file node isExported:false). `isStatic` IS a hook +(isRubyClassMethod / `is_class_method`): every function/method node carries +it — true for a `singleton_method` (`def self.x`, `def Foo.x`) and for a +`method` whose parent, past one `body_statement`, is a `singleton_class` +(`class << self`), false for every other def (a nested def included). **no isConst means every `assignment` extracts kind `'variable'`, never `'constant'` — including `MAX = 3`** (value-ref targeting still works: kind variable is a target); `attr_accessor`/`attr_reader`/`attr_writer` synthesize NOTHING (no @@ -279,9 +283,9 @@ non-decorator sibling — always a no-op). | every node | visitNode hook first (943) | mixin calls + modules handled there (above); handled → scanFnRefSubtree + STOP | | every node | maybeCaptureFnRefs (990) | fires for `argument_list` / `pair` (the RUBY_SPEC dispatch keys) in visitNode context too — this is how class-body hook-DSL symbols are captured | | `method` | functionTypes:994 | inside class-like (class/module/…, isInsideClassLikeNode:1486 — **`module` counts**, 1498) AND in methodTypes → extractMethod:1737; else extractFunction:1517. skipChildren | -| `singleton_method` (`def self.x` / `def Foo.x` / `def obj.x`) | NOT functionTypes → methodTypes:1027 | no classifyMethodNode → extractMethod. At top level: not class-like, no methodsAreTopLevel, no receiver hook → gate 1747 sends it to extractFunction → a top-level `def self.x` is a plain **function** node named x. `object` field ignored everywhere — `def self.x` vs `def x` are indistinguishable in the graph (isStatic undefined) | +| `singleton_method` (`def self.x` / `def Foo.x` / `def obj.x`) | NOT functionTypes → methodTypes:1027 | no classifyMethodNode → extractMethod. At top level: not class-like, no methodsAreTopLevel, no receiver hook → gate 1747 sends it to extractFunction → a top-level `def self.x` is a plain **function** node named x. `object` field ignored everywhere; `def self.x` and `def x` share a qualified name, and only isStatic (true for the singleton_method) tells them apart | | `class` | classTypes:1005 | no classifyClassNode → extractClass:1679 (kind 'class') | -| `singleton_class` (`class << self`) | NO branch | recursed → its `body` body_statement's defs extract as methods of the OUTER class, indistinguishable from instance methods (probed) | +| `singleton_class` (`class << self`) | NO branch | recursed → its `body` body_statement's defs extract as methods of the OUTER class, under the same qualified names as instance methods, with isStatic true (probed) | | `assignment` (top level / class / module) | variableTypes:1098 | gate: `!isInsideClassLikeNode() \|\| isClassScopeConstantAssignment` (1508: type==='assignment' AND (`left` field ?? namedChild(0)).type === `'constant'`). File scope: identifier AND constant LHS both extract. Class/module scope: ONLY constant-LHS. Then extractVariable:2538 + scanFnRefSubtree (1110) + skipChildren — **the RHS is never walked: top-level/class-level `X = Foo.new` emits NO instantiates/calls** | | `operator_assignment` (`+=`, `\|\|=`) | no branch | not `assignment` → nothing extracted at any scope; recursed (children emit nothing either at non-body scope) | | `call` (top level / class / module body) | **importTypes:1209 — NEVER callTypes** | extractImport:3170 (below). require/require_relative → import node + refs; **every other call → NOTHING** (hook null → 3350 `if (this.extractor.extractImport) return;`). skipChildren stays false → children ARE visited: nested calls also land here (nothing), argument_lists get fn-ref capture. This kills `attr_accessor`, `has_many`, `define_method` (+ its do_block body!), `get '/x' do…end` route blocks, bare DSL calls — **all invisible to extraction at non-body scope** except fn-ref candidates + the rails regex extractor | @@ -330,7 +334,7 @@ extractFilePackage:1397 (no packageTypes → no namespace node); `value=` — keep the suffix in the name). `` path (1549) can't trigger (grammar requires a name). No misparse hook. Node extras: docstring (getPrecedingDocstring), signature undefined, visibility (sibling scan!), - isExported/isAsync/isStatic/returnType undefined. extractTypeAnnotations → + isExported/isAsync/returnType undefined, isStatic from the hook (above). extractTypeAnnotations → no-op (ruby ∉ TYPE_ANNOTATION_LANGUAGES:5752). extractDecoratorsFor → no-op. Push, walk `body` field (`body_statement`) via visitFunctionBody, pop. **`parameters` (method_parameters) are NEVER walked — a call in a @@ -429,13 +433,17 @@ field_expression machinery, which ruby therefore NEVER runs. Steps: normalized to `()`**, unlike java/php chain encodings); literal receivers are NOT filtered (`"literal".upcase` → `"literal".upcase`, `5.times` → `5.times` — unresolvable noise, PRESERVE). -6. Receiver node TYPE exactly `constant` (and not skipped) → ADDITIONAL - `references` ref {name: receiverName, line/col: **the RECEIVER's** - position} — `Klass.static_call` emits calls `Klass.static_call` + - references `Klass`; fires for VALUE constants too (`RETRY_MAX.times {…}` - → calls `RETRY_MAX.times` + references `RETRY_MAX` — pinned). A - `scope_resolution` receiver (`Foo::Bar.baz`) gets NO references ref - (type ≠ constant). PRESERVE. +6. Receiver a CONSTANT PATH (and not skipped) → ADDITIONAL `references` + ref {name: receiverName, line/col: **the RECEIVER's** position} — + `Klass.static_call` emits calls `Klass.static_call` + references `Klass`; + fires for VALUE constants too (`RETRY_MAX.times {…}` → calls + `RETRY_MAX.times` + references `RETRY_MAX` — pinned). A constant path is + a `constant`, or a `scope_resolution` whose `name` is a `constant` and + whose `scope` is absent or itself a constant path (#2503): `Foo::Bar.baz` + → references `Foo::Bar`, `::Foo.baz` → references `::Foo` (the leading + `::` kept — the resolver reads it as "from the top level"). A path + through a value (`obj::Thing.go`, `self.class::LIMIT.times`) gets NONE. + Both engines: `is_constant_path` / `isRubyConstantPath`. 7. return — inner receiver calls are ALSO visited afterwards (the body walker recurses children at 5277 — extractCall does not consume its subtree), so `chained.first_call.second_call` emits BOTH diff --git a/docs/viewer-launch-changelog.md b/docs/viewer-launch-changelog.md index 2d86369084..554edc45d8 100644 --- a/docs/viewer-launch-changelog.md +++ b/docs/viewer-launch-changelog.md @@ -129,6 +129,8 @@ These describe `codegraph ui` and its screens. They were taken out of `## [Unrel - **Routes served by a component are listed.** Entry points and the Steps picker listed a route only when a function, method or class served it. A Vue Router, Nuxt, Svelte or Astro screen is served by a component, so those apps' routes weren't listed at all. They are now, each with the component that serves it. +- **One busy inline handler no longer hides the other routes.** Entry points and the Steps picker filled their list by the symbols each route reaches rather than by routes, so an inline handler with dozens of calls could use up the list by itself, and the routes after it went unlisted. They now count routes. Thanks @danusha2345. (#2370) + - **Where the app goes after login is a fork, not two always-es.** A navigation whose destination comes back from a helper — `router.replace(await resolvePostLoginRoute())` over `return (await hasSeenWelcome(…)) ? '/home/' : '/welcome/'` — drew both screens with no condition, reading as if the welcome screen always shows. The two arms share a line, and only a column can tell them apart; each synthesized edge now carries its literal's own position, so the guard reader says which arm it is: `WHEN await hasSeenWelcome(…)` → home, and its negation → welcome. And the scan starts at the helper's body, so a literal-union return type — `Promise<'/welcome/' | '/home/'>`, whose routes are string literals too, written first — no longer stands in for the navigation itself. Re-index after upgrading to pick the positions up. - **A screen that talks to native code keeps its own navigations.** In a React Native or Expo app, a `router.push` written inside a listener for a native event was credited to whichever screen had *started* that round trip, not to the screen the push is written on. In one app that moved seven transitions off the capture screen and onto the review screen it opens — leaving the review screen looking as though nothing in the app could reach it, stranded in the "no transition reaches this" band at the bottom of the Screens tab, and printing Swift conditions like `Thread.isMainThread` on a JavaScript navigation. A navigation now belongs to the screen whose file it is written in; an event arriving from native code, from an HTTP call or off a queue is no longer read backwards as if it were a caller. @@ -215,6 +217,10 @@ These describe `codegraph ui` and its screens. They were taken out of `## [Unrel - A viewer trail now holds up to 64 hops everywhere — the trail bar, saved trails and "Read as flow". Thanks @inth3shadows for the report and @danusha2345. (#1976) +- **Saved trails stay inside your project.** When `.codegraph/ui` was a link to a folder somewhere else, saving a trail wrote it into that folder, even though the save then reported an error, and a trail file that was itself a link to a file outside the project was listed and opened like one of yours. Both are now refused, and links that stay inside the project keep working. Thanks @danusha2345. + +- **Saved trails record the right author when one process serves several projects.** An app embedding the viewer for more than one project wrote the first project's git user name on every trail it saved; each project now uses its own. Thanks @danusha2345. + - **`codegraph ui` on Windows moves on from a port another program holds.** Windows refuses a port that another program keeps for itself, or one inside a range it reserves, with a different error than a port that is simply in use, so the viewer gave up at the first such port instead of trying the next one, and blamed ports below 1024 needing elevated privileges — a rule Windows doesn't have. It now moves on to the next free port, and when none is left, or the port you pinned with `--port` is refused, it says what Windows did and how to list the reserved ranges. Thanks @ijbranch for the report and @sx4im. (#2299) - **A file's outside imports are listed in more languages.** The file screen lists the imports that name nothing in your project, like a third-party package or a system header. In C and C++, Objective-C, JavaScript and TypeScript, Python, C#, Lua, R, Solidity, Svelte and Vue, an import that named no file had been linked to its own import statement, so it was listed nowhere. Re-index after upgrading. diff --git a/scripts/build-bundle.sh b/scripts/build-bundle.sh index 8bbb870343..70d4fde455 100755 --- a/scripts/build-bundle.sh +++ b/scripts/build-bundle.sh @@ -13,7 +13,7 @@ # scripts/build-bundle.sh [node-version] # target: darwin-arm64 | darwin-x64 | linux-x64 | linux-arm64 # | win32-x64 | win32-arm64 -# node-version: e.g. v24.16.0 (default below; pin for reproducible builds) +# node-version: e.g. v24.22.1 (default below; pin for reproducible builds) # # Output: # unix: release/codegraph-.tar.gz (launcher: bin/codegraph) @@ -21,7 +21,14 @@ set -euo pipefail TARGET="${1:?usage: build-bundle.sh [node-version]}" -NODE_VERSION="${2:-v24.16.0}" +# The release workflow passes no version, so this default is the Node every +# release bundles. Keep it on 24.x (the CLI refuses Node 25+) and at 24.22.1 or +# later. 24.22.1 is Node's October 2026 security release (issue #2780). Before +# 24.20.0, Node 24 can abort on Windows just as the process exits, after its +# work is done, with 0xC0000409 and "Assertion failed: !(handle->flags & +# UV_HANDLE_CLOSING), file src\win\async.c" (nodejs/node#56645, fixed by +# nodejs/node#61999; issue #2617). +NODE_VERSION="${2:-v24.22.1}" ROOT="$(cd "$(dirname "$0")/.." && pwd)" OUT="$ROOT/release" @@ -33,24 +40,41 @@ OSFAM="${TARGET%-*}" # darwin | linux | win32 echo "[bundle] target=${TARGET} node=${NODE_VERSION}" -# 1. Download + extract the official Node runtime for the target platform. +# 1. Download + extract the official Node runtime for the target platform. The +# archive must match the SHA-256 that nodejs.org publishes for the release +# before anything is unpacked, so a bundle ships exactly that release's +# official build, never a truncated or substituted download (issue #2780). if [ "$OSFAM" = "win32" ]; then NODE_DIST="node-${NODE_VERSION}-win-${ARCH}" - NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.zip" - echo "[bundle] downloading ${NODE_URL}" - curl -fsSL "$NODE_URL" -o "$WORK/node.zip" + NODE_ARCHIVE="${NODE_DIST}.zip" +else + NODE_DIST="node-${NODE_VERSION}-${TARGET}" + NODE_ARCHIVE="${NODE_DIST}.tar.gz" +fi +NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_ARCHIVE}" +echo "[bundle] downloading ${NODE_URL}" +curl -fsSL "$NODE_URL" -o "$WORK/$NODE_ARCHIVE" +curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/SHASUMS256.txt" -o "$WORK/SHASUMS256.txt" +EXPECTED_SHA="$(awk -v name="$NODE_ARCHIVE" '$2 == name { print $1 }' "$WORK/SHASUMS256.txt")" +if command -v sha256sum >/dev/null 2>&1; then + ACTUAL_SHA="$(sha256sum < "$WORK/$NODE_ARCHIVE" | cut -d' ' -f1)" +else + ACTUAL_SHA="$(shasum -a 256 < "$WORK/$NODE_ARCHIVE" | cut -d' ' -f1)" # macOS +fi +if [ -z "$EXPECTED_SHA" ] || [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then + echo "[bundle] error: ${NODE_ARCHIVE} does not match the SHA-256 that nodejs.org lists for ${NODE_VERSION} (listed: ${EXPECTED_SHA:-nothing}, downloaded: ${ACTUAL_SHA})" >&2 + exit 1 +fi +echo "[bundle] verified ${NODE_ARCHIVE} against nodejs.org's SHASUMS256.txt (sha256 ${ACTUAL_SHA})" +if [ "$OSFAM" = "win32" ]; then if command -v unzip >/dev/null 2>&1; then - unzip -q "$WORK/node.zip" -d "$WORK" + unzip -q "$WORK/$NODE_ARCHIVE" -d "$WORK" else - tar -xf "$WORK/node.zip" -C "$WORK" # bsdtar can read zip + tar -xf "$WORK/$NODE_ARCHIVE" -C "$WORK" # bsdtar can read zip fi NODE_BIN="$WORK/${NODE_DIST}/node.exe" else - NODE_DIST="node-${NODE_VERSION}-${TARGET}" - NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.tar.gz" - echo "[bundle] downloading ${NODE_URL}" - curl -fsSL "$NODE_URL" -o "$WORK/node.tar.gz" - tar -xzf "$WORK/node.tar.gz" -C "$WORK" + tar -xzf "$WORK/$NODE_ARCHIVE" -C "$WORK" NODE_BIN="$WORK/${NODE_DIST}/bin/node" fi [ -f "$NODE_BIN" ] || { echo "[bundle] error: node binary not found ($NODE_BIN)" >&2; exit 1; } diff --git a/scripts/gen-go-stdlib-methods.mjs b/scripts/gen-go-stdlib-methods.mjs new file mode 100644 index 0000000000..88f672b7a6 --- /dev/null +++ b/scripts/gen-go-stdlib-methods.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node +/** + * Regenerate the table in src/resolution/go-stdlib-methods.ts from Go's own + * source: + * + * npm run build + * node scripts/gen-go-stdlib-methods.mjs [--write] + * + * is a Go source tree (a release's checkout of golang/go, say). For + * every type the table lists now — add or drop one by editing the table, then + * rerun — the script reads /src//*.go, tests skipped, + * with the engine's Go grammar, and lists each method as + * `name:params/results`: + * - an interface's methods, and those of the interfaces it embeds; + * - a struct's or other named type's (receiver T or *T), and those its + * embedded fields promote; + * - an alias's, its target's. + * The counts come from the engine's own goSignatureArity, so a table method + * is held to what the arity gate reads off a project method's stored + * signature. Exported names are kept, and an unexported one only where a + * listed interface requires it (`testing.TB`'s `private`). + * + * Prints the table body; --write puts it in the module instead. Exits 1 when + * a listed type isn't found or a signature doesn't read. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const MODULE = path.join(ROOT, 'src', 'resolution', 'go-stdlib-methods.ts'); + +const goRoot = process.argv[2]; +const write = process.argv.includes('--write'); +if (!goRoot || !fs.existsSync(path.join(goRoot, 'src', 'io'))) { + console.error('usage: gen-go-stdlib-methods.mjs [--write] (a Go source tree, with src/io)'); + process.exit(2); +} +const wasm = path.join(ROOT, 'dist', 'extraction', 'wasm', 'tree-sitter-go.wasm'); +if (!fs.existsSync(wasm)) { + console.error('dist/ is missing: run `npm run build` first'); + process.exit(2); +} +const { goSignatureArity } = await import(pathToFileURL(path.join(ROOT, 'dist', 'resolution', 'callback-synthesizer.js')).href); +const { Parser, Language } = createRequire(path.join(ROOT, 'package.json'))('web-tree-sitter'); + +// The types the table lists, in its order. +const moduleText = fs.readFileSync(MODULE, 'utf8'); +const open = moduleText.indexOf('{', moduleText.indexOf('const GO_STDLIB_METHODS')); +const close = moduleText.indexOf('\n};', open); +const listed = new Function(`return ${moduleText.slice(open, close + 2)}`)(); + +await Parser.init(); +const parser = new Parser(); +parser.setLanguage(await Language.load(wasm)); + +const signature = (node) => { + const params = node.childForFieldName('parameters')?.text; + const result = node.childForFieldName('result')?.text; + return params && result ? `${params} ${result}` : params; +}; + +/** A package's top-level type declarations and the methods declared on each. */ +const packages = new Map(); +function pkg(importPath) { + let found = packages.get(importPath); + if (found) return found; + found = { types: new Map(), methods: new Map() }; + packages.set(importPath, found); + const dir = path.join(goRoot, 'src', importPath); + const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.go') && !f.endsWith('_test.go')) : []; + for (const file of files) { + const tree = parser.parse(fs.readFileSync(path.join(dir, file), 'utf8')); + const imports = new Map(); + for (const spec of tree.rootNode.descendantsOfType('import_spec')) { + const ip = spec.childForFieldName('path')?.text.replace(/^"|"$/g, ''); + if (ip) imports.set(spec.childForFieldName('name')?.text ?? ip.split('/').pop(), ip); + } + for (const spec of tree.rootNode.descendantsOfType(['type_spec', 'type_alias'])) { + if (spec.parent?.type !== 'type_declaration' || spec.parent.parent?.type !== 'source_file') continue; + const name = spec.childForFieldName('name')?.text; + if (name && !found.types.has(name)) { + found.types.set(name, { type: spec.childForFieldName('type'), imports, alias: spec.type === 'type_alias' }); + } + } + for (const m of tree.rootNode.namedChildren.filter((c) => c.type === 'method_declaration')) { + const receiver = /\(\s*(?:\w+\s+)?\*?\s*(\w+)/.exec(m.childForFieldName('receiver')?.text ?? '')?.[1]; + const name = m.childForFieldName('name')?.text; + if (!receiver || !name) continue; + const methods = found.methods.get(receiver) ?? found.methods.set(receiver, new Map()).get(receiver); + if (!methods.has(name)) methods.set(name, goSignatureArity(signature(m))); + } + } + return found; +} + +/** Method name → `params/results` for the type `name` the package at `importPath` declares. */ +function methodsOf(importPath, name, seen = new Set()) { + const methods = new Map(); + const key = `${importPath}.${name}`; + if (seen.has(key)) return methods; + seen.add(key); + const p = pkg(importPath); + for (const [m, arity] of p.methods.get(name) ?? []) methods.set(m, arity); + const declared = p.types.get(name); + if (!declared) return methods; + const take = (more) => { + for (const [m, arity] of more) if (!methods.has(m)) methods.set(m, arity); + }; + const embedded = (type, imports) => { + if (type?.type === 'pointer_type') type = type.namedChildren[0]; + if (type?.type === 'generic_type') type = type.childForFieldName('type'); + if (type?.type === 'qualified_type') { + take(methodsOf(imports.get(type.childForFieldName('package').text), type.childForFieldName('name').text, seen)); + } else if (type?.type === 'type_identifier') { + if (type.text === 'error') take(new Map([['Error', '0/1']])); + else take(methodsOf(importPath, type.text, seen)); + } + }; + const type = declared.type; + if (type?.type === 'interface_type') { + for (const element of type.namedChildren) { + if (element.type === 'method_elem') methods.set(element.childForFieldName('name').text, goSignatureArity(signature(element))); + else if (element.type === 'type_elem') { + const terms = element.namedChildren.filter((c) => c.type !== 'comment'); + if (terms.length === 1) embedded(terms[0], declared.imports); + } + } + } else if (type?.type === 'struct_type') { + const fields = type.namedChildren.find((c) => c.type === 'field_declaration_list'); + for (const field of fields?.namedChildren.filter((c) => c.type === 'field_declaration') ?? []) { + if (!field.namedChildren.some((c) => c.type === 'field_identifier')) embedded(field.childForFieldName('type'), declared.imports); + } + } else if (declared.alias) { + embedded(type, declared.imports); + } + return methods; +} + +const EXPORTED = /^\p{Lu}/u; +const generated = []; +const required = new Set(); +let failed = false; +for (const [importPath, types] of Object.entries(listed)) { + for (const name of Object.keys(types)) { + const declared = pkg(importPath).types.get(name); + if (!declared) { + console.error(`not found: ${importPath}.${name}`); + failed = true; + continue; + } + const methods = methodsOf(importPath, name); + generated.push({ importPath, name, methods }); + if (declared.type?.type === 'interface_type') for (const m of methods.keys()) if (!EXPORTED.test(m)) required.add(m); + } +} + +const lines = []; +let current = null; +for (const { importPath, name, methods } of generated) { + if (importPath !== current) { + if (current !== null) lines.push(' },'); + lines.push(` ${/^[a-z]+$/.test(importPath) ? importPath : `'${importPath}'`}: {`); + current = importPath; + } + const kept = [...methods].filter(([m]) => EXPORTED.test(m) || required.has(m)).sort(([a], [b]) => (a < b ? -1 : 1)); + for (const [m, arity] of kept) { + if (arity === null) { + console.error(`signature doesn't read: ${importPath}.${name}.${m}`); + failed = true; + } + } + lines.push(` ${name}: '${kept.map(([m, arity]) => `${m}:${arity}`).join(' ')}',`); +} +lines.push(' },'); +if (failed) process.exit(1); + +if (!write) { + console.log(lines.join('\n')); +} else { + const eol = moduleText.includes('\r\n') ? '\r\n' : '\n'; + const start = moduleText.indexOf('\n', open) + 1; + const end = close + 1; + fs.writeFileSync(MODULE, moduleText.slice(0, start) + lines.join(eol) + eol + moduleText.slice(end)); + console.error(`wrote ${generated.length} types to ${path.relative(ROOT, MODULE)}`); +} diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index cf6963d595..a90b207f9d 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -571,9 +571,11 @@ async function offerIndexIgnoredRepos( opts: { interactive: boolean }, ): Promise { let repos: string[]; + let patterns: string[]; try { - const { findUnindexedIgnoredRepos } = await import('../extraction'); + const { findUnindexedIgnoredRepos, includeIgnoredPatternFor } = await import('../extraction'); repos = findUnindexedIgnoredRepos(projectPath); + patterns = repos.map(includeIgnoredPatternFor); } catch { return; // detection is advisory — never let it break the command } @@ -584,7 +586,7 @@ async function offerIndexIgnoredRepos( const SHOWN = 6; const names = repos.slice(0, SHOWN).map((r) => r.replace(/\/$/, '')); const extra = repos.length > SHOWN ? ` (+${formatNumber(repos.length - SHOWN)} more)` : ''; - const snippet = `{ "includeIgnored": [${repos.map((p) => JSON.stringify(p)).join(', ')}] }`; + const snippet = `{ "includeIgnored": [${patterns.map((p) => JSON.stringify(p)).join(', ')}] }`; clack.log.warn( `Your .gitignore excludes ${isOne ? 'a nested git repository' : `${formatNumber(repos.length)} nested git repositories`} here, ` + @@ -615,7 +617,7 @@ async function offerIndexIgnoredRepos( let added: number; try { const { addIncludeIgnoredPatterns } = await import('../project-config'); - added = addIncludeIgnoredPatterns(projectPath, repos); + added = addIncludeIgnoredPatterns(projectPath, patterns); } catch (err) { clack.log.error(`Could not update ${PROJECT_CONFIG_FILENAME}: ${err instanceof Error ? err.message : String(err)}`); manualHint(); @@ -957,10 +959,12 @@ program try { // A live MCP daemon keeps SQLite handles open. Verify it by its socket // before stopping it so a stale pidfile can never signal another process. - const { stopDaemonAt } = await import('../mcp/daemon-registry'); - const daemonStop = await stopDaemonAt(fs.realpathSync(projectPath), { preserveUnverified: true }); + // A lock left naming a process that isn't CodeGraph is cleared (#2770). + const { stopDaemonAt, daemonStopFailureMessage } = await import('../mcp/daemon-registry'); + const realRoot = fs.realpathSync(projectPath); + const daemonStop = await stopDaemonAt(realRoot, { preserveUnverified: true }); if (daemonStop.outcome === 'unverified' || daemonStop.outcome === 'still-running') { - throw new Error('Could not verify that the active CodeGraph daemon has stopped. Run `codegraph daemon stop` to stop it, then retry `codegraph index`.'); + throw new Error(await daemonStopFailureMessage(realRoot, daemonStop)); } // Keep the writer slot through recreation AND indexing. A reconnecting @@ -2862,17 +2866,25 @@ program ? [opts.location as 'global' | 'local'] : ['global', 'local']; let changed = 0; + let failed = 0; for (const loc of locs) { for (const report of refreshTargets(ALL_TARGETS, loc)) { + if (report.status === 'failed') { + failed += 1; + error(`${report.displayName}: could not refresh — ${report.error}`); + } for (const p of report.changedPaths) { changed += 1; console.log(` ${report.displayName}: refreshed ${p}`); } } } - if (changed === 0) { + if (changed === 0 && failed === 0) { console.log('All configured agent surfaces are already current.'); } + // Non-zero so `codegraph upgrade` tells the user to look; every other + // agent was still refreshed. + if (failed > 0) process.exitCode = 1; return; } diff --git a/src/bin/prompt-hook-gate.ts b/src/bin/prompt-hook-gate.ts index ef89333568..972a24e5d8 100644 --- a/src/bin/prompt-hook-gate.ts +++ b/src/bin/prompt-hook-gate.ts @@ -12,8 +12,10 @@ import { extractCodeTokens, hasStructuralKeyword, isAgentMessage, - isHostNotification, + isCrossSessionMessage, + isTaskNotification, planFrontload, + stripSystemReminders, type FrontloadPlan, } from '../directory'; import { extractProseCandidates } from '../search/identifier-segments'; @@ -77,10 +79,13 @@ export function promptHookDisabled(): boolean { export function gatePromptHook(raw: string): PromptHookInput | null { let input: { prompt?: string; cwd?: string } = {}; try { input = JSON.parse(raw); } catch { return null; } - const prompt = String(input.prompt || ''); + // The desktop app's blocks are host status, not what the + // user typed: gate on the rest (#2635). + const prompt = stripSystemReminders(String(input.prompt || '')); - // Host notifications and subagent hand-backs (#2184) are not user prompts. - if (isHostNotification(prompt) || isAgentMessage(prompt)) { recordPromptHookGate('noop-notification'); return null; } + // Host notifications, subagent hand-backs (#2184) and messages from other + // sessions (#2635) are not user prompts, and neither is nothing. + if (!prompt || isTaskNotification(prompt) || isAgentMessage(prompt) || isCrossSessionMessage(prompt)) { recordPromptHookGate('noop-notification'); return null; } const keyworded = hasStructuralKeyword(prompt); // "review my changes", `main..HEAD`: explore answers these from the diff, // so a confirmed change question gets the full injection like a keyword. diff --git a/src/codegraph.ts b/src/codegraph.ts index fa96029e2a..bdb01e0087 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -453,9 +453,19 @@ export class CodeGraph { throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`); } - // Open database + // Open database. The secondary indexes an interrupted bulk load dropped are + // rebuilt off this thread, and finished before the instance is returned + // (#1887): rebuilt in line on a large index, they blocked the thread longer + // than the liveness watchdog allows, and every restart was killed the same + // way. const dbPath = getDatabasePath(resolvedRoot); - const db = DatabaseConnection.open(dbPath, { readOnly: options.readOnly }); + const db = DatabaseConnection.open(dbPath, { readOnly: options.readOnly, deferIndexRecovery: true }); + try { + await db.recoverIndexesOffThread(); + } catch (err) { + db.close(); + throw err; + } const queries = new QueryBuilder(db.getDb()); const instance = new CodeGraph(db, queries, resolvedRoot); @@ -522,9 +532,14 @@ export class CodeGraph { } /** - * Open synchronously (without sync) + * Open synchronously (without sync). + * + * `deferIndexRecovery` leaves the secondary indexes an interrupted bulk load + * dropped for {@link recoverIndexes} to rebuild off this thread, instead of + * rebuilding them here (#1887): for a caller on a watched event loop, which + * then awaits {@link recoverIndexes} before it relies on the project. */ - static openSync(projectRoot: string, options: Pick = {}): CodeGraph { + static openSync(projectRoot: string, options: Pick & { deferIndexRecovery?: boolean } = {}): CodeGraph { const resolvedRoot = path.resolve(projectRoot); // Check if initialized @@ -540,12 +555,33 @@ export class CodeGraph { // Open database const dbPath = getDatabasePath(resolvedRoot); - const db = DatabaseConnection.open(dbPath, { readOnly: options.readOnly }); + const db = DatabaseConnection.open(dbPath, { readOnly: options.readOnly, deferIndexRecovery: options.deferIndexRecovery }); const queries = new QueryBuilder(db.getDb()); return new CodeGraph(db, queries, resolvedRoot); } + /** + * Rebuild, on a worker thread, the secondary indexes an interrupted bulk + * load dropped and `openSync({ deferIndexRecovery: true })` left (#1887). + * Resolves at once when nothing is left, which is every open but the first + * after such an interruption; rejects when the rebuild fails, and the next + * call tries again. Reads stay correct meanwhile, only slower. + */ + recoverIndexes(): Promise { + return this.db.recoverIndexesOffThread(); + } + + /** Whether {@link recoverIndexes} still has indexes to rebuild. */ + hasPendingIndexRecovery(): boolean { + return this.db.hasDeferredIndexRecovery(); + } + + /** The query tokens the index has never seen (no node, edge or file carries them). */ + getUnseenNames(names: readonly string[]): string[] { + return this.queries.getUnseenNames(names); + } + /** * Check if a directory has been initialized as a CodeGraph project */ @@ -2547,7 +2583,7 @@ export class CodeGraph { * via the `references` edge that framework resolvers emit. Returns * null when fewer than 3 valid (non-test) routes exist. */ - getRoutingManifest(limit?: number): { + getRoutingManifest(limit?: number, perRoute?: boolean): { entries: Array<{ url: string; handler: string; @@ -2562,7 +2598,7 @@ export class CodeGraph { topHandlerFileCount: number; totalRoutes: number; } | null { - return this.queries.getRoutingManifest(limit); + return this.queries.getRoutingManifest(limit, perRoute); } // =========================================================================== diff --git a/src/context/index.ts b/src/context/index.ts index f8bd828a62..d93899fbdf 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -37,6 +37,21 @@ import { describeSynthesizedHop } from '../graph/synthesized-hop'; */ const MAX_COPYBOOK_DECLARATION_ENTRIES = 3; +/** + * The most symbol names, and search terms, of one query that + * `findRelevantContext` searches the index for. A question or a bag of symbol + * names stays well below both: a name like `RefreshTokenService` is one + * symbol but several terms (`refreshtokenservice`, `refresh`, `token`, + * `service`, stems). A pasted log, document or source file runs to hundreds + * of words, and every word costs several searches of the index, so such a + * query took minutes on a large project and the prompt hook gave up on it + * (#2620). Both lists put code-shaped words (camelCase, snake_case, dotted + * names) first and keep the order they appear in, so a long query is searched + * by its identifiers and its opening words. + */ +export const MAX_QUERY_SYMBOLS = 64; +export const MAX_QUERY_TERMS = 128; + /** * Extract likely symbol names from a natural language query * @@ -228,10 +243,14 @@ export class ContextBuilder { * and its non-production budget cap are deliberately NOT joined: those REMOVE * content, and `deprioritize` is a ranking lever by definition — `exclude` is * the lever for taking things out of reach. + * + * Returned as a predicate with the patterns read once, for the candidate + * loops of one search: asking the live matcher per candidate re-checked + * `codegraph.json` hundreds of thousands of times on a long query (#2620). */ - private deprioritizedPathMatcher(): (filePath: string) => boolean { - // Reads the config, so take it once per call and reuse it for every path. - return this.queries.getDeprioritizedPathMatcher() ?? (() => false); + private deprioritizedPaths(): (filePath: string) => boolean { + const matcher = this.queries.getDeprioritizedPathSnapshot?.() ?? this.queries.getDeprioritizedPathMatcher?.(); + return (filePath) => matcher?.(filePath) ?? false; } /** @@ -479,12 +498,10 @@ export class ContextBuilder { return { nodes, edges, roots }; } - const isDeprioritized = this.deprioritizedPathMatcher(); - // === HYBRID SEARCH === // Step 1: Extract potential symbol names from query - const symbolsFromQuery = extractSymbolsFromQuery(query); + const symbolsFromQuery = extractSymbolsFromQuery(query).slice(0, MAX_QUERY_SYMBOLS); logDebug('Extracted symbols from query', { query, symbols: symbolsFromQuery }); // Step 2: Look up exact matches for extracted symbols @@ -679,7 +696,8 @@ export class ContextBuilder { // fell through to the fuzzy fallback and returned whatever program name // sat within edit distance (`CVACT01Y` → `CBACT01C`). const searchTerms = extractSearchTerms(query) - .filter((term) => !copybooks.members.has(term.toUpperCase())); + .filter((term) => !copybooks.members.has(term.toUpperCase())) + .slice(0, MAX_QUERY_TERMS); if (searchTerms.length > 0) { // Search each term individually to get broader coverage, // then boost results that match multiple terms @@ -799,7 +817,7 @@ export class ContextBuilder { // than nodes matching just one generic term. Without this, "ExecutionUtils" // (matches only "execution") fills budget slots meant for "ShardSearchRequest" // (matches "shard" + "search" + "request"). - const queryTermsForBoost = extractSearchTerms(query); + const queryTermsForBoost = extractSearchTerms(query).slice(0, MAX_QUERY_TERMS); if (queryTermsForBoost.length >= 2) { // Group terms that are substrings of each other (stem variants of the same // root word). "indexed", "indexe", "index" should count as ONE concept match, @@ -893,6 +911,18 @@ export class ContextBuilder { // SEPARATE LIKE batch so one hot single-word term can't crowd classes // out of the length-ordered 200-row batch. const camelCallableKinds: NodeKind[] = ['function', 'method', 'component']; + // A path's relevance to the query depends on nothing else, and the + // candidates of every term share their files: score each path once. + const isDeprioritized = this.deprioritizedPaths(); + const pathScores = new Map(); + const pathScoreOf = (filePath: string): number => { + let score = pathScores.get(filePath); + if (score === undefined) { + score = scorePathRelevance(filePath, query, undefined, isDeprioritized(filePath)); + pathScores.set(filePath, score); + } + return score; + }; const camelSearchedTerms = new Set(); const searchIdSet = new Set(searchResults.map(r => r.node.id)); // Track per-node term hits for multi-term boosting @@ -942,12 +972,7 @@ export class ContextBuilder { if (searchIdSet.has(r.node.id)) continue; if (isTestFile(r.node.filePath) && !isTestQuery) continue; - const pathScore = scorePathRelevance( - r.node.filePath, - query, - undefined, - isDeprioritized(r.node.filePath), - ); + const pathScore = pathScoreOf(r.node.filePath); const brevityBonus = Math.max(0, 6 - (name.length - titleCased.length) / 4); termCandidates.push({ node: r.node, score: 8 + brevityBonus + pathScore }); } @@ -1034,12 +1059,7 @@ export class ContextBuilder { const compoundResults: SearchResult[] = []; for (const [, entry] of compoundTermMap) { if (entry.terms.size >= 2) { - const pathScore = scorePathRelevance( - entry.node.filePath, - query, - undefined, - isDeprioritized(entry.node.filePath), - ); + const pathScore = pathScoreOf(entry.node.filePath); const brevityBonus = Math.max(0, 6 - entry.node.name.length / 8); compoundResults.push({ node: entry.node, @@ -1160,13 +1180,16 @@ export class ContextBuilder { if (hierarchyNodesAdded >= maxHierarchyNodes) break; if (typeHierarchyKinds.has(result.node.kind)) { const hierarchy = this.traverser.getTypeHierarchy(result.node.id); + // The budget holds per node: a widely implemented interface brings + // its whole descendant tree (IDisposable: hundreds of types). for (const [id, node] of hierarchy.nodes) { - if (!nodes.has(id)) { + if (!nodes.has(id) && hierarchyNodesAdded < maxHierarchyNodes) { nodes.set(id, node); hierarchyNodesAdded++; } } for (const edge of hierarchy.edges) { + if (!nodes.has(edge.source) || !nodes.has(edge.target)) continue; const exists = edges.some( (e) => e.source === edge.source && e.target === edge.target && e.kind === edge.kind ); diff --git a/src/db/index.ts b/src/db/index.ts index d491b76980..a3eb50ad1c 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -10,8 +10,11 @@ import * as path from 'path'; import { SchemaVersion } from '../types'; import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from './migrations'; import { getCodeGraphDir, statInode } from '../directory'; +import { toDatabaseOpenError } from './open-error'; export { SqliteDatabase, SqliteBackend } from './sqlite-adapter'; +export { DatabaseOpenError } from './open-error'; +export { NodeSqliteUnavailableError } from './node-sqlite'; /** * Apply connection-level PRAGMAs. Shared by `initialize` and `open` so the two @@ -44,6 +47,22 @@ function configureConnection(db: SqliteDatabase, readOnly = false): void { db.pragma(`journal_size_limit = ${WAL_HEAL_THRESHOLD_BYTES}`); } +/** + * Run the rest of an open on the connection `db`. Its first statements read + * the file, so a database SQLite cannot use (locked, damaged, not a database, + * an I/O error) fails here as well as when the connection is made. Such a + * failure names SQLite's code and the path ({@link DatabaseOpenError}), and + * leaves no connection open. + */ +function withOpenFailures(db: SqliteDatabase, dbPath: string, finish: () => T): T { + try { + return finish(); + } catch (err) { + try { db.close(); } catch { /* the open's failure is the one to report */ } + throw toDatabaseOpenError(err, dbPath); + } +} + /** * WAL size past which `healOversizedWal` (run at every `open`) checkpoints and * truncates the file, and to which `journal_size_limit` clips the WAL after any @@ -107,9 +126,12 @@ export class DatabaseConnection { fs.mkdirSync(dir, { recursive: true }); } - // Create and configure database const { db, backend } = createDatabase(dbPath); + return withOpenFailures(db, dbPath, () => DatabaseConnection.initializeOn(db, backend, dbPath)); + } + /** The rest of {@link initialize}: configure the connection and create the schema. */ + private static initializeOn(db: SqliteDatabase, backend: SqliteBackend, dbPath: string): DatabaseConnection { configureConnection(db); // Run schema initialization, splitting FTS5 from the rest so @@ -160,15 +182,31 @@ export class DatabaseConnection { } /** - * Open an existing database + * Open an existing database. + * + * `deferIndexRecovery` leaves the secondary indexes an interrupted bulk load + * dropped for {@link recoverIndexesOffThread} instead of rebuilding them + * here. Rebuilding one scans and sorts its whole table: on a large index + * that is minutes of synchronous work, and on a watched event loop the + * liveness watchdog kills the process before the index commits, every time + * it starts (#1887). Reads stay correct meanwhile, only slower. */ - static open(dbPath: string, options: { readOnly?: boolean } = {}): DatabaseConnection { + static open(dbPath: string, options: { readOnly?: boolean; deferIndexRecovery?: boolean } = {}): DatabaseConnection { if (!fs.existsSync(dbPath)) { throw new Error(`Database not found: ${dbPath}`); } const { db, backend } = createDatabase(dbPath, options); + return withOpenFailures(db, dbPath, () => DatabaseConnection.openOn(db, backend, dbPath, options)); + } + /** The rest of {@link open}: configure the connection, then migrate and repair a writable one. */ + private static openOn( + db: SqliteDatabase, + backend: SqliteBackend, + dbPath: string, + options: { readOnly?: boolean; deferIndexRecovery?: boolean }, + ): DatabaseConnection { configureConnection(db, options.readOnly); // Detect FTS5 availability for search fallback (#1532) @@ -194,7 +232,8 @@ export class DatabaseConnection { // beginBulkNodeLoad and endBulkNodeLoad): the FTS triggers are missing and // nodes_fts is stale. Rebuild + recreate so search stays in sync. conn.healBulkNodeLoad(); - conn.healBulkSecondaryIndexes(); + if (options.deferIndexRecovery) conn.deferredIndexDdls = conn.missingBulkSecondaryIndexDdls(); + else conn.healBulkSecondaryIndexes(); // Self-heal a killed session's leftover oversized WAL (#1431) — one // statSync when healthy, off-thread checkpoint+truncate when not. @@ -424,6 +463,14 @@ export class DatabaseConnection { /** Recreate every secondary index a killed bulk parse/ref/edge window may leave dropped. */ private healBulkSecondaryIndexes(): void { + for (const ddl of this.missingBulkSecondaryIndexDdls()) this.db.exec(ddl); + } + + /** + * The DDL that recreates every secondary index a killed bulk parse/ref/edge + * window may leave dropped; empty, after one count, when none is missing. + */ + private missingBulkSecondaryIndexDdls(): string[] { const names = [...new Set([ ...DatabaseConnection.BULK_PARSE_INDEX_NAMES, ...DatabaseConnection.BULK_REF_INDEX_NAMES, @@ -433,15 +480,99 @@ export class DatabaseConnection { const row = this.db .prepare(`SELECT count(*) AS c FROM sqlite_master WHERE type = 'index' AND name IN (${placeholders})`) .get(...names) as { c: number } | undefined; - if ((row?.c ?? 0) >= names.length) return; + if ((row?.c ?? 0) >= names.length) return []; const schemaPath = path.join(__dirname, 'schema.sql'); const schema = fs.readFileSync(schemaPath, 'utf-8'); - for (const idx of names) { + return names.map((idx) => { const m = schema.match(new RegExp(`CREATE INDEX IF NOT EXISTS ${idx}\\b[^;]*;`)); if (!m) throw new Error(`schema.sql: index ${idx} not found for crash recovery`); - this.db.exec(m[0]); + return m[0]; + }); + } + + /** Index DDL {@link open} left for {@link recoverIndexesOffThread}. */ + private deferredIndexDdls: string[] = []; + private indexRecovery: Promise | null = null; + + /** Whether {@link open} left dropped secondary indexes for {@link recoverIndexesOffThread}. */ + hasDeferredIndexRecovery(): boolean { + return this.deferredIndexDdls.length > 0; + } + + /** + * Recreate the secondary indexes {@link open} deferred, on a worker thread + * with its own connection, so this thread's event loop, and the #850 + * liveness watchdog's heartbeat, keep turning however long the rebuild takes + * (#1887). Each index commits on its own, so an interrupted recovery keeps + * what it finished. Not best-effort like {@link runMaintenance}: rejects when + * a statement fails, and the next call tries again. Single-flight; resolves + * at once when nothing was deferred. + */ + recoverIndexesOffThread(): Promise { + if (this.deferredIndexDdls.length === 0) return Promise.resolve(); + this.indexRecovery ??= this.runRecoveryOffThread(this.deferredIndexDdls) + .then(() => { this.deferredIndexDdls = []; }) + .finally(() => { this.indexRecovery = null; }); + return this.indexRecovery; + } + + private async runRecoveryOffThread(statements: string[]): Promise { + let WorkerCtor: typeof import('node:worker_threads').Worker; + try { + WorkerCtor = (await import('node:worker_threads')).Worker; + } catch { + // No worker threads in this runtime: recover here, as open() would have. + for (const ddl of statements) this.db.exec(ddl); + return; } + // SQLite's default temp storage, not this connection's MEMORY: the sort + // behind a big index spills to disk instead of growing the heap. The + // worker collects its garbage before it ends, so it exits with no marking + // in flight (worker-teardown.ts), and nothing terminates it. + const workerSource = ` + const { workerData, parentPort } = require('node:worker_threads'); + let error = null; + try { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(workerData.dbPath); + try { + db.exec('PRAGMA busy_timeout = 30000'); + for (const sql of workerData.statements) db.exec(sql); + } finally { + db.close(); + } + } catch (e) { + error = String((e && e.message) || e); + } + try { + const v8 = require('v8'); + v8.setFlagsFromString('--expose-gc'); + const gc = require('vm').runInNewContext('gc'); + v8.setFlagsFromString('--no-expose-gc'); + gc(); + } catch {} + parentPort.postMessage({ error }); + `; + await new Promise((resolve, reject) => { + let settled = false; + const worker = new WorkerCtor(workerSource, { eval: true, workerData: { dbPath: this.dbPath, statements } }); + worker.once('message', (message: { error: string | null }) => { + settled = true; + if (message.error) reject(new Error(`Could not restore the indexes of ${this.dbPath}: ${message.error}`)); + else resolve(); + }); + worker.once('error', (err) => { + if (settled) return; + settled = true; + reject(err); + }); + worker.once('exit', (code) => { + if (settled) return; + settled = true; + reject(new Error(`Restoring the indexes of ${this.dbPath} stopped early (worker exit ${code}).`)); + }); + }); } /** diff --git a/src/db/name-substring-index.ts b/src/db/name-substring-index.ts new file mode 100644 index 0000000000..8ba3491e86 --- /dev/null +++ b/src/db/name-substring-index.ts @@ -0,0 +1,222 @@ +/** + * An in-memory answer to the substring lookup `codegraph_explore` runs for + * every word of its query (#2620): + * + * SELECT … FROM nodes WHERE name LIKE '%word%' [AND name NOT LIKE 'word%'] + * AND kind IN (…) ORDER BY length(name) LIMIT n + * + * A leading `%` can't use an index, so SQLite reads every node of those kinds + * for each call: about 80 ms on a 560k-node index, four calls a word, and a + * pasted log or document has hundreds of words. This index reads the names + * once and answers each call with one string search. + * + * It returns exactly what SQLite would. LIKE folds case for ASCII letters only, + * `_` matches any one character, and `length()` counts characters, not UTF-16 + * units, so all three are done that way here. SQLite leaves rows of equal + * length in the order its plan read them: through `idx_nodes_kind` (kind, + * file_path, start_line, id) on a large index, by rowid when it scans a small + * one. The index keeps both orders, and the caller says which plan SQLite + * chose. A pattern with a `%` is left to SQLite. + */ + +/** + * The kinds the index holds: the definitions and callables explore's + * camel-case and compound steps ask about, and the variables and constants its + * named-symbol seeding adds. + */ +export const NAME_SUBSTRING_KINDS: readonly string[] = [ + 'class', 'component', 'constant', 'enum', 'function', 'interface', 'method', + 'protocol', 'struct', 'trait', 'type_alias', 'union', 'variable', +]; + +/** Separates the names in {@link NameSubstringIndex.hay}; no name may contain it. */ +const SEPARATOR = '\u0000'; + +export interface NameSubstringIndex { + /** Kind codes: `kinds[entryKind[e]]` is entry `e`'s kind. */ + readonly kinds: readonly string[]; + /** Every distinct ASCII-lowercased name, each followed by {@link SEPARATOR}. */ + readonly hay: string; + /** Where each distinct name starts in `hay`, plus one past the end. */ + readonly starts: Int32Array; + /** The entries of distinct name `i` are `nameEntries[nameEntryStart[i] … nameEntryStart[i+1])`, ascending. */ + readonly nameEntryStart: Int32Array; + readonly nameEntries: Int32Array; + /** Per entry, in index order. */ + readonly entryRowid: Float64Array; + readonly entryKind: Uint8Array; + readonly entryLength: Int32Array; +} + +/** Lowercase ASCII letters only, as SQLite's LIKE compares them. */ +function asciiLower(text: string): string { + return /[A-Z]/.test(text) ? text.replace(/[A-Z]+/g, (run) => run.toLowerCase()) : text; +} + +/** `length(text)` as SQLite counts it: characters, so a surrogate pair is one. */ +function sqliteLength(text: string): number { + let length = text.length; + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code >= 0xd800 && code <= 0xdbff && i + 1 < text.length) { + const next = text.charCodeAt(i + 1); + if (next >= 0xdc00 && next <= 0xdfff) { length--; i++; } + } + } + return length; +} + +/** + * Build the index from rows of {@link NAME_SUBSTRING_KINDS} read in + * `idx_nodes_kind` order, which becomes the entries' order. Returns null when + * a name holds the separator, so the caller keeps asking SQLite. + */ +export function buildNameSubstringIndex( + rows: Iterable<{ rowid: number; kind: string; name: string }>, +): NameSubstringIndex | null { + const kindCodes = new Map(NAME_SUBSTRING_KINDS.map((kind, i) => [kind, i])); + const nameIds = new Map(); + const names: string[] = []; + const rowids: number[] = []; + const kinds: number[] = []; + const lengths: number[] = []; + const nameOf: number[] = []; + for (const row of rows) { + const code = kindCodes.get(row.kind); + if (code === undefined) continue; + const lowered = asciiLower(row.name); + if (lowered.includes(SEPARATOR)) return null; + let id = nameIds.get(lowered); + if (id === undefined) { + id = names.length; + nameIds.set(lowered, id); + names.push(lowered); + } + rowids.push(row.rowid); + kinds.push(code); + lengths.push(sqliteLength(row.name)); + nameOf.push(id); + } + + const starts = new Int32Array(names.length + 1); + let offset = 0; + for (let i = 0; i < names.length; i++) { + starts[i] = offset; + offset += names[i]!.length + SEPARATOR.length; + } + starts[names.length] = offset; + + // Group entries by name, keeping index order inside each group. + const nameEntryStart = new Int32Array(names.length + 1); + for (const id of nameOf) nameEntryStart[id + 1]!++; + for (let i = 0; i < names.length; i++) nameEntryStart[i + 1]! += nameEntryStart[i]!; + const fill = nameEntryStart.slice(0, names.length); + const nameEntries = new Int32Array(nameOf.length); + for (let entry = 0; entry < nameOf.length; entry++) nameEntries[fill[nameOf[entry]!]!++] = entry; + + return { + kinds: NAME_SUBSTRING_KINDS, + hay: names.join(SEPARATOR) + SEPARATOR, + starts, + nameEntryStart, + nameEntries, + entryRowid: Float64Array.from(rowids), + entryKind: Uint8Array.from(kinds), + entryLength: Int32Array.from(lengths), + }; +} + +/** The distinct name whose span in `hay` holds `offset`. */ +function nameAt(starts: Int32Array, offset: number): number { + let lo = 0; + let hi = starts.length - 2; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (starts[mid]! <= offset) lo = mid; else hi = mid - 1; + } + return lo; +} + +/** + * A matcher for a LIKE pattern without `%`: where the next match at or after + * `from` starts, and whether one starts exactly at `at`. Without `_` this is a + * plain string search; each `_` matches any one character but the separator. + */ +function patternMatcher(needle: string): { next(hay: string, from: number): number; startsAt(hay: string, at: number): boolean } { + if (!needle.includes('_')) { + return { + next: (hay, from) => hay.indexOf(needle, from), + startsAt: (hay, at) => hay.startsWith(needle, at), + }; + } + // `u`, so a wildcard takes a whole character, as SQLite's `_` does. + const source = needle + .split('_') + .map((part) => part.replace(/[\^$\\.*+?()[\]{}|/]/g, '\\$&')) + .join('[^\\u0000]'); + const anywhere = new RegExp(source, 'gu'); + const anchored = new RegExp(source, 'uy'); + return { + next(hay, from) { + anywhere.lastIndex = from; + const match = anywhere.exec(hay); + return match ? match.index : -1; + }, + startsAt(hay, at) { + anchored.lastIndex = at; + return anchored.test(hay); + }, + }; +} + +/** + * The order SQLite's plan reads candidate rows in, which its ORDER BY keeps + * for rows of equal length: the kind index's, or the table's by rowid. + */ +export type ScanOrder = 'index' | 'rowid'; + +/** + * The rowids SQLite's substring lookup returns, in its order, or null when the + * index can't answer exactly (a `%` in the pattern, an empty pattern, or a + * kind it doesn't hold). `order` is the order the plan reads rows in. + */ +export function searchNameSubstring( + index: NameSubstringIndex, + substring: string, + kinds: readonly string[], + excludePrefix: boolean, + limit: number, + order: ScanOrder, +): number[] | null { + const needle = asciiLower(substring); + if (needle.length === 0 || needle.includes('%') || needle.includes(SEPARATOR)) return null; + const wanted = new Uint8Array(index.kinds.length); + for (const kind of kinds) { + const code = index.kinds.indexOf(kind); + if (code < 0) return null; + wanted[code] = 1; + } + + const { hay, starts, nameEntryStart, nameEntries, entryKind, entryLength } = index; + const matcher = patternMatcher(needle); + const hits: number[] = []; + for (let at = matcher.next(hay, 0); at >= 0; ) { + const name = nameAt(starts, at); + if (!(excludePrefix && matcher.startsAt(hay, starts[name]!))) { + for (let k = nameEntryStart[name]!; k < nameEntryStart[name + 1]!; k++) { + const entry = nameEntries[k]!; + if (wanted[entryKind[entry]!]) hits.push(entry); + } + } + at = matcher.next(hay, starts[name + 1]!); + } + // Entry numbers follow the kind index, rowids the table: either breaks + // length ties the way SQLite's plan does. + const rowid = index.entryRowid; + hits.sort(order === 'index' + ? (a, b) => (entryLength[a]! - entryLength[b]!) || (a - b) + : (a, b) => (entryLength[a]! - entryLength[b]!) || (rowid[a]! - rowid[b]!)); + // A negative LIMIT is no limit in SQLite. + if (limit >= 0 && hits.length > limit) hits.length = limit; + return hits.map((entry) => index.entryRowid[entry]!); +} diff --git a/src/db/node-sqlite.ts b/src/db/node-sqlite.ts new file mode 100644 index 0000000000..788e9f13a3 --- /dev/null +++ b/src/db/node-sqlite.ts @@ -0,0 +1,36 @@ +/** + * Node's built-in `node:sqlite`, loaded in one place, so that a runtime + * without it fails with the one explanation that fits it: CodeGraph needs + * Node.js 22.5+, or its own bundled runtime. Any other failure to open a + * database is SQLite's own, and is reported as such (see ./open-error.ts). + */ + +/** The parts of `node:sqlite` CodeGraph uses. */ +export interface NodeSqliteModule { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + DatabaseSync: new (path: string, options?: { readOnly?: boolean }) => any; +} + +/** `node:sqlite` is missing or could not be loaded in this Node.js runtime. */ +export class NodeSqliteUnavailableError extends Error { + constructor(cause: unknown) { + const reason = cause instanceof Error ? cause.message : String(cause); + super( + `Node.js ${process.version} could not load its built-in SQLite module (${reason}).\n` + + 'CodeGraph requires node:sqlite (Node.js 22.5+). Install the self-contained CodeGraph ' + + 'release (it bundles a compatible Node), or run on Node 22.5+.', + { cause } + ); + this.name = 'NodeSqliteUnavailableError'; + } +} + +/** Load `node:sqlite`, or throw {@link NodeSqliteUnavailableError}. */ +export function loadNodeSqlite(): NodeSqliteModule { + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require('node:sqlite') as NodeSqliteModule; + } catch (err) { + throw new NodeSqliteUnavailableError(err); + } +} diff --git a/src/db/open-error.ts b/src/db/open-error.ts new file mode 100644 index 0000000000..345d7745d5 --- /dev/null +++ b/src/db/open-error.ts @@ -0,0 +1,93 @@ +/** + * Opening a CodeGraph database failed in SQLite itself: report what SQLite + * said, with the database's path and a short hint at the usual cause. Before, + * every such failure came out as "CodeGraph requires node:sqlite (Node.js + * 22.5+)", which sent users after their Node version when the file was locked, + * unreadable, or on a network share. That hint is now only for a runtime that + * cannot load `node:sqlite` at all (./node-sqlite.ts); an index Windows and WSL + * share keeps its own guidance (./wsl-shared-index.ts). + */ + +import { NodeSqliteUnavailableError } from './node-sqlite'; +import { toWslSharedIndexError, WslSharedIndexError } from './wsl-shared-index'; + +/** SQLite's primary result codes, by number. node:sqlite reports the extended code; its low byte is the primary one. */ +const SQLITE_CODE_NAMES: readonly string[] = [ + 'SQLITE_OK', 'SQLITE_ERROR', 'SQLITE_INTERNAL', 'SQLITE_PERM', 'SQLITE_ABORT', 'SQLITE_BUSY', + 'SQLITE_LOCKED', 'SQLITE_NOMEM', 'SQLITE_READONLY', 'SQLITE_INTERRUPT', 'SQLITE_IOERR', + 'SQLITE_CORRUPT', 'SQLITE_NOTFOUND', 'SQLITE_FULL', 'SQLITE_CANTOPEN', 'SQLITE_PROTOCOL', + 'SQLITE_EMPTY', 'SQLITE_SCHEMA', 'SQLITE_TOOBIG', 'SQLITE_CONSTRAINT', 'SQLITE_MISMATCH', + 'SQLITE_MISUSE', 'SQLITE_NOLFS', 'SQLITE_AUTH', 'SQLITE_FORMAT', 'SQLITE_RANGE', 'SQLITE_NOTADB', +]; + +/** What to check, for the failures an open meets in practice. */ +const HINTS: Readonly> = { + SQLITE_BUSY: 'Another process has the database locked; retry when it is done.', + SQLITE_PERM: 'Check that you can read and write the file and its folder.', + SQLITE_READONLY: 'The database or its folder is read-only, and CodeGraph has to write its index there.', + SQLITE_IOERR: + 'SQLite could not read or write the file. On a network share, or another filesystem without ' + + 'reliable file locking, keep the project and its index on a local disk.', + SQLITE_CORRUPT: 'The database is damaged; run "codegraph index" to rebuild it.', + SQLITE_FULL: 'The disk is full.', + SQLITE_CANTOPEN: + 'Check that the folder exists and that you can read and write it, and that no other program ' + + '(a backup or antivirus scan, say) holds the file open.', + SQLITE_NOTADB: 'The file is not a SQLite database; move it aside and run "codegraph init" to build a new index.', +}; + +/** A failure SQLite reported while opening the database at `dbPath`. */ +export class DatabaseOpenError extends Error { + /** node:sqlite's error code ("ERR_SQLITE_ERROR"), kept from the original. */ + readonly code: unknown; + /** SQLite's extended result code, kept from the original. */ + readonly errcode: unknown; + readonly errstr: unknown; + /** SQLite's name for the primary result code, e.g. "SQLITE_CANTOPEN". */ + readonly sqliteCode: string; + readonly dbPath: string; + + constructor(original: Error, dbPath: string) { + const codes = original as Error & { code?: unknown; errcode?: unknown; errstr?: unknown }; + const primary = typeof codes.errcode === 'number' ? codes.errcode & 0xff : -1; + const sqliteCode = SQLITE_CODE_NAMES[primary] ?? `SQLite error ${String(codes.errcode)}`; + const hint = HINTS[sqliteCode]; + super( + `Could not open the CodeGraph database at ${dbPath}: ${original.message} (${sqliteCode}).${hint ? ` ${hint}` : ''}`, + { cause: original } + ); + this.name = 'DatabaseOpenError'; + this.code = codes.code; + this.errcode = codes.errcode; + this.errstr = codes.errstr; + this.sqliteCode = sqliteCode; + this.dbPath = dbPath; + } +} + +/** + * The error to throw for `err`, raised while opening the database at + * `dbPath`: the Windows/WSL guidance for an index the two share, a + * {@link DatabaseOpenError} for any other failure SQLite reported (it carries + * a result code), and anything else unchanged, the missing-`node:sqlite` + * explanation included. + */ +export function toDatabaseOpenError(err: unknown, dbPath: string): unknown { + if (err instanceof DatabaseOpenError || err instanceof WslSharedIndexError || err instanceof NodeSqliteUnavailableError) { + return err; + } + const shared = toWslSharedIndexError(err, dbPath); + if (shared) return shared; + if (err instanceof Error && typeof (err as { errcode?: unknown }).errcode === 'number') { + return new DatabaseOpenError(err, dbPath); + } + return err; +} + +/** + * A project's index exists but could not be opened: what a caller reports as + * that, not as "no index here". The Windows/WSL case has its own guidance. + */ +export function isDatabaseOpenFailure(err: unknown): err is DatabaseOpenError | NodeSqliteUnavailableError { + return err instanceof DatabaseOpenError || err instanceof NodeSqliteUnavailableError; +} diff --git a/src/db/queries.ts b/src/db/queries.ts index 2376ced4b5..ad7a206a3c 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -23,6 +23,7 @@ import { parseQuery, boundedEditDistance } from '../search/query-parser'; import { isGeneratedFile } from '../extraction/generated-detection'; import { splitIdentifierSegments } from '../search/identifier-segments'; import { isModuleKey, referenceNameTail, MEMBER_TAIL_SQL, MODULE_PARKED_SQL } from './reference-tail'; +import { NAME_SUBSTRING_KINDS, buildNameSubstringIndex, searchNameSubstring, type NameSubstringIndex, type ScanOrder } from './name-substring-index'; /** * Files that should not be candidates for "dominant file" detection: tests, @@ -109,6 +110,15 @@ const THROUGH_NAMES = (column: string): string => `( */ export const DEPRIORITIZED_NAME_BONUS_SCALE = 0.75; +/** + * A `deprioritize` path predicate (#982) that can also hand out a copy with + * the project's patterns read once, for loops over many paths (#2620). + */ +export interface DeprioritizedPathMatcher { + (filePath: string): boolean; + snapshot?: () => (filePath: string) => boolean; +} + /** * The fields that tell one node of a file from another across a re-index of * that file, whose node ids change with every line shift (#2276). @@ -332,6 +342,12 @@ function rowToBinding(r: Record): Binding { * under-count nodes per file and pick that plan, and a few huge files (a * generated R.java) then cost ~100 s (#2571). Ties go to the path, so the * plan never decides between equal counts. + * + * Every file, not a top-N window: the test and generated files the caller + * drops can fill any window, and then the real dominant file, or its + * runner-up, was never among the rows (#2578). The window saved nothing + * either: ORDER BY needs every group, so the aggregate runs to the end + * whatever the LIMIT; only the rows handed back differ, one per file. */ export const DOMINANT_FILE_SQL = ` SELECT n.file_path AS file_path, COUNT(*) AS edge_count @@ -341,9 +357,15 @@ export const DOMINANT_FILE_SQL = ` WHERE n.file_path = m.file_path GROUP BY n.file_path ORDER BY edge_count DESC, n.file_path - LIMIT 20 `; +/** + * Ranked candidates the low-value filter is asked about at a time, in + * `computeDominantFile`: the generated flag is a database lookup, so it is + * made per batch rather than for every file with an in-file edge. + */ +const DOMINANT_FILE_BATCH = 20; + /** * Query builder for the knowledge graph database */ @@ -373,6 +395,12 @@ export class QueryBuilder { | { stamp: string; value: { filePath: string; edgeCount: number; nextEdgeCount: number } | null } | undefined; + // Query-independent reads that a search repeats for every word of its query + // — the name list the fuzzy fallback scans, the name-substring index — each + // tagged with the change stamp it was read under, like dominantFileMemo + // (#2620). See memoizedRead(). + private readMemos = new Map(); + // Prepared statements (lazily initialized) private stmts: { insertNode?: SqliteStatement; @@ -430,8 +458,11 @@ export class QueryBuilder { getDominantFile?: SqliteStatement; getChangeStamp?: SqliteStatement; getTopRouteFile?: SqliteStatement; + unseenName?: SqliteStatement; getRoutingManifest?: SqliteStatement; + getRoutingManifestPerRoute?: SqliteStatement; insertNameSegment?: SqliteStatement; + getCppIncludeNodes?: SqliteStatement; } = {}; // Names whose segments were already written this session — skips re-splitting @@ -521,6 +552,26 @@ export class QueryBuilder { // connection, or a worker following a rebuilt index keeps its answer (#1864). this.dominantFileMemo = undefined; this.allNodeNames = undefined; + this.readMemos.clear(); + } + + /** + * `compute()`'s answer, reused until the database changes (see + * getChangeStamp), or null inside a transaction, where a rollback could + * leave a stale answer under an unchanged stamp: the caller then reads the + * database directly, as it did before the memo. + */ + private memoizedRead(key: string, compute: () => T): T | null { + if (this.db.inTransaction !== false) { + this.readMemos.delete(key); + return null; + } + const stamp = this.getChangeStamp(); + const memo = this.readMemos.get(key); + if (memo?.stamp === stamp) return memo.value as T; + const value = compute(); + this.readMemos.set(key, { stamp, value }); + return value; } private shapeStmt(sql: string): SqliteStatement { @@ -578,6 +629,17 @@ export class QueryBuilder { return this.deprioritizedPathProvider?.(); } + /** + * The `deprioritize` predicate with the project's patterns read once, for a + * loop that asks about many paths. The plain predicate re-checks + * `codegraph.json` on every call, which a long query's candidate loops made + * hundreds of thousands of times (#2620). + */ + getDeprioritizedPathSnapshot(): ((filePath: string) => boolean) | undefined { + const matcher = this.getDeprioritizedPathMatcher() as DeprioritizedPathMatcher | undefined; + return matcher?.snapshot ? matcher.snapshot() : matcher; + } + // =========================================================================== // Node Operations // =========================================================================== @@ -1421,8 +1483,8 @@ export class QueryBuilder { private computeDominantFile(): { filePath: string; edgeCount: number; nextEdgeCount: number } | null { if (!this.stmts.getDominantFile) { - // Pull top 20 candidates; we then filter out test/generated files - // in code (regex-grade matching that SQL LIKE can't express). The + // Every file ranked; test/generated files are then filtered out in + // code (regex-grade matching that SQL LIKE can't express). The // generated-file filter is critical — without it, etcd's // `api/etcdserverpb/rpc.pb.go` (1916 in-file edges, generated // protobuf stub) outranks the real `server/etcdserver/server.go` @@ -1432,13 +1494,26 @@ export class QueryBuilder { this.stmts.getDominantFile = this.db.prepare(DOMINANT_FILE_SQL); } const rows = this.stmts.getDominantFile.all() as Array<{ file_path: string; edge_count: number }>; - const generated = this.getGeneratedPathsAmong(rows.map(r => r.file_path)); - const filtered = rows.filter(r => !isLowValueFile(r.file_path, generated)); - if (filtered.length === 0 || filtered[0]!.edge_count < 20) return null; + // The first two files the filter keeps, in rank order: the dominant file + // and its runner-up. Filtering a 20-row window instead returned null once + // 20 test files outranked the real code, and an understated runner-up + // (0) once 19 did, which passes the caller's "3x the next file" test for + // a file that is not dominant at all (#2578). + const kept: Array<{ file_path: string; edge_count: number }> = []; + for (let i = 0; i < rows.length && kept.length < 2; i += DOMINANT_FILE_BATCH) { + const batch = rows.slice(i, i + DOMINANT_FILE_BATCH); + const generated = this.getGeneratedPathsAmong(batch.map(r => r.file_path)); + for (const r of batch) { + if (isLowValueFile(r.file_path, generated)) continue; + kept.push(r); + if (kept.length === 2) break; + } + } + if (kept.length === 0 || kept[0]!.edge_count < 20) return null; return { - filePath: filtered[0]!.file_path, - edgeCount: filtered[0]!.edge_count, - nextEdgeCount: filtered[1]?.edge_count ?? 0, + filePath: kept[0]!.file_path, + edgeCount: kept[0]!.edge_count, + nextEdgeCount: kept[1]?.edge_count ?? 0, }; } @@ -1453,6 +1528,12 @@ export class QueryBuilder { * Excludes test/generated files from candidacy. Returns null if there * are fewer than 3 non-test routes total, or if no file holds at least * 30% of them (diffuse routing → no single answer file). + * + * Every route file is counted. A 20-row window, filtered afterwards, + * returned null once 20 test files held more routes than the app's own, + * and measured the 30% share against the routes of those 20 files rather + * than of the project (#2578). One row per file that has a route, so the + * window saved next to nothing. */ getTopRouteFile(): { filePath: string; routeCount: number; totalRoutes: number } | null { if (!this.stmts.getTopRouteFile) { @@ -1461,8 +1542,7 @@ export class QueryBuilder { FROM nodes WHERE kind = 'route' GROUP BY file_path - ORDER BY cnt DESC - LIMIT 20 + ORDER BY cnt DESC, file_path `); } const rows = this.stmts.getTopRouteFile.all() as Array<{ file_path: string; cnt: number }>; @@ -1486,8 +1566,12 @@ export class QueryBuilder { * Also returns the file with the most handler endpoints — used as the * "top handler file" to inline source for, so the agent has both the * mapping AND the handler implementations. + * + * `limit` caps rows, and a route has a row per symbol it reaches. With + * `perRoute` it caps routes instead and returns every row of those routes, + * so one inline handler's many calls can't take other routes' places. */ - getRoutingManifest(limit: number = 40): { + getRoutingManifest(limit: number = 40, perRoute: boolean = false): { entries: Array<{ url: string; handler: string; @@ -1503,12 +1587,15 @@ export class QueryBuilder { topHandlerFileCount: number; totalRoutes: number; } | null { - if (!this.stmts.getRoutingManifest) { + const key = perRoute ? 'getRoutingManifestPerRoute' : 'getRoutingManifest'; + if (!this.stmts[key]) { // Edge kind varies across framework resolvers: Spring/Rails/ // Laravel/Drupal emit `references`, Express emits `calls`. Accept // both — the semantic is the same (route → its handler). A screen in // a Vue / Svelte / Astro app is served by a `component`. - this.stmts.getRoutingManifest = this.db.prepare(` + const serves = `e.kind IN ('references', 'calls') + AND h.kind IN ('function', 'method', 'class', 'constant', 'variable', 'component')`; + this.stmts[key] = this.db.prepare(` SELECT r.name AS url, r.id AS route_id, @@ -1522,13 +1609,19 @@ export class QueryBuilder { JOIN edges e ON e.source = r.id JOIN nodes h ON e.target = h.id WHERE r.kind = 'route' - AND e.kind IN ('references', 'calls') - AND h.kind IN ('function', 'method', 'class', 'constant', 'variable', 'component') + AND ${serves} + ${perRoute ? `AND r.id IN ( + SELECT r2.id FROM nodes r2 + WHERE r2.kind = 'route' + AND EXISTS (SELECT 1 FROM edges e JOIN nodes h ON e.target = h.id WHERE e.source = r2.id AND ${serves}) + ORDER BY r2.file_path, r2.start_line + LIMIT ? + )` : ''} ORDER BY r.file_path, r.start_line - LIMIT ? + ${perRoute ? '' : 'LIMIT ?'} `); } - const rows = this.stmts.getRoutingManifest.all(limit) as Array<{ + const rows = this.stmts[key]!.all(limit) as Array<{ url: string; route_id: string; route_file: string; route_line: number; handler: string; handler_file: string; handler_line: number; handler_kind: string; }>; @@ -1908,7 +2001,7 @@ export class QueryBuilder { // Apply multi-signal scoring if (results.length > 0 && (text || query)) { const scoringQuery = text || query; - const isDeprioritizedPath = this.getDeprioritizedPathMatcher(); + const isDeprioritizedPath = this.getDeprioritizedPathSnapshot(); results = results.map(r => { // A path the project de-prioritized is saying its symbol NAMES are not // the answer, so the exact-name bonus has to be damped too. The -15 path @@ -1916,6 +2009,8 @@ export class QueryBuilder { // on #982's repro, a `usage()` helper sat at 74.8 vs 51.2 for the top // product symbol — -15 lands at 59.8, still ahead). Damped, not zeroed, // so the tree stays findable when it genuinely is what you asked for. + // Evaluated once per result and reused, from a snapshot that read the + // config once for the whole loop. const deprioritized = isDeprioritizedPath?.(r.node.filePath) ?? false; const nameBonus = nameMatchBonus(r.node.name, scoringQuery); return { @@ -1979,15 +2074,20 @@ export class QueryBuilder { const lowered = text.toLowerCase(); const maxDist = lowered.length <= 4 ? 1 : 2; - // Pull the distinct name list once. getAllNodeNames() caches it until the - // database changes; even on a 200k-node project the distinct - // name set is typically O(10k) because most names repeat. The - // candidate-cap below bounds memory regardless. - const allNames = this.getAllNodeNames(); + // The distinct name list, lowercased, read once until the database + // changes: this fallback runs for every query word that FTS and LIKE both + // miss, which is most words of a pasted log, and the DISTINCT scan alone + // took ~150 ms a call on a 560k-node index (#2620). Inside a transaction + // it is read fresh, as before. + const { names, lowered: loweredNames, starts } = this.fuzzyNames(); const candidates: Array<{ name: string; dist: number }> = []; - for (const name of allNames) { - const dist = boundedEditDistance(name.toLowerCase(), lowered, maxDist); - if (dist <= maxDist) candidates.push({ name, dist }); + for (let i = 0; i < names.length; i++) { + // An edit distance is at least the length difference, so a name outside + // this window could never pass the bound: skipping it changes nothing. + const end = starts[i + 1]! - 1; + if (Math.abs(end - starts[i]! - lowered.length) > maxDist) continue; + const dist = boundedEditDistance(loweredNames.slice(starts[i]!, end), lowered, maxDist); + if (dist <= maxDist) candidates.push({ name: names[i]!, dist }); } candidates.sort((a, b) => a.dist - b.dist); @@ -2073,6 +2173,42 @@ export class QueryBuilder { }; } + /** + * Which of `names` the index has never seen: no node is called that, in any + * case, and no indexed name, qualified name, signature or docstring has a + * word starting with it. A file's path is its qualified name, so a folder or + * file named that counts as seen too. Explore uses this to say plainly that + * a name the query spelled is not in the index (#2462). + * + * A qualified name (`app.handle`, `pkg::run`) is asked about by its last + * segment: the index does not always record the receiver it was written on + * — express's `app.handle = function handle(…)` is indexed as `handle` — so + * only a member it has never seen makes the whole name unseen. + * + * Bounded like `getExploreMissDiagnostics`: two index probes per name (the + * lower(name) index and FTS postings), never a source scan, at most 16 names. + */ + getUnseenNames(names: readonly string[]): string[] { + const checked = [...new Set(names)].filter((n) => n.length > 0 && n.length <= 128).slice(0, 16); + if (checked.length === 0) return []; + if (!this.stmts.unseenName) { + this.stmts.unseenName = this.db.prepare(` + SELECT EXISTS (SELECT 1 FROM nodes WHERE lower(name) = lower(?)) + OR EXISTS (SELECT 1 FROM nodes_fts WHERE nodes_fts MATCH ?) AS seen + `); + } + const stmt = this.stmts.unseenName; + return checked.filter((name) => { + const member = name.split(/::|\./).pop()!; + // FTS splits the phrase on everything but letters and digits, so a + // snake_case name is matched as the words it is made of. + const phrase = member.replace(/"/g, ''); + if (phrase.length === 0) return false; + const row = stmt.get(member, `{name qualified_name signature docstring} : "${phrase}"*`) as { seen: number }; + return !row.seen; + }); + } + /** * FTS5 search with prefix matching */ @@ -2161,6 +2297,21 @@ export class QueryBuilder { private searchNodesLike(query: string, options: SearchOptions, hard?: HardFilters): SearchResult[] { const { kinds, languages, limit = 100, offset = 0 } = options; + // This runs when FTS found nothing, and a word FTS can't find is usually + // in no name at all: most words of a pasted log or document. Each scan + // read every node, about 120 ms on a 560k-node index (#2620). A word of + // ASCII letters and digits can only match inside a name, or inside one + // run of such characters in a qualified name, and lowercasing keeps those + // runs whole, so when neither lowercased list holds it there is nothing + // to find. Any other word is scanned as before. + if (/^[A-Za-z0-9]+$/.test(query)) { + const runs = this.memoizedRead('qualifiedNameRuns', () => this.readQualifiedNameRuns()); + if (runs !== null) { + const word = query.toLowerCase(); + if (!this.fuzzyNames().lowered.includes(word) && !runs.includes(word)) return []; + } + } + let sql = ` SELECT nodes.*, CASE @@ -2355,13 +2506,99 @@ export class QueryBuilder { sql += ' ORDER BY length(name) ASC, kind ASC, nodes.rowid ASC LIMIT ?'; params.push(limit); - const rows = this.shapeStmt(sql).all(...params) as (NodeRow & { score: number })[]; + // Explore asks this four times for every word of its query, and the + // leading `%` makes each one a scan of every node of those kinds (#2620). + // The in-memory index answers with the same rows in the same order: ties + // in length come out in the order SQLite's plan for this very statement + // reads rows, the kind index's or the table's. Anything it can't answer + // exactly (no kind filter, a language filter, a `%` in the word, another + // plan, a transaction) is asked of SQLite. + if (kinds && kinds.length > 0 && !(languages && languages.length > 0)) { + const index = this.memoizedRead('nameSubstringIndex', () => this.readNameSubstringIndex()); + const order = index ? this.substringScanOrder(index, sql, params, excludePrefix ? 2 : 1) : null; + const rowids = order ? searchNameSubstring(index!.index, substring, kinds, excludePrefix === true, limit, order) : null; + if (rowids) { + return this.getNodeRowsByRowid(rowids).map((row) => ({ node: rowToNode(row), score: 1.0 })); + } + } + + const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; return rows.map((row) => ({ node: rowToNode(row), score: row.score, })); } + /** + * The order SQLite's plan for a substring lookup reads candidate rows in, + * which is the order its ORDER BY leaves ties of equal length in: the kind + * index's (kind, file_path, start_line, id), or the table's (rowid). Which + * one depends on the statistics — a small index is scanned whole — so it is + * asked of SQLite once per statement shape. Null for any other plan. + */ + private substringScanOrder( + memo: { index: NameSubstringIndex; plans: Map }, + sql: string, + params: readonly (string | number)[], + patternCount: number, + ): ScanOrder | null { + // The patterns (the first `patternCount` parameters) can't change the + // plan, since a leading `%` uses no index; the kinds and the limit can. + const key = `${sql}\u0000${params.slice(patternCount).join('\u0000')}`; + let order = memo.plans.get(key); + if (order === undefined) { + order = null; + try { + const details = (this.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...params) as Array<{ detail: string }>) + .map((row) => row.detail) + .filter((detail) => /^(SCAN|SEARCH) (TABLE )?nodes\b/.test(detail)); + if (details.length === 1) { + if (/^SEARCH (TABLE )?nodes USING (COVERING )?INDEX idx_nodes_kind \(/.test(details[0]!)) order = 'index'; + else if (/^SCAN (TABLE )?nodes$/.test(details[0]!)) order = 'rowid'; + } + } catch { + order = null; + } + memo.plans.set(key, order); + } + return order; + } + + /** + * Every node of the kinds the name-substring index holds, read in + * idx_nodes_kind order (kind, file_path, start_line, id): the order its + * results break length ties by. + */ + private readNameSubstringIndex(): { index: NameSubstringIndex; plans: Map } | null { + const rows = this.db + .prepare( + `SELECT rowid AS rowid, kind, name FROM nodes WHERE kind IN (${NAME_SUBSTRING_KINDS.map(() => '?').join(',')})` + + ' ORDER BY kind, file_path, start_line, id' + ) + .all(...NAME_SUBSTRING_KINDS) as Array<{ rowid: number; kind: string; name: string }>; + const index = buildNameSubstringIndex(rows); + return index ? { index, plans: new Map() } : null; + } + + /** Whole node rows for `rowids`, in that order; a rowid no longer present is skipped. */ + private getNodeRowsByRowid(rowids: readonly number[]): NodeRow[] { + if (rowids.length === 0) return []; + const byRowid = new Map(); + for (let i = 0; i < rowids.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = rowids.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const rows = this.db + .prepare(`SELECT rowid AS rowid_, nodes.* FROM nodes WHERE rowid IN (${chunk.map(() => '?').join(',')})`) + .all(...chunk) as Array; + for (const row of rows) byRowid.set(row.rowid_, row); + } + const out: NodeRow[] = []; + for (const rowid of rowids) { + const row = byRowid.get(rowid); + if (row) out.push(row); + } + return out; + } + // =========================================================================== // Edge Operations // =========================================================================== @@ -3241,6 +3478,42 @@ export class QueryBuilder { })); } + /** + * {@link getUnresolvedReferencesFrom} for many nodes at once, limited to + * `kinds`: a Go type's embeddings of types outside the index, say. Chunked + * probe over `idx_unresolved_from_node`. + */ + getUnresolvedReferencesFromNodes(nodeIds: Iterable, kinds: readonly string[]): UnresolvedReference[] { + const unique = [...new Set(nodeIds)]; + const found: UnresolvedReference[] = []; + if (unique.length === 0 || kinds.length === 0) return found; + const kindPlaceholders = kinds.map(() => '?').join(','); + for (let i = 0; i < unique.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = unique.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT * FROM unresolved_refs + WHERE from_node_id IN (${placeholders}) + AND reference_kind IN (${kindPlaceholders})` + ) + .all(...chunk, ...kinds) as UnresolvedRefRow[]; + for (const row of rows) { + found.push({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + filePath: row.file_path, + language: row.language as Language, + rowId: row.id, + }); + } + } + return found; + } + /** * Find all edges where both source and target are in the given node set. * Useful for recovering inter-node connectivity after BFS. @@ -3509,6 +3782,37 @@ export class QueryBuilder { })); } + /** + * The module edges (`imports` edges into a file node) into any file under + * the folder `dir` (projectRoot-relative, no trailing slash), with the + * source file and language a resurrection needs: where an import of a + * package by its name can land, whichever file of the package its manifest + * or the folder's index makes the entry. + */ + getModuleImportsUnder( + dir: string + ): Array { + // `dir/` up to `dir0` ('0' is the byte after '/'). + const rows = this.db + .prepare( + `SELECT e.*, tgt.file_path AS target_file_path, src.file_path AS source_file_path, src.language AS source_language + FROM nodes tgt + JOIN edges e ON e.target = tgt.id AND e.kind = 'imports' + JOIN nodes src ON src.id = e.source + WHERE tgt.kind = 'file' AND tgt.file_path >= ? AND tgt.file_path < ?` + ) + .all(`${dir}/`, `${dir}0`) as Array< + EdgeRow & { target_file_path: string; source_file_path: string; source_language: Language } + >; + return rows.map((row) => ({ + ...rowToEdge(row), + edgeId: row.id, + targetFilePath: row.target_file_path, + sourceFilePath: row.source_file_path, + sourceLanguage: row.source_language, + })); + } + /** * The module edges (`imports` edges into a file node) into the files an * import of `dir/name` can find instead of a file added there: `dir/name.*` @@ -3628,11 +3932,12 @@ export class QueryBuilder { /** * Which of `filePaths` the index flagged as tool-generated (schema v9+). * - * Bounded-lookup by design: every consumer already holds a short candidate - * list (a ranked file group, an FTS result page, a LIMIT-20 aggregate), so - * this stays a partial-index probe over a handful of paths — no whole-repo - * set to materialize, and no cache to invalidate, which means a ranking call - * can never serve a verdict the last sync already replaced. + * Bounded-lookup by design: every consumer already holds a candidate list + * (a ranked file group, an FTS result page, a batch of the dominant-file + * ranking, the files that hold a route), so this stays a partial-index + * probe over those paths — no whole-repo set to materialize, and no cache + * to invalidate, which means a ranking call can never serve a verdict the + * last sync already replaced. * * Returns ONLY the content/index signal; callers union it with * {@link isGeneratedFile} so pre-v9 databases (column present, all zeros @@ -4191,6 +4496,77 @@ export class QueryBuilder { return names; } + /** + * The distinct node names, in {@link getAllNodeNames} order, and the same + * names lowercased and newline-joined: name `i` lowercased is + * `lowered.slice(starts[i], starts[i + 1] - 1)`. Read once until the + * database changes (fresh inside a transaction). The fuzzy fallback compares + * against it, and the LIKE fallback asks it whether any name holds a word. + */ + private fuzzyNames(): { names: string[]; lowered: string; starts: Int32Array } { + const read = (): { names: string[]; lowered: string; starts: Int32Array } => { + const names = this.getAllNodeNames(); + const lowered = names.map((name) => name.toLowerCase()); + const starts = new Int32Array(names.length + 1); + for (let i = 0; i < lowered.length; i++) starts[i + 1] = starts[i]! + lowered[i]!.length + 1; + return { names, lowered: lowered.join('\n') + '\n', starts }; + }; + return this.memoizedRead('fuzzyNames', read) ?? read(); + } + + /** + * The runs of ASCII letters and digits in the distinct qualified names, + * lowercased and newline-joined: where a word LIKE could find in a + * qualified name has to be. + */ + private readQualifiedNameRuns(): string { + const runs = new Set(); + const rows = this.db.prepare('SELECT DISTINCT qualified_name AS name FROM nodes').all() as Array<{ name: string }>; + for (const row of rows) { + for (const run of row.name.toLowerCase().split(/[^a-z0-9]+/)) { + if (run) runs.add(run); + } + } + return [...runs].join('\n'); + } + + /** The names of a language's nodes of the given kinds (repeats included), decoding nothing else. */ + getNodeNamesOfKinds(language: Language, kinds: readonly NodeKind[]): string[] { + // `+language`: the kind index narrows further than the language one. + const rows = this.db + .prepare(`SELECT name FROM nodes WHERE kind IN (${kinds.map(() => '?').join(', ')}) AND +language = ?`) + .all(...kinds, language) as Array<{ name: string }>; + return rows.map((r) => r.name); + } + + /** + * The import node of every C / C++ `#include`, narrowed to the path it + * spells and where it is written: the include graph reads nothing else, and + * decoding every import node of a large C++ project costs several times more. + */ + getCppIncludeNodes(): Array> { + if (!this.stmts.getCppIncludeNodes) { + this.stmts.getCppIncludeNodes = this.db.prepare( + `SELECT id, name, file_path AS filePath, language, start_line AS startLine, start_column AS startColumn + FROM nodes WHERE kind = 'import' AND language IN ('c', 'cpp')` + ); + } + return this.stmts.getCppIncludeNodes.all() as Array>; + } + + /** + * Stream the distinct node names one row at a time — the incremental + * counterpart to {@link getAllNodeNames} for callers that need to yield + * to the event loop mid-scan (resolver cache warm-up on multi-million-node + * indexes). Fresh statement per call: the iterator holds an open cursor. + */ + *iterateNodeNames(): IterableIterator { + const stmt = this.db.prepare('SELECT DISTINCT name FROM nodes'); + for (const row of stmt.iterate()) { + yield (row as { name: string }).name; + } + } + /** * Get unresolved references scoped to specific file paths. * Uses the idx_unresolved_file_path index for efficient lookup. diff --git a/src/db/sqlite-adapter.ts b/src/db/sqlite-adapter.ts index 39a9555395..9fee708355 100644 --- a/src/db/sqlite-adapter.ts +++ b/src/db/sqlite-adapter.ts @@ -7,9 +7,11 @@ * * CodeGraph ships with a bundled Node runtime, so `node:sqlite` (real SQLite, * with WAL + FTS5) is always available — there is no native build step and no - * wasm fallback. When run from source instead, it requires Node >= 22.13 or Bun >= 1.4. + * wasm fallback. When run from source instead, it requires Node >= 22.5. */ +import { loadNodeSqlite } from './node-sqlite'; +import { toDatabaseOpenError } from './open-error'; import { toWslSharedIndexError } from './wsl-shared-index'; export interface SqliteStatement { @@ -57,8 +59,7 @@ class NodeSqliteAdapter implements SqliteDatabase { private readonly _dbPath: string; constructor(dbPath: string, opts?: { readOnly?: boolean }) { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { DatabaseSync } = require('node:sqlite'); + const { DatabaseSync } = loadNodeSqlite(); this._dbPath = dbPath; this._db = opts?.readOnly ? new DatabaseSync(dbPath, { readOnly: true }) : new DatabaseSync(dbPath); // The search `path:` / `name:` filters' exact test where SQLite's ASCII-only @@ -209,16 +210,8 @@ export function createDatabase(dbPath: string, opts?: { readOnly?: boolean }): { try { return { db: new NodeSqliteAdapter(dbPath, opts), backend: 'node-sqlite' }; } catch (error) { - // node:sqlite loaded and SQLite itself failed — not a missing-module case. - const shared = toWslSharedIndexError(error, dbPath); - if (shared) throw shared; - const msg = error instanceof Error ? error.message : String(error); - throw new Error( - 'Failed to open SQLite via the built-in node:sqlite module.\n' + - 'CodeGraph requires node:sqlite (Node.js 22.13+ or Bun 1.4+). Install the\n' + - 'self-contained CodeGraph release (it bundles a compatible Node), or run\n' + - 'on Node 22.13+ or Bun 1.4+.\n' + - `Underlying error: ${msg}` - ); + // A runtime without node:sqlite says so (NodeSqliteUnavailableError); + // anything SQLite itself reported names its code and the path. + throw toDatabaseOpenError(error, dbPath); } } diff --git a/src/directory.ts b/src/directory.ts index 4a7ef3818c..29cd973cce 100644 --- a/src/directory.ts +++ b/src/directory.ts @@ -8,6 +8,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { isWslWindowsDrive } from './sync/watch-policy'; +import { loadNodeSqlite } from './db/node-sqlite'; /** The default per-project data directory name. */ export const DEFAULT_CODEGRAPH_DIR = '.codegraph'; @@ -235,10 +236,11 @@ function hasCodeGraphSchema(dbPath: string, st: fs.Stats): boolean { * second connection opened and closed here leaves the first one's locks alone. */ function probeSchema(dbPath: string): 'schema' | 'no-schema' | 'not-sqlite' | 'unknown' { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { DatabaseSync } = require('node:sqlite'); let db: any = null; try { + // A runtime without node:sqlite can't inspect the file: `unknown`, so the + // command goes on to open it and reports why that is impossible. + const { DatabaseSync } = loadNodeSqlite(); db = new DatabaseSync(dbPath, { readOnly: true }); const row = db.prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'nodes'").get(); return row !== undefined ? 'schema' : 'no-schema'; @@ -739,13 +741,14 @@ const STRUCTURAL_UNSEGMENTED = /如何|怎么|怎麼|在哪|哪里|哪裡|追踪 const DOC_DATA_EXT = /\.(md|markdown|txt|rst|json|ya?ml|toml|lock|csv|tsv|log|ini|cfg|conf|env|xml|html?|png|jpe?g|gif|svg|pdf)$/i; /** - * Is `prompt` a message the host wrote rather than the user? Claude Code - * delivers a finished background task as a `` user turn, and - * its task paths, ids and summary trip the keyword gate, so every finished - * command front-loaded unrelated source into the conversation. + * Claude Code injects `…` blocks as + * `user` messages when a background agent finishes, and UserPromptSubmit + * hooks receive them exactly like typed prompts (#1832). The whole prompt + * must be that single envelope; a user question that merely mentions the + * marker is still a prompt. */ -export function isHostNotification(prompt: string): boolean { - return /^\s*<(?:task-notification|system-reminder)>/.test(prompt); +export function isTaskNotification(prompt: string): boolean { + return /^\s*[\s\S]*<\/task-notification>\s*$/.test(prompt); } /** @@ -1151,3 +1154,41 @@ export function validateDirectory(projectRoot: string): { export function isAgentMessage(prompt: string): boolean { return /^\s*]*)?>[\s\S]*<\/agent-message>\s*$/.test(prompt); } + +/** + * A message another local Claude Code session sends to this one arrives as a + * `…` prompt, which + * UserPromptSubmit hooks also receive (#2635). Same rule as + * {@link isTaskNotification}: only a prompt that is entirely that envelope is + * skipped. + */ +export function isCrossSessionMessage(prompt: string): boolean { + return /^\s*]*)?>[\s\S]*<\/cross-session-message>\s*$/.test(prompt); +} + +const SYSTEM_REMINDER_CLOSE = ''; +/** One complete reminder block. The attribute run stops at `<`, so a tag left + * open in pasted text can't make the scan quadratic. */ +const SYSTEM_REMINDER_BLOCK = /]*)?>[\s\S]*?<\/system-reminder>/g; + +/** + * The prompt the user typed, without the `…` + * blocks Claude Code's desktop app puts in front of it, such as which other + * local sessions are linked. UserPromptSubmit hooks receive them as + * part of the prompt (#2635), but they are the host's status, not the user's + * words: gated and explored on, they made every turn, a bare "yes" included, + * inject context about the reminder, and they hid a task notification, a + * subagent hand-back or a cross-session message behind them from + * {@link isTaskNotification}, {@link isAgentMessage} and + * {@link isCrossSessionMessage}. Every complete block is removed and the rest + * trimmed; a `` the user only mentions, with no closing tag, + * stays. + */ +export function stripSystemReminders(prompt: string): string { + // Only the text up to the last closing tag can hold a complete block, and + // bounding the scan there keeps an unclosed tag from rescanning the tail. + const end = prompt.lastIndexOf(SYSTEM_REMINDER_CLOSE); + if (end < 0) return prompt.trim(); + const head = prompt.slice(0, end + SYSTEM_REMINDER_CLOSE.length); + return (head.replace(SYSTEM_REMINDER_BLOCK, '\n') + prompt.slice(head.length)).trim(); +} diff --git a/src/extraction/cfml-extractor.ts b/src/extraction/cfml-extractor.ts index 16d3b5dc24..92c537a20a 100644 --- a/src/extraction/cfml-extractor.ts +++ b/src/extraction/cfml-extractor.ts @@ -410,6 +410,11 @@ export class CfmlExtractor { case 'cf_function_tag': if (declScope) this.extractFunctionTag(node, parentClassName ? containerId : undefined, containerId, parentClassName); return; + case 'cf_selfclose_tag': + // ``, wherever it is written (#2651). Its + // attributes are still walked below for `#hash#` expressions. + if (this.selfCloseTagName(node) === 'include') this.extractIncludeTag(node, containerId); + break; case 'cf_tag': { const name = this.cfTagName(node); if (name === 'interface' && declScope && !parentClassName) { @@ -581,6 +586,60 @@ export class CfmlExtractor { return nameNode ? this.source.substring(nameNode.startIndex, nameNode.endIndex).toLowerCase() : undefined; } + /** + * A void tag's name, lowercased, without the `cf` prefix (`` → + * `include`). The grammar keeps no node for it: it is the text after `` (#2651) includes the template + * as cfscript's `include "formlistshared.cfm";` does, so it is extracted the + * same way: an import node in the scope it sits in (the function, the + * component, or the file of a template) and an `imports` reference from + * that scope, which the resolver links to the file. A template built at + * runtime (`template="#dir#/x.cfm"`) names no file the index can link. + */ + private extractIncludeTag(tag: SyntaxNode, scopeId: string): void { + const valueNode = this.tagAttrValueNode(tag, 'template', true); + const template = valueNode ? this.source.substring(valueNode.startIndex, valueNode.endIndex).trim() : ''; + if (!template) return; + // Qualified like the generic extractor's import nodes: under the scope's + // own name, without the file path (`baseform::formlistshared.cfm`). + const scope = this.nodes.find((n) => n.id === scopeId); + const prefix = !scope || scope.kind === 'file' ? '' : scope.kind === 'method' ? scope.qualifiedName : scope.name; + const line = tag.startPosition.row + 1; + const column = tag.startPosition.column; + const id = this.nodeIds.generate(this.filePath, 'import', template, line, column); + this.nodes.push({ + id, + kind: 'import', + name: template, + qualifiedName: prefix ? `${prefix}::${template}` : template, + filePath: this.filePath, + language: this.language, + startLine: line, + endLine: tag.endPosition.row + 1, + startColumn: column, + endColumn: tag.endPosition.column, + signature: this.source.substring(tag.startIndex, tag.endIndex).trim(), + updatedAt: Date.now(), + }); + this.edges.push({ source: scopeId, target: id, kind: 'contains' }); + this.unresolvedReferences.push({ + fromNodeId: scopeId, + referenceName: template, + referenceKind: 'imports', + filePath: this.filePath, + line, + column, + language: this.language, + }); + } + /** * Delegate a `...` tag body to the cfscript grammar. * With `parentClassName` set (the block sits at component scope), functions @@ -662,8 +721,32 @@ export class CfmlExtractor { } this.unresolvedReferences.push(ref); } - for (const error of result.errors) { - if (error.line) error.line += startLine; + this.addBlockErrors(result.errors, scriptTag, 'cfscript', startLine); + } + + /** + * Record a delegated block's errors as the file's (#2654). The block's own + * extraction warns "parse produced no symbols … contributes nothing to the + * graph" when its code has a syntax error and declares nothing — which a + * `` body never does, so any SQL its grammar can't read earned it. + * That is about the block, not the file, whose symbols come from the tag + * walk: it is recorded as a block of the file that failed to parse, by its + * tag and line, under its own code, so `init` and `status` don't report the + * file as one that contributes nothing. Other errors pass through. + */ + private addBlockErrors(errors: ExtractionError[], tag: SyntaxNode, tagName: 'cfscript' | 'cfquery', lineOffset: number): void { + for (const error of errors) { + if (error.severity === 'warning' && error.code === 'parse_error') { + const line = tag.startPosition.row + 1; + this.errors.push({ + message: `${this.filePath}: <${tagName}> at line ${line} has a syntax error — the rest of the file is indexed, but calls inside that block may be missing from the graph`, + line, + severity: 'warning', + code: 'partial_parse', + }); + continue; + } + if (error.line) error.line += lineOffset; this.errors.push(error); } } @@ -696,10 +779,7 @@ export class CfmlExtractor { if ((!ref.fromNodeId || ref.fromNodeId === innerFileNodeId) && parentId) ref.fromNodeId = parentId; this.unresolvedReferences.push(ref); } - for (const error of result.errors) { - if (error.line) error.line += startLine; - this.errors.push(error); - } + this.addBlockErrors(result.errors, queryTag, 'cfquery', startLine); } /** Read a `cf_attribute`'s value by name from a tag node's direct `cf_attribute`/`cf_tag_attributes` children. */ @@ -713,11 +793,13 @@ export class CfmlExtractor { /** * The `attribute_value` node of a tag attribute — `undefined` when the tag * has no such attribute, `null` when its value isn't plain text (empty, or - * a `#hash#` expression). + * a `#hash#` expression). With `wholeValue`, also `null` when only part of + * the value is plain text (`template="#dir#/x.cfm"`). */ - private tagAttrValueNode(tag: SyntaxNode, attrName: string): SyntaxNode | null | undefined { + private tagAttrValueNode(tag: SyntaxNode, attrName: string, wholeValue = false): SyntaxNode | null | undefined { const wrapper = this.tagAttrValueWrapper(tag, attrName); if (wrapper === undefined) return undefined; + if (wholeValue && wrapper?.namedChildCount !== 1) return null; return wrapper?.namedChildren.find((c: SyntaxNode) => c.type === 'attribute_value') ?? null; } diff --git a/src/extraction/function-ref.ts b/src/extraction/function-ref.ts index 411a81d816..efe1d6dc69 100644 --- a/src/extraction/function-ref.ts +++ b/src/extraction/function-ref.ts @@ -358,6 +358,11 @@ const PASCAL_SPEC: FnRefSpec = { ['assignment', { mode: 'rhs', field: 'rhs' }], // OnClick := Handler ]), unwrap: new Map([['exprUnary', 'operand']]), // @Handler + // A bare name (`OnClick := Handler`, `Register(Handler)`) is a paren-less + // name the extractor records itself, linked by scope in the kernel's + // pascal_scope port (#2514); only the explicit `@Handler` is a function + // value here. + addressOfOnly: true, }; /** @@ -881,3 +886,163 @@ function rubyEnclosingCall(node: SyntaxNode): SyntaxNode | null { } return null; } + +// --------------------------------------------------------------------------- +// Go: a local that shadows the function a value names (#2596) +// --------------------------------------------------------------------------- + +/** + * Go nodes that open a scope a local can be declared in: a block, the implicit + * block of an `if` / `for` / `switch` / `select` statement, and each clause of + * a `switch` or `select`. + */ +const GO_SCOPE_TYPES = new Set([ + 'block', + 'if_statement', + 'for_statement', + 'expression_switch_statement', + 'type_switch_statement', + 'select_statement', + 'expression_case', + 'default_case', + 'type_case', + 'communication_case', +]); + +/** + * Where a Go function-value candidate sits: the innermost scope node around + * it, and the unit it is written in: the function or method declaration, or + * for a func literal at file scope (`var h = func(…) {…}`) the outermost + * literal, else null. Mirrored in the kernel (go.rs `go_fn_ref_site`). + */ +export interface GoFnRefSite { + scope: number; + unit: SyntaxNode | null; +} + +/** {@link GoFnRefSite} of the value container `node`. */ +export function goFnRefSite(node: SyntaxNode): GoFnRefSite { + let scope = -1; + let literal: SyntaxNode | null = null; + for (let cur: SyntaxNode | null = node; cur; cur = cur.parent) { + if (scope < 0 && GO_SCOPE_TYPES.has(cur.type)) scope = cur.id; + if (cur.type === 'function_declaration' || cur.type === 'method_declaration') return { scope, unit: cur }; + if (cur.type === 'func_literal') literal = cur; + } + return { scope, unit: literal }; +} + +/** + * The names among `names` that a Go function binds itself, keyed + * `|`: its func literals' parameters, `:=`, `var` and `const`, + * range and type-switch variables, and a select case's received value. Such a + * local shadows a package function of the same name for the rest of its + * scope, which the resolver reads per use (`isGoLocallyBound`). A name the + * function's own receiver, parameters or results bind is left out: those + * cover its whole body, so every use is the local and one reference is + * enough. Mirrored in the kernel (go.rs `go_local_bindings`). + */ +export function goLocalBindings( + units: Iterable, + names: ReadonlySet, + source: string +): Set { + const out = new Set(); + for (const unit of units) { + const bind = (id: SyntaxNode | null | undefined): void => { + if (id?.type !== 'identifier') return; + const name = getNodeText(id, source); + if (names.has(name)) out.add(`${unit.id}|${name}`); + }; + const declares = (n: SyntaxNode): boolean => { + for (let i = 0; i < n.childCount; i++) { + const c = n.child(i); + if (c && !c.isNamed && c.type === ':=') return true; + } + return false; + }; + const stack: SyntaxNode[] = [unit]; + while (stack.length > 0) { + const n = stack.pop()!; + switch (n.type) { + case 'short_var_declaration': + for (const c of getChildByField(n, 'left')?.namedChildren ?? []) bind(c); + break; + case 'range_clause': + case 'receive_statement': + if (declares(n)) for (const c of getChildByField(n, 'left')?.namedChildren ?? []) bind(c); + break; + case 'type_switch_statement': + for (const c of getChildByField(n, 'alias')?.namedChildren ?? []) bind(c); + break; + case 'var_spec': + case 'const_spec': + case 'parameter_declaration': + case 'variadic_parameter_declaration': + for (const c of n.childrenForFieldName('name')) bind(c); + break; + } + for (let i = n.namedChildCount - 1; i >= 0; i--) { + const c = n.namedChild(i); + if (c) stack.push(c); + } + } + for (const field of ['receiver', 'parameters', 'result']) { + const list = getChildByField(unit, field); + if (list?.type !== 'parameter_list') continue; + for (const p of list.namedChildren) { + if (p.type !== 'parameter_declaration' && p.type !== 'variadic_parameter_declaration') continue; + for (const c of p.childrenForFieldName('name')) { + if (c.type === 'identifier') out.delete(`${unit.id}|${getNodeText(c, source)}`); + } + } + } + } + return out; +} + +/** + * The Go function-value candidates that need one reference per scope instead + * of one per function (#2596), each with the id of the scope it sits in. A + * function records a value-use of a name once, at its first use; but when the + * function also binds that name itself, the first use can be the local while + * a later one, in another scope, is the package function. The resolver drops + * a use the local shadows, so the later use needs a reference of its own. One + * per scope node is enough: within one scope a local declared before the + * first use still covers every later use. Only a name a function uses at + * least twice is looked up, so a candidate's position is read from the tree + * just for those. Mirrored in the kernel (go.rs `go_scoped_fn_ref_scopes`). + */ +export function goScopedFnRefScopes( + candidates: ReadonlyArray, + source: string +): Map { + const groups = new Map(); + for (const c of candidates) { + if (!c.goNode || c.name.includes('.')) continue; + const key = `${c.fromNodeId}|${c.name}`; + const group = groups.get(key); + if (group) group.push(c); + else groups.set(key, [c]); + } + const sites = new Map(); + const units = new Map(); + const names = new Set(); + for (const group of groups.values()) { + if (group.length < 2) continue; + for (const c of group) { + const site = goFnRefSite(c.goNode!); + if (!site.unit) continue; + sites.set(c, site); + units.set(site.unit.id, site.unit); + names.add(c.name); + } + } + const out = new Map(); + if (units.size === 0) return out; + const bound = goLocalBindings(units.values(), names, source); + for (const [c, site] of sites) { + if (bound.has(`${site.unit!.id}|${c.name}`)) out.set(c, site.scope); + } + return out; +} diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index ad700cd63c..5279414c72 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -8,6 +8,7 @@ */ import type { ExtractionError, Language } from '../types'; +import { parseSourceTreeSync, type TreeNode } from './parse-tree'; import * as fs from 'fs'; import * as path from 'path'; @@ -355,6 +356,19 @@ export async function loadGrammarsForLanguages(_languages: Language[]): Promise< export async function loadAllGrammars(): Promise {} +/** + * A parser handle for `language`, shaped as upstream's web-tree-sitter + * `Parser` ({ parse(source) → { rootNode } }). The kernel is the only + * parser on this fork — this is a shim over `parseSourceTreeSync` + * (parse-tree.ts), kept so upstream tests and callers that inspect a + * tree directly keep working. Returns null for a language with no + * tree-sitter grammar. + */ +export function getParser(language: Language): { parse(source: string): { rootNode: TreeNode; delete(): void } | null } | null { + if (!hasTreeSitterGrammar(language)) return null; + return { parse: (source: string) => parseSourceTreeSync(source, language) }; +} + /** * Detect language from file extension. * diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 70a22ea3a9..8c3fa8a761 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -518,9 +518,12 @@ function readGitExcludeExtraPatterns(rootDir: string): string { /** * Directories `git ls-files -o -i --exclude-standard --directory` reports as - * ignored-untracked. Seeded into {@link ScopeIgnore} so nested `.gitignore` - * effects (and any exclude-standard rule the flat matcher might miss) prune the - * watcher the same way the indexer skips them (#1728). + * ignored-untracked, relative to `rootDir` (git prints paths relative to its + * cwd, also when `rootDir` is a subfolder of the repo) and trailing-slashed. + * Seeded into {@link ScopeIgnore} so nested `.gitignore` effects (and any + * exclude-standard rule the flat matcher might miss) prune the watcher the + * same way the indexer skips them (#1728). Each one is a path, not a pattern: + * see {@link ScopeIgnore}. */ function listGitIgnoredDirectories(rootDir: string): string[] { try { @@ -536,8 +539,11 @@ function listGitIgnoredDirectories(rootDir: string): string[] { }, ); const dirs: string[] = []; - for (const entry of out.split('\0')) { - if (!entry) continue; + for (const raw of out.split('\0')) { + // `./` is git's "this whole directory" for a root that is itself ignored + // (#936); the root's own files stay in scope. + if (isWholeCwdEntry(raw)) continue; + const entry = normalizePath(raw); dirs.push(entry.endsWith('/') ? entry : `${entry}/`); } return dirs; @@ -959,11 +965,29 @@ export class ScopeIgnore { */ private include: Ignore | null = null, private includeRoots: string[] = [], + /** + * The directories `git ls-files --exclude-standard` reports as ignored + * (see listGitIgnoredDirectories): root-relative, trailing-slashed paths. + * Each names one directory and is matched as a literal path prefix, never + * as a gitignore pattern. As a pattern, `src/` matched every `src/` below + * the root too, hiding a whole Maven monorepo's sources (#2849), and a + * folder named `[draft]` or `a[b` read as a glob (or did not compile). + */ + private gitIgnoredDirs: ReadonlySet = new Set(), ) { // Longest root first so paths in nested embedded repos hit the innermost matcher. this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length); } + /** True when `rel` is one of {@link gitIgnoredDirs} or lies inside one. */ + private inGitIgnoredDir(rel: string): boolean { + if (this.gitIgnoredDirs.size === 0) return false; + for (let slash = rel.indexOf('/'); slash !== -1; slash = rel.indexOf('/', slash + 1)) { + if (this.gitIgnoredDirs.has(rel.slice(0, slash + 1))) return true; + } + return false; + } + ignores(rel: string): boolean { // User `exclude` (#999) is checked first and against the full root-relative // path: it must drop git-TRACKED paths (which `.gitignore` can't) and apply @@ -995,7 +1019,7 @@ export class ScopeIgnore { if (rel.endsWith('/') && this.embedded.some(({ root }) => root.startsWith(rel))) { return false; } - return this.rootMatcher.ignores(rel); + return this.rootMatcher.ignores(rel) || this.inGitIgnoredDir(rel); } } @@ -1007,19 +1031,17 @@ export class ScopeIgnore { export function buildScopeIgnore(rootDir: string, embeddedRoots?: Iterable): ScopeIgnore { const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir); const include = loadIncludeMatcher(rootDir); - // Root matcher already has defaults + root `.gitignore` + info/exclude + - // core.excludesFile. Seed ignored-untracked directories from git so nested - // `.gitignore` effects prune the watcher identically to the indexer (#1728). - const rootMatcher = buildDefaultIgnore(rootDir); - for (const dir of listGitIgnoredDirectories(rootDir)) { - rootMatcher.add(dir); - } + // The root matcher has defaults + root `.gitignore` + info/exclude + + // core.excludesFile. The ignored-untracked directories git reports join + // them, so nested `.gitignore` effects prune the watcher identically to the + // indexer (#1728) — as literal paths, not patterns (#2849). return new ScopeIgnore( - rootMatcher, + buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots(loadIncludePatterns(rootDir)) : [], + new Set(listGitIgnoredDirectories(rootDir)), ); } @@ -1168,6 +1190,18 @@ export function findUnindexedIgnoredRepos(rootDir: string): string[] { return repos; } +/** + * The `codegraph.json` `includeIgnored` pattern that opts in exactly the repo + * at `repoDir`, a root-relative, trailing-slashed path from + * {@link findUnindexedIgnoredRepos}. Patterns are gitignore syntax, so the + * path is anchored to the root (a bare `child/` also matches every `child/` + * below it) and its `*` and `[` are escaped (#2849). The `ignore` matcher has + * no working escape for `?`, so a `?` stays a one-character wildcard. + */ +export function includeIgnoredPatternFor(repoDir: string): string { + return '/' + repoDir.replace(/[*[]/g, '\\$&'); +} + /** * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each * gitignored directory, search for nested `.git` roots. Returns repo paths @@ -2020,6 +2054,7 @@ export class ExtractionOrchestrator { getNodesByName: () => [], getNodesByQualifiedName: () => [], getNodesByKind: () => [], + getNodesByLowerName: () => [], getImportMappings: () => [], getAllFiles: () => files, getProjectRoot: () => rootDir, diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index b291e7a1e2..8424f2676d 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -28,6 +28,91 @@ import { export { getKernel, kernelSupports, resetKernelForTests } from './loader'; export { decodeExtractBuffers } from './decode'; +/** + * Languages routed to the kernel by default (gate-passed only — see the + * per-language tracker in docs/design/rust-kernel-migration-plan.md §4). + * Per-file safety valve regardless of routing: a file whose parse tree + * contains ERRORS defers to the wasm extractor (error recovery differs + * between UTF-8 and UTF-16 parsing — wasm's recovery is canonical). + */ +const DEFAULT_ROUTED: ReadonlySet = new Set([ + 'typescript', + 'tsx', + 'javascript', + 'jsx', + 'java', + 'python', + 'go', + // R7a (2026-07-17): parity swept 0-diff on redis/git/fmt/protobuf/ALS + // (2,389 files compared) + full-init dump-diffs byte-identical; erroring + // files defer per-file to wasm (routine for macro-heavy C/C++ — see + // scripts/kernel-parity.mjs --max-deferral). + 'c', + 'cpp', + // R7b (2026-07-20): parity swept 0-diff on ripgrep/tokio/rust-analyzer + // (2,108 files byte-parity) + full-init dump-diffs byte-identical. Rust + // deferral is ~0% on normal repos; token-macro-table sources (rust-analyzer's + // parser crates, 18%) error on BOTH arms — grammar-inherent, not a walker + // signal. + 'rust', + // R7b (2026-07-20): parity swept 0-diff on serilog/Newtonsoft.Json/jellyfin + // (3,229 files byte-parity) + full-init dump-diffs byte-identical ×3. + // Deferral 0.05–3.3% — both-branches-kept `#if` damage that errors on BOTH + // arms (the preParse blanking hoist keeps the kernel's input identical). + 'csharp', + // R7b (2026-07-20): parity swept 0-diff on sinatra/jekyll/rails (3,763 + // files byte-parity, 0 deferrals — ruby error incidence is 0.00%) + + // full-init dump-diffs byte-identical ×3. Any deferral on a ruby sweep is + // a walker-bug signal, not grammar reality. + 'ruby', + // R7b (2026-07-20): parity swept 0-diff on monolog/laravel-framework/ + // symfony (13,950 files byte-parity) + full-init dump-diffs byte-identical + // ×3. Deferral ≈0–0.1% (genuinely-broken fixtures) — default sweep guard. + 'php', + // R7b (2026-07-20): parity swept 0-diff on Alamofire/vapor/swift-nio (720 + // clean files byte-parity; Alamofire's 348 #1020 property nodes reproduced + // exactly) + full-init dump-diffs byte-identical ×3. Swift error incidence + // is structurally 9–27% on BOTH arms (heavy #if conditionalization) — + // sweeps run --max-deferral 0.3; a deferral-rate JUMP is the bug signal. + 'swift', + // R7b (2026-07-20): parity swept 0-diff on okio/okhttp/kotlinx.coroutines + // (1,861 clean files byte-parity; KMP expect/actual synthesis identical — + // 412 edges both arms on kotlinx.coroutines) + full-init dump-diffs + // byte-identical ×3. Deferral 4.7–8.5% both-arm grammar reality + // (fun-interface misparses + PHANTOM hasError files) — default sweep guard + // holds; a JUMP past ~10% is the bug signal. + 'kotlin', + // R7b batch 4 (2026-07-20): parity swept 0-diff on dplyr/ggplot2/shiny + + // AnomalyDetection + full-init dump-diffs byte-identical ×3. R error + // incidence is ~0% (deferrals 0/0/0/1 across the gate repos — the 1 is a + // moustache-template pseudo-R file, both-arm): any deferral on an R sweep + // is a walker-bug signal, not grammar reality. + 'r', + // R7b batch 4 (2026-07-20): one walker, two dialects. Parity swept 0-diff + // on kong/lazy.nvim/lua-resty-core (lua) + lune/Fusion (luau) + full-init + // dump-diffs byte-identical. Lua error incidence ~0% (any deferral is a + // walker-bug signal); luau 1.4–7.1% both-arm grammar reality (generic type + // packs, default type params) — a JUMP past ~10% is the bug signal. + 'lua', + 'luau', + // R7b batch 4 (2026-07-20): parity swept 0-diff on os-lib/cats + scala3 + // compiler/src + library/src + full-init dump-diffs byte-identical. + // Error incidence is bimodal: mainstream Scala-2-style repos ~0-2% + // (os-lib 0%, cats 1.8%) but scala-3-frontier code runs 10-18% with + // PHANTOM-dominated error sets (hasError=true, zero ERROR nodes — + // capture-checking `^` types; defer on the FLAG). Sweeps over + // scala3-style repos use --max-deferral 0.3 (swift precedent); a + // deferral JUMP on cats/os-lib-style code is the bug signal. + 'scala', + // R7b batch 4 (2026-07-20): parity swept 0-diff on shelf/bloc/flutter + + // full-init dump-diffs byte-identical. Error incidence 3.4-20.7% both-arm + // grammar reality (empty object patterns — the sealed-class idiom — and + // unnamed `library;` dominate; flutter sits at ~21%) — sweeps use + // --max-deferral 0.3; the RATE, not the count, is the signal (flutter + // HEAD drifts). + 'dart', +]); + /** * Per-language TS post-pass over the decoded result — the escape hatch for * logic `.scm` queries can't express (macro salvage, dialect sniffing, @@ -56,14 +141,81 @@ function preParsedSource(filePath: string, source: string, language: Language): return pre ? pre(source, filePath) : source; } -/** True when `language` has a bespoke kernel walker (contractInfo.languages). */ +/** + * Whether the file's re-exports keep their statements as written while the + * kernel reads the preParsed bytes (`reExportAsWritten`: TypeScript's + * `export type *`, #2474). The decoded result then takes those import nodes' + * signatures from `source`, as TreeSitterExtractor does, and the undecoded + * fast path steps aside for it. + */ +function keepsReExportsAsWritten(source: string, pre: string, language: Language): boolean { + return pre !== source && EXTRACTORS[language]?.reExportAsWritten?.(source) === true; +} + +/** Each re-export import node's signature, read from `source` by the node's span. */ +function restoreReExportSignatures(result: ExtractionResult, source: string): void { + let lineStarts: number[] | null = null; + for (const node of result.nodes) { + if (node.kind !== 'import' || !node.signature?.startsWith('export')) continue; + if (!lineStarts) { + lineStarts = [0]; + for (let i = source.indexOf('\n'); i !== -1; i = source.indexOf('\n', i + 1)) lineStarts.push(i + 1); + } + const start = lineStarts[node.startLine - 1]! + node.startColumn; + const end = lineStarts[node.endLine - 1]! + node.endColumn; + node.signature = source.slice(start, end).trim(); + } +} + +function isRouted(language: Language): boolean { + const env = process.env.CODEGRAPH_KERNEL_LANGS; + if (env === undefined || env === '') return DEFAULT_ROUTED.has(language); + if (env === 'all') return true; + return env + .split(',') + .map((s) => s.trim()) + .includes(language); +} + +/** True when `language` would be extracted by the kernel right now. */ export function kernelRoutes(language: Language): boolean { - return kernelSupports(language); + return isRouted(language) && kernelSupports(language); } /** Warned-once registry so a broken language logs a single line, not one per file. */ const warned = new Set(); +/** + * One-slot defer memo. A file the kernel defers (parse errors → the generic + * extractor) used to pay the full pipeline again at every seam: the worker's + * raw try blanked + native-parsed it, extractFromSource's kernel try blanked + + * native-parsed it AGAIN, and the generic extractor then re-applied preParse a + * third time. On a high-deferral tree (the Linux kernel defers ~79% of files) + * that waste dominated the arm's parse phase. The slot remembers the LAST + * deferred (file, source, language) so (a) a repeat kernel attempt for the + * same file short-circuits to null, and (b) the generic extractor can reuse + * the already-blanked source instead of re-running preParse. Source is matched + * by string identity — the worker passes the same string through every seam. + */ +let deferSlot: { filePath: string; source: string; language: Language; pre: string } | null = null; + +/** The hoisted preParse output for a just-deferred file, if it matches. */ +export function takeDeferredPreParse( + filePath: string, + source: string, + language: Language +): string | null { + if ( + deferSlot && + deferSlot.filePath === filePath && + deferSlot.source === source && + deferSlot.language === language + ) { + return deferSlot.pre; + } + return null; +} + /** The raw table buffers + the cheap facts the orchestrator needs pre-decode. */ export interface KernelRawResult { buffers: NonNullable; @@ -84,7 +236,9 @@ function sourceParsedClean(meta: Buffer, pre: string, source: string): boolean { * so the main thread never materializes per-node objects. Returns null under * exactly the conditions tryKernelExtract does, PLUS when the language has a * registered post() pass (post passes operate on decoded results, so those - * languages keep the decoded path). + * languages keep the decoded path), and for a file whose re-exports keep their + * statements as written (keepsReExportsAsWritten), which the decoded path + * restores. */ export function tryKernelExtractRaw( filePath: string, @@ -94,7 +248,10 @@ export function tryKernelExtractRaw( if (!kernelRoutes(language) || POST_PASSES[language]) return null; const kernel = getKernel(); if (!kernel) return null; + if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred const pre = preParsedSource(filePath, source, language); + // Its signatures come from the source as written, which needs the decoded path. + if (keepsReExportsAsWritten(source, pre, language)) return null; try { const buffers = kernel.extractFile(filePath, pre, language); const meta = buffers.meta; @@ -119,7 +276,10 @@ export function tryKernelExtractRaw( const message = err instanceof Error ? err.message : String(err); // `defer:` is the stack guard's signal; the generic extractor takes the // file (parse_tree is iterative). Silent by design. - if (message.includes('defer:')) return null; + if (message.includes('defer:')) { + deferSlot = { filePath, source, language, pre }; + return null; + } if (!warned.has(language)) { warned.add(language); process.stderr.write( @@ -170,11 +330,13 @@ export function tryKernelExtract( if (!kernelRoutes(language)) return null; const kernel = getKernel(); if (!kernel) return null; + if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred const t0 = Date.now(); const pre = preParsedSource(filePath, source, language); try { const buffers = kernel.extractFile(filePath, pre, language); const result = decodeExtractBuffers(buffers, filePath, language); + if (keepsReExportsAsWritten(source, pre, language)) restoreReExportSignatures(result, source); POST_PASSES[language]?.(result, source); // Literal seeds are a pass over source text and the node list, never over // the tree (literal-capture.ts), so they need no Rust mirror — the same @@ -194,7 +356,10 @@ export function tryKernelExtract( const message = err instanceof Error ? err.message : String(err); // `defer:` is the stack guard's signal; the generic extractor takes the // file (parse_tree is iterative). Silent by design. - if (message.includes('defer:')) return null; + if (message.includes('defer:')) { + deferSlot = { filePath, source, language, pre }; + return null; + } if (!warned.has(language)) { warned.add(language); process.stderr.write( diff --git a/src/extraction/languages/c-cpp.ts b/src/extraction/languages/c-cpp.ts index 3ed5e88515..76cb7ed184 100644 --- a/src/extraction/languages/c-cpp.ts +++ b/src/extraction/languages/c-cpp.ts @@ -101,9 +101,11 @@ function findDeclaratorQualifiedId(declarator: SyntaxNode): SyntaxNode | undefin if (current.type === 'qualified_identifier') return current; for (let i = 0; i < current.namedChildCount; i++) { const child = current.namedChild(i); - // Don't descend into parameters or the trailing return type — their types - // (`const std::string&`, `-> std::string`) aren't the function name. - if (child && !['parameter_list', 'trailing_return_type', 'noexcept', 'throw_specifier', 'requires_clause'].includes(child.type)) { + // Don't descend into parameters, the trailing return type or the + // clauses after them — `const std::string&`, `-> std::string`, + // nlohmann/json's `noexcept(noexcept(JSONSerializer::to_json(…)))`, + // `throw(std::bad_alloc)`, `requires std::integral` name no function. + if (child && !DECLARATOR_NON_NAME_PARTS.has(child.type)) { queue.push(child); } } @@ -111,6 +113,11 @@ function findDeclaratorQualifiedId(declarator: SyntaxNode): SyntaxNode | undefin return undefined; } +/** The parts of a function declarator that hold types or expressions, never its name. */ +const DECLARATOR_NON_NAME_PARTS: ReadonlySet = new Set([ + 'parameter_list', 'trailing_return_type', 'noexcept', 'throw_specifier', 'requires_clause', +]); + /** * Single-argument macros are ambiguous without their definition (#1373). * Require a preceding local #define whose replacement uses its sole parameter @@ -1124,15 +1131,6 @@ export function maskCppRawStrings(source: string): { source: string; restore: (b }; } -/** - * A C definition whose return type a column-0 macro supplies: `LOCAL(int)` - * then `SRE(at)(SRE_STATE *state)` or `tf_q(long long c)` and its `{`. The - * grammar reads the macro as the type (`macro_type_specifier`); blanking it - * leaves a definition with no type, which C parses as `(at)` or as no - * function at all. - */ -const C_MACRO_TYPED_DEFINITION_RE = /^[A-Za-z_]\w*\s*\([^()]*\)(?:\s*\([^()]*\))?\s*\{/; - /** * Blank annotation-style macro invocations that decorate a declaration but carry * NO terminating semicolon — the pervasive Unreal-Engine reflection markup @@ -1165,13 +1163,20 @@ const C_MACRO_TYPED_DEFINITION_RE = /^[A-Za-z_]\w*\s*\([^()]*\)(?:\s*\([^()]*\)) * an operator (`MAKE(a) + 1`) — all rejected. String/char literals inside the * args are skipped so an embedded `)` can't mis-close the balance. * - * Shared by C and C++. C passes `keepReturnTypeMacros`, which keeps a column-0 - * macro that a definition follows (see C_MACRO_TYPED_DEFINITION_RE). A blanked - * macro inside a block comment is harmless (comments don't parse), and the rare - * line-leading no-semicolon ALL-CAPS call that isn't markup only loses that one - * annotation, never a whole class. + * C runs it too (redis's `REDIS_NO_SANITIZE("bounds")` above a definition), + * with `keepCReturnTypes`: there a macro that wraps the return type of the + * function declared after it is no markup, it IS the type — cJSON's + * `CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item)`, libgit2's + * `GIT_EXTERN(int) git_blob_dup(…);` — and tree-sitter-c reads it as one (a + * `macro_type_specifier`). Blanked with its argument, it left `cJSON_Print(…)` + * and a block, which parse as a call: the function had no node (#2470). See + * isCReturnTypeMacro for how such a macro is told from markup. + * + * A blanked macro inside a block comment is harmless (comments don't parse), + * and the rare line-leading no-semicolon ALL-CAPS call that isn't markup only + * loses that one annotation, never a whole class. */ -export function blankCppAnnotationMacroCalls(source: string, keepReturnTypeMacros = false): string { +export function blankCppAnnotationMacroCalls(source: string, options: { keepCReturnTypes?: boolean } = {}): string { if (!/^[ \t]*[A-Z][A-Z0-9_]{2,}\s*\(/m.test(source)) return source; const rawStrings = maskCppRawStrings(source); source = rawStrings.source; @@ -1180,7 +1185,8 @@ export function blankCppAnnotationMacroCalls(source: string, keepReturnTypeMacro let m: RegExpExecArray | null; while ((m = re.exec(source)) !== null) { const macroStart = m.index + (m[1] ?? '').length; // skip leading indent - let i = m.index + m[0].length - 1; // index of the opening '(' + const open = m.index + m[0].length - 1; // index of the opening '(' + let i = open; let depth = 0; let end = -1; for (; i < source.length; i++) { @@ -1207,7 +1213,7 @@ export function blankCppAnnotationMacroCalls(source: string, keepReturnTypeMacro // Only markup is followed by the declaration it decorates; a statement call // (`;`), init-list item (`,`/`{`), or expression fragment (operator) is not. if (!after || !/[A-Za-z_~#]/.test(after)) continue; - if (keepReturnTypeMacros && macroStart === m.index && C_MACRO_TYPED_DEFINITION_RE.test(source.slice(j, j + 4096))) continue; + if (options.keepCReturnTypes && macroStart === m.index && isCReturnTypeMacro(source, m.index, open, end, j)) continue; for (let k = macroStart; k < end; k++) { if (chars[k] !== '\n' && chars[k] !== '\r') chars[k] = ' '; } @@ -1216,25 +1222,104 @@ export function blankCppAnnotationMacroCalls(source: string, keepReturnTypeMacro return rawStrings.restore(chars.join('')); } +/** A macro argument that reads as a C type: `char *`, `const cJSON *`, `unsigned long`, `git_repository *`. */ +const C_RETURN_TYPE_ARG_RE = /^\s*[A-Za-z_][\w\s*]*$/; +/** A function declarator's name and the `(` of its parameter list. */ +const C_DECLARATOR_HEAD_RE = /[A-Za-z_]\w*\s*\(/y; +/** + * What may end a function declarator: attributes (`__attribute__((format(printf, 1, 2)))`, + * an ALL-CAPS attribute macro, with or without arguments), then the body's `{` or a prototype's `;`. + */ +const C_DECLARATOR_TAIL_RE = /\s*(?:(?:__attribute__\s*\(\((?:[^()]|\([^()]*\))*\)\)|[A-Z][A-Z0-9_]*(?![\w$])(?:\s*\((?:[^()]|\([^()]*\))*\))?)\s*)*[{;]/y; + +/** + * Whether the column-0 macro call `MACRO(…)` starting at `lineStart` (its + * argument list from `open` to `end`, the next code at `next`) is the return + * type of the function declared right after it, not markup: + * - its one argument reads as a type, so `ATTR_FORMAT(1, 2)` and + * `REDIS_NO_SANITIZE("bounds")` are markup; + * - only the function's declarator follows: a name, its parameter list, + * attributes, then `{` or `;`. Markup is followed by a declaration with a + * type of its own (`ATTR(x) void f(void);`); + * - the line above leaves no declaration open. jemalloc writes the type + * there: `size_t JEMALLOC_NOTHROW\nJEMALLOC_ATTR(pure)\nje_sallocx(…) {`, + * and that macro is the attribute it looks like. + * Column 0 is where a file-scope declaration starts; an indented statement + * prefix in a body (`FOREACH(n) visit(n);`) is no return type. + */ +function isCReturnTypeMacro(source: string, lineStart: number, open: number, end: number, next: number): boolean { + if (!C_RETURN_TYPE_ARG_RE.test(source.slice(open + 1, end - 1))) return false; + C_DECLARATOR_HEAD_RE.lastIndex = next; + const head = C_DECLARATOR_HEAD_RE.exec(source); + if (!head) return false; + let close = -1; + let depth = 0; + for (let i = next + head[0].length - 1; i < source.length && close < 0; i++) { + const c = source[i]; + if (c === '(') depth++; + else if (c === ')') { if (--depth === 0) close = i; } + else if (c === ';' || c === '{' || c === '}') return false; + } + if (close < 0) return false; + C_DECLARATOR_TAIL_RE.lastIndex = close + 1; + if (C_DECLARATOR_TAIL_RE.test(source)) return !opensCDeclaration(source, lineStart); + // `MACRO(t) NAME(a)(b) {`: a macro-shaped name whose declarator carries a + // second parameter list (`SRE(at)(SRE_STATE *state)`) — the first pass of + // C_MACRO_TYPED_DEFINITION_RE's `(?:\s*\([^()]*\))?` group. The second group + // must end at a body; a `;` makes this markup's ordinary declaration. + let i = close + 1; + while (i < source.length && /\s/.test(source[i] as string)) i++; + if (source[i] !== '(') return false; + let innerClose = -1; + let innerDepth = 0; + for (; i < source.length; i++) { + const c = source[i]; + if (c === '(') innerDepth++; + else if (c === ')') { if (--innerDepth === 0) { innerClose = i; break; } } + else if (c === ';' || c === '{' || c === '}') return false; + } + if (innerClose < 0) return false; + i = innerClose + 1; + while (i < source.length && /\s/.test(source[i] as string)) i++; + return source[i] === '{' && !opensCDeclaration(source, lineStart); +} + +/** + * Whether the line right above `lineStart` leaves a declaration open: its + * code ends in a type's last token (`unsigned long`, `char *`, `size_t + * JEMALLOC_NOTHROW`). A blank line, a directive, or a comment closes none. + */ +function opensCDeclaration(source: string, lineStart: number): boolean { + if (lineStart === 0) return false; + const from = source.lastIndexOf('\n', lineStart - 2) + 1; + const line = source.slice(from, lineStart - 1).replace(/\/\/.*$/, '').trim(); + return line !== '' && !line.startsWith('#') && /[\w*]$/.test(line); +} + /** * Blank a macro that is the ONLY token on its line — no parens, no semicolon: * namespace-management macros (`FMT_BEGIN_NAMESPACE`, `FMT_END_EXPORT`, * `JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS`), Qt's `Q_OBJECT`, and friends. A * bare identifier is not a statement or declaration in C or C++, so - * tree-sitter drops into error recovery at every one, which can mis-nest - * scopes in header-heavy trees. Replacing the token with equal-length spaces + * tree-sitter drops into error recovery at every one — and since the kernel + * path defers ANY erroring file to wasm, this single idiom deferred 13/73 fmt + * files and a comparable share of jemalloc, forfeiting the native-parse win + * on exactly the header-heavy trees it targets (the wasm path also mis-nests + * scopes around them today). Replacing the token with equal-length spaces * preserves every byte offset and the surrounding declarations parse clean. * * Matched tightly so a real identifier can never be touched — ALL of: - * - the line consists of ONE ALL-CAPS token (≥4 chars after any leading `_`, - * with `_`; `_TESTFUNC_IMPL` counts), optionally + * - the line consists of ONE ALL-CAPS token (≥4 chars, with `_`), optionally * followed by a same-line comment — a lone lowercase identifier or any * second token disqualifies; * - the PREVIOUS non-blank line does not end in a continuation character * (`=`, an operator, `,`, `(`, `?`, `:`, or a `\` macro-definition * continuation) — so an ALL-CAPS operand split onto its own line inside a * multi-line expression (`int x =\n SOME_CONST\n | OTHER;`) is left - * alone; and + * alone. A preprocessor directive ends at its own line, even one that + * ends in `>`: GLib's `G_BEGIN_DECLS` below `#include `, and + * Objective-C's `NS_ASSUME_NONNULL_BEGIN` below `#import + * `, are blanked; and * - the NEXT line of code starts like a declaration/scope token * (letter, `_`, `#`, `{`, `}`, or `~`) or the file ends — an operator, * string literal, or `;` continuation rejects the match. Comments in @@ -1243,36 +1328,30 @@ export function blankCppAnnotationMacroCalls(source: string, keepReturnTypeMacro * Shared by C and C++ (the idiom is identical in both), and by Objective-C * through OBJC_LONE_MACROS. */ -const LONE_MACRO_LINE_RE = /^[ \t]*(_*[A-Z][A-Z0-9_]{3,})[ \t]*(?:\/\/[^\n\r]*|\/\*[^\n\r]*\*\/[ \t]*)?\r?$/; +const LONE_MACRO_LINE_RE = /^[ \t]*([A-Z][A-Z0-9_]{3,})[ \t]*(?:\/\/[^\n\r]*|\/\*[^\n\r]*\*\/[ \t]*)?\r?$/; const LONE_MACRO_CONTINUATION_END_RE = /[=+\-*/%&|^<>?:,(\\]$/; /** How a language's code reads around a macro alone on its line. */ interface LoneMacroRules { /** What the code after the macro may start with. */ nextCode: RegExp; - /** Whether a preprocessor directive above the macro ends at its own line. */ - directivesEnd: boolean; /** Whether a macro that is a body's only statement, between `{` and `}`, stays. */ keepsSoleStatement: boolean; } -const C_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~]/, directivesEnd: false, keepsSoleStatement: false }; +const C_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~]/, keepsSoleStatement: false }; /** * Objective-C's: the code after the macro may be a directive, like the * `@interface` below `NS_ASSUME_NONNULL_BEGIN` or the `@end` below - * `OF_SINGLETON_METHODS`. A preprocessor line above it continues nothing, - * even one that ends in `>`, like the `#import ` - * above most headers' `NS_ASSUME_NONNULL_BEGIN`. Blanking that macro but not - * another before the same `@interface` (`OF_SUBCLASSING_RESTRICTED`) left - * `OF_ASSUME_NONNULL_BEGIN @interface …`, which reads as a function. + * `OF_SINGLETON_METHODS`. * * A macro that is a method's whole body, like ObjFW's `{ * OF_UNRECOGNIZED_SELECTOR }`, stays: the parser's error there stays inside * that body, but blanking it changed how the parser recovered from an * earlier misread, and OFMutableString.m lost 11 calls in the methods below. */ -export const OBJC_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~@]/, directivesEnd: true, keepsSoleStatement: true }; +export const OBJC_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~@]/, keepsSoleStatement: true }; export function blankLoneMacroLines(source: string, rules: LoneMacroRules = C_LONE_MACROS): string { if (!/^[ \t]*_*[A-Z][A-Z0-9_]{3,}[ \t]*\r?$/m.test(source)) return source; @@ -1774,8 +1853,12 @@ export function blankCLeadingAttrMacros(source: string): string { * * is NOT valid C — `extern "C" {` (and any other C++-only line under the * guard) drops tree-sitter-c into error recovery, so effectively every public - * C header carries parse errors. A C compiler never sees the guarded lines - * (`__cplusplus` is only defined for C++), so blanking the region BODY mirrors the + * C header carries parse errors. The wasm path shrugs (recovery keeps the + * rest); the kernel path defers EVERY erroring file to wasm by policy — so + * this one idiom pushed C-header deferral to ~32% on redis (vs the <10% + * gate) and forfeited the native-parse win exactly where C repos have the + * most files. A C compiler never sees the guarded lines (`__cplusplus` is + * only defined for C++), so blanking the region BODY mirrors the * preprocessor's own view of the file. * * Matched conservatively, line-based and offset-preserving: @@ -1826,10 +1909,11 @@ export function blankCCplusplusGuardBodies(source: string): string { * &list) { … }` (git), `list_for_each_entry(pos, head, member) { … }` (the * Linux kernel's core iteration idiom). A call followed by a brace block is * not a C statement, so tree-sitter-c drops into error recovery at every use — - * these macros cause parse errors in macro-heavy C trees. + * these macros are the single largest source of parse errors in macro-heavy C + * trees (git: ~39% of files error; the kernel path defers each one to wasm). * Blanking JUST the macro call leaves the brace block as a bare compound - * statement — valid C — so the body's calls/locals extract normally with both - * the bespoke and generic extractors instead of relying on error recovery. + * statement — valid C — so the body's calls/locals extract normally on both + * arms instead of riding error recovery. * * C-ONLY, and matched tightly: * - the call must be INDENTED (statement position; file-scope definitions @@ -2450,7 +2534,7 @@ export function blankCNamedVariadicDefineDots(source: string): string { * cleanly — at the cost of the references inside the initializer, which the * broken parse was not yielding either. Statement-level only (`);` follows), * macro-cased name only, offsets preserved. Runs before the kernel route - * point, so both the bespoke and generic C extractors see the same bytes. + * point, so both the wasm and kernel C arms see the same bytes. */ export function blankCDesignatedMacroArgs(source: string): string { if (source.indexOf('=') === -1) return source; @@ -2500,7 +2584,7 @@ function preParseCSource(source: string): string { ) ) ), - true + { keepCReturnTypes: true } ) ) ) @@ -2582,23 +2666,20 @@ export const cppExtractor: LanguageExtractor = { return parameters ? getNodeText(parameters, source) + (node.type === 'declaration' ? ';' : '') : undefined; }, getVisibility: (node) => { - // The nearest access specifier before the node among its parent's - // children; none means undefined, not the class or struct default. + // Check for access specifier in parent const parent = node.parent; - let visibility: 'public' | 'private' | 'protected' | undefined; if (parent) { for (let i = 0; i < parent.childCount; i++) { const child = parent.child(i); - if (!child || child.startIndex >= node.startIndex) break; - if (child.type === 'access_specifier') { + if (child?.type === 'access_specifier') { const text = child.text; - if (text.includes('public')) visibility = 'public'; - else if (text.includes('private')) visibility = 'private'; - else if (text.includes('protected')) visibility = 'protected'; + if (text.includes('public')) return 'public'; + if (text.includes('private')) return 'private'; + if (text.includes('protected')) return 'protected'; } } } - return visibility; + return undefined; }, resolveTypeAliasKind: (node, _source) => { // C++ typedef: `typedef enum { ... } name;`, `typedef struct { ... } name;`, diff --git a/src/extraction/languages/cfscript.ts b/src/extraction/languages/cfscript.ts index 87ec48e220..8b79504b63 100644 --- a/src/extraction/languages/cfscript.ts +++ b/src/extraction/languages/cfscript.ts @@ -17,6 +17,95 @@ function cfmlVisibility(node: SyntaxNode): 'public' | 'private' | 'protected' | return undefined; } +/** + * A string literal's text when it is written out in full, or null when a + * `#…#` expression builds it at runtime (`"#dir#/x.cfm"`, `"#dir#.Batch"`): + * such a path names no file, and such a name no component, the index can + * link. + */ +function literalString(node: SyntaxNode | null | undefined, source: string): string | null { + if (node?.type !== 'string' || node.namedChildren.some((c: SyntaxNode) => c.type !== 'string_fragment')) return null; + return getNodeText(node, source).replace(/^["']|["']$/g, '').trim() || null; +} + +/** The value of a `template=…` argument or attribute, when it is a literal string. */ +function templateArgument(node: SyntaxNode, source: string): string | null { + if (node.type !== 'assignment_expression') return null; + const left = getChildByField(node, 'left'); + if (left?.type !== 'identifier' || getNodeText(left, source).toLowerCase() !== 'template') return null; + return literalString(getChildByField(node, 'right'), source); +} + +/** + * Whether a call is `cfinclude(…)` — the function form of `include` and of + * `` (ACF 11+, Lucee): an include of a template, not a call (#2651). + */ +export function isCfIncludeCall(node: SyntaxNode, source: string): boolean { + const fn = getChildByField(node, 'function'); + return fn?.type === 'identifier' && getNodeText(fn, source).toLowerCase() === 'cfinclude'; +} + +/** The literal template `cfinclude(template="x.cfm")` or `cfinclude("x.cfm")` names. */ +function cfIncludeCallTemplate(node: SyntaxNode, source: string): string | null { + const args = getChildByField(node, 'arguments'); + if (!args) return null; + for (const arg of args.namedChildren) { + const named = templateArgument(arg, source); + if (named) return named; + } + return literalString(args.namedChildren[0], source); +} + +/** A component name or dotted component path (`Batch`, `contenscms.core.queue.Batch`). */ +const COMPONENT_PATH = /^[A-Za-z_][\w$-]*(?:\.[A-Za-z_][\w$-]*)*$/; + +/** + * The component `createObject("component", "a.b.Batch")` makes — or the + * one-argument `createObject("a.b.Batch")` (Lucee, ACF), or with named + * arguments, `createObject(type="component", component="a.b.Batch")` — as + * written (#2652). Null for any other call, for another type of object + * (`createObject("java", …)`), and for a name built at runtime. + */ +export function cfmlCreatedComponent(call: SyntaxNode, source: string): string | null { + if (call.type !== 'call_expression') return null; + const fn = getChildByField(call, 'function'); + if (fn?.type !== 'identifier' || getNodeText(fn, source).toLowerCase() !== 'createobject') return null; + const args = getChildByField(call, 'arguments')?.namedChildren ?? []; + let type: string | null | undefined; + let name: string | null | undefined; + if (args.length > 0 && args.every((a: SyntaxNode) => a.type === 'assignment_expression')) { + for (const arg of args) { + const key = getNodeText(getChildByField(arg, 'left') ?? arg, source).toLowerCase(); + if (key === 'type') type = literalString(getChildByField(arg, 'right'), source); + else if (key === 'component') name = literalString(getChildByField(arg, 'right'), source); + } + } else if (args.length === 1) { + name = literalString(args[0], source); + } else if (args.length >= 2) { + type = literalString(args[0], source); + name = literalString(args[1], source); + } + if (type !== undefined && type?.toLowerCase() !== 'component') return null; + return name && COMPONENT_PATH.test(name) ? name : null; +} + +/** + * The component an expression constructs, when the expression is a + * construction a call is chained onto: `new a.b.Batch()` or a component + * `createObject(…)`, parenthesized or not (#2652). + */ +export function cfmlConstructedComponent(node: SyntaxNode, source: string): string | null { + let expr: SyntaxNode | null = node; + while (expr?.type === 'parenthesized_expression') expr = expr.namedChild(0); + if (!expr) return null; + if (expr.type === 'new_expression') { + const ctor = getChildByField(expr, 'constructor'); + const name = ctor ? getNodeText(ctor, source).replace(/\s+/g, '') : ''; + return COMPONENT_PATH.test(name) ? name : null; + } + return cfmlCreatedComponent(expr, source); +} + export const cfscriptExtractor: LanguageExtractor = { functionTypes: ['function_declaration', 'function_expression', 'arrow_function'], classTypes: ['component'], @@ -45,10 +134,18 @@ export const cfscriptExtractor: LanguageExtractor = { const importText = source.substring(node.startIndex, node.endIndex).trim(); if (node.type === 'include_statement') { - // `include "path/to/file.cfm";` — the included template path. + // `include "path/to/file.cfm";` — the included template path — or the + // attribute form, `include template="path/to/file.cfm";` (#2651). A + // path built at runtime (`include "#view#";`) is no import. const expr = node.namedChildren.find((c: SyntaxNode) => c.type === 'string'); - if (!expr) return null; - const moduleName = getNodeText(expr, source).replace(/^["']|["']$/g, ''); + const attr = expr ? undefined : node.namedChildren.find((c: SyntaxNode) => c.type === 'assignment_expression'); + const moduleName = expr ? literalString(expr, source) : attr ? templateArgument(attr, source) : null; + return moduleName ? { moduleName, signature: importText } : null; + } + + if (node.type === 'call_expression') { + // `cfinclude(template="path/to/file.cfm");` (#2651). + const moduleName = isCfIncludeCall(node, source) ? cfIncludeCallTemplate(node, source) : null; return moduleName ? { moduleName, signature: importText } : null; } diff --git a/src/extraction/languages/dart.ts b/src/extraction/languages/dart.ts index b748f53352..d174884ed8 100644 --- a/src/extraction/languages/dart.ts +++ b/src/extraction/languages/dart.ts @@ -43,6 +43,82 @@ export function pushDartTypeRefs( for (const child of node.namedChildren) pushDartTypeRefs(child, fromNodeId, push); } +/** + * The name a typedef declares. The grammar gives `type_alias` no fields, and + * the first type name is not always it: the legacy form writes its return + * type first (`typedef Report LegacyMaker(int n);`). The name is the last type + * name before the `=`, the type parameters or the parameter list. + */ +export function dartTypeAliasName(node: SyntaxNode): SyntaxNode | undefined { + let name: SyntaxNode | undefined; + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i); + if (!child || child.type === '=' || child.type === 'type_parameters' || child.type === 'formal_parameter_list') break; + if (child.type === 'type_identifier') name = child; + } + return name; +} + +/** `scope` and the type parameters `node` declares, when it declares any (`Mapper`, `T Function(T a)`). */ +function dartTypeParameterScope(node: SyntaxNode, scope: ReadonlySet): ReadonlySet { + const parameters = node.namedChildren.find((c: SyntaxNode) => c.type === 'type_parameters'); + if (!parameters) return scope; + const inner = new Set(scope); + for (const parameter of parameters.namedChildren) { + const name = parameter.type === 'type_parameter' + ? parameter.namedChildren.find((c: SyntaxNode) => c.type === 'type_identifier') + : undefined; + if (name) inner.add(name.text); + } + return inner; +} + +/** pushDartTypeRefs for a typedef's aliased type: a type parameter in `scope` is no type of its own. */ +function pushDartAliasedTypeRefs( + node: SyntaxNode, + fromNodeId: string, + scope: ReadonlySet, + push: (ref: UnresolvedReference) => void, +): void { + if (node.type === 'type_identifier') { + // `p.T` is the type an import names, whatever is declared as `T` here. + const parameter = scope.has(node.text) && node.previousSibling?.type !== '.'; + if (!parameter && isDartTypeName(node)) { + push({ + fromNodeId, + referenceName: node.text, + referenceKind: 'references', + line: node.startPosition.row + 1, + column: node.startPosition.column, + }); + } + return; + } + const inner = dartTypeParameterScope(node, scope); + for (const child of node.namedChildren) pushDartAliasedTypeRefs(child, fromNodeId, inner, push); +} + +/** + * A `references` ref from the typedef `node` for every type its aliased type + * names (#2327): a function type's return and parameter types, generic + * arguments, record fields, a type parameter's bound, and the legacy form's + * return and parameter types. A type parameter, the typedef's own or a + * generic function type's (`T Function(T a)`), means that parameter and no + * type of the project's. The typedef's annotations are not its type. + */ +export function pushDartTypeAliasRefs( + node: SyntaxNode, + fromNodeId: string, + push: (ref: UnresolvedReference) => void, +): void { + const name = dartTypeAliasName(node); + const scope = dartTypeParameterScope(node, new Set()); + for (const child of node.namedChildren) { + if (child.type === 'annotation' || (name && child.equals(name))) continue; + pushDartAliasedTypeRefs(child, fromNodeId, scope, push); + } +} + /** The bodies a Dart field `declaration` can sit in: a class's or mixin's, an extension's, an enum's. */ const DART_MEMBER_BODIES: ReadonlySet = new Set(['class_body', 'extension_body', 'enum_body']); @@ -646,6 +722,9 @@ export const dartExtractor: LanguageExtractor = { return false; }, resolveName: (node) => { + // A typedef has no `name` field, and a legacy one writes its return type + // before its name (`typedef Report LegacyMaker(int n);`). + if (node.type === 'type_alias') return dartTypeAliasName(node)?.text; // `class A = B with C;` — a mixin application — names its class inside // the `mixin_application_class`, not in a `name` field. if (node.type === 'class_definition') { diff --git a/src/extraction/languages/erlang.ts b/src/extraction/languages/erlang.ts index 8b283994a4..d705160594 100644 --- a/src/extraction/languages/erlang.ts +++ b/src/extraction/languages/erlang.ts @@ -1,5 +1,10 @@ import type { TreeNode as SyntaxNode } from '../parse-tree'; -import { getNodeText, getChildByField, getPrecedingDocstring } from '../tree-sitter-helpers'; +import { + getNodeText, + getChildByField, + getPrecedingDocstring, + namedArgCount, +} from '../tree-sitter-helpers'; import type { LanguageExtractor, ExtractorContext } from '../tree-sitter-types'; // Node names follow the vendored WhatsApp/tree-sitter-erlang grammar (0.19, @@ -95,10 +100,9 @@ function moduleExports(node: SyntaxNode, source: string, filePath: string): Set< return result; } -/** Argument count of a clause/sig: the `args` (expr_args) field's named-child count. */ +/** Argument count of a clause/sig: the `args` (expr_args) field's arguments. */ function nodeArity(withArgs: SyntaxNode): number { - const args = getChildByField(withArgs, 'args'); - return args ? args.namedChildCount : 0; + return namedArgCount(getChildByField(withArgs, 'args')); } /** diff --git a/src/extraction/languages/go.ts b/src/extraction/languages/go.ts index ce7c16c15b..fba3726dcd 100644 --- a/src/extraction/languages/go.ts +++ b/src/extraction/languages/go.ts @@ -42,13 +42,15 @@ const GO_PREDECLARED_TYPES: ReadonlySet = new Set([ * embed `Base`. A qualified type yields its name, not its package: the package * stays in the source text, and resolution reads it back from the reference's * position (#2322). Undefined for any other type (a literal, `~T`, a union's - * term) and for a predeclared one. + * term) and for a predeclared one but `error`: an interface with a method, + * which an embedding brings in like any type's from outside the index. */ export function goEmbeddedTypeName(type: SyntaxNode | null | undefined, source: string): SyntaxNode | undefined { if (type?.type === 'generic_type') type = getChildByField(type, 'type'); if (type?.type === 'qualified_type') return getChildByField(type, 'name') ?? undefined; if (type?.type !== 'type_identifier') return undefined; - return GO_PREDECLARED_TYPES.has(getNodeText(type, source)) ? undefined : type; + const name = getNodeText(type, source); + return name === 'error' || !GO_PREDECLARED_TYPES.has(name) ? type : undefined; } /** diff --git a/src/extraction/languages/java.ts b/src/extraction/languages/java.ts index 38f5ea24ec..9a86cbd56f 100644 --- a/src/extraction/languages/java.ts +++ b/src/extraction/languages/java.ts @@ -1,6 +1,7 @@ import type { TreeNode as SyntaxNode } from '../parse-tree'; import { getNodeText, getChildByField } from '../tree-sitter-helpers'; import type { ExtractorContext, LanguageExtractor } from '../tree-sitter-types'; +import { IMPLICIT_RECORD_ACCESSOR_DOC } from '../../types'; /** * Tree-sitter-java node types for a method's `type` (return) field that can @@ -250,10 +251,74 @@ function synthesizeLombokMembers(classNode: SyntaxNode, ctx: ExtractorContext): } } +// --------------------------------------------------------------------------- +// Record component members +// --------------------------------------------------------------------------- +// A record header `record Point(int x, int y)` declares, per component, a +// private final field and a public accessor `x()` — neither appears in the +// body, so without them `p.x()` resolves to nothing and a component's type +// can't type a receiver. Each is anchored on the component's name token (a +// leaf, like the Lombok members). The canonical constructor is not +// synthesized: `new Point(…)` already links to the record via `instantiates`. +// Varargs components (`String... rest`) are skipped. +function synthesizeRecordComponents(recordNode: SyntaxNode, ctx: ExtractorContext): void { + const params = getChildByField(recordNode, 'parameters'); + if (!params) return; + const recordId = ctx.nodeStack[ctx.nodeStack.length - 1]; + const recordQN = ctx.nodes.find((n) => n.id === recordId)?.qualifiedName; + const takenMethods = new Set(); + const takenFields = new Set(); + if (recordQN) { + for (const n of ctx.nodes) { + if (n.filePath === ctx.filePath && n.qualifiedName === `${recordQN}::${n.name}`) { + if (n.kind === 'method' || n.kind === 'function') takenMethods.add(n.name); + else if (n.kind === 'field' || n.kind === 'variable' || n.kind === 'constant' || n.kind === 'property') { + takenFields.add(n.name); + } + } + } + } + for (const param of params.namedChildren) { + if (param.type !== 'formal_parameter') continue; + const nameNode = getChildByField(param, 'name'); + const typeNode = getChildByField(param, 'type'); + if (!nameNode || !typeNode) continue; + const name = getNodeText(nameNode, ctx.source).trim(); + const typeText = getNodeText(typeNode, ctx.source).trim(); + if (!name) continue; + if (!takenFields.has(name)) { + takenFields.add(name); + ctx.createNode('field', name, nameNode, { + visibility: 'private', + isStatic: false, + signature: `${typeText} ${name}`, + }); + } + if (!takenMethods.has(name)) { + takenMethods.add(name); + ctx.createNode('method', name, nameNode, { + visibility: 'public', + isStatic: false, + signature: `${typeText} ${name}()`, + docstring: IMPLICIT_RECORD_ACCESSOR_DOC, + returnType: normalizeJavaType(typeNode, ctx.source), + }); + } + } +} + +function synthesizeJavaMembers(classNode: SyntaxNode, ctx: ExtractorContext): void { + if (classNode.type === 'record_declaration') synthesizeRecordComponents(classNode, ctx); + synthesizeLombokMembers(classNode, ctx); +} + export const javaExtractor: LanguageExtractor = { functionTypes: [], - classTypes: ['class_declaration'], - methodTypes: ['method_declaration', 'constructor_declaration'], + // `record_declaration` (Java 16+) is a class: its body holds methods, + // constructors, static fields and nested types like any class body. + classTypes: ['class_declaration', 'record_declaration'], + // `compact_constructor_declaration` is a record's `Point { … }` constructor. + methodTypes: ['method_declaration', 'constructor_declaration', 'compact_constructor_declaration'], // `annotation_type_declaration` is `@interface Foo { … }` — an annotation // definition. Without it, annotation types (`@SerializedName`, `@GetMapping`, // JPA/Spring annotations) aren't nodes, so the `@Foo` usages that DO get @@ -272,7 +337,7 @@ export const javaExtractor: LanguageExtractor = { paramsField: 'parameters', returnField: 'type', getReturnType: extractJavaReturnType, - synthesizeMembers: synthesizeLombokMembers, + synthesizeMembers: synthesizeJavaMembers, getSignature: (node, source) => { const params = getChildByField(node, 'parameters'); const returnType = getChildByField(node, 'type'); diff --git a/src/extraction/languages/javascript.ts b/src/extraction/languages/javascript.ts index 899e1530a3..17920e7fde 100644 --- a/src/extraction/languages/javascript.ts +++ b/src/extraction/languages/javascript.ts @@ -1,10 +1,11 @@ import { getNodeText, getChildByField } from '../tree-sitter-helpers'; import type { LanguageExtractor } from '../tree-sitter-types'; -import { classifyTsClassMember } from './typescript'; +import { classifyTsClassMember, hasStaticModifier } from './typescript'; export const javascriptExtractor: LanguageExtractor = { functionTypes: ['function_declaration', 'generator_function_declaration', 'arrow_function', 'function_expression', 'generator_function'], - classTypes: ['class_declaration'], + // A named class expression too, as in TypeScript (#2499). + classTypes: ['class_declaration', 'class'], methodTypes: ['method_definition', 'field_definition'], // JS `field_definition` ≙ TS `public_field_definition`: plain fields are // properties, function-valued fields are methods (#808). @@ -77,6 +78,9 @@ export const javascriptExtractor: LanguageExtractor = { } return false; }, + // A `static` method or field, as in TypeScript (#2761): without it every JS + // class member read as an instance one. + isStatic: hasStaticModifier, isConst: (node) => { if (node.type === 'lexical_declaration') { for (let i = 0; i < node.childCount; i++) { diff --git a/src/extraction/languages/pascal-preparse.ts b/src/extraction/languages/pascal-preparse.ts new file mode 100644 index 0000000000..8e3675b209 --- /dev/null +++ b/src/extraction/languages/pascal-preparse.ts @@ -0,0 +1,1083 @@ +/** + * Offset-preserving source fixes the Pascal grammar needs before it parses a + * Delphi unit (the extractor's `preParse`). Each one blanks syntax the grammar + * has no rule for with spaces, or writes a word it knows in its place, keeping + * every newline, so node positions and text stay where they are. + * + * - An attribute on a routine parameter, before or after its modifier: + * `const [ref] AValue: Integer`, `[ref] const Source: TVarData`, + * `const [MVCFromBody] Person: TPerson`, `[MVCFromQueryString('q', '')] Q: string`. + * The grammar's parameter rule takes no attribute, so the parse fell apart + * at the `[` and every member declared after it in the class was lost. + * The extractor reads nothing from an attribute, and a signature still + * shows it: Pascal signatures read the source as written. + * - The inline variable of a `for … in` loop (Delphi 10.3): `for var lItem in + * AList do`, `for var lHandle: THandle in lHandles do`. The grammar's for-in + * rule takes a plain variable, so the whole loop became an error and no + * call in its body was read. Blanking `var` and the type leaves the loop + * the grammar knows. The resolver still finds the type: it reads a + * receiver's declaration from the source as written. + * - A property override, which names an inherited property without its type + * to publish it or change a specifier: `property OwnsObjects;`, `property + * Cursor default crIBeam;`. The grammar's property rule needs the type, so + * the class fell apart at the first override and every member declared + * after it was lost. An override declares nothing new to index. + * - A conditional right after a routine's header that doesn't hold whole + * bodies: `procedure KeyUp(…); {$IFDEF LINUX} var Code: Byte; {$ENDIF}`, + * `function Get: Integer; {$IFDEF HASINLINE}inline;{$ENDIF}`. The grammar + * reads a conditional there as alternative bodies only, so any other one + * broke the whole unit. Blanking its directives leaves its branches as + * plain code, which is how the grammar reads a conditional anywhere else. + * - A branch holding only a `begin` that another branch of its conditional + * opens too: `{$IFDEF FOLDING} Index: Integer; begin … {$ELSE} begin + * {$ENDIF}`. Read together, the branches open one block more than the code + * closes, so every routine after it was lost. The other branch's `begin` + * stays. An assembler version that opens the body beside a Pascal one, + * `{$IFDEF WIN32_ASM} asm … {$ELSE} begin … {$ENDIF} end;`, is blanked + * whole, so the Pascal version is the body. + * - A field, variable or typed constant of an anonymous record type: + * `AlgData: record case … end`, `Stack: array[1..32] of record First, Last: + * LongInt; end;`. The grammar knows a record only as a named type, so the + * record's `end` closed the class around it and the members declared after + * it were lost. The `record` (or the `packed` before it) is written over by + * `string` and the fields are blanked, so the declaration keeps its name: + * the fields belong to no type the graph could index them under. + * - A re-raise, `raise` with no exception (`except Log; raise; end`). The + * grammar's `raise` takes an exception, so it read the next word (often + * the `end`) as one and the unit broke. The `raise` is blanked, or written + * over by `;` right before an `end` (`else raise end`), where the grammar + * needs a statement to end the list. + * - A label that ends its block, right before `end`, `until`, `except` or + * `finally` (`quit: end;`). The grammar takes a label only before a + * statement, so it read the `end` as a name and the routine never closed, + * taking the routines after it along. The graph reads no labels. + * - Alternative statements where the grammar takes one: `OP_X: {$IFDEF U} + * begin … end; {$ELSE} while … do Inc(N); {$ENDIF}` after a case label, or + * `then {$IFDEF X} A := 1 {$ELSE} A := 2 {$ENDIF} else …`. Read together, + * the branches are two statements, so the case arm broke and took the + * routines after it along. The conditional's directives become `begin`, `;` + * and `end`, which makes its branches one block and keeps the code of each. + */ + +const MODIFIERS: ReadonlySet = new Set(['const', 'var', 'out', 'constref']); +const ROUTINE_KEYWORDS: ReadonlySet = new Set(['procedure', 'function', 'constructor', 'destructor', 'operator']); +/** An attribute group longer than this is not one (and keeps the scan linear). */ +const MAX_ATTRIBUTE_LENGTH = 2000; +/** A for-in loop variable's type longer than this is not one. */ +const MAX_LOOP_TYPE_LENGTH = 500; +/** Words that end a for-in variable's type without an `in`: not the for-in form. */ +const NOT_A_FOR_IN: ReadonlySet = new Set(['do', 'to', 'downto', 'begin', 'end']); +/** + * Whether a file can hold anything to fix at all (the scan below decides): a directive, an attribute, a `for var` + * loop, a property override, an anonymous record, a re-raise or a label (only a `goto` needs one). + */ +const MAY_NEED_FIXES = /\{\$|\[|\bfor\s+var\b|\bproperty\b|\brecord\b|\braise\b|\bgoto\b/i; +/** A property override longer than this, up to its `;`, is not one. */ +const MAX_OVERRIDE_LENGTH = 500; +/** A routine header's rest after its parameters, or a hint after it (`message WM_SIZE;`), longer than this is not one. */ +const MAX_HEADER_PART_LENGTH = 500; +/** An anonymous record longer than this, `record` to `end`, is left alone. */ +const MAX_RECORD_LENGTH = 20000; +/** The hints a routine's header can carry after its `;`: the grammar's procedure attributes. */ +const ROUTINE_HINTS: ReadonlySet = new Set([ + 'static', 'virtual', 'dynamic', 'abstract', 'override', 'overload', 'reintroduce', 'inline', + 'stdcall', 'cdecl', 'pascal', 'register', 'safecall', 'assembler', 'noreturn', 'local', 'far', + 'near', 'default', 'nodefault', 'deprecated', 'experimental', 'message', 'platform', + 'unimplemented', 'cppdecl', 'cvar', 'mwpascal', 'nostackframe', 'interrupt', 'iocheck', + 'hardfloat', 'softfloat', 'ms_abi_default', 'ms_abi_cdecl', 'saveregisters', 'sysv_abi_default', + 'sysv_abi_cdecl', 'vectorcall', 'varargs', 'winapi', 'public', 'export', 'alias', 'dispid', +]); +/** Words that end a block's statements, so a label right before one labels no statement, and a `raise` re-raises. */ +const LABEL_ENDS: ReadonlySet = new Set(['end', 'until', 'except', 'finally']); +/** Words that open a block an `end` closes, for telling whether a conditional branch leaves one open. */ +const BLOCK_OPENERS: ReadonlySet = new Set(['begin', 'asm', 'try', 'case', 'record']); +/** Words an anonymous record's fields can't hold: a `record` before them is something else. */ +const NOT_IN_RECORD_FIELDS: ReadonlySet = new Set([ + 'begin', 'class', 'object', 'interface', 'implementation', 'procedure', 'function', 'constructor', + 'destructor', 'property', 'type', 'var', 'const', 'private', 'protected', 'public', 'published', 'strict', +]); +/** Words that start a statement, for telling a conditional's branch holds statements rather than a type. */ +const STATEMENT_KEYWORDS: ReadonlySet = new Set([ + 'begin', 'if', 'while', 'for', 'repeat', 'case', 'try', 'with', 'raise', 'inherited', 'goto', 'asm', +]); +/** Words a type can start with, so a name before `(` that is one of them starts no call. */ +const TYPE_KEYWORDS: ReadonlySet = new Set([ + 'procedure', 'function', 'reference', 'class', 'record', 'object', 'interface', 'array', 'set', 'file', + 'packed', 'string', 'type', 'const', 'var', 'out', +]); +/** Words a statement can't end with: a branch ending in one holds a statement's head or a part of an expression. */ +const STATEMENT_CONTINUES: ReadonlySet = new Set([ + 'then', 'do', 'else', 'of', 'and', 'or', 'xor', 'not', 'div', 'mod', 'shl', 'shr', 'in', 'is', 'as', 'to', + 'downto', 'until', 'goto', 'on', +]); + +const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_SPACE = 32, CH_DOLLAR = 36, CH_AMP = 38, CH_QUOTE = 39; +const CH_LPAREN = 40, CH_RPAREN = 41, CH_STAR = 42, CH_COMMA = 44, CH_DOT = 46, CH_SLASH = 47; +const CH_COLON = 58, CH_SEMI = 59, CH_LT = 60, CH_EQ = 61, CH_GT = 62, CH_AT = 64; +const CH_LBRACKET = 91, CH_RBRACKET = 93, CH_LBRACE = 123, CH_RBRACE = 125; + +/** Identifier characters, `&` (Delphi's keyword escape) and any non-ASCII letter. */ +function isIdentStart(c: number): boolean { + return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || c === CH_AMP || c >= 128; +} +function isIdentPart(c: number): boolean { + return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || (c >= 48 && c <= 57) || c >= 128; +} +function isSpace(c: number): boolean { + return c === CH_SPACE || c === CH_TAB || c === CH_LF || c === CH_CR; +} + +function wordEnd(s: string, i: number): number { + while (i < s.length && isIdentPart(s.charCodeAt(i))) i++; + return i; +} + +/** + * The index after the comment or string that starts at `i`, or -1 when none + * does. Pascal comments are `{…}` (compiler directives included), `(*…*)` and + * `//…`; a string is `'…'` with `''` for a quote and ends at its line's end + * when unterminated. + */ +function commentOrStringEnd(s: string, i: number): number { + const c = s.charCodeAt(i); + if (c === CH_LBRACE) { + const e = s.indexOf('}', i + 1); + return e < 0 ? s.length : e + 1; + } + if (c === CH_LPAREN && s.charCodeAt(i + 1) === CH_STAR) { + const e = s.indexOf('*)', i + 2); + return e < 0 ? s.length : e + 2; + } + if (c === CH_SLASH && s.charCodeAt(i + 1) === CH_SLASH) { + const e = s.indexOf('\n', i + 2); + return e < 0 ? s.length : e; + } + if (c === CH_QUOTE) { + let j = i + 1; + while (j < s.length) { + const d = s.charCodeAt(j); + if (d === CH_QUOTE) { + if (s.charCodeAt(j + 1) !== CH_QUOTE) return j + 1; + j += 2; + continue; + } + if (d === CH_LF) return j; + j++; + } + return j; + } + return -1; +} + +/** Whether a comment starts at `i`. */ +function isCommentStart(s: string, i: number): boolean { + const c = s.charCodeAt(i); + return c === CH_LBRACE || (c === CH_LPAREN && s.charCodeAt(i + 1) === CH_STAR) || (c === CH_SLASH && s.charCodeAt(i + 1) === CH_SLASH); +} + +/** The index of the first character from `i` that is neither whitespace nor in a comment. */ +function skipTrivia(s: string, i: number): number { + while (i < s.length) { + if (isSpace(s.charCodeAt(i))) i++; + else if (isCommentStart(s, i)) i = commentOrStringEnd(s, i); + else break; + } + return i; +} + +/** The index after the last character before `i` that is neither whitespace nor in a `{…}` or `(*…*)` comment. */ +function tokenEndBefore(s: string, i: number): number { + for (;;) { + while (i > 0 && isSpace(s.charCodeAt(i - 1))) i--; + if (i > 0 && s.charCodeAt(i - 1) === CH_RBRACE) { + const open = s.lastIndexOf('{', i - 2); + if (open < 0) return i; + i = open; + } else if (i > 1 && s.charCodeAt(i - 1) === CH_RPAREN && s.charCodeAt(i - 2) === CH_STAR) { + const open = s.lastIndexOf('(*', i - 3); + if (open < 0) return i; + i = open; + } else { + return i; + } + } +} + +/** The start of the identifier that ends at `i`, or -1 when none does. */ +function wordStartBefore(s: string, i: number): number { + let k = i; + while (k > 0 && isIdentPart(s.charCodeAt(k - 1))) k--; + if (k === i) return -1; + if (k > 0 && s.charCodeAt(k - 1) === CH_AMP) k--; + return isIdentStart(s.charCodeAt(k)) ? k : -1; +} + +/** The index after the first `;` from `i` outside parentheses and brackets, or -1 when none comes within `max` characters. */ +function semicolonEnd(s: string, i: number, max: number): number { + const limit = Math.min(s.length, i + max); + let depth = 0; + while (i < limit) { + const k = commentOrStringEnd(s, i); + if (k >= 0) { + i = k; + continue; + } + const c = s.charCodeAt(i); + if (c === CH_LPAREN || c === CH_LBRACKET) { + depth++; + } else if (c === CH_RPAREN || c === CH_RBRACKET) { + depth--; + } else if (c === CH_SEMI && depth <= 0) { + return i + 1; + } + i++; + } + return -1; +} + +/** + * The index after the `]` closing the attribute group whose `[` is at `i`, + * or -1 when the group doesn't close the way an attribute does: nested + * brackets and parentheses balance, strings and comments are skipped, and an + * attribute's arguments hold no `;`. + */ +function attributeEnd(s: string, i: number): number { + const limit = Math.min(s.length, i + MAX_ATTRIBUTE_LENGTH); + let depth = 0; + let j = i; + while (j < limit) { + const k = commentOrStringEnd(s, j); + if (k >= 0) { + j = k; + continue; + } + const c = s.charCodeAt(j); + if (c === CH_LBRACKET || c === CH_LPAREN) { + depth++; + } else if (c === CH_RBRACKET || c === CH_RPAREN) { + depth--; + if (depth === 0) return c === CH_RBRACKET ? j + 1 : -1; + } else if (c === CH_SEMI) { + return -1; + } + j++; + } + return -1; +} + +/** + * The index after a routine's name — `Name`, `TFoo.Bar`, `TFoo.Bar`, + * `Get` — read from `i`, or `i` itself (trivia + * skipped) when no name starts there (`procedure(…) of object`). + */ +function routineNameEnd(s: string, i: number): number { + for (;;) { + i = skipTrivia(s, i); + if (!isIdentStart(s.charCodeAt(i))) return i; + i = wordEnd(s, i + 1); + let j = skipTrivia(s, i); + if (s.charCodeAt(j) === CH_LT) { + let depth = 0; + for (; j < s.length; j++) { + const c = s.charCodeAt(j); + if (c === CH_LT) { + depth++; + } else if (c === CH_GT) { + if (--depth === 0) { + j++; + break; + } + } else if (c === CH_SEMI || c === CH_LPAREN || c === CH_RPAREN || c === CH_LF) { + return i; // not a generic parameter list + } + } + i = j; + j = skipTrivia(s, i); + } + if (s.charCodeAt(j) !== CH_DOT) return i; + i = j + 1; + } +} + +/** + * Walk the parameter list whose `(` is at `open`, pushing the `[from, to)` + * range of each attribute group a parameter starts with — before or after its + * modifier — onto `ranges`. Returns the index after the list's `)`. + */ +function scanParameterList(s: string, open: number, ranges: number[]): number { + let depth = 0; + let i = open; + let atParameterStart = false; + while (i < s.length) { + if (atParameterStart) { + atParameterStart = false; + for (;;) { + i = skipTrivia(s, i); + const c = s.charCodeAt(i); + if (c === CH_LBRACKET) { + const end = attributeEnd(s, i); + if (end < 0) break; + ranges.push(i, end); + i = end; + } else if (isIdentStart(c)) { + const e = wordEnd(s, i + 1); + if (!MODIFIERS.has(s.slice(i, e).toLowerCase())) break; + i = e; + } else { + break; + } + } + continue; + } + const k = commentOrStringEnd(s, i); + if (k >= 0) { + i = k; + continue; + } + const c = s.charCodeAt(i); + if (c === CH_LPAREN || c === CH_LBRACKET) { + if (++depth === 1) atParameterStart = true; + } else if (c === CH_RPAREN || c === CH_RBRACKET) { + if (--depth <= 0) return i + 1; + } else if (c === CH_SEMI && depth === 1) { + atParameterStart = true; + } + i++; + } + return i; +} + +/** The word starting at `i`, lowercased, or '' when no word starts there. */ +function lowerWordAt(s: string, i: number): string { + return isIdentStart(s.charCodeAt(i)) ? s.slice(i, wordEnd(s, i + 1)).toLowerCase() : ''; +} + +/** + * After a `for` keyword ending at `forEnd`, recognize `var Name in` or + * `var Name: Type in` and push the `var` range and the `: Type` range onto + * `ranges`. The type runs to the `in` at nesting depth 0, so generic types + * (`TPair>`) and dotted names are whole. Any other + * shape (`for var i := 0 to …`, which the grammar parses, or no `in`) adds + * nothing. Returns the index to resume scanning from. + */ +function scanForInVariable(s: string, forEnd: number, ranges: number[]): number { + const varStart = skipTrivia(s, forEnd); + if (lowerWordAt(s, varStart) !== 'var') return forEnd; + const varEnd = varStart + 3; + const nameStart = skipTrivia(s, varEnd); + if (!isIdentStart(s.charCodeAt(nameStart))) return forEnd; + let i = skipTrivia(s, wordEnd(s, nameStart + 1)); + let typeStart = -1; + if (s.charCodeAt(i) === CH_COLON && s.charCodeAt(i + 1) !== CH_EQ) { + typeStart = i; + const limit = Math.min(s.length, i + MAX_LOOP_TYPE_LENGTH); + let depth = 0; + i++; + for (;;) { + if (i >= limit) return forEnd; + const k = commentOrStringEnd(s, i); + if (k >= 0) { + i = k; + continue; + } + const c = s.charCodeAt(i); + if (c === CH_LT || c === CH_LPAREN || c === CH_LBRACKET) { + depth++; + } else if (c === CH_GT || c === CH_RPAREN || c === CH_RBRACKET) { + if (--depth < 0) return forEnd; + } else if (c === CH_SEMI || c === CH_EQ) { + return forEnd; // `for var i: Integer := 0 to …`, or no loop at all + } else if (isIdentStart(c)) { + const word = lowerWordAt(s, i); + if (depth === 0 && word === 'in') break; + if (NOT_A_FOR_IN.has(word)) return forEnd; + i = wordEnd(s, i + 1); + continue; + } + i++; + } + } + if (lowerWordAt(s, i) !== 'in') return forEnd; + ranges.push(varStart, varEnd); + if (typeStart >= 0) ranges.push(typeStart, i); + return i; +} + +/** + * After a `property` keyword running from `start` to `propertyEnd`, + * recognize a property override, which names an inherited property without + * its type to publish it or change a specifier — `property OwnsObjects;`, + * `property Cursor default crIBeam;`, `property Color stored IsColorStored;` + * — and push its range, keyword (or the `class` before it) to `;`, onto + * `ranges`. A name followed by `:` or by an index list (`property + * Items[Index: Integer]: T`) declares a property the grammar knows. Returns + * the index to resume scanning from. + */ +function scanPropertyOverride(s: string, start: number, propertyEnd: number, ranges: number[]): number { + const nameStart = skipTrivia(s, propertyEnd); + if (!isIdentStart(s.charCodeAt(nameStart))) return propertyEnd; + const i = skipTrivia(s, wordEnd(s, nameStart + 1)); + const next = s.charCodeAt(i); + if (next === CH_COLON || next === CH_LBRACKET) return propertyEnd; + const end = semicolonEnd(s, i, MAX_OVERRIDE_LENGTH); + if (end < 0) return propertyEnd; + const classStart = classBefore(s, start); + ranges.push(classStart >= 0 ? classStart : start, end); + return end; +} + +/** + * A conditional compiler directive group, from its `{$IF…}`, `{$IFDEF…}`, + * `{$IFNDEF…}` or `{$IFOPT…}` to its `{$ENDIF}` or `{$IFEND}`. `bounds` + * holds the `[start, end)` offsets of the group's own directives in order, + * its `{$ELSE}`s and `{$ELSEIF…}`s between, so branch `b` runs from + * `bounds[2b + 1]` to `bounds[2b + 2]`; `nested[b]` says whether another + * group sits in that branch. + */ +interface ConditionalGroup { + bounds: number[]; + nested: boolean[]; +} + +const COND_NONE = 0, COND_OPEN = 1, COND_ELSE = 2, COND_CLOSE = 3; + +/** What the `{$…}` directive at `i` does to a conditional: opens one, starts its next branch, closes it, or none of these. */ +function conditionalKind(s: string, i: number): number { + let j = i + 2; + while (j < s.length) { + const c = s.charCodeAt(j) | 32; + if (c < 97 || c > 122) break; + j++; + } + switch (s.slice(i + 2, j).toLowerCase()) { + case 'if': case 'ifdef': case 'ifndef': case 'ifopt': return COND_OPEN; + case 'else': case 'elseif': return COND_ELSE; + case 'endif': case 'ifend': return COND_CLOSE; + default: return COND_NONE; + } +} + +/** + * Every closed conditional group in `s`, innermost first. Directives in + * strings and comments don't count, nor do `(*$…*)` ones, which the grammar + * reads as comments. + */ +function conditionalGroups(s: string): ConditionalGroup[] { + const groups: ConditionalGroup[] = []; + const open: ConditionalGroup[] = []; + const n = s.length; + let i = 0; + while (i < n) { + const c = s.charCodeAt(i); + const k = c === CH_LBRACE || c === CH_QUOTE || c === CH_LPAREN || c === CH_SLASH ? commentOrStringEnd(s, i) : -1; + if (k < 0) { + i++; + continue; + } + if (c === CH_LBRACE && s.charCodeAt(i + 1) === CH_DOLLAR) { + const kind = conditionalKind(s, i); + const top = open[open.length - 1]; + if (kind === COND_OPEN) { + if (top) top.nested[top.nested.length - 1] = true; + open.push({ bounds: [i, k], nested: [false] }); + } else if (kind === COND_ELSE && top) { + top.bounds.push(i, k); + top.nested.push(false); + } else if (kind === COND_CLOSE && top) { + top.bounds.push(i, k); + groups.push(open.pop()!); + } + } + i = k; + } + return groups; +} + +/** + * Whether the code from `from` to `to` is a routine's locals and body, the + * one conditional the grammar reads right after a routine's header: it ends + * with the `end;` of a block that no routine declared in it owns. Code that + * closes a block it didn't open (a local class type) counts as a body too, + * so it stays the way the grammar reads it. + */ +function holdsBody(s: string, from: number, to: number): boolean { + const blocks: string[] = []; + let pending = 0; // routines declared at the top whose body hasn't come yet + let own = false; // whether the last top-level block is no declared routine's + let last = ''; + let beforeLast = ''; + let i = from; + while (i < to) { + const k = commentOrStringEnd(s, i); + if (k >= 0) { + if (s.charCodeAt(i) === CH_QUOTE) { + beforeLast = last; + last = "'"; + } + i = k; + continue; + } + const c = s.charCodeAt(i); + if (isIdentStart(c)) { + const e = wordEnd(s, i + 1); + const word = e - i <= 11 ? s.slice(i, e).toLowerCase() : 'word'; + if (blocks.length === 0) { + if (ROUTINE_KEYWORDS.has(word) && (last === '' || last === ';')) pending++; + else if ((word === 'forward' || word === 'external') && pending > 0) pending--; + } + if (word === 'end') { + if (blocks.pop() === undefined) return true; + } else if (word === 'begin' || word === 'asm') { + if (blocks.length === 0) { + own = pending === 0; + if (!own) pending--; + } + blocks.push(word); + } else if (word === 'try' || word === 'record' || (word === 'case' && blocks[blocks.length - 1] !== 'record')) { + blocks.push(word); + } + beforeLast = last; + last = word; + i = e; + continue; + } + if (!isSpace(c)) { + beforeLast = last; + last = s[i]!; + } + i++; + } + return blocks.length === 0 && own && beforeLast === 'end' && last === ';'; +} + +/** Whether the code from `from` to `to` holds a `begin` or an `asm`: a body, or a part of one. */ +function holdsBlock(s: string, from: number, to: number): boolean { + let i = from; + while (i < to) { + const k = commentOrStringEnd(s, i); + if (k >= 0) { + i = k; + continue; + } + if (isIdentStart(s.charCodeAt(i))) { + const e = wordEnd(s, i + 1); + if (e - i === 3 || e - i === 5) { + const word = s.slice(i, e).toLowerCase(); + if (word === 'begin' || word === 'asm') return true; + } + i = e; + continue; + } + i++; + } + return false; +} + +/** The index of the first character from `i` that is neither whitespace nor in a comment, stopping at a conditional group's opening directive. */ +function skipTriviaToGroup(s: string, i: number, groupAt: ReadonlyMap): number { + while (i < s.length) { + if (isSpace(s.charCodeAt(i))) i++; + else if (isCommentStart(s, i) && !groupAt.has(i)) i = commentOrStringEnd(s, i); + else break; + } + return i; +} + +/** + * From `i`, the index after a routine header's `;`, blank the directives of + * each conditional that follows the header, past the hints it carries + * (`overload;`, `inline;`), unless every branch of the conditional holds a + * whole body: the one conditional the grammar reads there. When the + * conditional holds no block at all (only locals or hints), the scan goes on + * in its first branch, now plain code, so a conditional nested at its start + * comes right after the header too. + */ +function blankConditionalsAfterHeader(s: string, i: number, groupAt: ReadonlyMap, ranges: number[]): void { + for (;;) { + i = skipTriviaToGroup(s, i, groupAt); + const group = groupAt.get(i); + if (group) { + const { bounds } = group; + let bodies = true; + for (let b = 1; b < bounds.length - 1 && bodies; b += 2) bodies = holdsBody(s, bounds[b]!, bounds[b + 1]!); + if (bodies) return; // the grammar reads these + ranges.push(...bounds); + i = holdsBlock(s, bounds[1]!, bounds[bounds.length - 2]!) ? bounds[bounds.length - 1]! : bounds[1]!; + continue; + } + if (!ROUTINE_HINTS.has(lowerWordAt(s, i))) return; + i = semicolonEnd(s, i, MAX_HEADER_PART_LENGTH); + if (i < 0) return; + } +} + +const OPEN_BEGIN = 0, OPEN_ASM = 1, OPEN_OTHER = 2; + +/** + * The blocks the code from `from` to `to` leaves open, in order, each as its + * opener's offset and kind (OPEN_BEGIN, OPEN_ASM, or OPEN_OTHER for `try`, + * `case` and `record`), or null when it closes a block it didn't open. + */ +function openBlocks(s: string, from: number, to: number): Array<[number, number]> | null { + const open: Array<[number, number]> = []; + let i = from; + while (i < to) { + const k = commentOrStringEnd(s, i); + if (k >= 0) { + i = k; + continue; + } + if (isIdentStart(s.charCodeAt(i))) { + const e = wordEnd(s, i + 1); + if (e - i >= 3 && e - i <= 6) { + const word = s.slice(i, e).toLowerCase(); + if (word === 'end') { + if (open.pop() === undefined) return null; + } else if (BLOCK_OPENERS.has(word)) { + open.push([i, word === 'begin' ? OPEN_BEGIN : word === 'asm' ? OPEN_ASM : OPEN_OTHER]); + } + } + i = e; + continue; + } + i++; + } + return open; +} + +/** + * In every conditional (holding no conditional of its own) whose branches + * each leave blocks open for the code after it to close. When they all leave + * the same number of `begin`s open, the `begin`s each branch after the first + * leaves open are blanked: read together, they would open that many blocks + * more than the code closes, and a branch's statements can sit in the first + * branch's block, as the graph doesn't read nesting. When each opens one + * body, some with `asm` and some with `begin` (an assembler and a Pascal + * version of a routine sharing one `end`), each assembler branch is blanked + * whole, so the Pascal one is the body; the graph reads nothing in assembler. + */ +function blankDuplicateBegins(s: string, groups: readonly ConditionalGroup[], ranges: number[]): void { + for (const { bounds, nested } of groups) { + if (nested.length < 2 || nested.includes(true)) continue; + const open: Array> = []; + for (let b = 0; b < nested.length; b++) { + const blocks = openBlocks(s, bounds[2 * b + 1]!, bounds[2 * b + 2]!); + if (!blocks || blocks.length === 0) break; + open.push(blocks); + } + if (open.length < nested.length) continue; + const kinds = (kind: number) => open.filter((o) => o.every(([, k]) => k === kind)).length; + if (kinds(OPEN_BEGIN) === open.length && open.every((o) => o.length === open[0]!.length)) { + for (let b = 1; b < open.length; b++) for (const [at] of open[b]!) ranges.push(at, at + 5); + } else if (open.every((o) => o.length === 1) && kinds(OPEN_ASM) > 0 && kinds(OPEN_BEGIN) > 0 && kinds(OPEN_BEGIN) + kinds(OPEN_ASM) === open.length) { + let kept = false; + for (let b = 0; b < open.length; b++) { + const [at, kind] = open[b]![0]!; + if (kind === OPEN_ASM) ranges.push(bounds[2 * b + 1]!, bounds[2 * b + 2]!); + else if (kept) ranges.push(at, at + 5); + else kept = true; + } + } + } +} + +const SHAPE_NONE = -1, SHAPE_STATEMENTS = 0, SHAPE_STATEMENTS_SEMI = 1, SHAPE_EMPTY = 2; +const START_NONE = 0, START_STATEMENT = 1, START_CALL = 2, START_NAME = 3; + +/** + * Whether the code from `from` to `to`, a conditional's branch, holds whole statements: its blocks and brackets + * close, every `if` has its `then` and every `while`, `for`, `with` and `on` its `do`, and it ends where a + * statement can. It must start with a statement's keyword, assign at its top level or call a routine (`Foo(…)`, + * `A.B(…)`); with `bare`, a lone name counts too (`Exit`, `List.Clear;`), which after a case label could just as + * well be a type. Returns SHAPE_STATEMENTS (SHAPE_STATEMENTS_SEMI when it ends with `;`), SHAPE_EMPTY for no code, + * or SHAPE_NONE. + */ +function statementsShape(s: string, from: number, to: number, bare: boolean): number { + const blocks: boolean[] = []; // an open block each: true for a `repeat`, which `until` closes + let depth = 0; + let heads = 0; // `if`, `while`, `for`, `with` and `on` still waiting for their `then` or `do` + let tokens = 0; + let start = START_NONE; + let name = 0; // reading the first dotted name: 1 after a part of it, 2 after its `.`, 0 when done + let last = ''; // the last token: a word lowercased ('word' when long), '0' for a number, else its character + let i = from; + while (i < to) { + const k = commentOrStringEnd(s, i); + if (k >= 0) { + if (s.charCodeAt(i) === CH_QUOTE) { + tokens++; + name = 0; + last = "'"; + } + i = k; + continue; + } + const c = s.charCodeAt(i); + if (isSpace(c)) { + i++; + continue; + } + tokens++; + if (isIdentStart(c)) { + const e = wordEnd(s, i + 1); + const word = e - i <= 9 ? s.slice(i, e).toLowerCase() : 'word'; + if (word === 'begin' || word === 'try' || word === 'case' || word === 'asm') blocks.push(false); + else if (word === 'repeat') blocks.push(true); + else if (word === 'end' && blocks.pop() !== false) return SHAPE_NONE; + else if (word === 'until' && blocks.pop() !== true) return SHAPE_NONE; + else if (word === 'if' || word === 'while' || word === 'for' || word === 'with' || word === 'on') heads++; + else if (word === 'then' || word === 'do') heads--; + if (tokens === 1) { + if (STATEMENT_KEYWORDS.has(word)) start = START_STATEMENT; + else if (!TYPE_KEYWORDS.has(word) && !STATEMENT_CONTINUES.has(word) && word !== 'end') name = 1; + } else { + name = name === 2 ? 1 : 0; + } + last = word; + i = e; + continue; + } + if (c >= 48 && c <= 57) { + name = 0; + last = '0'; + i = wordEnd(s, i + 1); + continue; + } + if (c === CH_COLON && s.charCodeAt(i + 1) === CH_EQ) { + if (depth === 0 && start !== START_CALL) start = START_STATEMENT; + name = 0; + last = ':='; + i += 2; + continue; + } + if (c === CH_LPAREN || c === CH_LBRACKET) { + if (c === CH_LPAREN && name === 1 && start === START_NONE) start = START_CALL; + depth++; + } else if ((c === CH_RPAREN || c === CH_RBRACKET) && --depth < 0) { + return SHAPE_NONE; + } else if (c === CH_SEMI && name === 1 && start === START_NONE) { + start = START_NAME; + } + name = c === CH_DOT && name === 1 ? 2 : 0; + last = s[i]!; + i++; + } + if (tokens === 0) return SHAPE_EMPTY; + if (depth !== 0 || blocks.length > 0 || heads !== 0) return SHAPE_NONE; + if (name === 1 && start === START_NONE) start = START_NAME; + if (start === START_NONE || (start === START_NAME && !bare)) return SHAPE_NONE; + const ends = last === ';' || last === ')' || last === ']' || last === '^' || last === "'" || last === '0' + || (isIdentStart(last.charCodeAt(0)) && !STATEMENT_CONTINUES.has(last)); + if (!ends) return SHAPE_NONE; + return last === ';' ? SHAPE_STATEMENTS_SEMI : SHAPE_STATEMENTS; +} + +/** Whether a blank range in `ranges` overlaps `[from, to)`. */ +function overlapsRange(ranges: readonly number[], from: number, to: number): boolean { + for (let r = 0; r < ranges.length; r += 2) if (ranges[r]! < to && ranges[r + 1]! > from) return true; + return false; +} + +/** Whether `[at, at + length)` holds no line break, so a word written there keeps every line where it is. */ +function onOneLine(s: string, at: number, length: number): boolean { + for (let i = at; i < at + length; i++) { + const c = s.charCodeAt(i); + if (c === CH_LF || c === CH_CR) return false; + } + return true; +} + +/** + * Read as one statement each conditional whose branches are alternative statements where the grammar takes just + * one: after a case label, `then`, `else` or `do` (`label` maps each such group's opening directive to whether a + * case label's `:` comes before it). Its opening directive becomes `begin`, each `{$ELSE}` a `;`, and its closing + * directive `end`, or `end;` when its branches end with `;`. Read together, the branches of one after a case label + * always broke the arm (`OP_X: begin … end; while … do Inc(N);`). After the others they break only when a branch + * before the last has no `;` (`then A {$ELSE} B {$ENDIF} else C`), since `then A; B;` parses as it is. A group + * holding another conditional is left alone, as is one whose branches end in a statement's head (`if … then`) for a + * body after them. + */ +function wrapAlternativeStatements( + s: string, + groups: readonly ConditionalGroup[], + label: ReadonlyMap, + ranges: readonly number[], + words: Array<[number, number, string]>, +): void { + for (const { bounds, nested } of groups) { + const afterLabel = label.get(bounds[0]!); + if (afterLabel === undefined || nested.length < 2 || nested.includes(true)) continue; + let filled = 0; + let semi = false; + let breaks = afterLabel; + let ok = true; + for (let b = 0; b < nested.length && ok; b++) { + const shape = statementsShape(s, bounds[2 * b + 1]!, bounds[2 * b + 2]!, !afterLabel); + if (shape === SHAPE_EMPTY) continue; + if (shape === SHAPE_NONE) ok = false; + if (filled > 0 && !semi) breaks = true; + filled++; + semi = shape === SHAPE_STATEMENTS_SEMI; + } + if (!ok || filled < 2 || !breaks || !onOneLine(s, bounds[0]!, 5)) continue; + let clear = true; + for (let d = 0; d < bounds.length && clear; d += 2) clear = !overlapsRange(ranges, bounds[d]!, bounds[d + 1]!); + if (!clear) continue; + words.push([bounds[0]!, bounds[1]!, 'begin']); + for (let d = 2; d < bounds.length - 2; d += 2) words.push([bounds[d]!, bounds[d + 1]!, ';']); + words.push([bounds[bounds.length - 2]!, bounds[bounds.length - 1]!, semi ? 'end;' : 'end']); + } +} + +/** + * The kind of statement position the token from `at` to `end` makes for what follows it: 1 for a case label's + * `:`, 0 for `then`, `else` or `do`, -1 for anything else. + */ +function statementPosition(s: string, at: number, end: number): number { + const length = end - at; + if (length === 1) return s.charCodeAt(at) === CH_COLON ? 1 : -1; + if (length < 2 || length > 4) return -1; + const word = s.slice(at, end).toLowerCase(); + return word === 'then' || word === 'else' || word === 'do' ? 0 : -1; +} + +/** + * The start of the declaration that the `record` keyword at `i` gives an + * anonymous type to: the first of the `Names` in `Names: [packed] record` or + * `Names: [packed] array[…] of [packed] record`, or the `TName` in `TName = + * array[…] of record`. -1 for a named record type (`TName = record`) and + * anything else. + */ +function anonymousRecordDeclStart(s: string, i: number): number { + let j = tokenEndBefore(s, i); + let w = wordStartBefore(s, j); + if (w >= 0 && lowerWordAt(s, w) === 'packed') { + j = tokenEndBefore(s, w); + w = wordStartBefore(s, j); + } + let array = false; + if (w >= 0 && lowerWordAt(s, w) === 'of') { + j = tokenEndBefore(s, w); + if (s.charCodeAt(j - 1) === CH_RBRACKET) { + const open = s.lastIndexOf('[', j - 1); + if (open < 0) return -1; + j = tokenEndBefore(s, open); + } + w = wordStartBefore(s, j); + if (w < 0 || lowerWordAt(s, w) !== 'array') return -1; + j = tokenEndBefore(s, w); + w = wordStartBefore(s, j); + if (w >= 0 && lowerWordAt(s, w) === 'packed') j = tokenEndBefore(s, w); + array = true; + } + const c = s.charCodeAt(j - 1); + if (c !== CH_COLON && !(c === CH_EQ && array)) return -1; + let start = wordStartBefore(s, tokenEndBefore(s, j - 1)); + if (start < 0 || c === CH_EQ) return start; + for (;;) { + const k = tokenEndBefore(s, start); + if (s.charCodeAt(k - 1) !== CH_COMMA) return start; + const name = wordStartBefore(s, tokenEndBefore(s, k - 1)); + if (name < 0) return -1; + start = name; + } +} + +/** + * The index after the `end` closing a record whose fields start at `i`, or -1 when they hold what a record's fields + * can't. A word like `procedure` or `object` is part of a field's type after `:`, `of` or `to` (`OnIdle: procedure + * of object`, `Get: reference to function: Integer`, `Args: array of const`), and words in brackets are a + * routine type's parameters. + */ +function recordEnd(s: string, i: number): number { + const limit = Math.min(s.length, i + MAX_RECORD_LENGTH); + let depth = 1; + let brackets = 0; + let prev = ''; // the token before: a word lowercased, or its character + while (i < limit) { + const k = commentOrStringEnd(s, i); + if (k >= 0) { + if (s.charCodeAt(i) === CH_QUOTE) prev = "'"; + i = k; + continue; + } + const c = s.charCodeAt(i); + if (isIdentStart(c)) { + const e = wordEnd(s, i + 1); + const word = s.slice(i, e).toLowerCase(); + if (brackets === 0) { + if (word === 'end') { + if (--depth === 0) return e; + } else if (word === 'record') { + depth++; + } else if (NOT_IN_RECORD_FIELDS.has(word) && prev !== ':' && prev !== 'of' && prev !== 'to') { + return -1; + } + } + prev = word; + i = e; + continue; + } + if (c === CH_LPAREN || c === CH_LBRACKET) brackets++; + else if (c === CH_RPAREN || c === CH_RBRACKET) brackets--; + else if (c === CH_EQ && brackets === 0) return -1; + if (!isSpace(c)) prev = s[i]!; + i++; + } + return -1; +} + +/** + * At a `record` keyword from `start` to `end`, recognize the declaration of a field, variable or typed constant of + * an anonymous record type, followed by its `;`, a typed constant's `=`, or the `)` or `end` after a last field. The + * `record` (or the `packed` before it) is written over by `string`, a type the grammar knows, and the fields after + * it are blanked through its `end`, so the declaration keeps its name and a typed constant its value. Returns the + * index to resume from. + */ +function scanAnonymousRecord(s: string, start: number, end: number, ranges: number[], words: Array<[number, number, string]>): number { + if (anonymousRecordDeclStart(s, start) < 0) return end; + const first = lowerWordAt(s, skipTrivia(s, end)); + if (first === '' || NOT_IN_RECORD_FIELDS.has(first)) return end; // ``, `R: record;` + const close = recordEnd(s, end); + if (close < 0) return end; + const next = skipTrivia(s, close); + const c = s.charCodeAt(next); + const last = lowerWordAt(s, next) === 'end'; // a last field, whose `;` Pascal lets go but the grammar needs + if (c !== CH_SEMI && c !== CH_EQ && c !== CH_RPAREN && !last) return end; + const packed = wordStartBefore(s, tokenEndBefore(s, start)); + const at = packed >= 0 && lowerWordAt(s, packed) === 'packed' ? packed : start; + words.push([at, at + 6, 'string']); + if (last) { + ranges.push(at + 6, close - 3); + words.push([close - 3, close, ';']); + } else { + ranges.push(at + 6, close); + } + return close; +} + +/** + * `s` with every `[from, to)` range in `ranges` blanked to spaces, newlines kept, and every `[from, to, word]` in + * `words` (which overlap neither each other nor a range) written over by its word, padded with spaces. Ranges may + * come in any order and overlap. + */ +function applyEdits(s: string, ranges: readonly number[], words: ReadonlyArray): string { + if (ranges.length === 0 && words.length === 0) return s; + const edits: Array = words.slice(); + for (let r = 0; r < ranges.length; r += 2) edits.push([ranges[r]!, ranges[r + 1]!, '']); + edits.sort((a, b) => a[0] - b[0]); + let out = ''; + let last = 0; + for (const [start, to, word] of edits) { + const from = Math.max(start, last); + if (to <= from) continue; + const written = from === start && word.length <= to - from ? word : ''; + out += s.slice(last, from) + written + s.slice(from + written.length, to).replace(/[^\r\n]/g, ' '); + last = to; + } + return out + s.slice(last); +} + +/** The start of the `class` word before index `i`, past whitespace, or -1. */ +function classBefore(s: string, i: number): number { + let j = i; + while (j > 0 && isSpace(s.charCodeAt(j - 1))) j--; + const w = j - 5; + return w >= 0 && s.slice(w, j).toLowerCase() === 'class' && (w === 0 || !isIdentPart(s.charCodeAt(w - 1))) ? w : -1; +} + +/** + * Blank what the Pascal grammar can't parse (see the file comment). Returns + * the source unchanged when there is nothing to blank. + */ +export function preParsePascal(source: string): string { + if (!MAY_NEED_FIXES.test(source)) return source; + const directives = source.indexOf('{$') >= 0; + const ranges: number[] = []; + const words: Array<[number, number, string]> = []; + const groups = directives ? conditionalGroups(source) : []; + const groupAt = new Map(); + for (const group of groups) groupAt.set(group.bounds[0]!, group); + // Each conditional's opening directive that comes where one statement goes: whether a case label is before it. + const label = new Map(); + // The last token read, start and end, for that; -1 right after a declaration the scan read past. + let lastAt = -1; + let lastEnd = -1; + const n = source.length; + let i = 0; + while (i < n) { + const k = commentOrStringEnd(source, i); + if (k >= 0) { + if (source.charCodeAt(i) === CH_QUOTE) { + lastAt = i; + lastEnd = k; + } else if (lastAt >= 0 && source.charCodeAt(i + 1) === CH_DOLLAR && groupAt.has(i)) { + const position = statementPosition(source, lastAt, lastEnd); + if (position >= 0) label.set(i, position === 1); + } + i = k; + continue; + } + const c = source.charCodeAt(i); + if (isIdentStart(c)) { + const e = wordEnd(source, i + 1); + const len = e - i; + const word = len >= 6 && len <= 11 ? source.slice(i, e).toLowerCase() : ''; + // `operator` is no reserved word in Delphi (a call `Operator([1, 2])` is + // code), so it starts a header only after `class`. + if (ROUTINE_KEYWORDS.has(word) && (word !== 'operator' || classBefore(source, i) >= 0)) { + const open = skipTrivia(source, routineNameEnd(source, e)); + const next = source.charCodeAt(open) === CH_LPAREN ? scanParameterList(source, open, ranges) : e; + if (groupAt.size > 0) { + const headerEnd = semicolonEnd(source, next, MAX_HEADER_PART_LENGTH); + if (headerEnd >= 0) blankConditionalsAfterHeader(source, headerEnd, groupAt, ranges); + } + i = next; + lastAt = -1; + continue; + } else if (word === 'property') { + i = scanPropertyOverride(source, i, e, ranges); + lastAt = -1; + continue; + } else if (word === 'record') { + i = scanAnonymousRecord(source, i, e, ranges, words); + lastAt = -1; + continue; + } else if (len === 3 && source.slice(i, e).toLowerCase() === 'for') { + i = scanForInVariable(source, e, ranges); + lastAt = -1; + continue; + } else if (len === 5 && source.slice(i, e).toLowerCase() === 'raise') { + // A re-raise: blanked, or an empty statement where one must end a list (`else raise end`). + const next = skipTrivia(source, e); + const after = lowerWordAt(source, next); + if (source.charCodeAt(next) === CH_SEMI || after === 'else') ranges.push(i, e); + else if (LABEL_ENDS.has(after)) words.push([i, e, ';']); + } else { + let colon = e; + while (source.charCodeAt(colon) === CH_SPACE || source.charCodeAt(colon) === CH_TAB) colon++; + // A label ending its block (`quit: end;`), not an assembler one (`@z:`). + if (source.charCodeAt(colon) === CH_COLON && source.charCodeAt(colon + 1) !== CH_EQ && source.charCodeAt(i - 1) !== CH_AT + && LABEL_ENDS.has(lowerWordAt(source, skipTrivia(source, colon + 1)))) ranges.push(i, colon + 1); + } + lastAt = i; + lastEnd = e; + i = e; + } else if (c >= 48 && c <= 57) { + lastAt = i; + i = wordEnd(source, i + 1); // a number + lastEnd = i; + } else if (isSpace(c)) { + i++; + } else { + lastAt = i; + i += c === CH_COLON && source.charCodeAt(i + 1) === CH_EQ ? 2 : 1; + lastEnd = i; + } + } + if (groups.length > 0) { + blankDuplicateBegins(source, groups, ranges); + if (label.size > 0) wrapAlternativeStatements(source, groups, label, ranges, words); + } + return applyEdits(source, ranges, words); +} diff --git a/src/extraction/languages/pascal-system.ts b/src/extraction/languages/pascal-system.ts new file mode 100644 index 0000000000..14daeabb03 --- /dev/null +++ b/src/extraction/languages/pascal-system.ts @@ -0,0 +1,31 @@ +/** + * The names Delphi's compiler and System unit give meaning to, lowercased: + * intrinsics (`SizeOf`, `TypeInfo`, `High`, `Inc`, `Assigned`, `Exit`), the + * System and SysUtils routines everyone calls (`Move`, `FillChar`, `Copy`, + * `Format`, `WriteLn`), and the built-in types a typecast names (`Pointer(P)`, + * `Integer(X)`). Pascal names ignore case: `length(S)` is `Length(S)`. A call + * to one is the System unit's unless a declaration in its scope shadows it + * (see resolution/pascal-scope). + */ +export const PASCAL_SYSTEM_NAMES: ReadonlySet = new Set([ + // Intrinsics and System routines + 'abs', 'addr', 'arctan', 'assert', 'assigned', 'assignfile', 'atomiccmpexchange', 'atomicdecrement', + 'atomicexchange', 'atomicincrement', 'blockread', 'blockwrite', 'break', 'chdir', 'chr', 'closefile', 'concat', + 'continue', 'copy', 'cos', 'dec', 'default', 'delete', 'dispose', 'eof', 'eoln', 'exclude', 'exit', 'exp', + 'fail', 'filepos', 'filesize', 'fillchar', 'finalize', 'frac', 'freemem', 'getdir', 'getmem', 'gettypekind', + 'halt', 'hasweakref', 'hi', 'high', 'inc', 'include', 'initialize', 'insert', 'int', 'ioresult', + 'isconstvalue', 'ismanagedtype', 'length', 'ln', 'lo', 'low', 'mkdir', 'move', 'new', 'odd', 'ord', + 'paramcount', 'paramstr', 'pi', 'pred', 'ptr', 'random', 'randomize', 'read', 'readln', 'reallocmem', + 'returnaddress', 'rmdir', 'round', 'runerror', 'setlength', 'setstring', 'sin', 'sizeof', 'slice', 'sqr', + 'sqrt', 'str', 'succ', 'swap', 'trunc', 'typehandle', 'typeinfo', 'typeof', 'uniquestring', 'upcase', 'val', + 'vararrayredim', 'write', 'writeln', + // SysUtils routines the resolver has always taken for the RTL's + 'abort', 'date', 'datetostr', 'floattostr', 'format', 'freeandnil', 'inttostr', 'lowercase', 'now', 'pos', + 'strtodate', 'strtofloat', 'strtoint', 'time', 'trim', 'uppercase', + // Built-in types, called as typecasts + 'ansichar', 'ansistring', 'boolean', 'byte', 'bytebool', 'cardinal', 'char', 'comp', 'currency', 'double', + 'extended', 'int16', 'int32', 'int64', 'int8', 'integer', 'longbool', 'longint', 'longword', 'nativeint', + 'nativeuint', 'olevariant', 'pansichar', 'pchar', 'pointer', 'pwidechar', 'rawbytestring', 'real', + 'shortint', 'shortstring', 'single', 'smallint', 'string', 'uint16', 'uint32', 'uint64', 'uint8', + 'unicodestring', 'utf8string', 'variant', 'widechar', 'widestring', 'word', 'wordbool', +]); diff --git a/src/extraction/languages/pascal.ts b/src/extraction/languages/pascal.ts index a7a9ba18c7..f37be6a780 100644 --- a/src/extraction/languages/pascal.ts +++ b/src/extraction/languages/pascal.ts @@ -1,8 +1,14 @@ import type { TreeNode as SyntaxNode } from '../parse-tree'; import { getNodeText, getChildByField } from '../tree-sitter-helpers'; import type { LanguageExtractor } from '../tree-sitter-types'; +import { preParsePascal } from './pascal-preparse'; export const pascalExtractor: LanguageExtractor = { + // Blank Delphi syntax the grammar has no rule for, such as an attribute on a + // parameter (`const [ref] AValue: Integer`), offsets kept (#2545). Signatures + // still read the attribute from the source as written. + preParse: preParsePascal, + signatureAsWritten: true, functionTypes: ['declProc'], classTypes: ['declClass'], methodTypes: ['declProc'], diff --git a/src/extraction/languages/ruby.ts b/src/extraction/languages/ruby.ts index ba0db5398f..6f7beed717 100644 --- a/src/extraction/languages/ruby.ts +++ b/src/extraction/languages/ruby.ts @@ -2,6 +2,20 @@ import type { TreeNode as SyntaxNode } from '../parse-tree'; import { getNodeText, getChildByField } from '../tree-sitter-helpers'; import type { LanguageExtractor } from '../tree-sitter-types'; +/** + * A class method: `def self.x` or `def Widget.x` (a singleton_method), or a + * `def` inside `class << self`. It shares its qualified name with an instance + * method of the same name (`Widget::build` for both), so only this tells the + * two apart. + */ +function isRubyClassMethod(node: SyntaxNode): boolean { + if (node.type === 'singleton_method') return true; + if (node.type !== 'method') return false; + let parent = node.parent; + if (parent?.type === 'body_statement') parent = parent.parent; + return parent?.type === 'singleton_class'; +} + export const rubyExtractor: LanguageExtractor = { functionTypes: ['method'], classTypes: ['class'], @@ -103,6 +117,7 @@ export const rubyExtractor: LanguageExtractor = { return name; }, + isStatic: isRubyClassMethod, getVisibility: (node) => { // Ruby visibility is based on preceding visibility modifiers let sibling = node.previousNamedSibling; diff --git a/src/extraction/languages/typescript.ts b/src/extraction/languages/typescript.ts index 22fbcdebc3..aca50b2a2f 100644 --- a/src/extraction/languages/typescript.ts +++ b/src/extraction/languages/typescript.ts @@ -3,6 +3,18 @@ import type { LanguageExtractor } from '../tree-sitter-types'; import type { TreeNode as SyntaxNode } from '../parse-tree'; import { hasFlowPragma } from '../grammars'; +/** + * Whether a class member is written `static`: its own `static` keyword, never + * a member named `static()`. Shared by the TypeScript and JavaScript + * extractors, which parse class bodies alike. + */ +export function hasStaticModifier(node: SyntaxNode): boolean { + for (let i = 0; i < node.childCount; i++) { + if (node.child(i)?.type === 'static') return true; + } + return false; +} + /** * A TS/JS class field (`public_field_definition` / `field_definition`) is a * METHOD only when its value is callable — an arrow function, a function @@ -57,6 +69,42 @@ export function blankFlowSyntax(source: string, filePath?: string): string { .replace(/\.\.\.(?=\s*[},])/g, ' '); } +/** Whether the file writes `export type *` anywhere: most files don't, and skip the rest. */ +const EXPORT_TYPE_STAR_HINT = /export\s+type\s*\*/; +/** + * `export type *` where a statement starts (a line's first word, or after `;`, + * `{`, `}` or the end of a block comment), followed by `from` or `as`. + */ +const EXPORT_TYPE_STAR = /(^|[;{}]|\*\/)([ \t]*export\s+)type(?=\s*\*\s*(?:from|as)\b)/gm; + +/** + * TypeScript 5.0's type-only re-export of a whole module, `export type * from + * './types'` and `export type * as ns from './types'`, with its `type` blanked + * to spaces (offsets survive), so it reads as the `export *` it is to the + * graph. The grammar has no rule for it (nor does upstream's): `type` parsed + * as an error, so a barrel of re-exports was reported as producing no symbols + * and counted by `codegraph status` as a file it could not parse, and the + * kernel handed every such file to the wasm parser (#2474). Only a statement + * start is blanked, so a mention in a comment or a doc keeps its text. The + * import resolver reads the statement as written, `type` included, from the + * file itself (`extractReExports`), and so does the import node the statement + * makes ({@link blanksExportTypeStar}). Applied before both engines parse (the + * kernel's preParse hoist), so they agree. + */ +export function blankExportTypeStar(source: string): string { + if (!EXPORT_TYPE_STAR_HINT.test(source)) return source; + return source.replace(EXPORT_TYPE_STAR, '$1$2 '); +} + +/** + * Whether {@link blankExportTypeStar} blanks anything in `source`. Such a + * file's re-exports keep their statements as written in the graph, `type` + * included (`reExportAsWritten`). + */ +export function blanksExportTypeStar(source: string): boolean { + return EXPORT_TYPE_STAR_HINT.test(source) && blankExportTypeStar(source) !== source; +} + /** * Whether `node` is declared in a `declare module 'x' { … }` or `declare * global { … }` body, which exports it without an `export` keyword. @@ -111,10 +159,14 @@ export function isAmbientFunctionSignature(node: SyntaxNode, filePath: string): } export const typescriptExtractor: LanguageExtractor = { - preParse: blankFlowSyntax, - isAmbientFunction: isAmbientFunctionSignature, + preParse: (source, filePath) => blankExportTypeStar(blankFlowSyntax(source, filePath)), + reExportAsWritten: blanksExportTypeStar, functionTypes: ['function_declaration', 'generator_function_declaration', 'arrow_function', 'function_expression', 'generator_function'], - classTypes: ['class_declaration', 'abstract_class_declaration'], + isAmbientFunction: isAmbientFunctionSignature, + // `class` is a class expression, `register(class Widget extends Base {…})`: + // a named one is a class with its members and supertypes (#2499), and an + // anonymous one is passed over (TreeSitterExtractor.isClassTypeNode). + classTypes: ['class_declaration', 'abstract_class_declaration', 'class'], // `method_signature` is the interface/type-literal form of a method; without it // an interface's members never enter the graph, so a `.d.ts` platform API has // no declaration node for call sites to attach to (#1638). Java/C# don't need @@ -206,13 +258,7 @@ export const typescriptExtractor: LanguageExtractor = { } return false; }, - isStatic: (node) => { - for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); - if (child?.type === 'static') return true; - } - return false; - }, + isStatic: hasStaticModifier, isConst: (node) => { // For lexical_declaration, check if it's 'const' or 'let' // For variable_declaration, it's always 'var' diff --git a/src/extraction/liquid-extractor.ts b/src/extraction/liquid-extractor.ts index f6d9aee60a..4de1826c83 100644 --- a/src/extraction/liquid-extractor.ts +++ b/src/extraction/liquid-extractor.ts @@ -1,3 +1,4 @@ +import { parseTree, findNodeAtLocation, type ParseError } from 'jsonc-parser'; import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference } from '../types'; import { generateNodeId, NodeIdAllocator } from './tree-sitter-helpers'; @@ -6,9 +7,12 @@ import { generateNodeId, NodeIdAllocator } from './tree-sitter-helpers'; * * Liquid is a templating language (used by Shopify, Jekyll, etc.) that doesn't * have traditional functions or classes. Instead, we extract: - * - Section references ({% section 'name' %}) + * - Section references ({% section 'name' %}) and section group references + * ({% sections 'header-group' %} → `sections/header-group.json`) * - Snippet references ({% render 'name' %} and {% include 'name' %}) * - Schema blocks ({% schema %}...{% endschema %}) + * - Variables declared with {% assign %} and {% capture %} + * - The sections a Shopify JSON template or section group lists */ export class LiquidExtractor { private filePath: string; @@ -18,6 +22,8 @@ export class LiquidExtractor { private edges: Edge[] = []; private unresolvedReferences: UnresolvedReference[] = []; private errors: ExtractionError[] = []; + /** Where Liquid runs nothing — see `silentRegions`. Computed once per file. */ + private silent?: Array<[start: number, end: number]>; constructor(filePath: string, source: string) { this.filePath = filePath; @@ -50,7 +56,7 @@ export class LiquidExtractor { // Extract schema block this.extractSchema(fileNode.id); - // Extract assign statements as variables + // Extract assign and capture declarations as variables this.extractAssignments(fileNode.id); } } catch (error) { @@ -100,31 +106,68 @@ export class LiquidExtractor { * mapping an id → `{ "type": "", ... }`; the `type` names a * `sections/.liquid` file. Emit a `references` edge to each, so a section * used only from a JSON template (the OS 2.0 norm) is no longer orphaned. + * + * The theme editor writes a block comment ("IMPORTANT: The contents of this + * file are auto-generated…") at the top of every template and section group + * it saves, and Shopify's theme JSON allows comments and trailing commas. + * Anything else that is not JSON, like a file cut off partway, links nothing. + * Each reference sits where its `type` value is written. */ private extractShopifyJsonSections(fromNodeId: string): void { - let parsed: unknown; - try { - parsed = JSON.parse(this.source); - } catch { - return; // not valid JSON (or a partial) — nothing to link - } - const sections = (parsed as { sections?: Record })?.sections; - if (!sections || typeof sections !== 'object') return; + const errors: ParseError[] = []; + const root = parseTree(this.source, errors, { allowTrailingComma: true }); + if (errors.length > 0 || root?.type !== 'object') return; + const sections = findNodeAtLocation(root, ['sections']); + if (sections?.type !== 'object') return; const seen = new Set(); - for (const key of Object.keys(sections)) { - const type = sections[key]?.type; - if (typeof type !== 'string' || seen.has(type)) continue; - seen.add(type); + for (const section of sections.children ?? []) { + const value = section.children?.[1]; + const type = value?.type === 'object' ? findNodeAtLocation(value, ['type']) : undefined; + if (type?.type !== 'string' || typeof type.value !== 'string' || seen.has(type.value)) continue; + seen.add(type.value); + const line = this.getLineNumber(type.offset); this.unresolvedReferences.push({ fromNodeId, - referenceName: `sections/${type}.liquid`, + referenceName: `sections/${type.value}.liquid`, referenceKind: 'references', - line: 1, - column: 0, + line, + column: type.offset - this.getLineStart(line), }); } } + /** + * The regions Liquid runs nothing in: `{% comment %}…{% endcomment %}` and + * `{% raw %}…{% endraw %}`, each from its opening tag to the end of its + * closing one (or to the end of the file, when it is never closed). The tags, + * schemas and declarations written inside one are examples, not code. + */ + private silentRegions(): Array<[start: number, end: number]> { + if (this.silent) return this.silent; + const regions: Array<[number, number]> = []; + const tags = /\{%[-]?\s*(\w+|#)([\s\S]*?)[-]?%\}/g; + let open: { name: string; start: number } | undefined; + let tag; + while ((tag = tags.exec(this.source)) !== null) { + const name = tag[1]!; + if (open) { + if (name === `end${open.name}`) { + regions.push([open.start, tags.lastIndex]); + open = undefined; + } + } else if (name === 'comment' || name === 'raw') { + open = { name, start: tag.index }; + } + } + if (open) regions.push([open.start, this.source.length]); + return (this.silent = regions); + } + + /** The silent region (see `silentRegions`) holding `index`, if any. */ + private silentRegionAt(index: number): [start: number, end: number] | undefined { + return this.silentRegions().find(([start, end]) => index >= start && index < end); + } + /** * Every occurrence of a Liquid tag, in BOTH spellings it can be written in. * @@ -152,18 +195,14 @@ export class LiquidExtractor { const blocks: Array<{ bodyStart: number; body: string }> = []; const tags = /\{%[-]?\s*(\w+|#)([\s\S]*?)[-]?%\}/g; const braced = new RegExp(`^\\{%[-]?\\s*(${tagPattern})\\s+${argPattern}`); - let suppressed: string | undefined; let tag; while ((tag = tags.exec(this.source)) !== null) { - const name = tag[1]!; - if (suppressed) { - if (name === `end${suppressed}`) suppressed = undefined; - continue; - } - if (name === 'comment' || name === 'raw') { - suppressed = name; + const silent = this.silentRegionAt(tag.index); + if (silent) { + tags.lastIndex = silent[1]; continue; } + const name = tag[1]!; if (name === 'liquid') { blocks.push({ bodyStart: tag.index + tag[0].indexOf(name) + name.length, @@ -280,13 +319,16 @@ export class LiquidExtractor { } /** - * Extract {% section 'name' %} references + * Extract {% section 'name' %} and {% sections 'group' %} references. A + * section is `sections/.liquid`; a section group, which a layout renders + * with the plural tag, is the JSON file `sections/.json` that lists its + * sections. */ private extractSectionReferences(fileNodeId: string): void { // Both spellings, as for render/include above. - for (const match of this.findTagOccurrences('section', `['"]([^'"]+)['"]`)) { + for (const match of this.findTagOccurrences('sections?', `['"]([^'"]+)['"]`)) { const fullMatch = match.fullMatch; - const sectionName = match.groups[1]; + const [tagType, sectionName] = match.groups; const line = this.getLineNumber(match.index); // Create an import node for searchability @@ -315,13 +357,13 @@ export class LiquidExtractor { }); // Create a component node for the section reference - const nodeId = this.nodeIds.generate(this.filePath, 'component', `section:${sectionName}`, line, match.index - this.getLineStart(line)); + const nodeId = this.nodeIds.generate(this.filePath, 'component', `${tagType}:${sectionName}`, line, match.index - this.getLineStart(line)); const node: Node = { id: nodeId, kind: 'component', name: sectionName!, - qualifiedName: `${this.filePath}::section:${sectionName}`, + qualifiedName: `${this.filePath}::${tagType}:${sectionName}`, filePath: this.filePath, language: 'liquid', startLine: line, @@ -340,10 +382,10 @@ export class LiquidExtractor { kind: 'contains', }); - // Add unresolved reference to the section file + // Add unresolved reference to the section file, or the group's JSON this.unresolvedReferences.push({ fromNodeId: fileNodeId, - referenceName: `sections/${sectionName}.liquid`, + referenceName: `sections/${sectionName}.${tagType === 'sections' ? 'json' : 'liquid'}`, referenceKind: 'references', line, column: match.index - this.getLineStart(line), @@ -360,6 +402,13 @@ export class LiquidExtractor { let match; while ((match = schemaRegex.exec(this.source)) !== null) { + // A schema inside {% comment %} or {% raw %} is an example, not the + // section's: look again after that region ends. + const silent = this.silentRegionAt(match.index); + if (silent) { + schemaRegex.lastIndex = silent[1]; + continue; + } const [fullMatch, schemaContent] = match; const startLine = this.getLineNumber(match.index); const endLine = this.getLineNumber(match.index + fullMatch.length); @@ -411,40 +460,53 @@ export class LiquidExtractor { } /** - * Extract {% assign var = value %} statements + * Extract {% assign var = value %} and {% capture var %}…{% endcapture %} + * declarations as variables */ private extractAssignments(fileNodeId: string): void { // Both spellings. Most of a modern theme's assigns are the bare kind. for (const match of this.findTagOccurrences('assign', `(\\w+)\\s*=`)) { - const variableName = match.groups[1]; - const line = this.getLineNumber(match.index); - - // Create a variable node - const nodeId = this.nodeIds.generate(this.filePath, 'variable', variableName!, line, match.index - this.getLineStart(line)); + this.addVariable(fileNodeId, match.groups[1]!, match); + } + // A capture declares its name as an assign does, and Liquid reads a quoted + // name (`{% capture 'title' %}`) as the name inside the quotes. A name that + // goes on with `-` or `.`, which the assign rule doesn't read either, gets + // no node rather than a cut-short one. Captures come after assigns, so an + // assign keeps its id when a capture of the same name shares its line. + for (const match of this.findTagOccurrences('capture', `(?:'(\\w+)'|"(\\w+)"|(\\w+))(?=\\s|-?%\\}|$)`)) { + const [, singleQuoted, doubleQuoted, bare] = match.groups; + this.addVariable(fileNodeId, (singleQuoted ?? doubleQuoted ?? bare)!, match); + } + } - const node: Node = { - id: nodeId, - kind: 'variable', - name: variableName!, - qualifiedName: `${this.filePath}::${variableName}`, - filePath: this.filePath, - language: 'liquid', - startLine: line, - endLine: line, - startColumn: match.index - this.getLineStart(line), - endColumn: match.index - this.getLineStart(line) + match.fullMatch.length, - updatedAt: Date.now(), - }; + /** A `variable` node, contained by the file, for a declaration that `findTagOccurrences` found. */ + private addVariable(fileNodeId: string, variableName: string, match: { fullMatch: string; index: number }): void { + const line = this.getLineNumber(match.index); + const column = match.index - this.getLineStart(line); + const nodeId = this.nodeIds.generate(this.filePath, 'variable', variableName, line, column); + + const node: Node = { + id: nodeId, + kind: 'variable', + name: variableName, + qualifiedName: `${this.filePath}::${variableName}`, + filePath: this.filePath, + language: 'liquid', + startLine: line, + endLine: line, + startColumn: column, + endColumn: column + match.fullMatch.length, + updatedAt: Date.now(), + }; - this.nodes.push(node); + this.nodes.push(node); - // Add containment edge from file - this.edges.push({ - source: fileNodeId, - target: nodeId, - kind: 'contains', - }); - } + // Add containment edge from file + this.edges.push({ + source: fileNodeId, + target: nodeId, + kind: 'contains', + }); } /** diff --git a/src/extraction/sfc-script.ts b/src/extraction/sfc-script.ts index 9a78e45c5c..eec289c400 100644 --- a/src/extraction/sfc-script.ts +++ b/src/extraction/sfc-script.ts @@ -3,6 +3,27 @@ import type { Binding, Edge, ExtractionError, ExtractionResult, Language, Node, UnresolvedReference } from '../types'; import * as path from 'path'; +const TS_LANG_ATTR = /lang\s*=\s*["'](ts|typescript)["']/; +const TSX_LANG_ATTR = /lang\s*=\s*["']tsx["']/; + +/** + * The grammar a `