diff --git a/.claude/skills/codegraph-build-validation/SKILL.md b/.claude/skills/codegraph-build-validation/SKILL.md index 9871cb1c6c..1ba65ec9d1 100644 --- a/.claude/skills/codegraph-build-validation/SKILL.md +++ b/.claude/skills/codegraph-build-validation/SKILL.md @@ -35,6 +35,8 @@ Under Bun the suite runs via `__tests__/bun-homedir.setup.ts` (an `os.homedir` s The native kernel (`codegraph-kernel/`, Rust) is the **only parser**: every grammar is compiled into it (crates.io pins plus the vendored C under `codegraph-kernel/grammars/`, provenance in `grammars/PROVENANCE.md`). There is no wasm fallback. A source checkout needs a prebuild at `codegraph-kernel/prebuilds/-/codegraph-kernel.node`; `npm test` builds it through `scripts/ensure-kernel.mjs` when it is missing or was built from other kernel sources (needs Rust 1.89 or newer), release bundles ship it, and a platform without a prebuild is unsupported (the CLI says so at startup). A host release build of committed kernel sources is saved to a per-user cache (`scripts/kernel-cache.mjs`; `CODEGRAPH_KERNEL_CACHE` moves it, `=off` disables it), so a new worktree whose kernel sources match a recent build restores it instead of compiling; `build:kernel --no-cache` compiles anyway. `CARGO_TARGET_DIR` moves the build output. Languages with a bespoke walker are extracted in Rust; the rest are parsed by the kernel and walked by the generic TypeScript extractor over the serialized tree (`src/extraction/parse-tree.ts`). The whole-graph golden dumps (`__tests__/kernel-golden-dumps.test.ts`) are the regression gate for any extraction change; see `docs/design/kernel-only-extraction-plan.md`. +The kernel also builds as a plain Rust library with no Node-API dependency: `cargo build --manifest-path codegraph-kernel/Cargo.toml --no-default-features --release --example extract-native` produces an internal probe (`target/release/examples/extract-native `) that writes the six extraction buffer tables through the same code the addon uses. It is a development check, not a distributed command. The default `addon` feature keeps the Node-API exports; Rust unit tests that build a `KernelResolver` outside Node need `--features addon-tests` (dynamic Node-API symbols). `lib.rs` allows dead code in builds without the addon because the exports are reached only through it. + One other build step writes into `dist/` and is subject to the same rule: `build:ui` builds the browser viewer into `dist/viewer/` (never `dist/ui/` — that's the terminal ui). `scripts/check-ui-build.mjs` asserts `dist/viewer/` after every build and inside every release diff --git a/.github/workflows/contained-source.yml b/.github/workflows/contained-source.yml index 6c5ee47446..6d6cde65a6 100644 --- a/.github/workflows/contained-source.yml +++ b/.github/workflows/contained-source.yml @@ -51,5 +51,5 @@ jobs: - run: node -e "if (process.versions.node.split('.')[0] !== '24') process.exit(1)" - run: npm ci --ignore-scripts - run: npm run build:kernel - - run: cargo test --manifest-path codegraph-kernel/Cargo.toml --release contained_source::tests -- --test-threads=1 + - run: cargo test --manifest-path codegraph-kernel/Cargo.toml --release --features addon-tests contained_source::tests -- --test-threads=1 - run: npx vitest run __tests__/source-scan.test.ts __tests__/security.test.ts __tests__/explore-unmatched-names.test.ts -t "scanIndexedSource|native source reader|unmatched names" --maxWorkers=2 --minWorkers=1 diff --git a/__tests__/ensure-kernel-stamp.test.ts b/__tests__/ensure-kernel-stamp.test.ts index 90becae80c..63307ac522 100644 --- a/__tests__/ensure-kernel-stamp.test.ts +++ b/__tests__/ensure-kernel-stamp.test.ts @@ -268,12 +268,12 @@ describe('kernel cache', () => { describe.runIf(process.platform !== 'win32')('build-kernel with the kernel cache', () => { const lib = process.platform === 'darwin' ? 'libcodegraph_kernel.dylib' : 'libcodegraph_kernel.so'; - // A fake cargo: `build` writes a library naming its profile and logs the call; + // A fake cargo: `rustc` (how the script builds the cdylib alone) writes a library naming its profile and logs the call; // FAKE_EDIT makes it change a kernel source while "compiling". const fakeCargo = () => { write('build-bin/cargo', `#!/bin/sh case "$1" in - build) + rustc) echo build >> "$CALLS" profile=release; [ "$2" = "--profile" ] && profile=$3 dir=\${CARGO_TARGET_DIR:-target}/$profile diff --git a/codegraph-kernel/Cargo.toml b/codegraph-kernel/Cargo.toml index 40e7a2e784..d1a4dd5495 100644 --- a/codegraph-kernel/Cargo.toml +++ b/codegraph-kernel/Cargo.toml @@ -8,11 +8,16 @@ publish = false description = "Native extraction kernel for CodeGraph — tree-sitter parse+extract with one JS boundary crossing per file" [lib] -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] + +[features] +default = ["addon"] +addon = ["dep:napi", "dep:napi-derive", "dep:napi-build"] +addon-tests = ["addon", "napi/dyn-symbols"] [dependencies] -napi = { version = "3", default-features = false, features = ["napi8"] } -napi-derive = "3" +napi = { version = "3", default-features = false, features = ["napi8"], optional = true } +napi-derive = { version = "3", optional = true } tree-sitter = "0.25" sha2 = "0.10" memchr = "2" @@ -83,14 +88,10 @@ tree-sitter-language = "0.1" libc = "0.2" [build-dependencies] -napi-build = "2" +napi-build = { version = "2", optional = true } cc = "1" -# Unit tests run outside Node: load the N-API symbols at run time instead of -# linking them, so a test that builds a KernelResolver links. Release builds -# keep the features above. -[dev-dependencies] -napi = { version = "3", default-features = false, features = ["napi8", "dyn-symbols"] } +# Addon unit tests outside Node use --features addon-tests for dynamic symbols. [profile.release] # Never `panic = "abort"`: lib.rs walk_file turns a walker panic into a diff --git a/codegraph-kernel/build.rs b/codegraph-kernel/build.rs index 0bd5d34ded..13d84d00dc 100644 --- a/codegraph-kernel/build.rs +++ b/codegraph-kernel/build.rs @@ -1,4 +1,5 @@ fn main() { + #[cfg(feature = "addon")] napi_build::setup(); // Kotlin grammar — vendored C, compiled here instead of a crate dep: the diff --git a/codegraph-kernel/examples/extract-native.rs b/codegraph-kernel/examples/extract-native.rs new file mode 100644 index 0000000000..6fba1e714c --- /dev/null +++ b/codegraph-kernel/examples/extract-native.rs @@ -0,0 +1,89 @@ +//! Internal extraction/parity probe, not a distributed CodeGraph CLI. +use codegraph_kernel::{contract_info, extract_native, NativeExtractBuffers}; +use sha2::{Digest, Sha256}; +use std::{error::Error, fs, path::Path, time::Instant}; + +fn tables(output: &NativeExtractBuffers) -> [(&str, &[u8]); 6] { + [ + ("meta", &output.meta), + ("nodes", &output.nodes), + ("edges", &output.edges), + ("refs", &output.refs), + ("bindings", &output.bindings), + ("arena", &output.arena), + ] +} + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().skip(1).collect(); + if args.first().map(String::as_str) == Some("--bench") && args.len() == 2 { + // TSV rows are source path, language, and logical graph path. Load + // once before timing, just as the production-addon benchmark does. + let manifest = fs::read_to_string(&args[1])?; + let inputs = manifest + .lines() + .map(|line| { + let columns: Vec<_> = line.split('\t').collect(); + if columns.len() != 3 { + return Err("expected source, language, path".into()); + } + Ok(( + columns[2].to_owned(), + fs::read_to_string(columns[0])?, + columns[1].to_owned(), + )) + }) + .collect::, Box>>()?; + if inputs.is_empty() { + return Err("empty input manifest".into()); + } + let mut last: Vec> = (0..inputs.len()).map(|_| None).collect(); + let iterations = 100; + let mut bytes: usize = 0; + let start = Instant::now(); + for _ in 0..iterations { + for (index, (path, content, language)) in inputs.iter().enumerate() { + let output = extract_native(path, content, language)?; + bytes += tables(&output) + .iter() + .map(|(_, table)| table.len()) + .sum::(); + last[index] = Some(output); + } + } + let elapsed_ms = start.elapsed().as_secs_f64() * 1000.0; + let mut hash = Sha256::new(); + for output in last.into_iter().flatten() { + for (name, table) in tables(&output) { + if name == "meta" { + if table.len() != 40 { + return Err("unexpected metadata layout".into()); + } + let duration = f64::from_le_bytes(table[28..36].try_into()?); + if !duration.is_finite() || duration < 0.0 { + return Err("invalid duration".into()); + } + hash.update(&table[..28]); + hash.update(&table[36..]); + } else { + hash.update(table); + } + } + } + let contract = contract_info(); + println!("{{\"files\":{},\"iterations\":{},\"bytes\":{},\"elapsedMs\":{},\"outputHash\":\"{:x}\",\"contract\":{{\"abiVersion\":{},\"kernelVersion\":{:?},\"nodeKinds\":{:?},\"edgeKinds\":{:?},\"languages\":{:?}}}}}", + inputs.len() * iterations, iterations, bytes, elapsed_ms, hash.finalize(), + contract.abi_version, contract.kernel_version, contract.node_kinds, contract.edge_kinds, contract.languages); + return Ok(()); + } + if !(args.len() == 3 || args.len() == 4) { + return Err("usage: extract-native [logical-path]; or --bench ".into()); + } + let content = fs::read_to_string(&args[0])?; + let output = extract_native(args.get(3).unwrap_or(&args[0]), &content, &args[1])?; + fs::create_dir_all(&args[2])?; + for (name, table) in tables(&output) { + fs::write(Path::new(&args[2]).join(format!("{name}.bin")), table)?; + } + Ok(()) +} diff --git a/codegraph-kernel/src/cfnptr/napi.rs b/codegraph-kernel/src/cfnptr/napi.rs index 08aae0961c..3e91630cda 100644 --- a/codegraph-kernel/src/cfnptr/napi.rs +++ b/codegraph-kernel/src/cfnptr/napi.rs @@ -2,25 +2,26 @@ //! passes and receives, and the entry points for the sweep (stages A/B), //! the macro environment (stage C) and the link (stages D/E). +#[cfg(feature = "addon")] use napi_derive::napi; /// One struct node's extent for the cFnPtr sweep (mirror of the TS caller's /// `{ id, startLine, endLine }`, with `endLine ?? startLine` applied TS-side). -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrStructIn { pub id: String, pub start_line: u32, pub end_line: u32, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrFileIn { /// RAW file text, exactly as the resolver's readFile returned it. pub text: String, pub structs: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrPathIn { /// Absolute file path — the kernel reads the text itself (utf-8-lossy, the /// same bytes JS `readFileSync(path, 'utf-8')` produces) instead of taking @@ -29,16 +30,16 @@ pub struct CfnptrPathIn { pub structs: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrField { pub name: String, pub index: u32, pub ptr: bool, - #[napi(js_name = "type")] + #[cfg_attr(feature = "addon", napi(js_name = "type"))] pub ty: String, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrStructOut { pub id: String, pub parsed: bool, @@ -47,7 +48,7 @@ pub struct CfnptrStructOut { /// The cFnPtr extraction-sweep facts for one file — see cfnptr.rs (and the /// TS synthesizer's `FileFacts`) for field semantics. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrFacts { pub fn_ptr_typedefs: Vec, pub fn_type_typedefs: Vec, @@ -124,7 +125,7 @@ fn empty_cfnptr_facts() -> CfnptrFacts { /// of files and returns their collected facts, amortizing the NAPI boundary. /// Feature-detected by the TS loader — absent on older binaries, where the /// synthesizer keeps its JS sweep. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn cfnptr_scan_files(files: Vec) -> Vec { files .into_iter() @@ -150,7 +151,7 @@ fn read_lossy(path: &str) -> Option { /// itself and fans the batch across scoped threads. Output stays 1:1 with /// input order: an unreadable file or a per-file panic yields empty facts, /// and a chunk thread's own failure pads its outputs the same way. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn cfnptr_scan_paths(files: Vec) -> Vec { fn scan_one(f: &CfnptrPathIn) -> CfnptrFacts { let Some(text) = read_lossy(&f.path) else { @@ -172,28 +173,28 @@ pub fn cfnptr_scan_paths(files: Vec) -> Vec { /// Debug/differential hook: the native `stripCommentsForRegex(text, 'c')`. /// Exists so the strip differential oracle can pin the Rust stripper against /// the TS reference directly. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn cfnptr_strip_c(text: String) -> String { String::from_utf8(super::strip_c(text.as_bytes())).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) } // ---- cFnPtr stage C env extraction + stages D/E link ---- -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrFnMacro { pub name: String, pub params: Vec, pub expansion: String, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrObjMacro { pub name: String, pub value: String, } /// One file's `buildEnv` inputs — see `super::file_env`. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrFileEnv { pub fn_macros: Vec, pub obj_macros: Vec, @@ -213,7 +214,7 @@ pub struct CfnptrFileEnv { /// so a virtual FS still resolves) and per slot of a chunk thread that /// panicked. OPTIONAL: absent on older binaries, where the synthesizer keeps /// its lazy LRU-cached extractor path. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn cfnptr_file_envs(paths: Vec) -> Vec> { fn one(path: &str) -> Option { let text = read_lossy(path)?; @@ -239,22 +240,22 @@ pub fn cfnptr_file_envs(paths: Vec) -> Vec> { /// `{name, type, isFnPtr}` — the FieldInfo members stages D/E consult; `type` /// arrives null for fn-pointer fields (the TS side stores `''` there). -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLinkField { pub name: String, - #[napi(js_name = "type")] + #[cfg_attr(feature = "addon", napi(js_name = "type"))] pub ty: Option, pub is_fn_ptr: bool, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLinkFn { pub id: String, pub start_line: u32, pub end_line: u32, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLinkFile { /// Project-relative path (the node filePath — `registeredAt` uses it). pub rel: String, @@ -268,51 +269,51 @@ pub struct CfnptrLinkFile { pub fns: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrFieldStructs { pub field: String, pub structs: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLayout { pub name: String, pub fields: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLayoutVariants { pub name: String, pub variants: Vec>, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrVarType { pub var: String, - #[napi(js_name = "type")] + #[cfg_attr(feature = "addon", napi(js_name = "type"))] pub ty: String, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrRegEntry { pub key: String, pub ids: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrArrEntry { pub file: String, pub ids: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrArrReg { pub name: String, pub entries: Vec, } /// The registration tables verbatim — see `super::LinkTables`. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLinkTables { pub field_to_structs: Vec, pub struct_layout: Vec, @@ -322,7 +323,7 @@ pub struct CfnptrLinkTables { pub array_reg: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLinkEdge { pub source: String, pub target: String, @@ -331,7 +332,7 @@ pub struct CfnptrLinkEdge { pub registered_at: String, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct CfnptrLinkOut { pub edges: Vec, } @@ -339,7 +340,7 @@ pub struct CfnptrLinkOut { /// Stages D+E of the fn-pointer synthesis — field←field propagation to a /// fixpoint, then dispatch-site edges — internally threaded, file-order /// deterministic. OPTIONAL: absent on older binaries. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn cfnptr_link(files: Vec, tables: CfnptrLinkTables) -> CfnptrLinkOut { let files: Vec = files .into_iter() diff --git a/codegraph-kernel/src/contained_source.rs b/codegraph-kernel/src/contained_source.rs index 69376cf91f..907dc29347 100644 --- a/codegraph-kernel/src/contained_source.rs +++ b/codegraph-kernel/src/contained_source.rs @@ -167,28 +167,28 @@ fn opened_path(_file: &File) -> io::Result { )) } -#[napi_derive::napi(object)] +#[cfg_attr(feature = "addon", napi_derive::napi(object))] pub struct ContainedSourceRead { - pub content: Option, + pub content: Option, pub bytes_read: u32, pub limit_reached: bool, } -#[napi_derive::napi] +#[cfg_attr(feature = "addon", napi_derive::napi)] pub struct ContainedSourceReader { inner: SourceReader, } -#[napi_derive::napi] +#[cfg_attr(feature = "addon", napi_derive::napi)] impl ContainedSourceReader { - #[napi(constructor)] - pub fn new(root: String) -> napi::Result { + #[cfg_attr(feature = "addon", napi(constructor))] + pub fn new(root: String) -> crate::boundary::Result { let inner = SourceReader::new(Path::new(&root)) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; + .map_err(|e| crate::boundary::Error::from_reason(e.to_string()))?; Ok(Self { inner }) } - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn read( &self, relative: String, @@ -199,7 +199,7 @@ impl ContainedSourceReader { .inner .read(Path::new(&relative), max_file_bytes, remaining_bytes); ContainedSourceRead { - content: read.content.map(Into::into), + content: read.content.map(crate::boundary::buffer), bytes_read: read.bytes_read, limit_reached: read.limit_reached, } diff --git a/codegraph-kernel/src/lib.rs b/codegraph-kernel/src/lib.rs index 0d091be22c..9b854839d0 100644 --- a/codegraph-kernel/src/lib.rs +++ b/codegraph-kernel/src/lib.rs @@ -15,10 +15,10 @@ //! parity — checked by kernel-generic-extractor-tree.test.ts and golden dumps. #![deny(clippy::all)] -// napi-derive registers `#[napi]` exports only outside `cfg(test)`, so a test -// build sees every export, and all it alone reaches, as unused. The release -// build's `clippy --lib -D warnings` gate still reports real dead code. -#![cfg_attr(test, allow(dead_code))] +// Exports registered with napi-derive are reached only by the addon: a test +// build or a build without the `addon` feature sees them as unused. The +// addon build's `clippy --lib -D warnings` gate still reports real dead code. +#![cfg_attr(any(test, not(feature = "addon")), allow(dead_code))] /// First statement of every recursive walker function (see stack.rs, #1581): /// once the stack pointer is inside the red zone, stop descending — the @@ -376,6 +376,20 @@ macro_rules! markdown_refs_impl { } mod buffers; +pub mod native; +mod boundary { + pub fn buffer(bytes: Vec) -> Buffer { + #[cfg(feature = "addon")] + { bytes.into() } + #[cfg(not(feature = "addon"))] + { bytes } + } + #[cfg(feature = "addon")] + pub use napi::{bindgen_prelude::Buffer, Error, Result}; + #[cfg(not(feature = "addon"))] + pub use crate::native::{Buffer, Error, Result}; +} +pub use buffers::EmitOut as NativeExtractBuffers; mod ccpp; mod cfnptr; mod csharp; @@ -404,12 +418,16 @@ mod walker; mod python; mod tsjs; +#[cfg(feature = "addon")] use napi::bindgen_prelude::*; +#[cfg(not(feature = "addon"))] +use crate::native::{Buffer, Error, Result}; +#[cfg(feature = "addon")] use napi_derive::napi; /// The six flat tables for one file. See buffers.rs for the byte layout; /// `src/extraction/kernel/layout.ts` is the TS mirror. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct ExtractBuffers { pub meta: Buffer, pub nodes: Buffer, @@ -423,7 +441,7 @@ pub struct ExtractBuffers { /// Wire-contract description — the TS loader verifies this against /// src/types.ts before routing anything to the kernel, so an out-of-date /// `.node` is rejected instead of mis-decoding. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct ContractInfo { pub abi_version: u32, pub kernel_version: String, @@ -435,7 +453,7 @@ pub struct ContractInfo { /// Grammar identity: exposes the native grammar's ABI and node-kind/field tables /// for grammar inspection and compatibility checks. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct GrammarInfo { pub abi_version: u32, pub node_kind_count: u32, @@ -444,7 +462,7 @@ pub struct GrammarInfo { pub field_names: Vec, } -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn contract_info() -> ContractInfo { ContractInfo { abi_version: buffers::KERNEL_ABI_VERSION as u32, @@ -455,7 +473,7 @@ pub fn contract_info() -> ContractInfo { } } -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn grammar_info(language: String) -> Option { let lang = langs::grammar_for(&language)?; let node_kind_count = lang.node_kind_count(); @@ -519,14 +537,14 @@ fn dispatch_walker(file_path: &str, content: &str, language: &str) -> std::resul } } -fn to_buffers(out: buffers::EmitOut) -> ExtractBuffers { +fn to_buffers(out: NativeExtractBuffers) -> ExtractBuffers { ExtractBuffers { - meta: out.meta.into(), - nodes: out.nodes.into(), - edges: out.edges.into(), - refs: out.refs.into(), - bindings: out.bindings.into(), - arena: out.arena.into(), + meta: boundary::buffer(out.meta), + nodes: boundary::buffer(out.nodes), + edges: boundary::buffer(out.edges), + refs: boundary::buffer(out.refs), + bindings: boundary::buffer(out.bindings), + arena: boundary::buffer(out.arena), } } @@ -537,14 +555,20 @@ fn to_buffers(out: buffers::EmitOut) -> ExtractBuffers { /// and refs are dropped and the rows come back nodeless for the TS side to /// attach by name and line. A file too deep for the walk yields `defer:` and /// no rows, like its nodes. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn bindings_file(file_path: String, content: String, language: String) -> Result { - let out = walk_file(&file_path, &content, &language).map_err(Error::from_reason)?; + let out = extract_native(&file_path, &content, &language).map_err(|e| Error::from_reason(e.to_string()))?; Ok(to_buffers(out.bindings_only())) } -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn extract_file(file_path: String, content: String, language: String) -> Result { - let out = walk_file(&file_path, &content, &language).map_err(Error::from_reason)?; + let out = extract_native(&file_path, &content, &language).map_err(|e| Error::from_reason(e.to_string()))?; Ok(to_buffers(out)) } + +/// Extract through the production walkers without any Node conversion or copy. +/// The caller owns every table. Stack exhaustion and walker panics remain errors. +pub fn extract_native(file_path: &str, content: &str, language: &str) -> native::Result { + walk_file(file_path, content, language).map_err(native::Error::from_reason) +} diff --git a/codegraph-kernel/src/native.rs b/codegraph-kernel/src/native.rs new file mode 100644 index 0000000000..445d7b5681 --- /dev/null +++ b/codegraph-kernel/src/native.rs @@ -0,0 +1,16 @@ +//! Owned data and errors for the runtime-independent kernel build. +pub type Buffer = Vec; +pub type Result = std::result::Result; +#[derive(Debug)] +pub struct Error(String); +impl Error { + pub fn from_reason(reason: impl Into) -> Self { + Self(reason.into()) + } +} +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} +impl std::error::Error for Error {} diff --git a/codegraph-kernel/src/resolve/go_asserted.rs b/codegraph-kernel/src/resolve/go_asserted.rs index 3b0ef67859..68c15daf02 100644 --- a/codegraph-kernel/src/resolve/go_asserted.rs +++ b/codegraph-kernel/src/resolve/go_asserted.rs @@ -261,25 +261,25 @@ fn local_declaration_in<'a>(scope: TsNode<'a>, name: &str, site: usize, text: &s } } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct GoLocalDeclaration { pub line: i64, pub column: i64, pub declared_type: Option } /// Source-only lexical lookup. Synthesis must not attach to the live WAL. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub struct GoSourceScope { source: String, tree: tree_sitter::Tree, } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl GoSourceScope { - #[napi(constructor)] + #[cfg_attr(feature = "addon", napi(constructor))] pub fn new(source: String) -> Result { let tree = crate::tree::parse_with_cached_parser(&source, "go")?; Ok(Self { source, tree }) } - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn declaration_at(&self, name: String, line: i64, column: i64) -> Option { local_at(&self.tree, &self.source, &name, line, column) } diff --git a/codegraph-kernel/src/resolve/mod.rs b/codegraph-kernel/src/resolve/mod.rs index 4dbac70143..ccbff475f0 100644 --- a/codegraph-kernel/src/resolve/mod.rs +++ b/codegraph-kernel/src/resolve/mod.rs @@ -22,7 +22,11 @@ //! baseline's reads would (WAL readers never pin a snapshot here — there is //! no spanning transaction). +#[cfg(feature = "addon")] use napi::bindgen_prelude::*; +#[cfg(not(feature = "addon"))] +use crate::native::{Error, Result}; +#[cfg(feature = "addon")] use napi_derive::napi; use regex::Regex; use rusqlite::{Connection, OpenFlags}; @@ -65,13 +69,13 @@ macro_rules! re { // napi input/output shapes // --------------------------------------------------------------------------- -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelKv { pub key: String, pub value: String, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelAliasPatternIn { pub prefix: String, pub suffix: String, @@ -79,7 +83,7 @@ pub struct KernelAliasPatternIn { pub replacements: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelAliasMapIn { pub base_url: Option, /// Whether the config (or one it extends) sets `compilerOptions.baseUrl`: @@ -90,13 +94,13 @@ pub struct KernelAliasMapIn { /// A nested tsconfig/jsconfig's aliases: files under `dir` (project-relative) /// use `map` in place of the root config's. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelScopedAliasIn { pub dir: String, pub map: KernelAliasMapIn, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelWorkspaceIn { /// Import-source path prefix → workspace package name (longest prefix wins). pub source_entries: Vec, @@ -108,13 +112,13 @@ pub struct KernelWorkspaceIn { pub local_link_names: Option>, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelGoModule { pub module_path: String, pub dir: String, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelResolverConfig { pub db_path: String, pub project_root: String, @@ -158,7 +162,7 @@ pub struct KernelResolverConfig { } /// One unresolved_refs row — mirrors the UnresolvedReference/rowId shape. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] #[derive(Clone)] pub struct ResolveRefIn { pub row_id: Option, @@ -189,7 +193,7 @@ impl ResolveRefIn { } } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct KernelCandidateOut { pub target_node_id: String, pub confidence: f64, @@ -198,15 +202,16 @@ pub struct KernelCandidateOut { /// One verdict per input ref. `status`: /// - `resolved` — verdict (gates + alias forwarding applied). `isFinal` -/// marks the import early-win; under frameworks it can -/// still be displaced by a ≥0.9 framework hit. +/// marks the import early-win; under frameworks it can +/// still be displaced by a ≥0.9 framework hit. /// - `unresolved` — a miss. With a candidate list (`[]` included) the -/// framework merge still runs; `reason` marks a ref the -/// deferred passes retry (`defer`, `defer-this`). +/// framework merge still runs; `reason` marks a ref the +/// deferred passes retry (`defer`, `defer-this`). +/// /// `candidates` is populated only when `frameworksActive` and the kernel /// produced a non-final verdict — the raw [import?, name?] list for the TS /// first-max merge. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct ResolveOutcome { pub status: String, pub edge_kind: Option, @@ -621,7 +626,7 @@ use self::names::*; use self::object_literal::LiteralProperty; use self::prefilter::*; -#[napi] +#[cfg_attr(feature = "addon", napi)] pub struct KernelResolver { // Option so close() can drop the Connection deterministically — see // close() for why GC-timed teardown is unsafe on a shared -shm. @@ -847,9 +852,9 @@ fn uri_path(db_path: &str) -> String { out } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl KernelResolver { - #[napi(constructor)] + #[cfg_attr(feature = "addon", napi(constructor))] pub fn new(config: KernelResolverConfig) -> Result { let snapshot = config.snapshot.unwrap_or(false); let conn = if snapshot { @@ -998,7 +1003,7 @@ impl KernelResolver { /// whenever V8 GCs the JS wrapper. A live-db conn is parked, not closed /// (live_conn): closing it would drop this process's POSIX locks on the /// db, node:sqlite's included. A snapshot conn closes. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn close(&mut self) { self.debug_stats("close"); self.release_conn(); @@ -1056,7 +1061,7 @@ impl KernelResolver { /// The `read` stage — same keyset cursor and prerequisite split as /// queries.getUnresolvedReferencesBatchAfter, plus the denormalized /// file/language fallback via the origin node that resolveOne applies. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn read_pending_batch( &mut self, after_row_id: i64, @@ -1131,7 +1136,7 @@ impl KernelResolver { } /// The `settle` stage — one verdict per ref in input order. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_chunk(&mut self, refs: Vec) -> Result> { let mut out = Vec::with_capacity(refs.len()); for r in refs { @@ -1159,7 +1164,7 @@ impl KernelResolver { } /// Scope checks shared with the framework name heuristics. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn is_name_candidate_visible(&mut self, node_id: String, r: ResolveRefIn) -> Result { match per_ref(&r, false, || { let Some(node) = self.node_by_id(&node_id)? else { return Ok(false) }; @@ -1180,7 +1185,7 @@ impl KernelResolver { /// resolveViaImport for one ref: what a framework resolver reaches through /// `context.resolveImport` (expo-modules' imported-receiver lookup). - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_via_import_ref(&mut self, r: ResolveRefIn) -> Result { match per_ref(&r, None, || self.resolve_via_import_member(&r)) { Ok(Some(c)) => Ok(ResolveOutcome::resolved(&c.node, c.confidence, c.resolved_by, true, None)), @@ -1191,7 +1196,7 @@ impl KernelResolver { /// resolveDeferredThisMemberRefs' per-ref match (matchDeferredThisMember), /// run after the main pass wrote every implements/extends edge. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_deferred_this_members(&mut self, refs: Vec) -> Result> { let mut out = Vec::with_capacity(refs.len()); for r in refs { @@ -1212,7 +1217,7 @@ impl KernelResolver { /// pass wrote every implements/extends edge: PHP `this->prop.method` /// through the method-call arm, Rust chains through bound factory returns, /// every other chain through the dotted arm, then the language gate. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_deferred_chains(&mut self, refs: Vec) -> Result> { let mut out = Vec::with_capacity(refs.len()); for r in refs { diff --git a/codegraph-kernel/src/resolve/python_globals.rs b/codegraph-kernel/src/resolve/python_globals.rs index 91d3ceb5d1..1ae37b39dd 100644 --- a/codegraph-kernel/src/resolve/python_globals.rs +++ b/codegraph-kernel/src/resolve/python_globals.rs @@ -1323,13 +1323,13 @@ impl KernelResolver { } } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl KernelResolver { - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn is_python_outside_module_receiver(&mut self, name: String, r: ResolveRefIn) -> Result { self.python_outside_module_receiver(&name,&r).map_err(Error::from) } - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_python_global_call(&mut self, receiver: String, member: String, r: ResolveRefIn) -> Result { match self.python_global_receiver_call(&receiver,&member,&r).map_err(Error::from)? { Some(c)=>Ok(ResolveOutcome::resolved(&c.node,c.confidence,c.resolved_by,true,None)), diff --git a/codegraph-kernel/src/resolve/python_modules.rs b/codegraph-kernel/src/resolve/python_modules.rs index aeec58f67b..8dbdf3975f 100644 --- a/codegraph-kernel/src/resolve/python_modules.rs +++ b/codegraph-kernel/src/resolve/python_modules.rs @@ -2,13 +2,13 @@ //! Root discovery and sync invalidation use the same indexed-file snapshot. use super::*; -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct PythonReopenScope { pub target_files: Vec, pub source_dirs: Vec, pub module_leaves: Vec, } -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct PythonImportBinding { pub file_path: String, pub local_name: String, @@ -356,15 +356,15 @@ impl KernelResolver { } } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl KernelResolver { - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_python_module(&mut self, spec: String, from: String) -> Result> { self.python_module_path(&spec, &from).map_err(Error::from) } /// Stable root-set serialization. The caller may hash it; sync compares it /// before/after extraction, using a fresh resolver snapshot each time. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn python_package_roots(&mut self) -> Result { let index = self.build_python_modules().map_err(Error::from)?; if index.files.is_empty() { @@ -384,11 +384,11 @@ impl KernelResolver { local.join("\n") )) } - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn clear_python_module_caches(&mut self) { self.py_globals = Default::default(); } - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn python_package_reopen_scope( &mut self, changed_files: Vec, @@ -556,11 +556,11 @@ mod tests { } } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl KernelResolver { /// Flat importer entries for grouping by file and local name. A binding is /// `file\0symbol`, `file\0` for a submodule, or null for a namespace. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn python_package_import_bindings( &mut self, init_files: Vec, @@ -641,11 +641,11 @@ impl KernelResolver { } } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl KernelResolver { /// The module ref of `from X import y` is emitted at the statement's /// start. Imported names have their own columns, even when one is also X. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn is_python_from_import_module(&mut self, r: ResolveRefIn) -> bool { if r.language != "python" || r.reference_kind != "imports" { return false; diff --git a/codegraph-kernel/src/resolve/python_types.rs b/codegraph-kernel/src/resolve/python_types.rs index 3fcde7af03..04555adf39 100644 --- a/codegraph-kernel/src/resolve/python_types.rs +++ b/codegraph-kernel/src/resolve/python_types.rs @@ -5,7 +5,7 @@ use super::*; use tree_sitter::Node as PyNode; -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct PythonAttributeResult { /// True for a proven type (including an outside/builtin type with no edge). /// False leaves the existing name strategies in charge. @@ -691,10 +691,10 @@ impl KernelResolver { Ok(result) } } -#[napi] +#[cfg_attr(feature = "addon", napi)] impl KernelResolver { /// Used both by the native pipeline and sync's failed-attribute retry filter. - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn resolve_python_attribute(&mut self, r: ResolveRefIn) -> Result { self.python_attribute_verdict(&r).map_err(Error::from) } diff --git a/codegraph-kernel/src/resolve/resolver_upstream.rs b/codegraph-kernel/src/resolve/resolver_upstream.rs index 58096871d7..a428a0a940 100644 --- a/codegraph-kernel/src/resolve/resolver_upstream.rs +++ b/codegraph-kernel/src/resolve/resolver_upstream.rs @@ -1014,7 +1014,7 @@ mod tests { /// the files under the directories in `CODEGRAPH_ORACLE_CORPUS` /// (colon-separated), and for the call search the eight lines from each /// line on, as `bare_call_receiver` reads them. Run with - /// `CODEGRAPH_ORACLE_CORPUS= cargo test --release -- --ignored split_matchers_agree_on_corpus`. + /// `CODEGRAPH_ORACLE_CORPUS= cargo test --release --features addon-tests -- --ignored split_matchers_agree_on_corpus`. #[test] #[ignore] fn split_matchers_agree_on_corpus() { diff --git a/codegraph-kernel/src/resolve/tables.rs b/codegraph-kernel/src/resolve/tables.rs index 451fe84992..4aedd9e378 100644 --- a/codegraph-kernel/src/resolve/tables.rs +++ b/codegraph-kernel/src/resolve/tables.rs @@ -418,7 +418,7 @@ pub(super) fn framework_claims_reference(framework: &str, name: &str) -> bool { /// Diagnostic/differential probe: `f.claimsReference(name)` for the resolver /// registered as `framework` — the same table `framework_claims` iterates, /// including the conservative claim for names the kernel does not know. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn framework_claims_name(framework: String, name: String) -> bool { framework_claims_reference(&framework, &name) } diff --git a/codegraph-kernel/src/tree.rs b/codegraph-kernel/src/tree.rs index 6ef348724c..add5aedfb2 100644 --- a/codegraph-kernel/src/tree.rs +++ b/codegraph-kernel/src/tree.rs @@ -57,7 +57,11 @@ //! offsets), and the child table is built by a counting pass over the parent //! array rather than one Vec per node. +#[cfg(feature = "addon")] use napi::bindgen_prelude::*; +#[cfg(not(feature = "addon"))] +use crate::native::{Buffer, Error, Result}; +#[cfg(feature = "addon")] use napi_derive::napi; use std::time::{Duration, Instant}; use tree_sitter::{Node, ParseOptions, Parser}; @@ -76,7 +80,7 @@ const FLAG_HAS_ERROR: u8 = 2; const FLAG_MISSING: u8 = 4; /// Whole-tree buffers for one parse. `src/extraction/kernel/tree.ts` mirrors. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct TreeBuffers { /// u32 LE: [abi, node_count, root_has_error] pub meta: Buffer, @@ -89,7 +93,7 @@ pub struct TreeBuffers { /// Kind and field names for one grammar (NUL-joined; kinds then fields), with /// the two counts. Fetched once per language by the TS facade. -#[napi(object)] +#[cfg_attr(feature = "addon", napi(object))] pub struct TreeNames { pub kind_count: u32, pub field_count: u32, @@ -336,12 +340,12 @@ fn finish(rows: Vec, root_has_error: bool) -> TreeBuffers { for v in [TREE_ABI_VERSION, n as u32, root_has_error as u32] { meta.extend_from_slice(&v.to_le_bytes()); } - TreeBuffers { meta: meta.into(), nodes: nodes.into(), children: children.into(), fields: fields.into() } + TreeBuffers { meta: crate::boundary::buffer(meta), nodes: crate::boundary::buffer(nodes), children: crate::boundary::buffer(children), fields: crate::boundary::buffer(fields) } } /// Kind and field names for `language`'s grammar. Field ids are 1-based in /// tree-sitter, so slot 0 of the field table is the empty "no field" name. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn tree_names(language: String) -> Option { let grammar = grammar_for(&language)?; let kind_count = grammar.node_kind_count() as u32; @@ -356,7 +360,7 @@ pub fn tree_names(language: String) -> Option { names.push_str(grammar.field_name_for_id(id).unwrap_or("")); names.push('\0'); } - Some(TreeNames { kind_count, field_count: field_count + 1, names: names.into_bytes().into() }) + Some(TreeNames { kind_count, field_count: field_count + 1, names: crate::boundary::buffer(names.into_bytes()) }) } /// Parse `content` with the native grammar for `language` and return the @@ -364,7 +368,7 @@ pub fn tree_names(language: String) -> Option { /// needs no stack guard; a missing grammar surfaces as a napi error. /// An optional request budget cancels parsing at progress callbacks. It does /// not interrupt external scanners or bound tree serialization. -#[napi] +#[cfg_attr(feature = "addon", napi)] pub fn parse_tree(content: String, language: String, budget_ms: Option) -> Result { parse_tree_inner(&content, &language, budget_ms) } diff --git a/codegraph-kernel/src/writer_lock.rs b/codegraph-kernel/src/writer_lock.rs index 50f9c8dc7d..f94c509ed9 100644 --- a/codegraph-kernel/src/writer_lock.rs +++ b/codegraph-kernel/src/writer_lock.rs @@ -1,14 +1,14 @@ use std::fs::{File, OpenOptions}; /// Keep this file in place: replacing its inode would split the lock domain. -#[napi_derive::napi] +#[cfg_attr(feature = "addon", napi_derive::napi)] pub struct WriterMutationLock { file: Option, } -#[napi_derive::napi] +#[cfg_attr(feature = "addon", napi_derive::napi)] impl WriterMutationLock { - #[napi] + #[cfg_attr(feature = "addon", napi)] pub fn release(&mut self) { self.file.take(); } @@ -16,18 +16,18 @@ impl WriterMutationLock { /// OS locks release on close or process exit, so a crashed writer cannot /// leave a stale coordination lock that another process must delete. -#[napi_derive::napi] -pub fn try_writer_mutation_lock(path: String) -> napi::Result> { +#[cfg_attr(feature = "addon", napi_derive::napi)] +pub fn try_writer_mutation_lock(path: String) -> crate::boundary::Result> { let file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(false) .open(path) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; + .map_err(|e| crate::boundary::Error::from_reason(e.to_string()))?; match file.try_lock() { Ok(()) => Ok(Some(WriterMutationLock { file: Some(file) })), Err(std::fs::TryLockError::WouldBlock) => Ok(None), - Err(std::fs::TryLockError::Error(e)) => Err(napi::Error::from_reason(e.to_string())), + Err(std::fs::TryLockError::Error(e)) => Err(crate::boundary::Error::from_reason(e.to_string())), } } diff --git a/scripts/build-kernel.mjs b/scripts/build-kernel.mjs index a6ad0b3527..735efbb9f8 100644 --- a/scripts/build-kernel.mjs +++ b/scripts/build-kernel.mjs @@ -78,10 +78,14 @@ if (cacheable && restoreKernel(platform, stamp, dest)) { console.log(`[kernel] building codegraph-kernel for ${platform}${target ? ` (target ${target})` : ''}${quick ? ' (quick profile)' : ''}`); const profile = quick ? ['--profile', 'quick'] : ['--release']; +// The crate also builds as an rlib for the internal native probe. An rlib among +// the crate types turns off the release profile's LTO, so the addon is built as +// a cdylib alone. +const ADDON_CRATE = ['--lib', '--crate-type', 'cdylib']; let outDir; if (target) { run('rustup', ['target', 'add', target], { allowFailure: true, quiet: true }); - run('cargo', ['build', '--release', '--target', target]); + run('cargo', ['rustc', '--release', '--target', target, ...ADDON_CRATE]); outDir = path.join(targetDir, target, 'release'); } else { // Lint gate: lib.rs denies clippy::all, so clippy must run where code is @@ -93,7 +97,7 @@ if (target) { } else { console.error('[kernel] warning: clippy unavailable; lint gate skipped'); } - run('cargo', ['build', ...profile]); + run('cargo', ['rustc', ...profile, ...ADDON_CRATE]); outDir = path.join(targetDir, quick ? 'quick' : 'release'); }