From 379038ca1e7b086e669520336db245721276851b Mon Sep 17 00:00:00 2001 From: bompus Date: Tue, 27 Oct 2026 14:17:19 -0600 Subject: [PATCH 1/2] perf(resolve): halve resolver-worker page cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each of the up-to-12 pool workers held a 32 MiB SQLite page cache — ~384 MiB summed inside the main process before any real working set. The workers do scattered point reads during resolution; -16384 (16 MiB) measured peak-RSS neutral-to-better on pretix (4.34 vs 4.24 GiB main) with identical-or-better wall time. --- src/resolution/resolver-worker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resolution/resolver-worker.ts b/src/resolution/resolver-worker.ts index 8213b5d23..429a35d74 100644 --- a/src/resolution/resolver-worker.ts +++ b/src/resolution/resolver-worker.ts @@ -57,7 +57,7 @@ port.on('message', (msg: InMessage) => { const created = createDatabase(msg.dbPath, { readOnly: true }); db = created.db; db.pragma('busy_timeout = 5000'); - db.pragma('cache_size = -32000'); + db.pragma('cache_size = -16000'); const tDb = Date.now(); queries = new QueryBuilder(db); resolver = new ReferenceResolver(msg.projectRoot, queries); @@ -102,7 +102,7 @@ port.on('message', (msg: InMessage) => { const reopened = createDatabase(dbPath, { readOnly: true }); db = reopened.db; db.pragma('busy_timeout = 5000'); - db.pragma('cache_size = -32000'); + db.pragma('cache_size = -16000'); queries.rebind(db); // A snapshot refreshed after the prerequisite phase: every supertype // edge the rest of the run reads is in it, so the kernel walks them. From c6a0672b2d0862b276cfccd3ef4dd29dc22140b6 Mon Sep 17 00:00:00 2001 From: bompus Date: Tue, 27 Oct 2026 14:20:10 -0600 Subject: [PATCH 2/2] perf(synth): reuse shingle sets between near-dup signing and scoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every body in a candidate pair was tokenized and shingled twice: once to write its MinHash signature, again to score the pair's exact Jaccard. Keep the Set computed at sign time and hand it to the pair pass — on pretix the phase drops ~38% (458 -> 284 ms); identical pairs, identical scores. --- src/graph/near-duplicates.ts | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/graph/near-duplicates.ts b/src/graph/near-duplicates.ts index a0aefff71..a16546362 100644 --- a/src/graph/near-duplicates.ts +++ b/src/graph/near-duplicates.ts @@ -99,10 +99,8 @@ export function jaccard(a: Set, b: Set): number { return common / (a.size + b.size - common); } -/** The MinHash signature of a body, or null when it is too small to compare. */ -export function signature(body: string, ownName?: string): Uint32Array | null { - const set = shingles(body, ownName); - if (!set) return null; +/** The MinHash signature of a shingle set. */ +export function signatureFromSet(set: Set): Uint32Array { const sig = new Uint32Array(HASHES).fill(0xffffffff); for (const x of set) { for (let k = 0; k < HASHES; k++) { @@ -113,6 +111,12 @@ export function signature(body: string, ownName?: string): Uint32Array | null { return sig; } +/** The MinHash signature of a body, or null when it is too small to compare. */ +export function signature(body: string, ownName?: string): Uint32Array | null { + const set = shingles(body, ownName); + return set ? signatureFromSet(set) : null; +} + /** The share of hashes two signatures agree on: an estimate of their shingle overlap. */ export function similarity(a: Uint32Array, b: Uint32Array): number { let same = 0; @@ -165,10 +169,15 @@ export function refreshNearDuplicates(deps: NearDuplicateDeps): { signed: number stale.set(c.filePath, list); } const rows: Array<{ nodeId: string; nodeUpdatedAt: number; sig: Uint8Array }> = []; + // The shingle sets just computed, keyed by node id: scoring reuses them + // instead of re-tokenizing every body that lands in a candidate pair. + const freshSets = new Map>(); for (const [filePath, bodies] of stale) { const lines = deps.readFile(filePath)?.split(/\r?\n/); for (const b of bodies) { - const sig = lines ? signature(lines.slice(b.startLine - 1, b.endLine).join('\n'), b.name) : null; + const set = lines ? shingles(lines.slice(b.startLine - 1, b.endLine).join('\n'), b.name) : null; + const sig = set ? signatureFromSet(set) : null; + if (set) freshSets.set(b.id, set); rows.push({ nodeId: b.id, nodeUpdatedAt: b.updatedAt, sig: sig ? new Uint8Array(sig.buffer) : new Uint8Array(0) }); } } @@ -177,7 +186,7 @@ export function refreshNearDuplicates(deps: NearDuplicateDeps): { signed: number queries.upsertMinhash(rows); queries.deleteMinhash(drop); if (rows.length === 0 && drop.length === 0 && !deps.force) return { signed: 0, pairs: null, rescored: 0 }; - const { kept, rescored } = pairNearDuplicates(deps); + const { kept, rescored } = pairNearDuplicates(deps, freshSets); return { signed: rows.length, pairs: kept, rescored }; } @@ -216,7 +225,7 @@ export function forEachBandBucket( } /** LSH over the stored signatures, one band at a time so memory stays linear in the bodies. */ -function pairNearDuplicates(deps: NearDuplicateDeps): { kept: number; rescored: number } { +function pairNearDuplicates(deps: NearDuplicateDeps, freshSets?: ReadonlyMap>): { kept: number; rescored: number } { const { queries } = deps; // Every signature in one scan (rowid order), then each band's buckets from // the bytes in memory: sixteen SQL scans with a per-row substr and base64 @@ -256,8 +265,11 @@ function pairNearDuplicates(deps: NearDuplicateDeps): { kept: number; rescored: const row = queries.minhashByRowid(rowid); if (!row) return null; const node = queries.getNodeById(row.nodeId); - const text = node ? lines(node.filePath)?.slice(node.startLine - 1, node.endLine).join('\n') : undefined; - hit = { nodeId: row.nodeId, sig: asSignature(row.sig), set: text ? shingles(text, node!.name) : null }; + // A set signed this refresh is already in memory; only unchanged bodies + // pay the tokenize pass. + const cached = node ? freshSets?.get(node.id) : undefined; + const text = !cached && node ? lines(node.filePath)?.slice(node.startLine - 1, node.endLine).join('\n') : undefined; + hit = { nodeId: row.nodeId, sig: asSignature(row.sig), set: cached ?? (text ? shingles(text, node!.name) : null) }; sigs.set(rowid, hit); } return hit;