Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions src/codegraph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1475,7 +1475,6 @@ export class CodeGraph {
const synthesisRuns = this.resolver.synthesisRuns;
if (this.synthesisDirty.size > 0) {
this.queries.setSynthesisPending(true);
this.queries.deleteAllSynthesizedEdges();
}
options.onProgress?.({
phase: 'resolving',
Expand Down Expand Up @@ -1503,7 +1502,15 @@ export class CodeGraph {
result.pendingRefsProcessed = recovery.stats.total;
result.pendingRefsResolved = recovery.stats.resolved;
result.pendingRefsUnresolved = recovery.stats.unresolved;
if (this.resolver.synthesisRuns > synthesisRuns) this.synthesisDirty.clear();
if (this.resolver.synthesisRuns > synthesisRuns) {
this.queries.reconcileSynthesizedEdges(this.resolver.lastSynthesizedEdges);
this.synthesisDirty.clear();
} else if (this.synthesisDirty.size > 0) {
// Refresh attempted and failed: drop the stale synthesized edges,
// the same end-state deleting them up front produced — readers
// see missing edges, never wrong ones, until the retry lands.
this.queries.deleteAllSynthesizedEdges();
}
}

// Python attribute types read from OTHER files — a factory's return, a
Expand Down Expand Up @@ -1894,12 +1901,16 @@ export class CodeGraph {
// Cleared by resynthesize once it completes. Until then the next sync,
// in this process or after a restart, retries the rebuild.
this.queries.setSynthesisPending(true);
const dropped = this.queries.deleteAllSynthesizedEdges();
const edges = await this.resolver.resynthesize((current, total) =>
onProgress?.({ phase: 'linking', current, total }));
const { deleted, updated } = this.queries.reconcileSynthesizedEdges(this.resolver.lastSynthesizedEdges);
this.lastSynthesisMs = Date.now() - t;
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] sync-resynthesis: ${Date.now() - t}ms (${files.length} files, ${dropped} dropped, ${edges} synthesized)`);
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] sync-resynthesis: ${Date.now() - t}ms (${files.length} files, ${deleted} pruned, ${updated} updated, ${edges} synthesized)`);
} catch (error) {
// A mid-run failure leaves stored edges stale and newly-inserted ones
// partial — drop them all, the same end-state delete-then-rebuild left
// behind on failure: readers see missing edges, never wrong ones.
this.queries.deleteAllSynthesizedEdges();
for (const file of files) this.synthesisDirty.add(file);
if (requireComplete) throw new Error('Reconciliation did not complete synthesized-edge refresh.', { cause: error });
logWarn('Synthesized-edge refresh failed; the next sync retries it', { error: String(error) });
Expand Down
48 changes: 47 additions & 1 deletion src/db/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5733,6 +5733,47 @@ export class QueryBuilder {
return Number(r.changes ?? 0);
}

/**
* Reconcile stored synthesized edges with the set a refresh just emitted,
* keyed on the idx_edges_identity columns. Kept edges stay put — unlike
* deleteAllSynthesizedEdges + reinsert, an unchanged refresh writes nothing,
* which on a one-file sync was ~800 identical edge rows deleted and
* re-created (and their index pages rewritten) for zero graph change.
* Emitted edges the table already holds get their metadata refreshed when it
* drifted, matching what the delete+reinsert produced. Returns rows deleted
* and updated. Call after the emitted edges were inserted (INSERT OR IGNORE
* already added whichever ones were missing).
*/
reconcileSynthesizedEdges(emitted: ReadonlyArray<{ source: string; target: string; kind: string; metadata?: unknown; line?: number | null; column?: number | null }>): { deleted: number; updated: number } {
const wanted = new Map<string, string | null>();
for (const e of emitted) {
wanted.set(
`${e.source}>${e.target}>${e.kind}>${e.line ?? -1}>${e.column ?? -1}`,
e.metadata ? JSON.stringify(e.metadata) : null,
);
}
const rows = this.db.prepare(
`SELECT id, source, target, kind, metadata, line, col FROM edges
WHERE provenance = 'heuristic' AND json_extract(metadata, '$.synthesizedBy') IS NOT NULL`,
).all() as Array<{ id: number; source: string; target: string; kind: string; metadata: string | null; line: number | null; col: number | null }>;
const toDelete: number[] = [];
const toUpdate: Array<[string | null, number]> = [];
for (const r of rows) {
const key = `${r.source}>${r.target}>${r.kind}>${r.line ?? -1}>${r.col ?? -1}`;
const want = wanted.get(key);
if (want === undefined) toDelete.push(r.id);
else if (want !== r.metadata) toUpdate.push([want, r.id]);
}
let updated = 0;
this.db.transaction(() => {
const del = this.db.prepare('DELETE FROM edges WHERE id = ?');
for (const id of toDelete) del.run(id);
const upd = this.db.prepare('UPDATE edges SET metadata = ? WHERE id = ?');
for (const [meta, id] of toUpdate) updated += Number(upd.run(meta, id).changes ?? 0);
})();
return { deleted: toDelete.length, updated };
}

// ===========================================================================
// Near-duplicate bodies (src/graph/near-duplicates.ts)
// ===========================================================================
Expand Down Expand Up @@ -5914,7 +5955,12 @@ export class QueryBuilder {
// Passes that skipped the same bytes earlier still hold; a different hash replaces them.
const prev = read.get(filePath) as { hash: string; passes: string } | undefined;
if (prev?.hash === hash) for (const name of prev.passes.split(' ')) passes.add(name);
write.run(filePath, hash, [...passes].sort().join(' '));
const next = [...passes].sort().join(' ');
// A resync re-records the same skips; the row already says exactly
// this, so the write would only churn pages (measured ~900 leaf pages
// on a one-file sync of a 20k-file repo). Skip it.
if (prev?.hash === hash && prev.passes === next) continue;
write.run(filePath, hash, next);
}
})();
}
Expand Down
15 changes: 12 additions & 3 deletions src/resolution/callback-synthesizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7300,12 +7300,15 @@ export async function synthesizeCallbackEdges(
// WAL-valve writer backstop (WalCheckpointValve.backpressure), called at
// pool-idle points in the edge-insert loops below — the passes themselves
// only read; every write in this function happens with the pool idle.
backpressure?: () => Promise<void> | null
backpressure?: () => Promise<void> | null,
// Forwarded to `synthesizeWith`: receives the emitted edge set so a caller
// can reconcile stored synthesized edges instead of deleting them all first.
emitted?: { edges?: Edge[] }
): Promise<number> {
const prevImports = importPathMemo;
importPathMemo = new Map();
try {
return await withStripMemo(() => synthesizeWith(queries, withRunCache(ctx), onProgress, pool, backpressure));
return await withStripMemo(() => synthesizeWith(queries, withRunCache(ctx), onProgress, pool, backpressure, emitted));
} finally {
importPathMemo = prevImports;
}
Expand All @@ -7322,7 +7325,12 @@ async function synthesizeWith(
// WAL-valve writer backstop (WalCheckpointValve.backpressure), called at
// pool-idle points in the edge-insert loops below — the passes themselves
// only read; every write in this function happens with the pool idle.
backpressure?: () => Promise<void> | null
backpressure?: () => Promise<void> | null,
// When given, receives the full emitted edge set (the same rows insertEdges
// just wrote, plus the Go pre-passes). Sync uses it to reconcile — keep the
// rows re-emitted, drop only the ones no pass produced — instead of deleting
// every synthesized edge first and rewriting the whole set.
emitted?: { edges?: Edge[] }
): Promise<number> {
// Each sub-pass below is a whole-graph scan, and there are ~30 of them, all
// running synchronously on the indexer's main thread. Their AGGREGATE can run
Expand Down Expand Up @@ -7539,5 +7547,6 @@ async function synthesizeWith(
await foldIfOver();
}
__mark('insertMergedEdges');
if (emitted) emitted.edges = [...goMethodContains, ...goImpl, ...merged];
return merged.length + goImpl.length + goMethodContains.length;
}
13 changes: 11 additions & 2 deletions src/resolution/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -410,9 +410,15 @@ export class ReferenceResolver {
*/
synthesisRuns = 0;

/** Edges the last completed synthesis emitted; sync reconciles stored
* synthesized rows against it instead of deleting them all first. */
lastSynthesizedEdges: Edge[] = [];

async resynthesize(onProgress?: (done: number, total: number) => void): Promise<number> {
this.clearCaches();
const edges = await synthesizeCallbackEdges(this.queries, this.context, onProgress);
const emitted: { edges?: Edge[] } = {};
const edges = await synthesizeCallbackEdges(this.queries, this.context, onProgress, undefined, undefined, emitted);
this.lastSynthesizedEdges = emitted.edges ?? [];
this.queries.setSynthesisPending(false);
return edges;
}
Expand Down Expand Up @@ -2833,14 +2839,17 @@ export class ReferenceResolver {
// loop. See docs/design/callback-edge-synthesis.md.
const tSynth = Date.now();
try {
const emitted: { edges?: Edge[] } = {};
aggregateStats.byMethod['callback-synthesis'] = await synthesizeCallbackEdges(
this.queries,
this.context,
onSynthesisProgress,
pool,
parallel?.backpressure
parallel?.backpressure,
emitted
);
this.synthesisRuns++;
this.lastSynthesizedEdges = emitted.edges ?? [];
this.queries.setSynthesisPending(false);
} catch {
// synthesis is additive and optional; ignore failures
Expand Down
Loading