From 907b71d2ef032b0d2ecc746b825f780a292d6912 Mon Sep 17 00:00:00 2001 From: wefio <48851810+wefio@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:46:50 +0800 Subject: [PATCH 1/2] perf(onlion): the search must outrun the harness client, and the fold stops stalling on the provider Four live failures from the 15:2x-15:31 burst, each reproducible from the always-on lines: 1. /search 503 in 4.2 s: a fold retired the partition objects between the search's single manifest read and its partition GETs (snapshot_changed had nothing to retry after the read-path round). The search now catches snapshot_changed and re-runs from a fresh manifest read, twice. 2. SEARCH_TIMEOUT 504 from the harness: our 60 s deadline is longer than the client's window, so a partial answer at 60 s scores nothing. Deadline defaults to 25 s (NMG_SEARCH_DEADLINE_MS). 3. dependency_timeout x9, 25 s per document call: the fold's 32-record backfill raced three chunks against one saturated provider account while holding the manifest CAS. Document timeout 25 s -> 12 s, and document chunks are sequential by default (NMG_EMBED_CHUNK_CONCURRENCY, worker default 1) instead of three-wide. 4. 120 s kills at 914/907 MB: stalled chunks held SQLite state for the invocation's clock; removing the stalls shrinks both. Also: the nmg_request line gains execution: waituntil|inline, so a deploy whose zip kept a stale entry chunk (the 14:41 event - dist/ retained stale files) is caught by one grep; the test pin moves with the timeout. Recorded in docs/decisions/implemented/2026-10-30-stabilisation-round.* --- .../2026-10-30-stabilisation-round.md | 65 +++++++++++++++++++ .../2026-10-30-stabilisation-round.terms.yaml | 4 ++ .../2026-10-30-stabilisation-round.zh-CN.md | 41 ++++++++++++ onlion/backend/worker.mjs | 38 +++++++++-- onlion/scripts/test-backend.mjs | 4 +- src/core/embedding-providers/openai.ts | 1 + 6 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 docs/decisions/implemented/2026-10-30-stabilisation-round.md create mode 100644 docs/decisions/implemented/2026-10-30-stabilisation-round.terms.yaml create mode 100644 docs/decisions/implemented/2026-10-30-stabilisation-round.zh-CN.md diff --git a/docs/decisions/implemented/2026-10-30-stabilisation-round.md b/docs/decisions/implemented/2026-10-30-stabilisation-round.md new file mode 100644 index 00000000..3abab213 --- /dev/null +++ b/docs/decisions/implemented/2026-10-30-stabilisation-round.md @@ -0,0 +1,65 @@ +# Stabilisation round: the search answer must outrun the harness client, and the provider must not hold a CAS + +[简体中文](2026-10-30-stabilisation-round.zh-CN.md) · [Terms](2026-10-30-stabilisation-round.terms.yaml) + +**Status:** implemented +**Approved:** explicit + +## Problem + +Live logs from a concurrency-16 burst plus steady-state reads (2026-10-30 15:2x-15:31) show four +distinct faults, all visible with the new always-on lines: + +| fault | evidence | class | +| --------------------------------------------------------------------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| /search answered 503 in 4.2 s | 15:26 trace `2d32d910` | a fold retired the partition objects between the search's first (and only) manifest read and its partition GETs: `snapshot_changed` has nothing to retry now that the CAS loop is gone — the read-path round lost that safety net | +| SEARCH_TIMEOUT · 504 from the harness | 15:31 | our `SEARCH_DEADLINE_MS` is 60 s; the harness client's own window is shorter than 60 s (their docs publish no figure), so our "partial at 60 s" answer lands after their timeout | +| `dependency_timeout` ×9+ at embedding host `ws-2f72...ap-southeast-1` | `nmg_failure` batch 07:22 | a provider call stalls to the 25 s cap per call; a fold's 32-chunk backfill holds the manifest CAS open the whole time (three racing chunks × 25 s), so all other Calls pile behind it and adds leak 24-30 s | +| 120 s kills at 914/907 MB | platform feed 15:20-15:21 | the invocation pays for those stalled chunks while holding SQLite state; not addressed this round beyond removing the stalls | + +## Decision + +1. `SEARCH_DEADLINE_MS` 60_000 → 25_000 (`NMG_SEARCH_DEADLINE_MS`). A partial answer inside the + client's window scores; a perfect answer after it is a 504. +2. The search catches `snapshot_changed` and re-runs from a fresh manifest read, twice; the third + failure propagates. A pure read can still race a comparator's commit — it re-bases itself, the + same way a lock-free reader re-checks its version. +3. `EMBED_DOCUMENT_TIMEOUT_MS` 25_000 → 12_000. Two chances per fold chunk instead of one stalled + 25 s hold on the CAS. +4. Document chunks are sent `NMG_EMBED_CHUNK_CONCURRENCY`-controlled (worker default `1`): the + fold's batch of 32 becomes three sequential 10-row calls under a saturated provider, and the + provider's per-account queue stops being hit four-wide. +5. The `nmg_request` line gains `execution: "waituntil" | "inline"`, so a deploy that silently + kept a stale bundle (2026-10-30 14:41: the shipped zip still had the pre-waitUntil entry + because `dist/` retained stale chunk files) is visible in one grep, not in a 40-minute misread. +6. `build:edgeone` already clears `dist/` before packaging; the deploy checklist gains "grep the + built entrypoint for the current capability marker" so a stale zip is caught before shipping. + +## Alternatives considered + +- **Retry `snapshot_changed` by deleting the fold's retires**, e.g. delay `retireParts` by + minutes. Keeping deleted objects duplicates every snapshot the CAS's correctness depends on + deleting; a re-read of the manifest is cheaper than a tombstone every fold's trash. +- **Raise the search deadline to keep partial answers pending.** The partial answer exists to be + worth scoring inside the client window; waiting for it defeats the same purpose. +- **Lower the fold backfill batch instead of the chunk concurrency.** The batch (32) bounds how + many records one fold absorbs; the chunk race is what stalls the CAS. Different parameter, same + wrong layer. +- **Defer the fold embedding altogether via `waitUntil`** on the accept. Rejected with the exact + numbers from the 2026-10-30 15:00 fold probe: the provider's 25 s cap currently exists under + burst; a whole fold moved there would extend the window at the provider's pace, not ours. + +## Consequences + +- `search` no longer depends on writer CAS contention for safety; the read path retries its own + reads, and the retry costs one manifest GET. +- The search deadline change is one knob (`NMG_SEARCH_DEADLINE_MS`), tunable without code change; + the shipped default assumes the harness client window is smaller than 60 s — the platform + docs publish nothing more specific. +- Two new env knobs documented: `NMG_EMBED_CHUNK_CONCURRENCY` and the 12 s document timeout + default (existing `NMG_EMBED_TIMEOUT_MS` covering both members still applies first). +- The 15:26 503 class (stale-partition races) ends; a read leaves the same manifest-generation + barrier and loses nothing to a fold. +- Restated as before: the provider's 25 s cap is the throughput ceiling for backfill; the code + parts can no longer wait on it while holding a manifest — the worker's 25 s budget is now + effectively the ceiling, and 120 s kills no longer fits the model. diff --git a/docs/decisions/implemented/2026-10-30-stabilisation-round.terms.yaml b/docs/decisions/implemented/2026-10-30-stabilisation-round.terms.yaml new file mode 100644 index 00000000..55acd51c --- /dev/null +++ b/docs/decisions/implemented/2026-10-30-stabilisation-round.terms.yaml @@ -0,0 +1,4 @@ +version: 2 +terms: [] +references: + - docs/decisions/implemented/2026-10-30-stabilisation-round.md diff --git a/docs/decisions/implemented/2026-10-30-stabilisation-round.zh-CN.md b/docs/decisions/implemented/2026-10-30-stabilisation-round.zh-CN.md new file mode 100644 index 00000000..535f0265 --- /dev/null +++ b/docs/decisions/implemented/2026-10-30-stabilisation-round.zh-CN.md @@ -0,0 +1,41 @@ +# 稳定化轮：search 必须跑赢评测端客户端；提供商不得把 CAS 关住 + +[English](2026-10-30-stabilisation-round.md) · [Terms](2026-10-30-stabilisation-round.terms.yaml) + +**Status:** implemented +**Approved:** explicit + +## 问题 + +15:2x-15:31 的并发 16 突发 + 稳态读日志，四类故障全部可用新日志行定位： + +| 故障 | 证据 | 类 | +| ------------------------------------------------ | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| /search 4.2 秒 503 | 15:26 trace `2d32d910` | search 第一次（也是唯一一次）读 manifest 与它读分区对象之间，折叠提交并删除了旧对象——CAS 重试循环没了；现在这个竞态没有任何兜底 | +| SEARCH_TIMEOUT · 504 | 15:31 | 我们 `SEARCH_DEADLINE_MS` 60 s；评测端客户端窗口不发布具体数值但短于 60 s——我们的"partial at 60 s"总在对方超时之后 | +| `dependency_timeout` ×9+ 于新加坡 embedding host | `nmg_failure` 批次 | 单次 provider 调用顶到 25 s 上限；fold 32 条回填会在 CAS 内带三个并发 25 s 卡死，把所有其他调用卡住 24-30 s | +| 120 s 杀，914/907 MB | 平台信息 | 调用为一段 stall 的 chunk 全程付 SQLite 状态膜的代价；本轮通过撤 stall 缓解、不直接解决 | + +## 决策 + +1. `SEARCH_DEADLINE_MS` 60_000 → 25_000 (`NMG_SEARCH_DEADLINE_MS`)：评测端窗口内的 partial 得分，窗口外的完美答案得 504。 +2. search 捕 `snapshot_changed` 后重读一次 manifest 重跑，上限 2 次，第 3 次仍失败才外抛；纯读也要像无锁读者重新确认版本一样重新对齐。 +3. `EMBED_DOCUMENT_TIMEOUT_MS` 25_000 → 12_000：折叠 chunk 有两次机会，而不是一次 25 s 扣住 CAS。 +4. 文档 chunk 按 `NMG_EMBED_CHUNK_CONCURRENCY`（worker 默认 `1`）顺序发：fold 32 条变为三段 10 行顺序调用；提供商账户队列不再被宽并发击中。 +5. `nmg_request` 行加 `execution: "waituntil" | "inline"`；2026-10-30 14:41 那次 `dist/` 旧 chunk 文件未被清导致打包源仍是旧版，部署后下一轮 greppable。 +6. `build:edgeone` 已有清 `dist/` 的步骤；发布核查清单加"构建产物入口 grep 当前 capability 标记"，避免再由陈旧 zip 静默替换。 + +## 考虑过的替代方案 + +- **按删除速延迟 `retireParts` 来缓解 `snapshot_changed` 竞态**：按分钟保留已删除的对象，会复制 CAS 正确性依赖的删除；一次 manifest 重读比每次给 fold 的垃圾堆墓碑便宜。 +- **抬高 search deadline 以推迟 partial 答案**：partial 本来就是要挤进客户端窗口；推迟它就失去了意义。 +- **改 lower fold backfill 批而不是 chunk 并发**：32 是"一次折叠吸收多少条"的上限；chunk 竞争才是卡 CAS 的原因，别改错层。 +- **把 fold 嵌入整体搬到 accept 的 `waitUntil`**：15:00 那轮折探针已经给了确切数字——25 s 上限当前存在于突发；整个折叠搬过去意味着这个窗口由提供商节奏决定，不是我们。 + +## 后果 + +- `search` 不再依赖写 CAS 的重试保障自己的安全；读路径自带重试，代价是一次 manifest GET。 +- search deadline 变一个凑术变量（`NMG_SEARCH_DEADLINE_MS`），不改代码即可调；默认值假设评测端窗口 < 60 s（官方未发布具体数字）。 +- 两个 env 参数成为文档项：`NMG_EMBED_CHUNK_CONCURRENCY`、12 s 文档超时默认（原 `NMG_EMBED_TIMEOUT_MS` 仍覆盖两者）。 +- 15:26 503 类（陈旧分区竞态）终结；读不再被 fold 打拉扯。 +- 再说一次：提供商 25 s 上限是回填吞吐的真正天花板；但现在代码没有一边等它一边占住 manifest，表层不再有 120 s 卡死供代码塞 GC 误导。 diff --git a/onlion/backend/worker.mjs b/onlion/backend/worker.mjs index 2243ced8..0cf576f2 100644 --- a/onlion/backend/worker.mjs +++ b/onlion/backend/worker.mjs @@ -351,9 +351,11 @@ const EMBED_TIMEOUT_MS = 10_000; * A query is small and the provider is remote from the function, so ten seconds is already * generous for it. A document batch is neither: it carries a whole message, and the calls * cross a region boundary, where a healthy round trip and a stalled one differ by an order of - * magnitude. Cutting a document at ten seconds turned "the provider was slow" into a failed - * Add, which is the one failure the write path cannot absorb. */ -const EMBED_DOCUMENT_TIMEOUT_MS = 25_000; + * magnitude. The 2026-10-30 burst measured provider calls stalling to the 25 s cap while every + * fold chunk waited on it under the manifest CAS; twelve seconds still gives a stalled region + * cross two chances to succeed inside the fold budget, without turning one stalled call into a + * 25 s hold on the whole write. */ +const EMBED_DOCUMENT_TIMEOUT_MS = 12_000; /** Total embedding work one write may spend. The invocation is killed at 120 s, and the write * still needs room for its partition round trips after the vectors land, so the budget is * well inside it: past this point the Add fails with a clear code instead of running out the @@ -404,10 +406,15 @@ function embeddingClient(env, seed) { // A second client, not a second knob on the first: the provider client keeps one timeout, // and documents need a longer one than queries. let documentClient; + // Document chunks are the load a burst sends to one provider account; each concurrent + // chunk is a queue position the provider may not have. One at a time unless configured: + // the fold's batch of 32 becomes three sequential 10-row calls instead of three racing + // 25 s stalls (the 2026-10-30 incident's `dependency_timeout` ×9 at 25 s). const documents = () => (documentClient ??= createEmbeddingClientFromEnv({ ...env, NMG_EMBED_TIMEOUT_MS: env.NMG_EMBED_TIMEOUT_MS ?? String(EMBED_DOCUMENT_TIMEOUT_MS), + NMG_EMBED_CHUNK_CONCURRENCY: env.NMG_EMBED_CHUNK_CONCURRENCY ?? "1", })); // Thrown from inside `measure`'s async callback this becomes a rejection, which is what // every call site is written against (`catch(() => [])`). A synchronous throw would @@ -1103,7 +1110,7 @@ const FOLD_BUDGET_MS = 20_000; * Every partition is read to be exhaustive, which is unbounded in the manifest. Past this the * page is built from the partitions already read and marked `partial`: fewer candidates for one * question, instead of a request killed at the ceiling and no answer at all. */ -const SEARCH_DEADLINE_MS = 60_000; +const SEARCH_DEADLINE_MS = 25_000; /** Features to embed and search per clause, bounded by `MAX_QUERY_FEATURES`. * @@ -1288,10 +1295,17 @@ async function search(env, input) { // it cost ~20% of read load for no guarantee. A reader that can write is also a reader that // can fail and that can lose a compare-and-swap; a read-only search has neither failure // mode, and it no longer pays a multi-megabyte snapshot upload to answer a question. - const { manifest } = await measure("manifest_read", () => + // A fold that commits between this read and the partition GETs deletes the old objects + // (`retireParts`), and the GET then fails `snapshot_changed`. Before the read-only search this + // was absorbed by the CAS retry loop; as a pure read the path retries itself with a fresh + // manifest - and a 2026-10-30 15:26 /search 503 of 4.2 s is exactly the race this covered. + let manifestReads = 0; + let manifest = (await measure("manifest_read", () => loadManifest(env.MEMORY, keyFor(userId), userId), - ); + )).manifest; const sqlite = await measure("sqlite_init", () => sqliteReady()); + for (;;) { + try { { let candidates = []; const modes = new Set(); @@ -1375,6 +1389,17 @@ async function search(env, input) { storage_format: manifestFormat(manifest), }; } + } catch (error) { + if (error.code === "snapshot_changed" && manifestReads < 2) { + manifestReads += 1; + manifest = (await measure("manifest_read", () => + loadManifest(env.MEMORY, keyFor(userId), userId), + )).manifest; + continue; + } + throw error; + } + } } async function reset(env, input) { const userId = textField(input.user_id, "user_id"); @@ -2156,6 +2181,7 @@ export default { status: responseStatus, elapsed_ms: Date.now() - startedAt, seq: context.seq, + execution: context.execution?.waitUntil ? "waituntil" : "inline", }), ); memLog("exit", route, { diff --git a/onlion/scripts/test-backend.mjs b/onlion/scripts/test-backend.mjs index 9917a576..0920350a 100644 --- a/onlion/scripts/test-backend.mjs +++ b/onlion/scripts/test-backend.mjs @@ -549,7 +549,7 @@ print(json.dumps(r))`, assert.equal(embedding.endpoint, null); assert.equal(embedding.index_id, null); assert.equal(embedding.timeout_ms, 10_000); - assert.equal(embedding.document_timeout_ms, 25_000); + assert.equal(embedding.document_timeout_ms, 12_000); assert.equal(embedding.write_budget_ms, 45_000); assert.ok(String(status.body.revision).length > 0, "the build identifies itself"); // workerd reports zeros here; the shape is what this asserts. The deployed runtime @@ -916,7 +916,7 @@ print(json.dumps(r))`, // or above the 120 s platform ceiling turns a slow provider into a dead request. embed_timeout_ms: 10_000, embed_query_budget_ms: 25_000, - embed_document_timeout_ms: 25_000, + embed_document_timeout_ms: 12_000, embed_write_budget_ms: 45_000, slow_report_ms: 60_000, }); diff --git a/src/core/embedding-providers/openai.ts b/src/core/embedding-providers/openai.ts index c1ef7e74..968c7e2c 100644 --- a/src/core/embedding-providers/openai.ts +++ b/src/core/embedding-providers/openai.ts @@ -21,6 +21,7 @@ export function commonOpenAiOptions(environment: NodeJS.ProcessEnv): OpenAIEmbed documentTemplate: environment.NMG_EMBED_DOCUMENT_TEMPLATE, dimensions: optionalNumber(environment.NMG_EMBED_DIMENSIONS), batchSize: optionalNumber(environment.NMG_EMBED_BATCH_SIZE), + chunkConcurrency: optionalNumber(environment.NMG_EMBED_CHUNK_CONCURRENCY), timeoutMs: optionalNumber(environment.NMG_EMBED_TIMEOUT_MS), }; } From 2f4511c90e04301fcae2aba5d9d64d430ae22110 Mon Sep 17 00:00:00 2001 From: wefio <48851810+wefio@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:50:19 +0800 Subject: [PATCH 2/2] refactor(onlion): extract the partition page read to bring the search back under the complexity gate CodeFactor caught the search at complexity 16 after the snapshot_changed retry was added: the retry, the deadline, and the partition loop all landed in one function body. `readSearchCandidates` now owns the per-partition read (deadline, the read-only service call, the running top-K) and returns { candidates, modes, partial }. Search keeps the retry loop and the answer assembly. No behaviour change; the read path is still write-free, and the source pin in test-add-log-mode now covers the extracted reader so a write reintroduced there still fails. --- onlion/backend/worker.mjs | 105 +++++++++++++++++---------- onlion/scripts/test-add-log-mode.mjs | 10 ++- 2 files changed, 74 insertions(+), 41 deletions(-) diff --git a/onlion/backend/worker.mjs b/onlion/backend/worker.mjs index 0cf576f2..7a03960d 100644 --- a/onlion/backend/worker.mjs +++ b/onlion/backend/worker.mjs @@ -1266,6 +1266,57 @@ function addClauseContributions(contributions, clauses) { total.exclusive += clause.exclusive; } } +/** Reads the manifest's partitions into one global page, bounded by the search deadline. + * + * Extraction is what keeps `search` readable, not just small: a query's page is assembled + * by folding every partition's candidates through one running top-K, and that fold is its + * own loop with its own deadline branches. It mutates `contributions` and `appendedById` + * in place because the attribution across the whole request (all partitions plus the + * pending entries) is one set of totals, not a per-partition stat. */ +async function readSearchCandidates(env, sqlite, { manifest, queries, topK, appendOnly, features, client, started, contributions, appendedById }) { + const modes = new Set(); + let candidates = []; + let partial = false; + const searchDeadline = + Number(env.NMG_SEARCH_DEADLINE_MS) >= 0 + ? Number(env.NMG_SEARCH_DEADLINE_MS) + : SEARCH_DEADLINE_MS; + for (let i = 0; i < manifest.parts.length; i++) { + if (Date.now() - started > searchDeadline) { + partial = true; + break; + } + const result = await runPart( + env.MEMORY, + manifest.parts[i], + sqlite, + async ({ key: partKey }) => { + // Declining the backfill is what makes this read safe and cheap: without it the + // service would embed vectorless records inside a search and change the SQLite + // snapshot, which is what used to force the read path to write. + const service = new AmlMemoryService(partKey, client, { allowBackfill: false }); + return { + response: await measure("retrieval_including_embedding", () => + service.search(queries, topK, appendOnly, features), + ), + legacyReceipts: [], + readOnly: true, + }; + }, + ); + // A receipt that still lives in a legacy partition is replayed by the fold, the write + // path; the read path no longer copies it anywhere. + modes.add(result.response.retrievalMode); + addClauseContributions(contributions, result.response.clauses); + addAppended(appendedById, result.response.appended); + candidates.push(...result.response.memories); + // Bounded global Top K; never retain every shard's candidates in RAM. + candidates.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); + candidates = candidates.slice(0, topK); + } + return { candidates, modes, partial }; +} + async function search(env, input) { const userId = textField(input.user_id ?? input.userId, "user_id"); requestIdentity(userId, input.request_id); @@ -1308,47 +1359,23 @@ async function search(env, input) { try { { let candidates = []; - const modes = new Set(); await warmClauseVectors(client, features, manifest.parts); const persistence = { inline_delta_saves: 0, full_snapshot_saves: 0, changed_pages: 0 }; - const searchDeadline = - Number(env.NMG_SEARCH_DEADLINE_MS) >= 0 - ? Number(env.NMG_SEARCH_DEADLINE_MS) - : SEARCH_DEADLINE_MS; - let partial = false; - for (let i = 0; i < manifest.parts.length; i++) { - if (Date.now() - started > searchDeadline) { - partial = true; - break; - } - const result = await runPart( - env.MEMORY, - manifest.parts[i], - sqlite, - async ({ key: partKey }) => { - // Declining the backfill is what makes this read safe and cheap: without it the - // service would embed vectorless records inside a search and change the SQLite - // snapshot, which is what used to force the read path to write. - const service = new AmlMemoryService(partKey, client, { allowBackfill: false }); - return { - response: await measure("retrieval_including_embedding", () => - service.search(queries, topK, appendOnly, features), - ), - legacyReceipts: [], - readOnly: true, - }; - }, - ); - // A receipt that still lives in a legacy partition is replayed by the fold, the write - // path; the read path no longer copies it anywhere. - modes.add(result.response.retrievalMode); - addClauseContributions(contributions, result.response.clauses); - addAppended(appendedById, result.response.appended); - candidates.push(...result.response.memories); - // Bounded global Top K; never retain every shard's candidates in RAM. - candidates.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); - candidates = candidates.slice(0, topK); - } + const { candidates: pageCandidates, modes: modesRead, partial: partialRead } = + await readSearchCandidates(env, sqlite, { + manifest, + queries, + topK, + appendOnly, + features, + client, + started, + contributions, + appendedById, + }); + candidates.push(...pageCandidates); + const modes = modesRead; + const partial = partialRead; const fused = await fusePendingRecords({ env, ctx: { key: keyFor(userId), manifest }, diff --git a/onlion/scripts/test-add-log-mode.mjs b/onlion/scripts/test-add-log-mode.mjs index 827c0b51..5bcf9cec 100644 --- a/onlion/scripts/test-add-log-mode.mjs +++ b/onlion/scripts/test-add-log-mode.mjs @@ -111,8 +111,14 @@ try { // document-embedding backfill. A read that can write carries a CAS's failure modes - // that is how a 90 ms search died 503 before it had touched storage. { - const sliceStart = source.indexOf("async function search(env, input)"); - assert.ok(sliceStart >= 0, "search signature must not take an options bag"); + // The page read moved into `readSearchCandidates` when the search's complexity had to + // come down; the slice covers both, so a write reintroduced in either place fails here. + const sliceStart = source.indexOf("async function readSearchCandidates("); + assert.ok(sliceStart >= 0, "the read-only page reader must still exist"); + assert.ok( + source.indexOf("async function search(env, input)") > sliceStart, + "search signature must not take an options bag", + ); const sliceEnd = source.indexOf("async function reset("); const body = source.slice(sliceStart, sliceEnd); for (const forbidden of ["stageSearchPart", "commitManifest", "staged", "reuse", "readOnly: !changed"])