From 816d8327dac696f2d313e972c777d96125d81aaf Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:21:46 +0000 Subject: [PATCH 1/2] fix(explore): bind measurements to the executed result Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- loopx/capabilities/explore/README.md | 7 ++++ loopx/capabilities/explore/measurements.py | 5 ++- loopx/capabilities/explore/result_log.py | 4 ++- .../capabilities/explore_measurements.ts | 21 ++++++++--- .../capabilities/explore_result_writeback.ts | 2 +- .../test_research_execution_loop.py | 35 +++++++++++++++++++ .../explore_measurements.test.ts | 33 ++++++++++++++++- 7 files changed, 99 insertions(+), 8 deletions(-) diff --git a/loopx/capabilities/explore/README.md b/loopx/capabilities/explore/README.md index da5fa850e2..971c9df759 100644 --- a/loopx/capabilities/explore/README.md +++ b/loopx/capabilities/explore/README.md @@ -31,6 +31,13 @@ attachments also require the same `input_revision`. Use a new finding id per evaluated candidate, and reuse an id only to correct that finding. Missing values are explicit `null`; zero and negative values remain valid measurements. +When an `execution_observation` declares a `result_ref`, each measurement must +cite that same result. This applies to manual findings and writeback attachments, +including shared comparison contexts. Listing both receipts in `evidence_refs` +does not bind a different result to the executed candidate. No result is inferred +when execution has not declared one. These consistency checks do not independently +verify a producer's calculator or turn `confirmed` into an evaluator attestation. + To reuse verified comparison conditions, save these three fields once in `conditions.json`: diff --git a/loopx/capabilities/explore/measurements.py b/loopx/capabilities/explore/measurements.py index 8e7c7f53e1..3490a20873 100644 --- a/loopx/capabilities/explore/measurements.py +++ b/loopx/capabilities/explore/measurements.py @@ -21,10 +21,13 @@ def _normalize(encoded: str) -> str: return json.dumps(result["measurements"], sort_keys=True, allow_nan=False) -def normalize_measurements(value, *, measurement_context=_UNSET): +def normalize_measurements(value, *, measurement_context=_UNSET, execution_observation=_UNSET): validate_public_safe_value(value, path="measurements") request = {"measurements": value} if measurement_context is not _UNSET: validate_public_safe_value(measurement_context, path="measurement_context") request["measurement_context"] = measurement_context + if execution_observation is not _UNSET: + validate_public_safe_value(execution_observation, path="execution_observation") + request["execution_observation"] = execution_observation return json.loads(_normalize(json.dumps(request, sort_keys=True, allow_nan=False))) diff --git a/loopx/capabilities/explore/result_log.py b/loopx/capabilities/explore/result_log.py index 11c6c5a7f5..bc7b0b088f 100644 --- a/loopx/capabilities/explore/result_log.py +++ b/loopx/capabilities/explore/result_log.py @@ -394,7 +394,9 @@ def build_explore_finding_event( if not node_id: raise ValueError("measurements require an explicit node") - measured = normalize_measurements(measurements) + measured = normalize_measurements( + measurements, execution_observation=event.get("execution_observation"), + ) if any(item["evidence_ref"] not in event.get("evidence_refs", []) for item in measured): raise ValueError("measurement evidence must be included in the finding's evidence_refs") event["measurements"] = measured diff --git a/loopx/control_plane/capabilities/explore_measurements.ts b/loopx/control_plane/capabilities/explore_measurements.ts index 2ed670896e..f051818ac6 100644 --- a/loopx/control_plane/capabilities/explore_measurements.ts +++ b/loopx/control_plane/capabilities/explore_measurements.ts @@ -4,6 +4,7 @@ import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {requireJsonObject, type JsonObject} from "../runtime_decode.ts"; import {isExploreEvidenceRef} from "./explore_evidence_ref.ts"; +import {normalizeExploreExecutionObservation} from "./explore_execution_observation.ts"; export interface ExploreMeasurement { metric: string; value: number | null; unit: string; higher_is_better: boolean; @@ -21,7 +22,7 @@ export interface ExploreMeasurementFeedback { improvement: number | null; measured_count: number; unknown_count: number; } -export function normalizeExploreMeasurements(value: unknown, context?: unknown): ExploreMeasurement[] { +export function normalizeExploreMeasurements(value: unknown, context?: unknown, executionObservation?: unknown): ExploreMeasurement[] { if (!Array.isArray(value) || !value.length || value.length > 8) { throw new EffectRuntimeRequestError("measurements requires one to eight explicit measurements"); } @@ -39,10 +40,10 @@ export function normalizeExploreMeasurements(value: unknown, context?: unknown): // Authoring only: persist the existing complete measurement, never a // mutable context reference or inferred comparison membership. return {...shared, ...observation}; - })); + }), undefined, executionObservation); } const seen = new Set(); - return value.map(item => { + const measurements = value.map(item => { const row = requireJsonObject(item, "measurement"); const fields = ["metric", "value", "unit", "higher_is_better", "comparison_scope", "input_revision", "evaluation_ref", "candidate_ref", "evidence_ref"]; @@ -76,6 +77,17 @@ export function normalizeExploreMeasurements(value: unknown, context?: unknown): seen.add(key); return result; }); + if (executionObservation != null) { + const execution = normalizeExploreExecutionObservation({observation: executionObservation}); + if (measurements.some(item => item.input_revision !== execution.input_revision)) { + throw new EffectRuntimeRequestError("measurements must retain the executed candidate's input revision"); + } + if (execution.result_ref !== undefined + && measurements.some(item => item.evidence_ref !== execution.result_ref)) { + throw new EffectRuntimeRequestError("measurements must cite the executed candidate's result reference"); + } + } + return measurements; } function comparisonKey(node: string, metric: ExploreMeasurement): string { @@ -123,5 +135,6 @@ export function projectExploreMeasurementFeedback(findings: readonly JsonObject[ } export function normalizeExploreMeasurementsRequest(params: JsonObject): JsonObject { - return {measurements: normalizeExploreMeasurements(params.measurements, params.measurement_context)}; + return {measurements: normalizeExploreMeasurements(params.measurements, params.measurement_context, + params.execution_observation)}; } diff --git a/loopx/control_plane/capabilities/explore_result_writeback.ts b/loopx/control_plane/capabilities/explore_result_writeback.ts index bf49b8cda2..be8b7af592 100644 --- a/loopx/control_plane/capabilities/explore_result_writeback.ts +++ b/loopx/control_plane/capabilities/explore_result_writeback.ts @@ -193,7 +193,7 @@ function normalizeAttachment(value: unknown, visionPacket?: unknown, linkedScope throw new EffectRuntimeRequestError("measurement_context requires measurements"); } const measurements = Object.hasOwn(row, "measurements") - ? normalizeExploreMeasurements(row.measurements, row.measurement_context) : null; + ? normalizeExploreMeasurements(row.measurements, row.measurement_context, execution) : null; if (measurements?.some(item => item.input_revision !== text(row.input_revision, "input_revision", 160) || !refs.includes(item.evidence_ref))) { throw new EffectRuntimeRequestError("measurements must retain this finding's input revision and evidence references"); diff --git a/tests/control_plane/test_research_execution_loop.py b/tests/control_plane/test_research_execution_loop.py index 9ea048dbd4..cee7f893ca 100644 --- a/tests/control_plane/test_research_execution_loop.py +++ b/tests/control_plane/test_research_execution_loop.py @@ -40,6 +40,41 @@ def test_real_probe_reports_early_exit_not_budget_exhaustion(tmp_path, monkeypat assert (tmp_path / "observation.json.stdout").read_text().strip() == "5" +def test_finding_measurements_keep_the_executed_result_identity(tmp_path, monkeypatch): + from loopx.capabilities.explore.result_log import ( + append_explore_result_event, build_explore_finding_event, + load_explore_result_events_strict, + ) + + observed = probe(tmp_path, monkeypatch) + observed.update(result_ref="result:executed", result_candidate_sha256=observed["candidate_sha256"]) + observed["evidence_refs"].append("result:executed") + observed["evidence_refs"].sort() + measurement = { + "metric": "quality", "value": 8, "unit": "points", "higher_is_better": True, + "comparison_scope": "validation:fixed", "input_revision": observed["input_revision"], + "evaluation_ref": "evaluator:v1", "candidate_ref": "candidate:executed", + "evidence_ref": "result:other", + } + fields = dict( + goal_id="research", title="The exact executed result", finding_id="measured", + node_id="question", status="tentative", execution_observation=observed, + evidence_refs=[*observed["evidence_refs"], "result:other"], + ) + log = tmp_path / "results.jsonl" + with pytest.raises(ValueError, match="executed candidate's result"): + build_explore_finding_event(**fields, measurements=[measurement]) + assert not log.exists(), "inconsistent evidence must fail before publication" + measurement["evidence_ref"] = "result:executed" + with pytest.raises(ValueError, match="executed candidate's input"): + build_explore_finding_event(**fields, measurements=[{**measurement, "input_revision": "other-input"}]) + event = build_explore_finding_event(**fields, measurements=[measurement]) + append_explore_result_event(log, event) + [retained] = load_explore_result_events_strict(log, goal_id="research") + assert retained["measurements"] == [measurement] + assert retained["execution_observation"] == observed + + def test_missing_solver_telemetry_stays_unknown_and_timeout_is_host_observed(tmp_path, monkeypatch): observation = probe(tmp_path, monkeypatch, source="import time; time.sleep(10)", limit_seconds=0.05) assert observation["search_iterations"] is None diff --git a/tests/control_plane_ts/explore_measurements.test.ts b/tests/control_plane_ts/explore_measurements.test.ts index 90551704e7..81588b40fd 100644 --- a/tests/control_plane_ts/explore_measurements.test.ts +++ b/tests/control_plane_ts/explore_measurements.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import {normalizeExploreMeasurements, projectExploreMeasurementFeedback} from "../../loopx/control_plane/capabilities/explore_measurements.ts"; +import {normalizeExploreMeasurements, normalizeExploreMeasurementsRequest, projectExploreMeasurementFeedback} from "../../loopx/control_plane/capabilities/explore_measurements.ts"; import {normalizeExploreResultAttachment} from "../../loopx/control_plane/capabilities/explore_result_writeback.ts"; import {projectExploreTurnContext} from "../../loopx/control_plane/capabilities/explore_turn_context.ts"; import {parseExploreHistoryPage, projectExploreHistory} from "../../loopx/control_plane/presentation/explore_history.ts"; @@ -11,6 +11,37 @@ const metric = {metric: "quality", value: 8, unit: "points", higher_is_better: t const finding = (id: string, value: number | null, extra = {}) => ({finding_id: id, node_id: "question-a", measurements: [{...metric, value, candidate_ref: `candidate:${id}`, evidence_ref: `result:${id}`, ...extra}]}); +test("measurements cannot cite another result than the attached executed candidate", () => { + const execution = {schema_version: "explore_execution_observation_v0", + candidate_sha256: "a".repeat(64), result_candidate_sha256: "a".repeat(64), + input_revision: metric.input_revision, planned_method_id: "probe", executed_method_id: "probe", + change_reason: null, elapsed_seconds: 1, limit_seconds: 10, search_iterations: null, + stop_reason: "process_exit:0", evidence_refs: ["result:executed"], result_ref: "result:executed"}; + const attachment = {schema_version: "explore_result_attachment_v0", question: "Which candidate improved?", + applicability: "Fixed task and budget", input_revision: metric.input_revision, + observation: "A measured result", interpretation: "Investigate the actual candidate", + status: "tentative", evidence_refs: [metric.evidence_ref, "result:executed"], + measurements: [metric], execution_observation: execution}; + assert.throws(() => normalizeExploreResultAttachment({attachment}), /executed candidate's result/); + assert.throws(() => normalizeExploreMeasurementsRequest({measurements: [metric], + execution_observation: execution}), /executed candidate's result/); + const matching = {...metric, evidence_ref: execution.result_ref}; + assert.deepEqual(normalizeExploreResultAttachment({attachment: {...attachment, + measurements: [matching]}}).measurements, [matching]); + assert.deepEqual(normalizeExploreMeasurementsRequest({measurements: [matching], + execution_observation: execution}).measurements, [matching]); + assert.throws(() => normalizeExploreMeasurementsRequest({measurements: [{...matching, + input_revision: "other-input"}], execution_observation: execution}), /executed candidate's input/); + const {comparison_scope, input_revision, evaluation_ref, ...observation} = matching; + assert.throws(() => normalizeExploreMeasurementsRequest({measurements: [{...observation, + evidence_ref: "result:other"}], measurement_context: {comparison_scope, input_revision, evaluation_ref}, + execution_observation: execution}), /executed candidate's result/); + assert.deepEqual(normalizeExploreMeasurements([metric]), [metric], "legacy measurements remain unchanged"); + const {result_ref: _result, result_candidate_sha256: _candidate, ...unfinished} = execution; + assert.deepEqual(normalizeExploreMeasurementsRequest({measurements: [metric], + execution_observation: unfinished}).measurements, [metric], "missing result is not inferred"); +}); + test("explicit comparison context is reused while each result retains its own candidate and receipt", () => { const {comparison_scope, input_revision, evaluation_ref, ...observation} = metric; const context = {comparison_scope, input_revision, evaluation_ref}; From c053027fb2f26c54cc3f5950ebee7021e87a5450 Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:03:49 +0000 Subject: [PATCH 2/2] fix(explore): preserve auditable history after attribution checks Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- loopx/capabilities/explore/README.md | 14 +++ loopx/capabilities/explore/result_log.py | 36 +++++++- .../capabilities/explore_measurements.ts | 28 +++--- .../capabilities/explore_turn_context.ts | 6 +- .../explore_measurement_history_v0.jsonl | 3 + .../test_explore_measurement_history.py | 89 +++++++++++++++++++ .../explore_measurements.test.ts | 18 ++++ 7 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 tests/control_plane/fixtures/explore_measurement_history_v0.jsonl create mode 100644 tests/control_plane/test_explore_measurement_history.py diff --git a/loopx/capabilities/explore/README.md b/loopx/capabilities/explore/README.md index 971c9df759..dcfc1d4b8e 100644 --- a/loopx/capabilities/explore/README.md +++ b/loopx/capabilities/explore/README.md @@ -1420,3 +1420,17 @@ outcomes. Native research completion retains the graph lock and, when inheriting, the device-configuration lock until its transaction finishes. Inherited planning requires the same exact typed experiment evidence as an explicit Goal mode. + +Historical compatibility: stricter execution/measurement attribution is a new-write +rule. Canonical readers preserve earlier records and their identities, including +an inconsistent original followed by a correction. Current finding views expose +`measurement_error` and omit contradictory quantities from comparison; raw history +retains the producer's original statement. A matching correction becomes usable +without deleting its prior version. Explicit ordered-prefix archival replay keeps +the same audit boundary; ordinary single/batch writes still reject contradictions. +Malformed, unsafe, unknown-field or forged canonical records remain rejected. + +历史兼容:执行与测量的一致性约束限制新写入,不能阻断旧记录及其后续更正的读取。 +当前 finding 若有归因矛盾,会返回 `measurement_error` 并排除该数量比较;历史仍保留 +生产者的原始声明。正确更正可恢复当前比较,不能删改旧行。显式按顺序恢复归档前缀也 +保留此审计边界;普通单条和批量写入仍拒绝新矛盾,结构损坏、不安全及伪造记录仍拒绝。 diff --git a/loopx/capabilities/explore/result_log.py b/loopx/capabilities/explore/result_log.py index bc7b0b088f..8647f42e27 100644 --- a/loopx/capabilities/explore/result_log.py +++ b/loopx/capabilities/explore/result_log.py @@ -352,6 +352,7 @@ def build_explore_finding_event( recorded_at: str | None = None, execution_observation: Mapping[str, Any] | None = None, measurements: Sequence[Mapping[str, Any]] | None = None, + _check_execution_measurements: bool = True, ) -> dict[str, Any]: safe_goal_id = _safe_goal_id(goal_id) safe_title = _compact_text(title, limit=TITLE_LIMIT, field="title") @@ -395,7 +396,8 @@ def build_explore_finding_event( if not node_id: raise ValueError("measurements require an explicit node") measured = normalize_measurements( - measurements, execution_observation=event.get("execution_observation"), + measurements, **({"execution_observation": event.get("execution_observation")} + if _check_execution_measurements else {}), ) if any(item["evidence_ref"] not in event.get("evidence_refs", []) for item in measured): raise ValueError("measurement evidence must be included in the finding's evidence_refs") @@ -458,6 +460,8 @@ def validate_explore_result_event( unknown fields, invalid ids, unsafe text, forged boundary flags, and stale event ids. Persisted text has its own bounded compatibility budget: it is validated and restored before hashing, never clipped to the writer default. + This structural read validation preserves older attribution contradictions; + writers reject new contradictions, and current views exclude their numbers. """ payload = dict(event) @@ -520,6 +524,10 @@ def validate_explore_result_event( supersedes=payload.get("supersedes"), execution_observation=payload.get("execution_observation"), measurements=payload.get("measurements"), + # Canonical history predates the stricter new-write rule. Keep + # original bytes/identities auditable; current views qualify the + # folded result separately, and append rechecks new writes below. + _check_execution_measurements=False, ) if summary: rebuilt["summary"] = summary @@ -529,8 +537,16 @@ def validate_explore_result_event( return rebuilt +def _check_finding_measurement_execution(event: Mapping[str, Any]) -> None: + if "measurements" in event: + from .measurements import normalize_measurements + + normalize_measurements(event["measurements"], execution_observation=event.get("execution_observation")) + + def append_explore_result_event(path: Path, event: Mapping[str, Any]) -> dict[str, Any]: validated = validate_explore_result_event(event) + _check_finding_measurement_execution(validated) log_path = path.expanduser() log_path.parent.mkdir(parents=True, exist_ok=True) with exclusive_file_lock(log_path): @@ -600,6 +616,11 @@ def append_explore_result_events( if require_ordered_prefix and create_only_node_ids: raise ValueError("ordered prefix replay cannot skip existing node revisions") validated = [validate_explore_result_event(event, expected_goal_id=expected_goal_id) for event in events] + # Ordered replay restores an audited historical prefix, not new evidence. + # Its old contradictions remain visible only as archival facts/diagnostics. + if not require_ordered_prefix: + for event in validated: + _check_finding_measurement_execution(event) requested_ids = [str(event["event_id"]) for event in validated] if len(requested_ids) != len(set(requested_ids)): raise ValueError("Explore result event batch contains duplicate event ids") @@ -843,10 +864,19 @@ def _materialized_parent_edges( def _finding_view(event: Mapping[str, Any]) -> dict[str, Any]: + measured: dict[str, Any] = {} + if "measurements" in event: + try: + _check_finding_measurement_execution(event) + except ValueError as error: + # Quarantine contradictory historical quantities in the current + # view, not the append-only history. Runtime failure propagates. + measured["measurement_error"] = str(error) + else: + measured.update(measurements=event["measurements"], recorded_order=event["_recorded_order"]) return { **({"execution_observation": event["execution_observation"]} if "execution_observation" in event else {}), - **({"measurements": event["measurements"], "recorded_order": event["_recorded_order"]} - if "measurements" in event else {}), + **measured, "finding_id": str(event.get("result_id") or ""), "finding": str(event.get("title") or ""), "summary": str(event.get("summary") or ""), diff --git a/loopx/control_plane/capabilities/explore_measurements.ts b/loopx/control_plane/capabilities/explore_measurements.ts index f051818ac6..dc73bbba05 100644 --- a/loopx/control_plane/capabilities/explore_measurements.ts +++ b/loopx/control_plane/capabilities/explore_measurements.ts @@ -77,19 +77,21 @@ export function normalizeExploreMeasurements(value: unknown, context?: unknown, seen.add(key); return result; }); - if (executionObservation != null) { - const execution = normalizeExploreExecutionObservation({observation: executionObservation}); - if (measurements.some(item => item.input_revision !== execution.input_revision)) { - throw new EffectRuntimeRequestError("measurements must retain the executed candidate's input revision"); - } - if (execution.result_ref !== undefined - && measurements.some(item => item.evidence_ref !== execution.result_ref)) { - throw new EffectRuntimeRequestError("measurements must cite the executed candidate's result reference"); - } - } + const mismatch = executionMeasurementError(measurements, executionObservation); + if (mismatch !== null) throw new EffectRuntimeRequestError(mismatch); return measurements; } +function executionMeasurementError(measurements: readonly ExploreMeasurement[], observation: unknown): string | null { + if (observation == null) return null; + const execution = normalizeExploreExecutionObservation({observation}); + if (measurements.some(item => item.input_revision !== execution.input_revision)) + return "measurements must retain the executed candidate's input revision"; + if (execution.result_ref !== undefined && measurements.some(item => item.evidence_ref !== execution.result_ref)) + return "measurements must cite the executed candidate's result reference"; + return null; +} + function comparisonKey(node: string, metric: ExploreMeasurement): string { return JSON.stringify([node, metric.metric, metric.unit, metric.higher_is_better, metric.comparison_scope, metric.input_revision, metric.evaluation_ref]); @@ -113,7 +115,11 @@ export function projectExploreMeasurementFeedback(findings: readonly JsonObject[ for (const finding of ordered) { const node = String(finding.node_id ?? ""), id = String(finding.finding_id ?? ""); if (!node || !id) throw new EffectRuntimeRequestError("measurements require their node and finding identities"); - for (const measurement of normalizeExploreMeasurements(finding.measurements)) { + const measurements = normalizeExploreMeasurements(finding.measurements); + // Old canonical history is auditable but is not a quantitative comparison + // source when its declared execution contradicts its measurements. + if (executionMeasurementError(measurements, finding.execution_observation) !== null) continue; + for (const measurement of measurements) { const key = comparisonKey(node, measurement), prior = groups.get(key); const previous = prior?.best ?? null; const point = measurement.value === null ? null : {finding_id: id, diff --git a/loopx/control_plane/capabilities/explore_turn_context.ts b/loopx/control_plane/capabilities/explore_turn_context.ts index 5329536e25..5b77bc3a5f 100644 --- a/loopx/control_plane/capabilities/explore_turn_context.ts +++ b/loopx/control_plane/capabilities/explore_turn_context.ts @@ -202,7 +202,7 @@ export function projectExploreTurnContext(params: JsonObject): JsonObject { // Manual quantitative capture uses the same finding owner. Keep its sources // available to the existing exact-result decision path, without requiring a // writeback-origin tag or admitting unrelated unmeasured notes. - const decisionResults = findings.filter(row => row.measurements !== undefined + const decisionResults = findings.filter(row => row.measurements !== undefined || row.measurement_error !== undefined || Array.isArray(row.tags) && row.tags.includes("writeback-result")); // Keep the same detail budget, but do not let unrelated newer results evict // the applicability of evidence linked to the next selected work item. @@ -344,13 +344,13 @@ export function projectExploreTurnContext(params: JsonObject): JsonObject { counts: projection.counts ?? {}, recent_nodes: nodes.slice(0, 3).map(row => compact(row, ["node_id", "title", "status", "blocked_reason"])), // The canonical evidence projection calls the finding's title `finding`. - recent_findings: findings.slice(0, 3).map(row => ({...compact({...row, title: row.finding}, ["finding_id", "node_id", "title", "status"]), + recent_findings: findings.slice(0, 3).map(row => ({...compact({...row, title: row.finding}, ["finding_id", "node_id", "title", "status", "measurement_error"]), ...findingDigest(row, findingSources)})), // Full scoped summaries; progressive reads can raise the page size. // These retain the observation AND applicability; clipping away conditions // could turn a bounded refutation into a blanket route ban. writeback_results: visibleResults - .map(row => ({...compact(row, ["finding_id", "node_id", "finding", "status", "evidence_refs"]), + .map(row => ({...compact(row, ["finding_id", "node_id", "finding", "status", "evidence_refs", "measurement_error"]), summary: String(row.summary ?? "").slice(0, EXPLORE_WRITEBACK_SUMMARY_LIMIT), ...(row.execution_observation ? {execution_observation: row.execution_observation} : {}), ...(row.measurements ? {measurements: row.measurements} : {}), diff --git a/tests/control_plane/fixtures/explore_measurement_history_v0.jsonl b/tests/control_plane/fixtures/explore_measurement_history_v0.jsonl new file mode 100644 index 0000000000..ee4e60ab54 --- /dev/null +++ b/tests/control_plane/fixtures/explore_measurement_history_v0.jsonl @@ -0,0 +1,3 @@ +{"boundary": {"absolute_paths_recorded": false, "credential_values_recorded": false, "raw_logs_recorded": false, "raw_session_transcript_recorded": false, "raw_task_text_recorded": false, "raw_trajectory_recorded": false}, "event_id": "cae05d192b3df6b2", "event_kind": "node", "goal_id": "research", "node_kind": "question", "recorded_at": "2026-01-01T00:00:00Z", "result_id": "question-a", "schema_version": "loopx_explore_result_event_v0", "status": "open", "title": "Which candidate improved?"} +{"boundary": {"absolute_paths_recorded": false, "credential_values_recorded": false, "raw_logs_recorded": false, "raw_session_transcript_recorded": false, "raw_task_text_recorded": false, "raw_trajectory_recorded": false}, "event_id": "5bd7df96eed54e81", "event_kind": "finding", "evidence_refs": ["result:a", "result:b"], "execution_observation": {"candidate_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "change_reason": null, "elapsed_seconds": 1, "evidence_refs": ["result:a", "result:b"], "executed_method_id": "probe", "input_revision": "dataset-v1", "limit_seconds": 10, "planned_method_id": "probe", "result_candidate_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "result_ref": "result:a", "schema_version": "explore_execution_observation_v0", "search_iterations": null, "stop_reason": "process_exit:0"}, "goal_id": "research", "measurements": [{"candidate_ref": "candidate:a", "comparison_scope": "validation:budget-v1", "evaluation_ref": "evaluator:v1", "evidence_ref": "result:b", "higher_is_better": true, "input_revision": "dataset-v1", "metric": "quality", "unit": "points", "value": 8}], "node_id": "question-a", "recorded_at": "2026-01-01T01:00:00Z", "result_id": "measured", "schema_version": "loopx_explore_result_event_v0", "status": "confirmed", "title": "Measured candidate"} +{"boundary": {"absolute_paths_recorded": false, "credential_values_recorded": false, "raw_logs_recorded": false, "raw_session_transcript_recorded": false, "raw_task_text_recorded": false, "raw_trajectory_recorded": false}, "event_id": "56e6425cc30feee1", "event_kind": "finding", "evidence_refs": ["result:a", "result:b"], "execution_observation": {"candidate_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "change_reason": null, "elapsed_seconds": 1, "evidence_refs": ["result:a", "result:b"], "executed_method_id": "probe", "input_revision": "dataset-v1", "limit_seconds": 10, "planned_method_id": "probe", "result_candidate_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "result_ref": "result:a", "schema_version": "explore_execution_observation_v0", "search_iterations": null, "stop_reason": "process_exit:0"}, "goal_id": "research", "measurements": [{"candidate_ref": "candidate:a", "comparison_scope": "validation:budget-v1", "evaluation_ref": "evaluator:v1", "evidence_ref": "result:a", "higher_is_better": true, "input_revision": "dataset-v1", "metric": "quality", "unit": "points", "value": 8}], "node_id": "question-a", "recorded_at": "2026-01-01T02:00:00Z", "result_id": "measured", "schema_version": "loopx_explore_result_event_v0", "status": "confirmed", "title": "Measured candidate"} diff --git a/tests/control_plane/test_explore_measurement_history.py b/tests/control_plane/test_explore_measurement_history.py new file mode 100644 index 0000000000..21aadfc9e1 --- /dev/null +++ b/tests/control_plane/test_explore_measurement_history.py @@ -0,0 +1,89 @@ +"""Base-writer history remains auditable after stricter new-write attribution.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from loopx.capabilities.explore.result_log import ( + append_explore_result_event, append_explore_result_events, build_explore_finding_event, + build_explore_result_projection, explore_result_log_path, load_explore_result_events_strict, +) +from loopx.control_plane.effect_runtime import effect_runtime_result +from loopx.presentation.explore_results_api import _finding_rows, _history_rows + +# Written by the real builders/append API at main 21367015cce45294dcf99f54a3211d66f5d842c5. +# Synthetic protocol records only: a node, mixed A/B finding, then A/A correction. +FIXTURE = Path(__file__).with_name("fixtures") / "explore_measurement_history_v0.jsonl" + + +def test_base_history_and_corrections_remain_readable_without_promoting_mixed_measurements(tmp_path): + original = FIXTURE.read_bytes() + assert hashlib.sha256(original).hexdigest() == "5473f0ef178e732be19dcc5fd6cb7c56de964a080bbe1fd7844c1dd4cd66a940" + events = [json.loads(line) for line in original.splitlines()] + runtime = tmp_path / "runtime" + log = explore_result_log_path(runtime, "research") + log.parent.mkdir(parents=True) + prefix = b"\n".join(original.splitlines()[:2]) + b"\n" + log.write_bytes(prefix) + retained = load_explore_result_events_strict(log, goal_id="research") + assert retained == events[:2] + assert _history_rows(runtime, "research")[1]["measurements"][0]["evidence_ref"] == "result:b" + [current] = _finding_rows(retained, [], goal_id="research") + assert "measurements" not in current + assert "executed candidate's result" in current["measurement_error"] + projection = build_explore_result_projection(retained, goal_id="research") + request = dict(goal_id="research", agent_id="worker", route=["loopx"], graph_enabled=True, + harness_gate={"enabled": True}, projection=projection, plan={}) + turn = effect_runtime_result("explore.turn_context", request)["graph"] + assert not turn.get("measurement_feedback") + assert turn["writeback_results"][0]["measurement_error"] == current["measurement_error"] + # New writes cannot bypass the builder by supplying a legacy canonical row. + for append in ( + lambda: append_explore_result_event(log, events[1]), + lambda: append_explore_result_events(log, [events[1]], expected_goal_id="research"), + ): + with pytest.raises(ValueError, match="executed candidate's result"): + append() + assert log.read_bytes() == prefix + corrected = events[2] + append_explore_result_event(log, corrected) + assert log.read_bytes() == original, "new correction appends without rewriting the old evidence" + assert load_explore_result_events_strict(log, goal_id="research") == events + history = _history_rows(runtime, "research") + assert len(history) == 3 and history[1]["measurements"][0]["evidence_ref"] == "result:b" + [current] = _finding_rows(events, [], goal_id="research") + assert "measurement_error" not in current + assert current["measurements"][0]["evidence_ref"] == "result:a" + request["projection"] = build_explore_result_projection(events, goal_id="research") + feedback = effect_runtime_result("explore.turn_context", request)["graph"]["measurement_feedback"] + assert feedback[0]["best"]["evidence_ref"] == "result:a" + assert feedback[0]["measured_count"] == 1 + # Correcting an already corrected finding remains an ordinary strict write. + fields = {key: corrected[key] for key in ("goal_id", "title", "node_id", "status", "evidence_refs", + "execution_observation", "measurements")} + again = build_explore_finding_event(**fields, finding_id="measured", recorded_at="2026-01-01T03:00:00Z") + append_explore_result_event(log, again) + assert len(load_explore_result_events_strict(log, goal_id="research")) == 4 + assert len(_history_rows(runtime, "research")) == 4 + assert log.read_bytes().startswith(original) + # Explicit archival prefix replay retains old facts and the correction; + # it grants neither execution nor quantitative validity to the old row. + replay = tmp_path / "replayed.jsonl" + restored = append_explore_result_events(replay, events, expected_goal_id="research", require_ordered_prefix=True) + assert restored["appended_event_count"] == 3 + assert replay.read_bytes() == original + repeated = append_explore_result_events(replay, events, expected_goal_id="research", require_ordered_prefix=True) + assert repeated["appended_event_count"] == 0 + assert repeated["reused_event_count"] == 3 + + +def test_historical_read_still_rejects_forged_canonical_records(tmp_path): + event = json.loads(FIXTURE.read_text().splitlines()[1]) + event["event_id"] = "forged" + log = tmp_path / "results.jsonl" + log.write_text(json.dumps(event) + "\n") + with pytest.raises(ValueError, match="canonical"): + load_explore_result_events_strict(log, goal_id="research") diff --git a/tests/control_plane_ts/explore_measurements.test.ts b/tests/control_plane_ts/explore_measurements.test.ts index 81588b40fd..b02b58f501 100644 --- a/tests/control_plane_ts/explore_measurements.test.ts +++ b/tests/control_plane_ts/explore_measurements.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; +import {readFileSync} from "node:fs"; import {normalizeExploreMeasurements, normalizeExploreMeasurementsRequest, projectExploreMeasurementFeedback} from "../../loopx/control_plane/capabilities/explore_measurements.ts"; import {normalizeExploreResultAttachment} from "../../loopx/control_plane/capabilities/explore_result_writeback.ts"; import {projectExploreTurnContext} from "../../loopx/control_plane/capabilities/explore_turn_context.ts"; @@ -260,3 +261,20 @@ test("measurement feedback pages retain every group and bind manual findings to assert.throws(() => projectExploreTurnContext({...corrected, result_offset: 3, result_revision: snapshot.result_page.revision}), /changed/); }); + + +test("base-produced mixed history remains auditable but only the correction enters comparison", () => { + const events = readFileSync(new URL("../control_plane/fixtures/explore_measurement_history_v0.jsonl", import.meta.url), "utf8") + .trim().split("\n").map(line => JSON.parse(line)); + const before = parseExploreHistoryPage({ok: true, goal_id: "research", items: events.slice(0, 2), total: 2, next_cursor: null}, "research"); + const historical = projectExploreHistory(before.items, "research"); + assert.equal(historical.events[1].measurements?.[0].evidence_ref, "result:b"); + assert.equal(historical.measurementFeedback, undefined, "the contradiction is not a best, latest, unknown or scored point"); + const corrected = parseExploreHistoryPage({ok: true, goal_id: "research", items: events, total: 3, next_cursor: null}, "research"); + const current = projectExploreHistory(corrected.items, "research"); + assert.equal(current.events.length, 3); + assert.equal(current.measurementFeedback?.[0].best?.evidence_ref, "result:a"); + assert.equal(current.measurementFeedback?.[0].measured_count, 1); + const raw = {...finding("legacy", 99), execution_observation: events[1].execution_observation}; + assert.deepEqual(projectExploreMeasurementFeedback([raw]), []); +});