From 1dbb6103518cd25918f0fc9af2aa146181157a1a Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sun, 1 Nov 2026 22:39:41 +0800 Subject: [PATCH 1/2] fix(benchmark): advance shared EdgeBench source Signed-off-by: huangruiteng --- benchmark/edgebench/README.md | 34 +++++++++--------------- benchmark/edgebench/source-default.json | 7 +---- benchmark/edgebench/source.py | 6 ++++- benchmark/tests/test_edgebench_source.py | 26 +++++++++--------- 4 files changed, 33 insertions(+), 40 deletions(-) diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index 8086b67f9c..a1bd85ea4c 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -7,7 +7,7 @@ blind retain native automatic evaluation. The CLI defaults to a reviewed commit from the shared [EdgeBench integration branch](https://github.com/huangruiteng/EdgeBench/tree/codex/edgebench-integration), maintained jointly by `huangruiteng` and `loopx-agent`. The canonical repository -and revision/compatibility-patch pair live in [`source-default.json`](source-default.json). +and reviewed revision live in [`source-default.json`](source-default.json). Run the controller on Linux with Docker and native SForge iptables permissions; a macOS Docker socket alone does not provide that boundary. @@ -24,25 +24,17 @@ never repin an active or historical cohort. For a **new cohort with no explicit source options**, `source-default.json` selects the reviewed integration commit -[`e278cda3f4d2da3cf06aabf2df4daa99b9749501`](https://github.com/huangruiteng/EdgeBench/commit/e278cda3f4d2da3cf06aabf2df4daa99b9749501) -and the versioned -[`nonfinite-checkpoint-v2` compatibility patch](compatibility/nonfinite-checkpoint-v2.patch). -This LoopX-owned patch fixes the native checkpoint representation of nonfinite -released scores and metrics. Native scoring, report values and selection remain -unchanged. Preparation verifies the asset digest and each touched file's exact -upstream blob and mode, applies the patch before provider import/installation, -and creates a clean, reproducible Git commit with the fetched commit as its -parent. The snapshot uses an explicit LoopX compatibility identity and the -upstream commit time; it does not attribute the patch to an upstream author. -The default revision and patch are one reviewed pair: advancing the integration -branch does not change a new default cohort. Updating the pair requires reviewing -the new source and qualifying the native checkpoint transport again. A changed -patch preimage, including a later upstream implementation of the fix, still stops -preparation without publishing a source directory. Explicit branch/revision -selection remains available and uses unmodified provider code; it does not -silently inherit the default patch or its qualification. - -Patched sources use `edgebench_source_v2` receipts. `upstream_commit` is the +[`2e29d24f44fece91dc240fc5459d17cad19c5fcd`](https://github.com/huangruiteng/EdgeBench/commit/2e29d24f44fece91dc240fc5459d17cad19c5fcd). +It includes the reviewed non-finite feedback serialization and serialized game +session lifecycle fixes in the shared branch itself, so the default no longer +needs a LoopX compatibility patch. Native scoring, report values and selection +remain provider-owned. Advancing the integration branch still does not change a +cohort: updating this pin requires reviewing and qualifying the new source. +Explicit branch/revision selection remains available and uses the selected +unmodified provider code. + +When a future default needs a compatibility patch, patched sources use +`edgebench_source_v2` receipts. `upstream_commit` is the fetched remote revision; `compatibility_patch` records the patch ID and SHA-256; `commit` is the **effective local derived revision** actually imported and bound by native checkpoint provenance. This derived SHA is not an upstream revision @@ -54,7 +46,7 @@ cohort. Preserve the complete `.edgebench-source` directory for offline reuse or exact patched-source replay. Older `edgebench_source_v1` cohorts retain their original unmodified source and receipt; preparation never upgrades them. -The bundled patch uses one-line unified context and Git's +Bundled patches use one-line unified context and Git's `diff.suppressBlankEmpty=true` representation. This keeps its full-index file identities and changed content readable while avoiding unrelated example text and trailing-space context markers in the containing repository. Validation diff --git a/benchmark/edgebench/source-default.json b/benchmark/edgebench/source-default.json index 8aee0f341a..1b39b7a25e 100644 --- a/benchmark/edgebench/source-default.json +++ b/benchmark/edgebench/source-default.json @@ -1,9 +1,4 @@ { "repository": "https://github.com/huangruiteng/EdgeBench.git", - "ref": "e278cda3f4d2da3cf06aabf2df4daa99b9749501", - "compatibility_patch": { - "id": "nonfinite-checkpoint-v2", - "relative_path": "compatibility/nonfinite-checkpoint-v2.patch", - "sha256": "02d21096a2e55f625d3c8beee2335728caf08aa6c75a5a27b1ab412bc46323bf" - } + "ref": "2e29d24f44fece91dc240fc5459d17cad19c5fcd" } diff --git a/benchmark/edgebench/source.py b/benchmark/edgebench/source.py index 177080fe36..66180015d7 100644 --- a/benchmark/edgebench/source.py +++ b/benchmark/edgebench/source.py @@ -175,7 +175,11 @@ def _validate(state: Path) -> dict: raise ValueError("Invalid EdgeBench source receipt; preserve it and use a new log directory") checkout = state / "checkout" if (_git(checkout, "rev-parse", "HEAD") != receipt["commit"] - or _git(checkout, "remote", "get-url", "origin") != receipt["repository"] + # `git remote get-url` applies url.*.insteadOf transport rewrites. + # The receipt owns the canonical URL written into this checkout, + # while a developer may legitimately fetch it through SSH. + or _git(checkout, "config", "--local", "--get", "remote.origin.url") + != receipt["repository"] or _git(checkout, "status", "--porcelain", "--untracked-files=normal")): raise ValueError("Frozen EdgeBench source changed; restore it or use a new log directory") if not (checkout / "sforge/__init__.py").is_file(): diff --git a/benchmark/tests/test_edgebench_source.py b/benchmark/tests/test_edgebench_source.py index d2400f7166..5fc1180893 100644 --- a/benchmark/tests/test_edgebench_source.py +++ b/benchmark/tests/test_edgebench_source.py @@ -67,6 +67,15 @@ def offline(checkout, *args): assert source.prepare_source(tmp_path / "logs")[1] == expected +def test_transport_rewrite_does_not_change_canonical_source_identity(repository, tmp_path): + checkout, receipt = source.prepare_source(tmp_path / "logs") + canonical = receipt["repository"] + git(checkout, "config", "url.ssh://git@example.invalid/source.insteadOf", canonical) + assert git(checkout, "remote", "get-url", "origin") != canonical + assert git(checkout, "config", "--local", "--get", "remote.origin.url") == canonical + assert source.prepare_source(tmp_path / "logs")[1] == receipt + + def test_parallel_worker_and_judge_share_one_source(repository, tmp_path): with ThreadPoolExecutor(max_workers=4) as pool: results = list(pool.map(lambda _: source.prepare_source(tmp_path / "logs")[1], range(4))) @@ -207,20 +216,13 @@ def install_test_patch(repository, other_changes=None, *, compact=True): return asset -def test_bundled_patch_passes_required_public_checks(tmp_path): - from loopx.contract import scan_public_boundary - +def test_production_default_is_a_reviewed_immutable_source(): defaults = json.loads(source.DEFAULT_FILE.read_text()) - # The production bundle is qualified with an immutable upstream revision. - # A moving branch can invalidate its full-index preimages between cohorts. assert re.fullmatch(r"[0-9a-f]{40}", defaults["ref"]) - _, patch = source._bundled_patch(defaults["compatibility_patch"]) - asset = tmp_path / "compatibility.patch" - asset.write_bytes(patch) - git(tmp_path, "init", "-q") - git(tmp_path, "add", "--", asset.name) - assert git(tmp_path, "diff", "--cached", "--check") == "" - assert scan_public_boundary([asset], registry={})["ok"] + assert defaults == { + "repository": "https://github.com/huangruiteng/EdgeBench.git", + "ref": "2e29d24f44fece91dc240fc5459d17cad19c5fcd", + } def test_pinned_default_and_patch_advance_only_as_a_reviewed_pair(repository, tmp_path): From a01d7ba9f01f3eda3f2c88d740ba5eff7430a19d Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Mon, 2 Nov 2026 01:08:14 +0800 Subject: [PATCH 2/2] fix(benchmark): retain qualified EdgeBench source contracts Signed-off-by: huangruiteng --- benchmark/edgebench/README.md | 38 ++++++++----- benchmark/edgebench/source-default.json | 7 ++- benchmark/tests/test_checkpoint_judge.py | 71 ++++++++++++++++++++++++ benchmark/tests/test_edgebench_source.py | 7 ++- 4 files changed, 107 insertions(+), 16 deletions(-) diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index a1bd85ea4c..551032bacc 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -7,7 +7,7 @@ blind retain native automatic evaluation. The CLI defaults to a reviewed commit from the shared [EdgeBench integration branch](https://github.com/huangruiteng/EdgeBench/tree/codex/edgebench-integration), maintained jointly by `huangruiteng` and `loopx-agent`. The canonical repository -and reviewed revision live in [`source-default.json`](source-default.json). +and revision/compatibility-patch pair live in [`source-default.json`](source-default.json). Run the controller on Linux with Docker and native SForge iptables permissions; a macOS Docker socket alone does not provide that boundary. @@ -23,18 +23,28 @@ before model/container entry. Use a new log directory to adopt a newer source; never repin an active or historical cohort. For a **new cohort with no explicit source options**, `source-default.json` -selects the reviewed integration commit -[`2e29d24f44fece91dc240fc5459d17cad19c5fcd`](https://github.com/huangruiteng/EdgeBench/commit/2e29d24f44fece91dc240fc5459d17cad19c5fcd). -It includes the reviewed non-finite feedback serialization and serialized game -session lifecycle fixes in the shared branch itself, so the default no longer -needs a LoopX compatibility patch. Native scoring, report values and selection -remain provider-owned. Advancing the integration branch still does not change a -cohort: updating this pin requires reviewing and qualifying the new source. -Explicit branch/revision selection remains available and uses the selected -unmodified provider code. - -When a future default needs a compatibility patch, patched sources use -`edgebench_source_v2` receipts. `upstream_commit` is the +selects the reviewed provider commit +[`118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2`](https://github.com/huangruiteng/EdgeBench/commit/118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2) +and the versioned +[`nonfinite-checkpoint-v2` compatibility patch](compatibility/nonfinite-checkpoint-v2.patch). +This LoopX-owned patch fixes the native checkpoint representation of nonfinite +released scores and metrics. Native scoring, report values and selection remain +unchanged. The upstream revision fixes ordinary HTTP feedback serialization; +checkpoint encoding is a separate boundary and still requires this patch. +Preparation verifies the asset digest and each touched file's exact +upstream blob and mode, applies the patch before provider import/installation, +and creates a clean, reproducible Git commit with the fetched commit as its +parent. The snapshot uses an explicit LoopX compatibility identity and the +upstream commit time; it does not attribute the patch to an upstream author. +The default revision and patch are one reviewed pair: advancing the integration +branch does not change a new default cohort. Updating the pair requires reviewing +the new source and qualifying the native checkpoint transport again. A changed +patch preimage, including a later upstream implementation of the fix, still stops +preparation without publishing a source directory. Explicit branch/revision +selection remains available and uses unmodified provider code; it does not +silently inherit the default patch or its qualification. + +Patched sources use `edgebench_source_v2` receipts. `upstream_commit` is the fetched remote revision; `compatibility_patch` records the patch ID and SHA-256; `commit` is the **effective local derived revision** actually imported and bound by native checkpoint provenance. This derived SHA is not an upstream revision @@ -46,7 +56,7 @@ cohort. Preserve the complete `.edgebench-source` directory for offline reuse or exact patched-source replay. Older `edgebench_source_v1` cohorts retain their original unmodified source and receipt; preparation never upgrades them. -Bundled patches use one-line unified context and Git's +The bundled patch uses one-line unified context and Git's `diff.suppressBlankEmpty=true` representation. This keeps its full-index file identities and changed content readable while avoiding unrelated example text and trailing-space context markers in the containing repository. Validation diff --git a/benchmark/edgebench/source-default.json b/benchmark/edgebench/source-default.json index 1b39b7a25e..793b91f255 100644 --- a/benchmark/edgebench/source-default.json +++ b/benchmark/edgebench/source-default.json @@ -1,4 +1,9 @@ { "repository": "https://github.com/huangruiteng/EdgeBench.git", - "ref": "2e29d24f44fece91dc240fc5459d17cad19c5fcd" + "ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2", + "compatibility_patch": { + "id": "nonfinite-checkpoint-v2", + "relative_path": "compatibility/nonfinite-checkpoint-v2.patch", + "sha256": "02d21096a2e55f625d3c8beee2335728caf08aa6c75a5a27b1ab412bc46323bf" + } } diff --git a/benchmark/tests/test_checkpoint_judge.py b/benchmark/tests/test_checkpoint_judge.py index 3025d6f1ca..7fcb0edae4 100644 --- a/benchmark/tests/test_checkpoint_judge.py +++ b/benchmark/tests/test_checkpoint_judge.py @@ -39,6 +39,77 @@ def test_disabled_extension_preserves_the_native_route_surface(native_config): second.state.judge.close() +@pytest.mark.parametrize("score,wire_score", [ + (float("inf"), "Infinity"), (float("-inf"), "-Infinity"), + (float("nan"), "NaN"), (0.0, 0.0), (None, None), +]) +def test_ordinary_feedback_keeps_failure_zero_and_missing_scores_distinct( + native_config, monkeypatch, score, wire_score): + import requests + from benchmark.edgebench.checkpoint_judge import create_app + from benchmark.edgebench.source import active_source_receipt + from benchmark.tests.test_checkpoint_controller import server + from sforge.harness.feedback_transport import JOURNAL_FILE, payload_sha256 + + active_source_receipt() + # Exercise ordinary solver feedback, not the checkpoint-only JSON codec. + # A broken proxy cannot explain an in-process Judge serialization failure. + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:1") + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:1") + app = create_app(native_config) + state = app.state.judge + try: + with server(app) as url, requests.Session() as client: + client.trust_env = False + token = state.register_session( + "fixture", "source", max_agent_submissions=3, submission_cooldown=3600) + state.consume_round(token) + report = {"score": score, "pass_rate": .5, "valid": False, + "metrics": {"nested": [score, 0.0, None]}} + state._record_submission("source", "accepted", "fixture", "agent-1", report, None) + state.submissions["accepted"] = { + "run_id": "source", "task_id": "fixture", "round": "agent-1", + "status": "completed", "report": report, "error": None} + before = state.resolve_token(token) + original = json.dumps(report, sort_keys=True) + relay = before["feedback_relay_id"] + delivered = [] + for operation in ("result/accepted", "history"): + host = client.get(url + "/api/v1/" + operation, + params={"token": token}, timeout=5) + response = client.get(url + "/api/v1/feedback/" + relay + "/" + operation, + timeout=5) + assert host.status_code == response.status_code == 200 + payload = response.json() + json.dumps(payload, allow_nan=False) + delivered.append(payload) + if operation.startswith("result/"): + assert host.json() == payload + assert payload["report"]["score"] == wire_score + assert payload["report"]["metrics"]["nested"] == [wire_score, 0.0, None] + else: + assert host.json()["entries"] == payload["entries"] + assert host.json()["best_score"] == payload["best_score"] + assert payload["entries"][0]["score"] == wire_score + for history in (host.json(), payload): + availability = history["agent_submission_availability"] + assert availability["remaining_submissions"] == 2 + assert 0 < availability["cooldown_remaining_seconds"] <= 3600 + + # Reading feedback must not consume another evaluation or rewrite + # the native numeric state used by ranking and checkpoint capture. + assert state.resolve_token(token) == before + assert json.dumps(state.submissions["accepted"]["report"], sort_keys=True) == original + records = [json.loads(line) for line in ( + native_config.log_dir / "runs/source/fixture" / JOURNAL_FILE + ).read_text().splitlines()] + responses = [row for row in records if row["record_kind"] == "response"] + assert [row["payload_sha256"] for row in responses] == [ + payload_sha256(payload) for payload in delivered] + finally: + state.close() + + def test_capture_retains_consumed_round_history_and_cooldown_without_mutation(native_config): from fastapi.testclient import TestClient from benchmark.edgebench.checkpoint_judge import create_app, ROUTE diff --git a/benchmark/tests/test_edgebench_source.py b/benchmark/tests/test_edgebench_source.py index 5fc1180893..679fe4c4c6 100644 --- a/benchmark/tests/test_edgebench_source.py +++ b/benchmark/tests/test_edgebench_source.py @@ -221,7 +221,12 @@ def test_production_default_is_a_reviewed_immutable_source(): assert re.fullmatch(r"[0-9a-f]{40}", defaults["ref"]) assert defaults == { "repository": "https://github.com/huangruiteng/EdgeBench.git", - "ref": "2e29d24f44fece91dc240fc5459d17cad19c5fcd", + "ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2", + "compatibility_patch": { + "id": "nonfinite-checkpoint-v2", + "relative_path": "compatibility/nonfinite-checkpoint-v2.patch", + "sha256": "02d21096a2e55f625d3c8beee2335728caf08aa6c75a5a27b1ab412bc46323bf", + }, }