diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index 8086b67f9c..551032bacc 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -23,13 +23,15 @@ before model/container entry. Use a new log directory to adopt a newer source; never repin an active or historical cohort. For a **new cohort with no explicit source options**, `source-default.json` -selects the reviewed integration commit -[`e278cda3f4d2da3cf06aabf2df4daa99b9749501`](https://github.com/huangruiteng/EdgeBench/commit/e278cda3f4d2da3cf06aabf2df4daa99b9749501) +selects the reviewed provider commit +[`118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2`](https://github.com/huangruiteng/EdgeBench/commit/118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2) and the versioned [`nonfinite-checkpoint-v2` compatibility patch](compatibility/nonfinite-checkpoint-v2.patch). This LoopX-owned patch fixes the native checkpoint representation of nonfinite released scores and metrics. Native scoring, report values and selection remain -unchanged. Preparation verifies the asset digest and each touched file's exact +unchanged. The upstream revision fixes ordinary HTTP feedback serialization; +checkpoint encoding is a separate boundary and still requires this patch. +Preparation verifies the asset digest and each touched file's exact upstream blob and mode, applies the patch before provider import/installation, and creates a clean, reproducible Git commit with the fetched commit as its parent. The snapshot uses an explicit LoopX compatibility identity and the diff --git a/benchmark/edgebench/source-default.json b/benchmark/edgebench/source-default.json index 8aee0f341a..793b91f255 100644 --- a/benchmark/edgebench/source-default.json +++ b/benchmark/edgebench/source-default.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/huangruiteng/EdgeBench.git", - "ref": "e278cda3f4d2da3cf06aabf2df4daa99b9749501", + "ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2", "compatibility_patch": { "id": "nonfinite-checkpoint-v2", "relative_path": "compatibility/nonfinite-checkpoint-v2.patch", diff --git a/benchmark/edgebench/source.py b/benchmark/edgebench/source.py index 177080fe36..66180015d7 100644 --- a/benchmark/edgebench/source.py +++ b/benchmark/edgebench/source.py @@ -175,7 +175,11 @@ def _validate(state: Path) -> dict: raise ValueError("Invalid EdgeBench source receipt; preserve it and use a new log directory") checkout = state / "checkout" if (_git(checkout, "rev-parse", "HEAD") != receipt["commit"] - or _git(checkout, "remote", "get-url", "origin") != receipt["repository"] + # `git remote get-url` applies url.*.insteadOf transport rewrites. + # The receipt owns the canonical URL written into this checkout, + # while a developer may legitimately fetch it through SSH. + or _git(checkout, "config", "--local", "--get", "remote.origin.url") + != receipt["repository"] or _git(checkout, "status", "--porcelain", "--untracked-files=normal")): raise ValueError("Frozen EdgeBench source changed; restore it or use a new log directory") if not (checkout / "sforge/__init__.py").is_file(): diff --git a/benchmark/tests/test_checkpoint_judge.py b/benchmark/tests/test_checkpoint_judge.py index 3025d6f1ca..7fcb0edae4 100644 --- a/benchmark/tests/test_checkpoint_judge.py +++ b/benchmark/tests/test_checkpoint_judge.py @@ -39,6 +39,77 @@ def test_disabled_extension_preserves_the_native_route_surface(native_config): second.state.judge.close() +@pytest.mark.parametrize("score,wire_score", [ + (float("inf"), "Infinity"), (float("-inf"), "-Infinity"), + (float("nan"), "NaN"), (0.0, 0.0), (None, None), +]) +def test_ordinary_feedback_keeps_failure_zero_and_missing_scores_distinct( + native_config, monkeypatch, score, wire_score): + import requests + from benchmark.edgebench.checkpoint_judge import create_app + from benchmark.edgebench.source import active_source_receipt + from benchmark.tests.test_checkpoint_controller import server + from sforge.harness.feedback_transport import JOURNAL_FILE, payload_sha256 + + active_source_receipt() + # Exercise ordinary solver feedback, not the checkpoint-only JSON codec. + # A broken proxy cannot explain an in-process Judge serialization failure. + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:1") + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:1") + app = create_app(native_config) + state = app.state.judge + try: + with server(app) as url, requests.Session() as client: + client.trust_env = False + token = state.register_session( + "fixture", "source", max_agent_submissions=3, submission_cooldown=3600) + state.consume_round(token) + report = {"score": score, "pass_rate": .5, "valid": False, + "metrics": {"nested": [score, 0.0, None]}} + state._record_submission("source", "accepted", "fixture", "agent-1", report, None) + state.submissions["accepted"] = { + "run_id": "source", "task_id": "fixture", "round": "agent-1", + "status": "completed", "report": report, "error": None} + before = state.resolve_token(token) + original = json.dumps(report, sort_keys=True) + relay = before["feedback_relay_id"] + delivered = [] + for operation in ("result/accepted", "history"): + host = client.get(url + "/api/v1/" + operation, + params={"token": token}, timeout=5) + response = client.get(url + "/api/v1/feedback/" + relay + "/" + operation, + timeout=5) + assert host.status_code == response.status_code == 200 + payload = response.json() + json.dumps(payload, allow_nan=False) + delivered.append(payload) + if operation.startswith("result/"): + assert host.json() == payload + assert payload["report"]["score"] == wire_score + assert payload["report"]["metrics"]["nested"] == [wire_score, 0.0, None] + else: + assert host.json()["entries"] == payload["entries"] + assert host.json()["best_score"] == payload["best_score"] + assert payload["entries"][0]["score"] == wire_score + for history in (host.json(), payload): + availability = history["agent_submission_availability"] + assert availability["remaining_submissions"] == 2 + assert 0 < availability["cooldown_remaining_seconds"] <= 3600 + + # Reading feedback must not consume another evaluation or rewrite + # the native numeric state used by ranking and checkpoint capture. + assert state.resolve_token(token) == before + assert json.dumps(state.submissions["accepted"]["report"], sort_keys=True) == original + records = [json.loads(line) for line in ( + native_config.log_dir / "runs/source/fixture" / JOURNAL_FILE + ).read_text().splitlines()] + responses = [row for row in records if row["record_kind"] == "response"] + assert [row["payload_sha256"] for row in responses] == [ + payload_sha256(payload) for payload in delivered] + finally: + state.close() + + def test_capture_retains_consumed_round_history_and_cooldown_without_mutation(native_config): from fastapi.testclient import TestClient from benchmark.edgebench.checkpoint_judge import create_app, ROUTE diff --git a/benchmark/tests/test_edgebench_source.py b/benchmark/tests/test_edgebench_source.py index d2400f7166..679fe4c4c6 100644 --- a/benchmark/tests/test_edgebench_source.py +++ b/benchmark/tests/test_edgebench_source.py @@ -67,6 +67,15 @@ def offline(checkout, *args): assert source.prepare_source(tmp_path / "logs")[1] == expected +def test_transport_rewrite_does_not_change_canonical_source_identity(repository, tmp_path): + checkout, receipt = source.prepare_source(tmp_path / "logs") + canonical = receipt["repository"] + git(checkout, "config", "url.ssh://git@example.invalid/source.insteadOf", canonical) + assert git(checkout, "remote", "get-url", "origin") != canonical + assert git(checkout, "config", "--local", "--get", "remote.origin.url") == canonical + assert source.prepare_source(tmp_path / "logs")[1] == receipt + + def test_parallel_worker_and_judge_share_one_source(repository, tmp_path): with ThreadPoolExecutor(max_workers=4) as pool: results = list(pool.map(lambda _: source.prepare_source(tmp_path / "logs")[1], range(4))) @@ -207,20 +216,18 @@ def install_test_patch(repository, other_changes=None, *, compact=True): return asset -def test_bundled_patch_passes_required_public_checks(tmp_path): - from loopx.contract import scan_public_boundary - +def test_production_default_is_a_reviewed_immutable_source(): defaults = json.loads(source.DEFAULT_FILE.read_text()) - # The production bundle is qualified with an immutable upstream revision. - # A moving branch can invalidate its full-index preimages between cohorts. assert re.fullmatch(r"[0-9a-f]{40}", defaults["ref"]) - _, patch = source._bundled_patch(defaults["compatibility_patch"]) - asset = tmp_path / "compatibility.patch" - asset.write_bytes(patch) - git(tmp_path, "init", "-q") - git(tmp_path, "add", "--", asset.name) - assert git(tmp_path, "diff", "--cached", "--check") == "" - assert scan_public_boundary([asset], registry={})["ok"] + assert defaults == { + "repository": "https://github.com/huangruiteng/EdgeBench.git", + "ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2", + "compatibility_patch": { + "id": "nonfinite-checkpoint-v2", + "relative_path": "compatibility/nonfinite-checkpoint-v2.patch", + "sha256": "02d21096a2e55f625d3c8beee2335728caf08aa6c75a5a27b1ab412bc46323bf", + }, + } def test_pinned_default_and_patch_advance_only_as_a_reviewed_pair(repository, tmp_path):