Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions benchmark/edgebench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ before model/container entry. Use a new log directory to adopt a newer source;
never repin an active or historical cohort.

For a **new cohort with no explicit source options**, `source-default.json`
selects the reviewed integration commit
[`e278cda3f4d2da3cf06aabf2df4daa99b9749501`](https://github.com/huangruiteng/EdgeBench/commit/e278cda3f4d2da3cf06aabf2df4daa99b9749501)
selects the reviewed provider commit
[`118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2`](https://github.com/huangruiteng/EdgeBench/commit/118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2)
and the versioned
[`nonfinite-checkpoint-v2` compatibility patch](compatibility/nonfinite-checkpoint-v2.patch).
This LoopX-owned patch fixes the native checkpoint representation of nonfinite
released scores and metrics. Native scoring, report values and selection remain
unchanged. Preparation verifies the asset digest and each touched file's exact
unchanged. The upstream revision fixes ordinary HTTP feedback serialization;
checkpoint encoding is a separate boundary and still requires this patch.
Preparation verifies the asset digest and each touched file's exact
upstream blob and mode, applies the patch before provider import/installation,
and creates a clean, reproducible Git commit with the fetched commit as its
parent. The snapshot uses an explicit LoopX compatibility identity and the
Expand Down
2 changes: 1 addition & 1 deletion benchmark/edgebench/source-default.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"repository": "https://github.com/huangruiteng/EdgeBench.git",
"ref": "e278cda3f4d2da3cf06aabf2df4daa99b9749501",
"ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2",
"compatibility_patch": {
"id": "nonfinite-checkpoint-v2",
"relative_path": "compatibility/nonfinite-checkpoint-v2.patch",
Expand Down
6 changes: 5 additions & 1 deletion benchmark/edgebench/source.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,11 @@ def _validate(state: Path) -> dict:
raise ValueError("Invalid EdgeBench source receipt; preserve it and use a new log directory")
checkout = state / "checkout"
if (_git(checkout, "rev-parse", "HEAD") != receipt["commit"]
or _git(checkout, "remote", "get-url", "origin") != receipt["repository"]
# `git remote get-url` applies url.*.insteadOf transport rewrites.
# The receipt owns the canonical URL written into this checkout,
# while a developer may legitimately fetch it through SSH.
or _git(checkout, "config", "--local", "--get", "remote.origin.url")
!= receipt["repository"]
or _git(checkout, "status", "--porcelain", "--untracked-files=normal")):
raise ValueError("Frozen EdgeBench source changed; restore it or use a new log directory")
if not (checkout / "sforge/__init__.py").is_file():
Expand Down
71 changes: 71 additions & 0 deletions benchmark/tests/test_checkpoint_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,77 @@ def test_disabled_extension_preserves_the_native_route_surface(native_config):
second.state.judge.close()


@pytest.mark.parametrize("score,wire_score", [
(float("inf"), "Infinity"), (float("-inf"), "-Infinity"),
(float("nan"), "NaN"), (0.0, 0.0), (None, None),
])
def test_ordinary_feedback_keeps_failure_zero_and_missing_scores_distinct(
native_config, monkeypatch, score, wire_score):
import requests
from benchmark.edgebench.checkpoint_judge import create_app
from benchmark.edgebench.source import active_source_receipt
from benchmark.tests.test_checkpoint_controller import server
from sforge.harness.feedback_transport import JOURNAL_FILE, payload_sha256

active_source_receipt()
# Exercise ordinary solver feedback, not the checkpoint-only JSON codec.
# A broken proxy cannot explain an in-process Judge serialization failure.
monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:1")
monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:1")
app = create_app(native_config)
state = app.state.judge
try:
with server(app) as url, requests.Session() as client:
client.trust_env = False
token = state.register_session(
"fixture", "source", max_agent_submissions=3, submission_cooldown=3600)
state.consume_round(token)
report = {"score": score, "pass_rate": .5, "valid": False,
"metrics": {"nested": [score, 0.0, None]}}
state._record_submission("source", "accepted", "fixture", "agent-1", report, None)
state.submissions["accepted"] = {
"run_id": "source", "task_id": "fixture", "round": "agent-1",
"status": "completed", "report": report, "error": None}
before = state.resolve_token(token)
original = json.dumps(report, sort_keys=True)
relay = before["feedback_relay_id"]
delivered = []
for operation in ("result/accepted", "history"):
host = client.get(url + "/api/v1/" + operation,
params={"token": token}, timeout=5)
response = client.get(url + "/api/v1/feedback/" + relay + "/" + operation,
timeout=5)
assert host.status_code == response.status_code == 200
payload = response.json()
json.dumps(payload, allow_nan=False)
delivered.append(payload)
if operation.startswith("result/"):
assert host.json() == payload
assert payload["report"]["score"] == wire_score
assert payload["report"]["metrics"]["nested"] == [wire_score, 0.0, None]
else:
assert host.json()["entries"] == payload["entries"]
assert host.json()["best_score"] == payload["best_score"]
assert payload["entries"][0]["score"] == wire_score
for history in (host.json(), payload):
availability = history["agent_submission_availability"]
assert availability["remaining_submissions"] == 2
assert 0 < availability["cooldown_remaining_seconds"] <= 3600

# Reading feedback must not consume another evaluation or rewrite
# the native numeric state used by ranking and checkpoint capture.
assert state.resolve_token(token) == before
assert json.dumps(state.submissions["accepted"]["report"], sort_keys=True) == original
records = [json.loads(line) for line in (
native_config.log_dir / "runs/source/fixture" / JOURNAL_FILE
).read_text().splitlines()]
responses = [row for row in records if row["record_kind"] == "response"]
assert [row["payload_sha256"] for row in responses] == [
payload_sha256(payload) for payload in delivered]
finally:
state.close()


def test_capture_retains_consumed_round_history_and_cooldown_without_mutation(native_config):
from fastapi.testclient import TestClient
from benchmark.edgebench.checkpoint_judge import create_app, ROUTE
Expand Down
31 changes: 19 additions & 12 deletions benchmark/tests/test_edgebench_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,15 @@ def offline(checkout, *args):
assert source.prepare_source(tmp_path / "logs")[1] == expected


def test_transport_rewrite_does_not_change_canonical_source_identity(repository, tmp_path):
checkout, receipt = source.prepare_source(tmp_path / "logs")
canonical = receipt["repository"]
git(checkout, "config", "url.ssh://git@example.invalid/source.insteadOf", canonical)
assert git(checkout, "remote", "get-url", "origin") != canonical
assert git(checkout, "config", "--local", "--get", "remote.origin.url") == canonical
assert source.prepare_source(tmp_path / "logs")[1] == receipt


def test_parallel_worker_and_judge_share_one_source(repository, tmp_path):
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(pool.map(lambda _: source.prepare_source(tmp_path / "logs")[1], range(4)))
Expand Down Expand Up @@ -207,20 +216,18 @@ def install_test_patch(repository, other_changes=None, *, compact=True):
return asset


def test_bundled_patch_passes_required_public_checks(tmp_path):
from loopx.contract import scan_public_boundary

def test_production_default_is_a_reviewed_immutable_source():
defaults = json.loads(source.DEFAULT_FILE.read_text())
# The production bundle is qualified with an immutable upstream revision.
# A moving branch can invalidate its full-index preimages between cohorts.
assert re.fullmatch(r"[0-9a-f]{40}", defaults["ref"])
_, patch = source._bundled_patch(defaults["compatibility_patch"])
asset = tmp_path / "compatibility.patch"
asset.write_bytes(patch)
git(tmp_path, "init", "-q")
git(tmp_path, "add", "--", asset.name)
assert git(tmp_path, "diff", "--cached", "--check") == ""
assert scan_public_boundary([asset], registry={})["ok"]
assert defaults == {
"repository": "https://github.com/huangruiteng/EdgeBench.git",
"ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2",
"compatibility_patch": {
"id": "nonfinite-checkpoint-v2",
"relative_path": "compatibility/nonfinite-checkpoint-v2.patch",
"sha256": "02d21096a2e55f625d3c8beee2335728caf08aa6c75a5a27b1ab412bc46323bf",
},
}


def test_pinned_default_and_patch_advance_only_as_a_reviewed_pair(repository, tmp_path):
Expand Down
Loading