Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions benchmark/edgebench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ before model/container entry. Use a new log directory to adopt a newer source;
never repin an active or historical cohort.

For a **new cohort with no explicit source options**, `source-default.json`
selects the reviewed integration commit
[`e278cda3f4d2da3cf06aabf2df4daa99b9749501`](https://github.com/huangruiteng/EdgeBench/commit/e278cda3f4d2da3cf06aabf2df4daa99b9749501)
selects the reviewed provider commit
[`118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2`](https://github.com/huangruiteng/EdgeBench/commit/118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2)
and the versioned
[`nonfinite-checkpoint-v2` compatibility patch](compatibility/nonfinite-checkpoint-v2.patch).
This LoopX-owned patch fixes the native checkpoint representation of nonfinite
released scores and metrics. Native scoring, report values and selection remain
unchanged. Preparation verifies the asset digest and each touched file's exact
unchanged. The upstream revision fixes ordinary HTTP feedback serialization;
checkpoint encoding is a separate boundary and still requires this patch.
Preparation verifies the asset digest and each touched file's exact
upstream blob and mode, applies the patch before provider import/installation,
and creates a clean, reproducible Git commit with the fetched commit as its
parent. The snapshot uses an explicit LoopX compatibility identity and the
Expand Down
2 changes: 1 addition & 1 deletion benchmark/edgebench/source-default.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"repository": "https://github.com/huangruiteng/EdgeBench.git",
"ref": "e278cda3f4d2da3cf06aabf2df4daa99b9749501",
"ref": "118f9257d39c71d0805bb5b9cd4eb85d13a1e6d2",
"compatibility_patch": {
"id": "nonfinite-checkpoint-v2",
"relative_path": "compatibility/nonfinite-checkpoint-v2.patch",
Expand Down
71 changes: 71 additions & 0 deletions benchmark/tests/test_checkpoint_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,77 @@ def test_disabled_extension_preserves_the_native_route_surface(native_config):
second.state.judge.close()


@pytest.mark.parametrize("score,wire_score", [
(float("inf"), "Infinity"), (float("-inf"), "-Infinity"),
(float("nan"), "NaN"), (0.0, 0.0), (None, None),
])
def test_ordinary_feedback_keeps_failure_zero_and_missing_scores_distinct(
native_config, monkeypatch, score, wire_score):
import requests
from benchmark.edgebench.checkpoint_judge import create_app
from benchmark.edgebench.source import active_source_receipt
from benchmark.tests.test_checkpoint_controller import server
from sforge.harness.feedback_transport import JOURNAL_FILE, payload_sha256

active_source_receipt()
# Exercise ordinary solver feedback, not the checkpoint-only JSON codec.
# A broken proxy cannot explain an in-process Judge serialization failure.
monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:1")
monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:1")
app = create_app(native_config)
state = app.state.judge
try:
with server(app) as url, requests.Session() as client:
client.trust_env = False
token = state.register_session(
"fixture", "source", max_agent_submissions=3, submission_cooldown=3600)
state.consume_round(token)
report = {"score": score, "pass_rate": .5, "valid": False,
"metrics": {"nested": [score, 0.0, None]}}
state._record_submission("source", "accepted", "fixture", "agent-1", report, None)
state.submissions["accepted"] = {
"run_id": "source", "task_id": "fixture", "round": "agent-1",
"status": "completed", "report": report, "error": None}
before = state.resolve_token(token)
original = json.dumps(report, sort_keys=True)
relay = before["feedback_relay_id"]
delivered = []
for operation in ("result/accepted", "history"):
host = client.get(url + "/api/v1/" + operation,
params={"token": token}, timeout=5)
response = client.get(url + "/api/v1/feedback/" + relay + "/" + operation,
timeout=5)
assert host.status_code == response.status_code == 200
payload = response.json()
json.dumps(payload, allow_nan=False)
delivered.append(payload)
if operation.startswith("result/"):
assert host.json() == payload
assert payload["report"]["score"] == wire_score
assert payload["report"]["metrics"]["nested"] == [wire_score, 0.0, None]
else:
assert host.json()["entries"] == payload["entries"]
assert host.json()["best_score"] == payload["best_score"]
assert payload["entries"][0]["score"] == wire_score
for history in (host.json(), payload):
availability = history["agent_submission_availability"]
assert availability["remaining_submissions"] == 2
assert 0 < availability["cooldown_remaining_seconds"] <= 3600

# Reading feedback must not consume another evaluation or rewrite
# the native numeric state used by ranking and checkpoint capture.
assert state.resolve_token(token) == before
assert json.dumps(state.submissions["accepted"]["report"], sort_keys=True) == original
records = [json.loads(line) for line in (
native_config.log_dir / "runs/source/fixture" / JOURNAL_FILE
).read_text().splitlines()]
responses = [row for row in records if row["record_kind"] == "response"]
assert [row["payload_sha256"] for row in responses] == [
payload_sha256(payload) for payload in delivered]
finally:
state.close()


def test_capture_retains_consumed_round_history_and_cooldown_without_mutation(native_config):
from fastapi.testclient import TestClient
from benchmark.edgebench.checkpoint_judge import create_app, ROUTE
Expand Down
Loading