Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ def __init__(
version: Optional[int],
criteria: list[dict],
prompt_id: str,
default_model: str = "claude-sonnet-4-6",
default_model: Optional[str] = None,
default_model_api_base: Optional[str] = None,
score_aggregator: Optional[ScoreAggregator] = None,
verifier_id: Optional[str] = None,
Expand Down Expand Up @@ -489,7 +489,9 @@ def __init__(
self.verifier_id = verifier_id or uuid.uuid4().hex
self.criteria = criteria
self.prompt_id = prompt_id
self.default_model = default_model
# A stored judge always holds a model: its author's, or the one the config names for judges, as for a step
# built in code, which the A2A validator's are.
self.default_model = default_model or get_config().get_model_for_role("judge") or "claude-sonnet-4-6"
self.default_model_api_base = default_model_api_base
if isinstance(score_aggregator, str):
score_aggregator = ScoreAggregator(score_aggregator)
Expand Down Expand Up @@ -593,15 +595,13 @@ def to_dict(self) -> dict:

@classmethod
def from_dict(cls, data: dict) -> RubricsVerifierTaskStep:
from agent_env.config import get_config

raw_agg = data.get("score_aggregator")
raw_tf = data.get("trajectory_filter")
return cls(
**cls._base_from_dict(data),
criteria=data["criteria"],
prompt_id=data["prompt_id"],
default_model=data.get("default_model") or get_config().get_model_for_role("judge") or "claude-sonnet-4-6",
default_model=data.get("default_model"),
default_model_api_base=data.get("default_model_api_base"),
score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None,
verifier_id=data.get("verifier_id"),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
"""A rubrics verifier built in code, as the A2A validator builds its own, judges with the configured ``judge`` model,
as one read from a task does."""

import pytest

from agent_env.config import reset_config
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep

CRITERIA = [{"id": "c1", "description": "Response says hello"}]


@pytest.fixture
def model_config(tmp_path, monkeypatch):
"""Install a config file holding ``model`` as its whole [model] table, so no developer config is read."""

def install(model: str) -> None:
config = tmp_path / "config.toml"
config.write_text(f"[model]\n{model}")
monkeypatch.setenv("AGENT_ENV_CONFIG", str(config))
reset_config()

yield install
reset_config()


@pytest.fixture
def judge_role(model_config):
model_config('default = "agent-model"\n\n[model.roles]\njudge = "judge-model"\n')


def _built(**kwargs) -> RubricsVerifierTaskStep:
return RubricsVerifierTaskStep(id="verify", version=None, criteria=CRITERIA, prompt_id="p1", **kwargs)


def test_a_verifier_built_in_code_judges_with_the_configured_judge_model(judge_role):
assert _built().default_model == "judge-model"


def test_a_verifier_read_from_a_task_judges_with_the_configured_judge_model(judge_role):
data = _built().to_dict()
del data["default_model"]

assert RubricsVerifierTaskStep.from_dict(data).default_model == "judge-model"


def test_a_model_the_step_names_wins_over_the_judge_role(judge_role):
assert _built(default_model="named-model").default_model == "named-model"


def test_without_a_judge_role_the_judge_is_the_configured_default_model(model_config):
model_config('default = "agent-model"\n')

assert _built().default_model == "agent-model"


def test_with_no_model_configured_the_judge_is_claude_sonnet(model_config):
model_config("")

assert _built().default_model == "claude-sonnet-4-6"
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ def issue_write_grant(self, object_url, *, media_type, max_bytes, expires_in=Non

def _use_store(monkeypatch, store, *, key_prefix=""):
config = type(
"Config", (), {"get_object_store": lambda self: store, "get_artifact_key_prefix": lambda self: key_prefix}
"Config", (), {"get_object_store": lambda self: store, "get_artifact_key_prefix": lambda self: key_prefix,
"get_model_for_role": lambda self, role: None}
)()
monkeypatch.setattr(f"{RV}.get_config", lambda: config)
monkeypatch.setattr(
Expand Down