diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 0c952b56..fc196e3f 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -434,7 +434,7 @@ def __init__( version: Optional[int], criteria: list[dict], prompt_id: str, - default_model: str = "claude-sonnet-4-6", + default_model: Optional[str] = None, default_model_api_base: Optional[str] = None, score_aggregator: Optional[ScoreAggregator] = None, verifier_id: Optional[str] = None, @@ -489,7 +489,9 @@ def __init__( self.verifier_id = verifier_id or uuid.uuid4().hex self.criteria = criteria self.prompt_id = prompt_id - self.default_model = default_model + # A stored judge always holds a model: its author's, or the one the config names for judges, as for a step + # built in code, which the A2A validator's are. + self.default_model = default_model or get_config().get_model_for_role("judge") or "claude-sonnet-4-6" self.default_model_api_base = default_model_api_base if isinstance(score_aggregator, str): score_aggregator = ScoreAggregator(score_aggregator) @@ -593,15 +595,13 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: - from agent_env.config import get_config - raw_agg = data.get("score_aggregator") raw_tf = data.get("trajectory_filter") return cls( **cls._base_from_dict(data), criteria=data["criteria"], prompt_id=data["prompt_id"], - default_model=data.get("default_model") or get_config().get_model_for_role("judge") or "claude-sonnet-4-6", + default_model=data.get("default_model"), default_model_api_base=data.get("default_model_api_base"), score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, verifier_id=data.get("verifier_id"), diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py new file mode 100644 index 00000000..643cfb5d --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py @@ -0,0 +1,59 @@ +"""A rubrics verifier built in code, as the A2A validator builds its own, judges with the configured ``judge`` model, +as one read from a task does.""" + +import pytest + +from agent_env.config import reset_config +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + +CRITERIA = [{"id": "c1", "description": "Response says hello"}] + + +@pytest.fixture +def model_config(tmp_path, monkeypatch): + """Install a config file holding ``model`` as its whole [model] table, so no developer config is read.""" + + def install(model: str) -> None: + config = tmp_path / "config.toml" + config.write_text(f"[model]\n{model}") + monkeypatch.setenv("AGENT_ENV_CONFIG", str(config)) + reset_config() + + yield install + reset_config() + + +@pytest.fixture +def judge_role(model_config): + model_config('default = "agent-model"\n\n[model.roles]\njudge = "judge-model"\n') + + +def _built(**kwargs) -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep(id="verify", version=None, criteria=CRITERIA, prompt_id="p1", **kwargs) + + +def test_a_verifier_built_in_code_judges_with_the_configured_judge_model(judge_role): + assert _built().default_model == "judge-model" + + +def test_a_verifier_read_from_a_task_judges_with_the_configured_judge_model(judge_role): + data = _built().to_dict() + del data["default_model"] + + assert RubricsVerifierTaskStep.from_dict(data).default_model == "judge-model" + + +def test_a_model_the_step_names_wins_over_the_judge_role(judge_role): + assert _built(default_model="named-model").default_model == "named-model" + + +def test_without_a_judge_role_the_judge_is_the_configured_default_model(model_config): + model_config('default = "agent-model"\n') + + assert _built().default_model == "agent-model" + + +def test_with_no_model_configured_the_judge_is_claude_sonnet(model_config): + model_config("") + + assert _built().default_model == "claude-sonnet-4-6" diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_trajectory.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_trajectory.py index 6a663fe1..73f6e81a 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_trajectory.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_trajectory.py @@ -91,7 +91,8 @@ def issue_write_grant(self, object_url, *, media_type, max_bytes, expires_in=Non def _use_store(monkeypatch, store, *, key_prefix=""): config = type( - "Config", (), {"get_object_store": lambda self: store, "get_artifact_key_prefix": lambda self: key_prefix} + "Config", (), {"get_object_store": lambda self: store, "get_artifact_key_prefix": lambda self: key_prefix, + "get_model_for_role": lambda self, role: None} )() monkeypatch.setattr(f"{RV}.get_config", lambda: config) monkeypatch.setattr(