From c3db863600d0b7dbbadb3bb9c73ab70c16d1e9a4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 16:07:06 -0700 Subject: [PATCH 01/11] feat(protocol): declare what a computer driver class serves on a platform ComputerDriverEnvironment.declaration(platform) builds the card the class serves on that platform, with default settings and under its own name, without running the subclass's __init__: what a computer_driver env records at put for each platform it supports. platform.supported(entry) gives the platforms a supports entry names. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/driver.py | 15 +++++++-- .../agentenv_protocol/computer/platform.py | 16 ++++++++++ .../tests/test_computer_driver.py | 31 ++++++++++++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 93cdf44b..1181505e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -5,9 +5,9 @@ import asyncio from typing import Any, Callable, ClassVar -from ..agent_env_environment import AgentEnvEnvironment, environment_card +from ..agent_env_environment import _CARD_CONFIG_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication, environment_card from ..tool_context import ToolContext -from ..types import EnvironmentExtension, EnvironmentTool +from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool from . import card, linux from .checks import check_features, check_names from .dataplane import DataPlane @@ -57,6 +57,17 @@ def __init__(self, settings: ComputerSettings | None = None, platform: Platform self.platform = platform if platform is not None else detect() self._computer = ComputerRuntime(self) + @classmethod + def declaration(cls, platform: Platform) -> EnvironmentCard: + """The card this driver class serves on ``platform`` with default settings, under its own name, built without + running its ``__init__``: what a ``computer_driver`` env records for each platform it supports. The card + hooks may read the class and the base class's state, not what a subclass's ``__init__`` sets.""" + driver = cls.__new__(cls) + ComputerDriverEnvironment.__init__(driver, settings=ComputerSettings(), platform=platform) + config = dict(getattr(cls, _CARD_CONFIG_ATTR, None) or {}) + card = EnvironmentCard(**{**config, "name": config.get("name") or cls.__name__}) + return AgentEnvFastMCPApplication(card, driver).environment_card + def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" return self._computer.has(capability) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py index 949d17e0..3f2587e7 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py @@ -46,6 +46,22 @@ def detect(environ: dict | None = None, os_release: Path = Path("/etc/os-release arch=_ARCH.get(machine, machine or None), display=display) +def supported(entry: dict) -> list[Platform]: + """The platforms a ``supports`` entry names: its ``os`` type, with each of its displays and arches, and its + distribution and version when it names one. ``{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}`` + names one platform.""" + os_ = entry.get("os") or {} + if not isinstance(os_, dict) or not isinstance(os_.get("type"), str): + raise ValueError(f"a supports entry names its os as {{'type': ...}}, got {entry!r}") + return [Platform(os=os_["type"], distribution=distribution, version=version, arch=arch, display=display) + for distribution in _values(os_.get("distribution")) for version in _values(os_.get("version")) + for display in _values(os_.get("display")) for arch in _values(entry.get("arch"))] + + +def _values(value) -> list: + return [None] if value is None else [value] if isinstance(value, str) else list(value) + + def _os_release(path: Path) -> dict: try: text = path.read_text() diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index e029f62b..a0e3331c 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -40,7 +40,7 @@ ) from agentenv_protocol.computer.keys import canonical # noqa: E402 from agentenv_protocol.computer.operations import declared, resolve # noqa: E402 -from agentenv_protocol.computer.platform import detect, normalize # noqa: E402 +from agentenv_protocol.computer.platform import detect, normalize, supported # noqa: E402 from agentenv_protocol.computer.settings import load_settings # noqa: E402 from agentenv_protocol.computer.tools import _modifiers # noqa: E402 @@ -384,3 +384,32 @@ def test_a_drivers_own_private_helpers_and_the_hooks_are_its_own(): Driver = _define({"_button": lambda self, d, b, down: None, "_move": lambda self: None, "caller_identity": lambda self, context: "x", "buttons": ("left", "right")}) assert Driver().caller_identity(None) == "x" + + +def test_a_driver_class_declares_what_it_serves_on_each_platform(monkeypatch): + monkeypatch.setenv("ENVIRONMENT_NAME", "deployed-name") + + @computer_driver(name="declared") + class Declared(FakeDriver): + def __init__(self): + raise AssertionError("declaration builds the card without the subclass's __init__") + + linux = Declared.declaration(LINUX) + elsewhere = MinimalDriver.declaration(Platform(os="macos")) + uris = lambda card: {e.uri for e in card.capabilities.extensions} # noqa: E731 + assert linux.name == "declared" and len(linux.capabilities.tools) == 14 + assert {"urn:agentenv:computer-pointer/v1", "urn:agentenv:computer-exec/v1"} <= uris(linux) + assert elsewhere.name == "MinimalDriver" and len(elsewhere.capabilities.tools) == 7 + assert "urn:agentenv:computer-exec/v1" not in uris(elsewhere) + computer = next(e for e in elsewhere.capabilities.extensions if e.uri == "urn:agentenv:computer/v1") + assert computer.params["platform"] == {"os": "macos"} + + +def test_a_supports_entry_names_each_platform_it_covers(): + entry = {"os": {"type": "linux", "distribution": "ubuntu", "version": ["22.04", "24.04"], "display": "x11"}, + "arch": ["amd64", "arm64"]} + assert [(p.version, p.arch) for p in supported(entry)] == [("22.04", "amd64"), ("22.04", "arm64"), + ("24.04", "amd64"), ("24.04", "arm64")] + assert supported({"os": {"type": "macos"}}) == [Platform(os="macos")] + with pytest.raises(ValueError, match="names its os as"): + supported({"os": "linux"}) From b1c9b938498efd56937c81751162e031f51ec1d1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:04:24 -0700 Subject: [PATCH 02/11] feat(protocol): add the computer conformance probe agentenv_protocol.computer.conformance checks a running driver against the contract through its card, tools and extensions only, and reports each of the driver design's checks P0 to P9 and each extension's own checks as passed, failed, unverified or not applicable, with each declared extension verified, unverified or failed. basic needs nothing on the computer; full stages the Linux fixture (the Fixture Desk, now with notes, a wheel pad, colour markers at odd and even pixels and a clipboard sentinel) with data/add, launches it with computer-exec and reads its state back with bounded waits. run_probe(url, level) returns a ProbeReport, the value M0.7 stores in metadata.validated_boot; python -m agentenv_protocol.computer.conformance prints it. The tests serve the fake driver on loopback with a fake fixture behind its input, drivers broken one way each fail the check for it, and stubs cover how a driver outside the base class can go wrong. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/__init__.py | 22 + .../computer/conformance/__main__.py | 42 + .../computer/conformance/checks.py | 719 ++++++++++++++++++ .../computer/conformance/client.py | 82 ++ .../computer/conformance/fixture.py | 209 +++++ .../computer/conformance/report.py | 43 ++ .../agentenv-protocol/tests/probe_fake.py | 211 +++++ .../tests/test_computer_conformance.py | 225 ++++++ .../tests/test_computer_conformance_checks.py | 283 +++++++ 9 files changed, 1836 insertions(+) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__init__.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__main__.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/client.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/report.py create mode 100644 packages/agentenv-protocol/tests/probe_fake.py create mode 100644 packages/agentenv-protocol/tests/test_computer_conformance.py create mode 100644 packages/agentenv-protocol/tests/test_computer_conformance_checks.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__init__.py new file mode 100644 index 00000000..58cb6c6d --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__init__.py @@ -0,0 +1,22 @@ +"""The conformance probe: checks a running computer driver against the contract through its card, tools and +extensions, and reports each check and each declared extension. ``basic`` needs nothing on the computer; ``full`` +stages the Linux fixture with data/add and reads it back with computer-exec. Run it as +``python -m agentenv_protocol.computer.conformance [--level basic|full] [--json report.json]``.""" +from __future__ import annotations + +from .checks import Probe +from .client import Driver +from .report import CheckResult, Level, ProbeReport, Status, Verdict + +__all__ = ["run_probe", "ProbeReport", "CheckResult", "Level", "Status", "Verdict"] + + +async def run_probe(url: str, level: Level = "full", *, timeout_s: float = 60, ready_timeout_s: float = 60) -> ProbeReport: + """Probe the driver serving its card at ``url``. ``ready_timeout_s`` is how long it may take to report ready.""" + driver = await Driver.connect(url, timeout_s) + async with driver.session() as session: + probe = Probe(driver, session, ready_timeout_s) + await probe.basic() + if level == "full": + await probe.full() + return ProbeReport.of(level, driver.name, probe.results, driver.extensions) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__main__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__main__.py new file mode 100644 index 00000000..06fd2ad5 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/__main__.py @@ -0,0 +1,42 @@ +"""``python -m agentenv_protocol.computer.conformance ``: probe a running driver, print each check, and +exit 1 when one failed or 2 when the driver couldn't be reached.""" +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +from . import run_probe + +_MARK = {"passed": "PASS", "failed": "FAIL", "unverified": "----", "not_applicable": "n/a "} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.computer.conformance", + description="Probe a running computer driver against the contract.") + parser.add_argument("url", help="where the driver serves its card, such as http://localhost:18765") + parser.add_argument("--level", choices=("basic", "full"), default="full") + parser.add_argument("--json", type=Path, help="also write the report to this file") + parser.add_argument("--timeout", type=float, default=60, help="seconds for each call, and to report ready") + args = parser.parse_args(argv) + try: + report = asyncio.run(run_probe(args.url, args.level, timeout_s=args.timeout, ready_timeout_s=args.timeout)) + except Exception as e: + print(f"couldn't probe {args.url}: {type(e).__name__}: {e}", file=sys.stderr) + return 2 + for result in report.checks: + detail = f": {result.detail}" if result.detail else "" + print(f"{_MARK[result.status]} {result.check} {result.name}{detail}") + counts = report.counts() + print(f"{report.driver}: {counts['passed']} passed, {counts['failed']} failed, {counts['unverified']} unverified, " + f"{counts['not_applicable']} not applicable at {report.level}") + for uri, verdict in report.extensions.items(): + print(f" {verdict:10} {uri}") + if args.json: + args.json.write_text(report.model_dump_json(indent=1)) + return 0 if report.passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py new file mode 100644 index 00000000..c8e91d55 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -0,0 +1,719 @@ +"""The checks a probe runs, named as the driver design names them: P0 to P9 against the contract, and X for one +extension's own methods. Everything goes through the driver's card, tools and extensions; ``full`` also stages the +fixture with data/add, launches it with computer-exec and reads its state back the same way.""" +from __future__ import annotations + +import asyncio +import json +import math +import time +from collections.abc import Awaitable, Callable +from importlib import resources + +from mcp import ClientSession +from mcp.types import CallToolResult + +from ... import client as protocol +from ...client import ExtensionCallError +from ...tool_access import DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI +from ...types import DATA_OBJECTS_EXTENSION_URI, INTAKE_EXTENSION_URI +from ..card import ( + COMPUTER, + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_POINTER, + COMPUTER_READY, + COMPUTER_SPACES, + GROUP_TOOLS, + POINT_TOOLS, +) +from .client import Driver, image +from .report import CheckResult + +MINIMUM_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", + "computer_key", "computer_wait") +DATA_PLANE = (INTAKE_EXTENSION_URI, DATA_OBJECTS_EXTENSION_URI) +#: Where the fixture is staged and keeps its state, relative to the session user's home. +STAGE = ".agentenv-probe" +#: The role the tool-access check hides a tool from. +PROBE_ROLE = "agentenv-probe" +#: How long the probe waits for an effect to reach the fixture, and how often it looks. +SETTLE_S = 5.0 +POLL_S = 0.15 +#: How long the fixture may take to start. +FIXTURE_START_S = 30.0 +NORM = 1000 +#: The checks only ``full`` runs, recorded unverified when the fixture can't be staged. +FULL_CHECKS = (("P3", "fixture colours land where the mapping says"), ("P4", "clicks show their effect"), + ("P5", "refusals leave the fixture as it was"), ("P6", "typed and pressed keys arrive"), + ("P7", "scrolls and drags arrive"), ("P9", "commands and data/add run as one unprivileged user")) + +Test = Callable[[], Awaitable[str | None]] + + +class Probe: + """One probe run: the driver, an MCP session on its tools, and the results so far. A test returns what went + wrong, or None when its check passes; a test that raises fails with what it raised.""" + + def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: float) -> None: + self.driver = driver + self.session = session + self.ready_timeout_s = ready_timeout_s + self.results: list[CheckResult] = [] + self.features = driver.params(COMPUTER) + self.layout: dict = {} + + async def basic(self) -> None: + for check, section in (("P0", self._ready), ("P1", self._card), ("P2", self._frame), ("P5", self._refusals), + ("X", self._display), ("X", self._spaces), ("X", self._keyboard_settings), + ("X", self._exec), ("X", self._data_get), ("X", self._tool_access), + ("X", self._pointer)): + await self._guarded(check, section) + self._grants() + + async def full(self) -> None: + missing = [uri for uri in (COMPUTER_EXEC,) if not self.driver.declares(uri)] + if not (protocol.intake_declaration(self.driver.card) or {}).get("add"): + missing.append("data/add") + if missing: + return self._skip_full(f"the fixture is staged with data/add and computer-exec; the driver lacks {missing}") + problem = await self._stage() + if problem: + return self._skip_full(f"the fixture didn't start: {problem}") + try: + for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), + ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), + ("X", self._pointer_input), ("X", self._data_add)): + await self._guarded(check, section) + finally: + await self._unstage() + + # Recording + + async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool: + try: + problem = await test() + except Exception as e: + problem = f"{type(e).__name__}: {e}" + self._record(check, name, "failed" if problem else "passed", problem or "", extensions) + return problem is None + + async def _guarded(self, check: str, section: Callable[[], Awaitable[None]]) -> None: + """Run a section of checks; one that raises outside its checks is a failed check of its own.""" + try: + await section() + except Exception as e: + self._record(check, f"{section.__name__.strip('_').replace('_', ' ')} could run", "failed", + f"{type(e).__name__}: {e}", ()) + + def _record(self, check: str, name: str, status: str, detail: str, extensions: tuple[str, ...]) -> None: + on = [COMPUTER, *extensions] if check.startswith("P") else list(extensions) + self.results.append(CheckResult(check=check, name=name, status=status, detail=detail[:500], extensions=on)) + + def _skip_full(self, why: str) -> None: + for check, name in FULL_CHECKS: + self._record(check, name, "unverified", why, ()) + + # Talking to the driver + + async def tool(self, name: str, args: dict | None = None) -> CallToolResult: + return await self.session.call_tool(name, args or {}) + + async def shot(self, **args) -> tuple[CallToolResult, object]: + result = await self.tool("computer_screenshot", args) + if result.isError: + raise AssertionError(f"computer_screenshot failed: {result.content[0].text}") + return result, image(result) + + async def act(self, name: str, args: dict) -> None: + result = await self.tool(name, args) + if result.isError: + raise AssertionError(f"{name} {args} failed: {result.content[0].text}") + + async def state(self) -> dict: + out = await self.driver.run("cat", f"{STAGE}/state/state.json") + return json.loads(out["stdout"]) + + async def until(self, holds: Callable[[dict], bool]) -> tuple[bool, dict]: + """Whether the fixture's state comes to satisfy ``holds`` within SETTLE_S, and the state it last had.""" + deadline = time.monotonic() + SETTLE_S + while True: + state = await self.state() + if holds(state) or time.monotonic() > deadline: + return holds(state), state + await asyncio.sleep(POLL_S) + + async def settled(self, holds: Callable[[dict], bool], show: Callable[[dict], object]) -> str | None: + ok, state = await self.until(holds) + return None if ok else f"the fixture shows {show(state)!r}" + + @staticmethod + def refused(result: CallToolResult, code: str) -> str | None: + data = result.structuredContent or {} + if not result.isError: + return f"it wasn't refused: {data.get('summary')!r}" + if data.get("code") != code: + return f"refused {data.get('code')!r}, not {code!r}: {data.get('message')}" + if data.get("applied") != "no": + return f"refused with applied {data.get('applied')!r}, not 'no'" + return None + + # P0 to P2: readiness, the card, frames + + async def _ready(self) -> None: + async def ready() -> str | None: + if not self.driver.declares(COMPUTER_READY): + return "the card doesn't declare computer-ready/v1" + deadline = time.monotonic() + self.ready_timeout_s + while True: + status = await self.driver.invoke(COMPUTER_READY, "status") + if status.get("ready"): + return None + if time.monotonic() > deadline: + failing = [c for c in status.get("checks", []) if not c.get("ok")] + return f"not ready after {self.ready_timeout_s:g} s: {failing}" + await asyncio.sleep(1) + + async def capture() -> str | None: + await self.shot() + return None + + await self.check("P0", "computer-ready reports ready", ready, COMPUTER_READY) + await self.check("P0", "a screenshot works", capture) + + async def _card(self) -> None: + tools = {t.name: t for t in (await self.session.list_tools()).tools} + + async def declared() -> str | None: + return None if self.driver.declares(COMPUTER) else "the card doesn't declare computer/v1" + + async def minimum() -> str | None: + missing = [n for n in MINIMUM_TOOLS if n not in tools] + bare = [n for n in MINIMUM_TOOLS if n in tools and not (tools[n].inputSchema and tools[n].outputSchema)] + if missing: + return f"missing {missing}" + return f"listed without an input or output schema: {bare}" if bare else None + + async def groups() -> str | None: + problems = [] + for uri, names in ((COMPUTER_POINTER, GROUP_TOOLS["pointer"]), (COMPUTER_KEYBOARD, GROUP_TOOLS["keyboard"])): + listed = [n for n in names if n in tools] + if self.driver.declares(uri) and len(listed) != len(names): + problems.append(f"{uri} is declared without {sorted(set(names) - set(listed))}") + if not self.driver.declares(uri) and listed: + problems.append(f"{listed} are listed without {uri}") + return "; ".join(problems) or None + + await self.check("P1", "the card declares computer/v1", declared) + await self.check("P1", "the seven minimum tools are listed with their schemas", minimum) + await self.check("P1", "each group's tools come with its extension", groups) + + async def _frame(self) -> None: + async def native() -> str | None: + result, img = await self.shot() + data, display = result.structuredContent, result.structuredContent["display"] + want = (round(display["width"] * display["scale"]), round(display["height"] * display["scale"])) + if img.size != (data["width"], data["height"]) or img.size != want: + return f"the image is {img.size}, the frame says {(data['width'], data['height'])}, the display {want}" + return None + + async def crop() -> str | None: + result, img = await self.shot() + w, h = img.size + box = [w // 4, h // 4, w // 4 + w // 3, h // 4 + h // 3] + cropped = await self.tool("computer_screenshot", {"region": box, "frame_id": result.structuredContent["frame_id"]}) + if cropped.isError: + return cropped.content[0].text + part = image(cropped) + if part.size != (box[2] - box[0], box[3] - box[1]): + return f"a {box} region came back {part.size}" + if cropped.structuredContent.get("parent_frame_id") != result.structuredContent["frame_id"]: + return f"its parent_frame_id is {cropped.structuredContent.get('parent_frame_id')!r}" + points = [(0, 0), (part.width - 1, part.height - 1), (part.width // 2, part.height // 2)] + wrong = [p for p in points if part.getpixel(p) != img.getpixel((box[0] + p[0], box[1] + p[1]))] + return f"its pixels at {wrong} aren't its frame's" if wrong else None + + async def jpeg() -> str | None: + result = await self.tool("computer_screenshot", {"format": "jpeg", "quality": 60}) + mime = next(c.mimeType for c in result.content if c.type == "image") + return None if mime == "image/jpeg" else f"it answered {mime}" + + await self.check("P2", "a screenshot is native: its image, frame and display agree", native) + await self.check("P2", "a region comes back as a native crop of its frame", crop) + await self.check("P2", "format jpeg answers a JPEG", jpeg) + + # P5: refusals + + async def _refusal_cases(self) -> list[tuple[str, str, dict, str]]: + """The calls a driver must refuse, aimed at a fresh frame, with the code each is refused with.""" + result, img = await self.shot() + frame, (w, _h) = result.structuredContent["frame_id"], img.size + over = int(self.features.get("max_click_count", 1)) + 1 + return [ + ("a point outside the frame", "computer_click", {"x": w, "y": 0, "frame_id": frame}, "out_of_bounds"), + ("an unknown frame", "computer_click", {"x": 1, "y": 1, "frame_id": "f999999999"}, "stale_frame"), + ("a key that isn't a W3C key value", "computer_key", {"keys": ["ctrl", "s"]}, "invalid_argument"), + ("control characters in text", "computer_type", {"text": "a\x1bb"}, "invalid_argument"), + ("a click count over the declared maximum", "computer_click", {"x": 1, "y": 1, "frame_id": frame, + "count": over}, "invalid_argument"), + ] + + async def _refusals(self) -> None: + for n, (name, _tool, _args, code) in enumerate(await self._refusal_cases()): + async def refused(n=n, code=code) -> str | None: + _name, tool, args, _code = (await self._refusal_cases())[n] + return self.refused(await self.tool(tool, args), code) + await self.check("P5", f"{name} is refused {code}", refused) + + # The extensions' own methods + + async def _display(self) -> None: + if not self.driver.declares(COMPUTER_DISPLAY): + return + + async def get() -> str | None: + got = await self.driver.invoke(COMPUTER_DISPLAY, "get") + result, _ = await self.shot() + frame = result.structuredContent["display"] + seen = (frame["width"], frame["height"], frame["scale"]) + return None if (got["width"], got["height"], got["scale"]) == seen else f"get says {got}, frames {frame}" + + async def same() -> str | None: + got = await self.driver.invoke(COMPUTER_DISPLAY, "get") + again = await self.driver.invoke(COMPUTER_DISPLAY, "set", {"width": got["width"], "height": got["height"]}) + return None if (again["width"], again["height"]) == (got["width"], got["height"]) else f"set gave {again}" + + await self.check("X", "computer-display get matches the frames", get, COMPUTER_DISPLAY) + if "set" in (self.driver.params(COMPUTER_DISPLAY).get("methods") or {}): + await self.check("X", "computer-display set to the current mode keeps it", same, COMPUTER_DISPLAY) + + async def _spaces(self) -> None: + if not self.driver.declares(COMPUTER_SPACES): + return + + async def rewrite() -> str | None: + before = await self.driver.invoke(COMPUTER_SPACES, "get") + other = "norm" if before["default_space"] == "frame" else "frame" + first = {t.name: t.description for t in (await self.session.list_tools()).tools} + await self.driver.invoke(COMPUTER_SPACES, "set", {"default_space": other, "default_norm_scale": NORM}) + try: + second = {t.name: t.description for t in (await self.session.list_tools()).tools} + now = await self.driver.invoke(COMPUTER_SPACES, "get") + finally: + await self.driver.invoke(COMPUTER_SPACES, "set", {"default_space": before["default_space"], + "default_norm_scale": before["default_norm_scale"]}) + if now["default_space"] != other: + return f"set {other}, then get said {now}" + changed = [n for n in POINT_TOOLS if n in first and first[n] != second.get(n)] + return None if changed else "no point tool's description changed" + + await self.check("X", "computer-spaces set changes the default and the point tools' descriptions", rewrite, + COMPUTER_SPACES) + + async def _keyboard_settings(self) -> None: + if not self.driver.declares(COMPUTER_KEYBOARD): + return + + async def keep() -> str | None: + got = await self.driver.invoke(COMPUTER_KEYBOARD, "get") + again = await self.driver.invoke(COMPUTER_KEYBOARD, "set", {k: got[k] for k in got if k != "layout"}) + return None if again == got else f"set {got}, then got {again}" + + await self.check("X", "computer-keyboard set to its own settings keeps them", keep, COMPUTER_KEYBOARD) + + async def _exec(self) -> None: + if not self.driver.declares(COMPUTER_EXEC): + return + cap = self.driver.params(COMPUTER_EXEC).get("max_output_bytes") + + async def output() -> str | None: + out = await self.driver.run("sh", "-c", "echo probe; echo oops >&2; exit 3") + want = {"exit_code": 3, "stdout": "probe\n", "stderr": "oops\n", "timed_out": False} + return None if {k: out.get(k) for k in want} == want else f"it answered {out}" + + async def timeout() -> str | None: + started = time.monotonic() + out = await self.driver.run("sh", "-c", "sleep 30", timeout_s=1) + took = time.monotonic() - started + return None if out.get("timed_out") and took < 15 else f"after {took:.1f} s it answered {out}" + + async def once() -> str | None: + request = {"argv": ["sh", "-c", "date +%s%N"], "request_id": f"{PROBE_ROLE}-once-{time.time_ns()}"} + first = await self.driver.invoke(COMPUTER_EXEC, "run", request) + again = await self.driver.invoke(COMPUTER_EXEC, "run", request) + return None if first == again else f"it ran twice: {first} then {again}" + + async def capped() -> str | None: + out = await self.driver.run("sh", "-c", f"head -c {cap + 1000} /dev/zero | tr '\\0' a") + return None if out.get("truncated") and len(out["stdout"]) == cap else ( + f"{len(out['stdout'])} bytes came back, truncated {out.get('truncated')}") + + async def missing() -> str | None: + try: + out = await self.driver.run("agentenv-probe-no-such-program") + except ExtensionCallError as e: + return None if e.code else f"refused without a code: {e}" + return f"it answered {out}" + + await self.check("X", "a command's exit code and output come back", output, COMPUTER_EXEC) + await self.check("X", "a command past its timeout is stopped", timeout, COMPUTER_EXEC) + await self.check("X", "a repeated request_id answers the first result", once, COMPUTER_EXEC) + if isinstance(cap, int): + await self.check("X", "output past max_output_bytes is cut there", capped, COMPUTER_EXEC) + await self.check("X", "a program that isn't there is refused with a code", missing, COMPUTER_EXEC) + + async def _data_get(self) -> None: + if not self.driver.declares(INTAKE_EXTENSION_URI): + return + + async def get() -> str | None: + got = await self.driver.rpc("data/get") + return None if isinstance((got.get("result") or {}).get("parts"), list) else f"it answered {got}" + + await self.check("X", "data/get answers its parts", get, *DATA_PLANE) + + async def _tool_access(self) -> None: + if not (self.driver.declares(DISABLE_TOOL_EXTENSION_URI) and self.driver.declares(ENABLE_TOOL_EXTENSION_URI)): + return + + async def per_role() -> str | None: + hidden = {"role": PROBE_ROLE, "tools": ["computer_wait"]} + await self.driver.invoke(DISABLE_TOOL_EXTENSION_URI, "disable", hidden) + try: + async with self.driver.session(PROBE_ROLE) as other: + listed = {t.name for t in (await other.list_tools()).tools} + refused = await other.call_tool("computer_wait", {"ms": 0}) + allowed = await self.tool("computer_wait", {"ms": 0}) + finally: + await self.driver.invoke(ENABLE_TOOL_EXTENSION_URI, "enable", hidden) + async with self.driver.session(PROBE_ROLE) as other: + back = await other.call_tool("computer_wait", {"ms": 0}) + if "computer_wait" in listed: + return "a disabled tool is still listed for its role" + if not refused.isError or (refused.structuredContent or {}).get("code") != "permission_denied": + return f"a disabled tool answered its role: {refused.content[0].text if refused.content else ''}" + if allowed.isError or back.isError: + return "the tool was refused to another role, or after it was enabled again" + return None + + await self.check("X", "a tool disabled for a role is hidden from it and refused, and only to it", per_role, + DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI) + + async def _pointer(self) -> None: + if not self.driver.declares(COMPUTER_POINTER): + return + + async def listed() -> str | None: + pointers = (await self.tool("computer_pointer_list")).structuredContent.get("pointers") or [] + return None if sum(p.get("default") is True for p in pointers) == 1 else f"it listed {pointers}" + + async def moved() -> str | None: + result, img = await self.shot() + frame, (x, y) = result.structuredContent["frame_id"], (img.width // 3, img.height // 3) + await self.act("computer_pointer_move", {"x": x, "y": y, "frame_id": frame}) + where = (await self.tool("computer_pointer_position", {"frame_id": frame})).structuredContent + return None if (where.get("x"), where.get("y"), where.get("inside_frame")) == (x, y, True) else ( + f"moved to ({x}, {y}); it reports {where}") + + await self.check("X", "computer_pointer_list names one default pointer", listed, COMPUTER_POINTER) + await self.check("X", "the pointer goes where computer_pointer_move sends it", moved, COMPUTER_POINTER) + + def _grants(self) -> None: + params = self.driver.params(COMPUTER_POINTER) if self.driver.declares(COMPUTER_POINTER) else {} + if "leasing" in params: + self._record("P8", "leases refuse other agents' input", "unverified", "the probe has no lease checks yet", + (COMPUTER_POINTER,)) + else: + self._record("P8", "leases refuse other agents' input", "not_applicable", "the driver declares no leasing", + ()) + + # full: the fixture + + async def _stage(self) -> str | None: + source = resources.files(__package__).joinpath("fixture.py").read_bytes() + added = await self.driver.add_file(f"{STAGE}/fixture.py", source) + if "error" in added: + return f"data/add refused it: {added['error'].get('message')}" + await self.driver.run("sh", "-c", f"rm -rf {STAGE}/state") + await self.driver.run("sh", "-c", f"python3 {STAGE}/fixture.py {STAGE}/state > {STAGE}/fixture.log 2>&1 &") + deadline = time.monotonic() + FIXTURE_START_S + while time.monotonic() < deadline: + layout = await self.driver.run("cat", f"{STAGE}/state/layout.json") + if layout.get("exit_code") == 0 and layout["stdout"].strip(): + self.layout = json.loads(layout["stdout"]) + return None + state = await self.driver.run("cat", f"{STAGE}/state/state.json") + if state.get("exit_code") == 0 and "error" in (failed := json.loads(state["stdout"] or "{}")): + return failed["error"] + await asyncio.sleep(0.5) + log = await self.driver.run("tail", "-n", "5", f"{STAGE}/fixture.log") + return f"no layout after {FIXTURE_START_S:g} s: {log['stdout'].strip() or log['stderr'].strip()}" + + async def _unstage(self) -> None: + if pid := self.layout.get("pid"): + await self.driver.run("kill", str(pid)) + await self.driver.run("rm", "-rf", STAGE) + + async def _base(self) -> tuple[str, float, object]: + """A fresh frame of the fixture: its id, the display's scale, and its image.""" + result, img = await self.shot() + return result.structuredContent["frame_id"], result.structuredContent["display"]["scale"], img + + def _at(self, box: str, scale: float, dx: float | None = None, dy: float | None = None) -> tuple[int, int]: + """A point of a fixture widget in frame pixels: its centre, or ``dx``, ``dy`` in from its corner.""" + x, y, w, h = self.layout[box] + return (math.floor((x + (w / 2 if dx is None else dx)) * scale), + math.floor((y + (h / 2 if dy is None else dy)) * scale)) + + async def _security(self) -> None: + async def one_user() -> str | None: + uid = (await self.driver.run("id", "-u"))["stdout"].strip() + owner = (await self.driver.run("stat", "-c", "%u", f"{STAGE}/fixture.py"))["stdout"].strip() + if uid == "0": + return "commands run as root" + return None if owner == uid else f"data/add wrote as uid {owner}, commands run as uid {uid}" + + await self.check("P9", "commands and data/add run as one unprivileged user", one_user, COMPUTER_EXEC, + *DATA_PLANE) + + async def _geometry(self) -> None: + frame, scale, img = await self._base() + markers = [(x, y, _rgb(colour)) for x, y, colour in self.layout["markers"]] + + async def pixels() -> str | None: + wrong = [(x, y, colour, img.getpixel((math.floor(x * scale), math.floor(y * scale)))) + for x, y, colour in markers + if img.getpixel((math.floor(x * scale), math.floor(y * scale))) != colour] + return f"markers read wrong (x, y, want, got): {wrong}" if wrong else None + + async def norm() -> str | None: + wrong = [] + for x, y, colour in markers: + fx, fy = math.floor(x * scale), math.floor(y * scale) + region = [(fx + 0.5) * NORM / img.width, (fy + 0.5) * NORM / img.height, + min((fx + 4.5) * NORM / img.width, NORM), min((fy + 4.5) * NORM / img.height, NORM)] + cropped = await self.tool("computer_screenshot", {"region": region, "frame_id": frame, "space": "norm", + "norm_scale": NORM}) + if cropped.isError: + return cropped.content[0].text + if (got := image(cropped).getpixel((0, 0))) != colour: + wrong.append((x, y, colour, got)) + return f"norm regions start on the wrong pixel (x, y, want, got): {wrong}" if wrong else None + + await self.check("P3", "markers at odd and even pixels read back in frame pixels", pixels) + if self.driver.declares(COMPUTER_SPACES): + await self.check("P3", "norm points land on the same pixels", norm, COMPUTER_SPACES) + + async def _effects(self) -> None: + frame, scale, img = await self._base() + size = self.layout["cell"] + cell = lambda row, col: self._at("grid", scale, col * size + size / 2, row * size + size / 2) # noqa: E731 + buttons = self.features.get("buttons") or ["left"] + + async def frame_space() -> str | None: + x, y = cell(2, 3) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) + problem = await self.settled(lambda s: s["cells"].get("2,3") == 1, lambda s: s["cells"]) + if problem: + return problem + _, img = await self.shot() + return None if img.getpixel((x, y)) == (255, 0, 0) else f"the next frame shows {img.getpixel((x, y))}" + + async def norm_space() -> str | None: + x, y = cell(5, 5) + await self.act("computer_click", {"x": (x + 0.5) * NORM / img.width, "y": (y + 0.5) * NORM / img.height, + "frame_id": frame, "space": "norm", "norm_scale": NORM}) + return await self.settled(lambda s: s["cells"].get("5,5") == 1, lambda s: s["cells"]) + + async def right() -> str | None: + x, y = cell(0, 0) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame, "button": "right"}) + return await self.settled(lambda s: s["right_clicks"] == 1, lambda s: s["right_clicks"]) + + async def double() -> str | None: + x, y = cell(7, 7) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame, "count": 2}) + return await self.settled(lambda s: s["double_clicks"] >= 1, lambda s: s["double_clicks"]) + + async def crop() -> str | None: + gx, gy = self._at("grid", scale, 0, 0) + gw, gh = math.floor(self.layout["grid"][2] * scale), math.floor(self.layout["grid"][3] * scale) + cropped = await self.tool("computer_screenshot", {"region": [gx, gy, gx + gw, gy + gh], "frame_id": frame}) + if cropped.isError: + return cropped.content[0].text + inner = cropped.structuredContent["frame_id"] + await self.act("computer_click", {"x": math.floor((4 * size + size / 2) * scale), + "y": math.floor((6 * size + size / 2) * scale), "frame_id": inner}) + return await self.settled(lambda s: s["cells"].get("6,4") == 1, lambda s: s["cells"]) + + async def together() -> str | None: + targets = [cell(row, col) for row in (1, 3) for col in range(8)] + results = await asyncio.gather(*(self.tool("computer_click", {"x": x, "y": y, "frame_id": frame}) + for x, y in targets)) + if any(r.isError for r in results): + return f"{sum(r.isError for r in results)} of {len(results)} were refused" + landed = lambda s: sum(s["cells"].get(f"{r},{c}", 0) >= 1 for r in (1, 3) for c in range(8)) # noqa: E731 + return await self.settled(lambda s: landed(s) == 16, lambda s: f"{landed(s)} of 16 cells") + + async def widgets() -> str | None: + await self.act("computer_click", {"x": self._at("save", scale)[0], "y": self._at("save", scale)[1], + "frame_id": frame}) + x0, y0, _, h = self.layout["subscribe"] + await self.act("computer_click", {"x": math.floor((x0 + 12) * scale), "y": math.floor((y0 + h / 2) * scale), + "frame_id": frame}) + return await self.settled(lambda s: s["saved_count"] == 1 and s["subscribed"], + lambda s: (s["saved_count"], s["subscribed"])) + + await self.check("P4", "a click in frame space hits its cell, and the next frame shows it", frame_space) + if self.driver.declares(COMPUTER_SPACES): + await self.check("P4", "a click in norm space hits its cell", norm_space, COMPUTER_SPACES) + if "right" in buttons: + await self.check("P4", "a right click arrives as one", right) + if int(self.features.get("max_click_count", 1)) >= 2: + await self.check("P4", "a double click arrives as one", double) + await self.check("P4", "a click in a crop's frame maps back to the screen", crop) + await self.check("P4", "16 clicks sent at once all land", together) + await self.check("P4", "a button and a checkbox take a click", widgets) + + async def _unchanged(self) -> None: + async def nothing() -> str | None: + before = await self.state() + for _name, tool, args, _code in await self._refusal_cases(): + await self.tool(tool, args) + await asyncio.sleep(SETTLE_S / 5) + after = await self.state() + changed = sorted(k for k in before if before[k] != after.get(k)) + return f"refused calls changed {changed}" if changed else None + + await self.check("P5", "refusals leave the fixture as it was", nothing) + + async def _typing(self) -> None: + frame, scale, _ = await self._base() + text = "Hello, Wörld! 東京 AbC" + + async def focus(box: str) -> None: + x, y = self._at(box, scale) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) + + async def clear() -> None: + await focus("name") + await self.act("computer_key", {"keys": ["Control", "/"]}) + await self.act("computer_key", {"keys": ["Backspace"]}) + + async def unicode() -> str | None: + await clear() + await self.act("computer_type", {"text": text}) + return await self.settled(lambda s: s["name"] == text, lambda s: s["name"]) + + async def events() -> str | None: + state = await self.state() + missing = [k for k in ("H", "W", "space", "comma") if k not in state["keys"]] + return f"no key events for {missing}" if missing else None + + async def chord() -> str | None: + await focus("name") + await self.act("computer_key", {"keys": ["Control", "/"]}) + await self.act("computer_key", {"keys": ["Backspace"]}) + return await self.settled(lambda s: s["name"] == "", lambda s: s["name"]) + + async def repeat() -> str | None: + await clear() + await self.act("computer_type", {"text": "abcd"}) + await self.act("computer_key", {"keys": ["Backspace"], "repeat": 2}) + return await self.settled(lambda s: s["name"] == "ab", lambda s: s["name"]) + + async def whitespace() -> str | None: + await focus("notes") + await self.act("computer_type", {"text": "one\ttwo\nthree"}) + return await self.settled(lambda s: s["notes"].endswith("one\ttwo\nthree"), lambda s: s["notes"]) + + async def clipboard() -> str | None: + state = await self.state() + return None if state["clipboard"] == self.layout["clipboard"] else f"the clipboard is {state['clipboard']!r}" + + async def held() -> str | None: + await clear() + await self.act("computer_keyboard_down", {"keys": ["Shift"]}) + await self.act("computer_type", {"text": "abc"}) + await self.act("computer_keyboard_up", {}) + await self.act("computer_type", {"text": "d"}) + return await self.settled(lambda s: s["name"] == "ABCd", lambda s: s["name"]) + + await self.check("P6", "Unicode and mixed-case text arrives whole", unicode) + await self.check("P6", "typed text arrives as key events", events) + await self.check("P6", "a chord selects all, and Backspace clears it", chord) + await self.check("P6", "a repeated key presses that many times", repeat) + await self.check("P6", "tabs and newlines arrive as typed", whitespace) + await self.check("P6", "typing leaves the clipboard as it was", clipboard) + if self.driver.declares(COMPUTER_KEYBOARD): + await self.check("P6", "a key held by computer_keyboard_down stays down while text is typed", held, + COMPUTER_KEYBOARD) + + async def _scroll_and_drag(self) -> None: + frame, scale, _ = await self._base() + + async def scroll_list() -> str | None: + x, y = self._at("list", scale) + await self.act("computer_scroll", {"x": x, "y": y, "direction": "down", "amount": 5, "frame_id": frame}) + return await self.settled(lambda s: s["scroll_top"] > 0, lambda s: s["scroll_top"]) + + async def directions() -> str | None: + x, y = self._at("wheel", scale) + want = {"up": 2, "down": 3, "left": 2, "right": 1} + for direction, amount in want.items(): + await self.act("computer_scroll", {"x": x, "y": y, "direction": direction, "amount": amount, + "frame_id": frame}) + return await self.settled(lambda s: s["wheel"] == want, lambda s: s["wheel"]) + + async def drag() -> str | None: + start, end, waypoint = (50, 50), (300, 250), (200, 100) + points = [start, waypoint, end] if int(self.features.get("max_drag_points", 2)) >= 3 else [start, end] + path = [list(self._at("pad", scale, x, y)) for x, y in points] + await self.act("computer_drag", {"path": path, "duration_ms": 400, "frame_id": frame}) + ok, state = await self.until(lambda s: bool(s["drags"])) + if not ok: + return "no drag reached the pad" + stroke = state["drags"][-1] + if math.dist(stroke["start"], start) > 2 or math.dist(stroke["end"], end) > 2 or stroke["points"] <= 3: + return f"the stroke was {stroke['start']} to {stroke['end']} in {stroke['points']} points" + if len(points) == 3 and min(math.dist(p, waypoint) for p in stroke["path"]) > 8: + return f"the stroke missed its waypoint {waypoint}" + return None + + await self.check("P7", "a scroll moves a list", scroll_list) + await self.check("P7", "scrolls up, down, left and right each arrive that many times", directions) + await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag) + + async def _pointer_input(self) -> None: + if not self.driver.declares(COMPUTER_POINTER): + return + frame, scale, _ = await self._base() + + async def down_up() -> str | None: + before = (await self.state())["saved_count"] + x, y = self._at("save", scale) + await self.act("computer_pointer_down", {"x": x, "y": y, "frame_id": frame}) + await self.act("computer_pointer_up", {}) + await self.act("computer_click", {}) + return await self.settled(lambda s: s["saved_count"] == before + 2, lambda s: s["saved_count"]) + + await self.check("X", "pointer down and up, then a click at the pointer, each press a button", down_up, + COMPUTER_POINTER) + + async def _data_add(self) -> None: + async def picked_up() -> str | None: + message = f"hello from data/add {time.time_ns()}" + added = await self.driver.add_file("inbox/message.txt", message.encode()) + if "error" in added: + return f"data/add refused it: {added['error'].get('message')}" + try: + return await self.settled(lambda s: s["loaded"] == message, lambda s: s["loaded"]) + finally: + await self.driver.run("rm", "-f", "inbox/message.txt") + + await self.check("X", "a file data/add writes is there for an app to read", picked_up, *DATA_PLANE) + + +def _rgb(colour: str) -> tuple[int, int, int]: + return int(colour[1:3], 16), int(colour[3:5], 16), int(colour[5:7], 16) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/client.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/client.py new file mode 100644 index 00000000..4a71ebd8 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/client.py @@ -0,0 +1,82 @@ +"""The probe's view of one running driver: its card, MCP sessions on its tools as a role, its framework extensions, +the data plane, and commands through computer-exec.""" +from __future__ import annotations + +import base64 +import contextlib +import io +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +from mcp import ClientSession +from mcp.client.streamable_http import streamable_http_client +from mcp.types import CallToolResult +from PIL import Image as PILImage + +from ... import client as protocol +from ...tool_context import ROLE_HEADER +from ..card import COMPUTER_EXEC + +#: How long an MCP session's event stream may stay quiet. +STREAM_READ_S = 300 + + +class Driver: + def __init__(self, url: str, card: dict, timeout_s: float) -> None: + self.url = url.rstrip("/") + self.card = card + self.timeout_s = timeout_s + + @classmethod + async def connect(cls, url: str, timeout_s: float) -> Driver: + return cls(url, await protocol.get_card(url.rstrip("/"), timeout=int(timeout_s)), timeout_s) + + @property + def name(self) -> str: + return self.card.get("name") or "" + + @property + def extensions(self) -> list[str]: + return [e["uri"] for e in (self.card.get("capabilities") or {}).get("extensions") or []] + + def declares(self, uri: str) -> bool: + return uri in self.extensions + + def params(self, uri: str) -> dict: + return protocol.extension_params(self.card, uri) + + @contextlib.asynccontextmanager + async def session(self, role: str | None = None) -> AsyncIterator[ClientSession]: + headers = {ROLE_HEADER: role} if role else {} + timeout = httpx.Timeout(self.timeout_s, read=max(self.timeout_s, STREAM_READ_S)) + async with httpx.AsyncClient(headers=headers, timeout=timeout) as http: + async with streamable_http_client(self.url + protocol.mcp_path(self.card), http_client=http) as streams: + async with ClientSession(streams[0], streams[1]) as session: + await session.initialize() + yield session + + async def invoke(self, uri: str, method: str, params: dict | None = None) -> Any: + return await protocol.invoke_extension(self.url, self.card, uri, params or {}, method=method, + timeout=int(self.timeout_s)) + + async def rpc(self, method: str, params: dict | None = None) -> dict: + async with httpx.AsyncClient(timeout=self.timeout_s) as http: + response = await http.post(self.url + (self.card.get("url") or "/agentenv"), + json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}) + return response.json() + + async def run(self, *argv: str, timeout_s: float = 30) -> dict: + return await self.invoke(COMPUTER_EXEC, "run", {"argv": list(argv), "request_id": uuid.uuid4().hex, + "timeout_s": timeout_s}) + + async def add_file(self, path: str, data: bytes) -> dict: + part = {"kind": "file", "file": {"bytes": base64.b64encode(data).decode("ascii")}, + "metadata": {"urn:agentenv:data-objects/v1": {"path": path}}} + return await self.rpc("data/add", {"parts": [part]}) + + +def image(result: CallToolResult) -> PILImage.Image: + block = next(c for c in result.content if c.type == "image") + return PILImage.open(io.BytesIO(base64.b64decode(block.data))).convert("RGB") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py new file mode 100644 index 00000000..c4e7441f --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py @@ -0,0 +1,209 @@ +"""The Linux fixture the conformance probe stages with data/add and launches with computer-exec: a Tk window, the +Fixture Desk, whose widgets' screen boxes are written to /layout.json and every effect that reaches it to +/state.json, so the probe can aim at it and read back what arrived. Run as +``python3 fixture.py ``; it needs a 1280x800 screen or larger.""" +from __future__ import annotations + +import json +import os +import sys +import tkinter as tk +from pathlib import Path + +#: The colours a grid cell cycles through, one more on each click. +COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff") +CELL = 50 +GRID = 8 +#: Single-pixel markers on a black field, at odd and even pixels: (x, y) within the field, then the colour. +MARKERS = ((11, 11, "#ff0000"), (12, 11, "#00ff00"), (11, 12, "#0000ff"), (12, 12, "#ffff00"), (37, 20, "#ff00ff"), + (20, 37, "#00ffff")) +CLIPBOARD = "agentenv-probe clipboard" +MIN_SIZE = (1280, 800) + + +def main(state_dir: Path) -> None: + state_dir.mkdir(parents=True, exist_ok=True) + root = tk.Tk() + root.title("Fixture Desk") + width, height = root.winfo_screenwidth(), root.winfo_screenheight() + root.geometry(f"{width}x{height}+0+0") + root.configure(bg="#f0f0f0") + state: dict = {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None, + "scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [], + "wheel": {"up": 0, "down": 0, "left": 0, "right": 0}, "clipboard": None, "loaded": None, + "screen": [width, height]} + + def save() -> None: + part = state_dir / ".state.json.part" + part.write_text(json.dumps(state)) + os.replace(part, state_dir / "state.json") + + if width < MIN_SIZE[0] or height < MIN_SIZE[1]: + state["error"] = f"the screen is {width}x{height}; the fixture needs {MIN_SIZE[0]}x{MIN_SIZE[1]}" + save() + root.destroy() + return + + font = ("DejaVu Sans", 16) + tk.Label(root, text="Name:", font=font, bg="#f0f0f0").place(x=40, y=40) + name = tk.Entry(root, font=font, width=30) + name.place(x=140, y=40) + + def key(event) -> None: + state["name"] = name.get() + state["keys"] = (state["keys"] + [event.keysym])[-100:] + state["clipboard"] = _clipboard(root) + save() + + name.bind("", key) + + def on_save() -> None: + state["saved_count"] += 1 + save() + + button = tk.Button(root, text="Save", font=font, command=on_save, width=8) + button.place(x=560, y=34) + + subscribed = tk.BooleanVar(value=False) + + def on_subscribe() -> None: + state["subscribed"] = subscribed.get() + save() + + subscribe = tk.Checkbutton(root, text="Subscribe", font=font, variable=subscribed, command=on_subscribe, + bg="#f0f0f0") + subscribe.place(x=40, y=100) + + notes = tk.Text(root, font=("DejaVu Sans", 14), width=28, height=3) + notes.place(x=760, y=40) + + def note(event) -> None: + state["notes"] = notes.get("1.0", "end-1c") + save() + + notes.bind("", note) + + frame = tk.Frame(root) + frame.place(x=40, y=160, width=300, height=400) + listbox = tk.Listbox(frame, font=("DejaVu Sans", 14), exportselection=False) + listbox.pack(side="left", fill="both", expand=True) + bar = tk.Scrollbar(frame, command=listbox.yview) + bar.pack(side="right", fill="y") + + def scrolled(first, last) -> None: + bar.set(first, last) + state["scroll_top"] = float(first) + save() + + listbox.configure(yscrollcommand=scrolled) + for i in range(1, 101): + listbox.insert("end", f"Item {i}") + + def select(event) -> None: + chosen = listbox.curselection() + state["selected"] = listbox.get(chosen[0]) if chosen else None + save() + + listbox.bind("<>", select) + + grid = tk.Canvas(root, width=GRID * CELL, height=GRID * CELL, bg="#ffffff", highlightthickness=0) + grid.place(x=400, y=160) + for row in range(GRID): + for col in range(GRID): + grid.create_rectangle(col * CELL, row * CELL, (col + 1) * CELL, (row + 1) * CELL, fill=COLOURS[0], + outline="#999999", tags=(f"cell-{row}-{col}",)) + + def cell(event, right: bool) -> None: + row, col = event.y // CELL, event.x // CELL + count = (state["cells"].get(f"{row},{col}", 0) + 1) % len(COLOURS) + state["cells"][f"{row},{col}"] = count + grid.itemconfigure(f"cell-{row}-{col}", fill=COLOURS[count]) + state["right_clicks"] += right + save() + + def double(event) -> None: + state["double_clicks"] += 1 + save() + + grid.bind("", lambda e: cell(e, False)) + grid.bind("", lambda e: cell(e, True)) + grid.bind("", double) + + pad = tk.Canvas(root, width=400, height=300, bg="#fffbe6", highlightthickness=0) + pad.place(x=840, y=160) + stroke: list = [] + + def press(event) -> None: + stroke[:] = [(event.x, event.y)] + + def motion(event) -> None: + if stroke: + pad.create_line(*stroke[-1], event.x, event.y, width=3) + stroke.append((event.x, event.y)) + + def release(event) -> None: + if stroke: + stroke.append((event.x, event.y)) + state["drags"] = (state["drags"] + [{"start": stroke[0], "end": stroke[-1], "points": len(stroke), + "path": stroke[::max(1, len(stroke) // 50)] + stroke[-1:]}])[-20:] + save() + stroke.clear() + + pad.bind("", press) + pad.bind("", motion) + pad.bind("", release) + + wheel = tk.Canvas(root, width=200, height=120, bg="#e6f0ff", highlightthickness=0) + wheel.place(x=840, y=500) + + def wheeled(event, button: int) -> None: + # Tk 8.6 delivers X buttons 6 and 7 (horizontal) as 4 and 5 with Shift held. + horizontal = bool(event.state & 1) + state["wheel"][("left" if horizontal else "up") if button == 4 else ("right" if horizontal else "down")] += 1 + save() + + wheel.bind("", lambda e: wheeled(e, 4)) + wheel.bind("", lambda e: wheeled(e, 5)) + + field = tk.Canvas(root, width=48, height=48, bg="#000000", highlightthickness=0, bd=0) + field.place(x=1080, y=500) + for x, y, colour in MARKERS: + field.create_rectangle(x, y, x + 1, y + 1, fill=colour, outline="", width=0) + + def poll_inbox() -> None: + message = Path.home() / "inbox" / "message.txt" + if message.exists() and (text := message.read_text()) != state["loaded"]: + state["loaded"] = text + save() + root.after(200, poll_inbox) + + def write_layout() -> None: + root.update_idletasks() + boxes = {key: [w.winfo_rootx(), w.winfo_rooty(), w.winfo_width(), w.winfo_height()] + for key, w in (("name", name), ("save", button), ("subscribe", subscribe), ("notes", notes), + ("list", listbox), ("grid", grid), ("pad", pad), ("wheel", wheel), ("field", field))} + boxes["list_row_height"] = listbox.bbox(0)[3] if listbox.bbox(0) else 0 + boxes["markers"] = [[field.winfo_rootx() + x, field.winfo_rooty() + y, colour] for x, y, colour in MARKERS] + boxes["cell"] = CELL + boxes["clipboard"] = CLIPBOARD + boxes["pid"] = os.getpid() + save() + (state_dir / "layout.json").write_text(json.dumps(boxes)) + + root.clipboard_clear() + root.clipboard_append(CLIPBOARD) + state["clipboard"] = CLIPBOARD + root.after(300, write_layout) + root.after(500, poll_inbox) + root.mainloop() + + +def _clipboard(root: tk.Tk) -> str | None: + try: + return root.clipboard_get() + except tk.TclError: + return None + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/report.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/report.py new file mode 100644 index 00000000..2d80a580 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/report.py @@ -0,0 +1,43 @@ +"""What a probe found: each check's status, and each declared extension verified, unverified or failed. The report is +the value M0.7 stores in an env's ``metadata.validated_boot``.""" +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + +Level = Literal["basic", "full"] +Status = Literal["passed", "failed", "unverified", "not_applicable"] +Verdict = Literal["verified", "unverified", "failed"] + + +class CheckResult(BaseModel): + #: The design's check id, P0 to P9, or X for a check of one extension's own methods. + check: str + name: str + status: Status + detail: str = "" + #: The extension URNs this result counts toward. + extensions: list[str] = Field(default_factory=list) + + +class ProbeReport(BaseModel): + level: Level + driver: str + passed: bool + checks: list[CheckResult] + extensions: dict[str, Verdict] + + @classmethod + def of(cls, level: Level, driver: str, checks: list[CheckResult], declared: list[str]) -> ProbeReport: + """The report for ``checks``: it passes when none failed, and each declared extension is failed when one of + its checks failed, verified when one passed, and unverified otherwise.""" + extensions: dict[str, Verdict] = {} + for uri in declared: + mine = [c.status for c in checks if uri in c.extensions] + extensions[uri] = "failed" if "failed" in mine else "verified" if "passed" in mine else "unverified" + return cls(level=level, driver=driver, passed=all(c.status != "failed" for c in checks), checks=checks, + extensions=extensions) + + def counts(self) -> dict[str, int]: + return {status: sum(c.status == status for c in self.checks) for status in Status.__args__} diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py new file mode 100644 index 00000000..ef3d5ae0 --- /dev/null +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -0,0 +1,211 @@ +"""The conformance fixture, faked on the fake driver: the Fixture Desk's layout, its markers painted on the fake screen, +and its state built from the input that reaches the screen, all served through the driver's data/add and +computer-exec as the real fixture is. Commands that aren't the fixture's run for real, through the Linux default. +``served`` puts a driver on loopback, so the probe uses its own HTTP and MCP clients.""" +from __future__ import annotations + +import contextlib +import json +import math +import socket +import threading +import time +from collections.abc import Iterator +from pathlib import Path + +import uvicorn +from computer_fake import FakeDriver + +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + ExecResult, + click, + drag, + pointer_up, + press, + run_in_session, + scroll, + type_text, +) +from agentenv_protocol.computer import linux +from agentenv_protocol.computer.conformance.checks import STAGE + +CLIPBOARD = "agentenv-probe clipboard" +FIELD = (1080, 500) +MARKERS = ((11, 11, "#ff0000"), (12, 11, "#00ff00"), (11, 12, "#0000ff"), (12, 12, "#ffff00"), (37, 20, "#ff00ff"), + (20, 37, "#00ffff")) +LAYOUT = {"name": [140, 40, 420, 34], "save": [560, 34, 120, 44], "subscribe": [40, 100, 160, 34], + "notes": [760, 40, 380, 90], "list": [40, 160, 280, 400], "grid": [400, 160, 400, 400], + "pad": [840, 160, 400, 300], "wheel": [840, 500, 200, 120], "field": [*FIELD, 48, 48], "list_row_height": 24, + "markers": [[FIELD[0] + x, FIELD[1] + y, c] for x, y, c in MARKERS], "cell": 50, "clipboard": CLIPBOARD, + "pid": 4242} +KEYSYMS = {" ": "space", ",": "comma", "!": "exclam", "\t": "Tab", "\n": "Return"} + + +class FakeDesk: + """What the Fixture Desk would record of the input the fake screen gets.""" + + def __init__(self, driver: FakeDriver) -> None: + self.driver = driver + self.started = False + self.focus: str | None = None + self.selected_all = False + self.state = {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None, + "scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [], + "wheel": {"up": 0, "down": 0, "left": 0, "right": 0}, "clipboard": CLIPBOARD, "loaded": None, + "screen": [1280, 800]} + + def start(self) -> None: + screen = self.driver.screen + for x in range(48): + for y in range(48): + screen.paint(FIELD[0] + x, FIELD[1] + y, (0, 0, 0)) + for x, y, colour in MARKERS: + screen.paint(FIELD[0] + x, FIELD[1] + y, (int(colour[1:3], 16), int(colour[3:5], 16), int(colour[5:7], 16))) + self.started = True + + def read(self) -> dict: + message = Path.home() / "inbox" / "message.txt" + self.state["loaded"] = message.read_text() if message.exists() else self.state["loaded"] + return self.state + + def _in(self, box: str, x: float, y: float) -> tuple[float, float] | None: + x0, y0, w, h = LAYOUT[box] + return (x - x0, y - y0) if x0 <= x < x0 + w and y0 <= y < y0 + h else None + + def click(self, x: float, y: float, button: str, count: int) -> None: + if not self.started: + return + if (at := self._in("grid", x, y)) is not None: + key = f"{int(at[1] // 50)},{int(at[0] // 50)}" + self.state["cells"][key] = (self.state["cells"].get(key, 0) + count) % 4 + self.state["right_clicks"] += button == "right" + self.state["double_clicks"] += count >= 2 + elif self._in("save", x, y) is not None: + self.state["saved_count"] += 1 + elif self._in("subscribe", x, y) is not None: + self.state["subscribed"] = not self.state["subscribed"] + for box in ("name", "notes"): + if self._in(box, x, y) is not None: + self.focus = box + + def type(self, text: str) -> None: + if self.focus is None: + return + held = "Shift" in self.driver.screen.held + text = text.upper() if held else text + self.state[self.focus] = ("" if self.selected_all else self.state[self.focus]) + text + self.selected_all = False + self.state["keys"] = (self.state["keys"] + [KEYSYMS.get(c, c) for c in text])[-100:] + + def press(self, keys: list[str]) -> None: + if self.focus is None: + return + if keys == ["Control", "/"]: + self.selected_all = True + elif keys == ["Backspace"]: + value = self.state[self.focus] + self.state[self.focus] = "" if self.selected_all else value[:-1] + self.selected_all = False + + def scroll(self, x: float, y: float, direction: str, amount: int) -> None: + if self._in("list", x, y) is not None and direction == "down": + self.state["scroll_top"] = min(1.0, self.state["scroll_top"] + 0.01 * amount) + elif self._in("wheel", x, y) is not None: + self.state["wheel"][direction] += amount + + def drag(self, path: list) -> None: + local = [self._in("pad", x, y) for x, y in path] + if None in local: + return + stroke = [local[0]] + for (x0, y0), (x1, y1) in zip(local, local[1:]): + steps = max(2, int(math.dist((x0, y0), (x1, y1)) // 10)) + stroke += [(x0 + (x1 - x0) * k / steps, y0 + (y1 - y0) * k / steps) for k in range(1, steps + 1)] + self.state["drags"].append({"start": list(stroke[0]), "end": list(stroke[-1]), "points": len(stroke), + "path": [list(p) for p in stroke]}) + + def release(self) -> None: + if self.started and self._in("save", *self.driver.screen.pointer) is not None: + self.state["saved_count"] += 1 + + +class FixtureDriver(FakeDriver): + """The fake driver with the fake desk behind its input, its data/add and its computer-exec.""" + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.desk = FakeDesk(self) + self.commands: list[list[str]] = [] + + @click + async def click_at(self, x, y, button, count, modifiers): + effect = await super().click_at(x, y, button, count, modifiers) + self.desk.click(x, y, button, count) + return effect + + @drag + async def drag_along(self, path, button, duration_ms, modifiers): + await super().drag_along(path, button, duration_ms, modifiers) + self.desk.drag(path) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): + await super().scroll_at(x, y, direction, amount, modifiers) + self.desk.scroll(x, y, direction, amount) + + @type_text + async def type_in(self, text): + await super().type_in(text) + self.desk.type(text) + + @press + async def press_keys(self, keys): + await super().press_keys(keys) + self.desk.press(list(keys)) + + @pointer_up + async def up(self, button): + await super().up(button) + self.desk.release() + + @run_in_session + async def run(self, argv, cwd, env, timeout_s): + self.commands.append(list(argv)) + answer = self._fixture(argv) + if answer is not None: + return answer + return await linux.run_in_session(argv, cwd, env, timeout_s, output_bytes=self.settings.exec_output_bytes) + + def _fixture(self, argv: list[str]) -> ExecResult | None: + text = " ".join(argv) + if argv[:1] == ["cat"] and argv[1].endswith("layout.json"): + return ExecResult(0, json.dumps(LAYOUT).encode()) if self.desk.started else ExecResult(1, b"", b"missing") + if argv[:1] == ["cat"] and argv[1].endswith("state.json"): + return ExecResult(0, json.dumps(self.desk.read()).encode()) if self.desk.started else ExecResult(1) + if f"python3 {STAGE}/fixture.py" in text: + self.desk.start() + return ExecResult(0) + if argv in (["id", "-u"], ["stat", "-c", "%u", f"{STAGE}/fixture.py"]): + return ExecResult(0, b"1000\n") + if argv[:1] in (["kill"], ["tail"]) or f"rm -rf {STAGE}/state" in text: + return ExecResult(0) + return None + + +@contextlib.contextmanager +def served(driver: ComputerDriverEnvironment) -> Iterator[str]: + """``driver`` on loopback in a thread of its own: the URL its card is served at.""" + app = (driver.mcp or driver.create_app()).streamable_http_app() + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + server = uvicorn.Server(uvicorn.Config(app, log_level="warning")) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + while not server.started: + time.sleep(0.01) + try: + yield f"http://127.0.0.1:{sock.getsockname()[1]}" + finally: + server.should_exit = True + thread.join(5) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance.py b/packages/agentenv-protocol/tests/test_computer_conformance.py new file mode 100644 index 00000000..aa329edf --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance.py @@ -0,0 +1,225 @@ +"""The conformance probe against fake drivers on loopback: the fake driver with the fake fixture passes every check; a +driver without what full needs gets those checks unverified; a driver broken one way fails the check for that way; +the report's verdicts and the CLI.""" +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeScreen, MinimalDriver # noqa: E402 +from mcp.types import CallToolResult, TextContent # noqa: E402 +from PIL import ImageChops # noqa: E402 +from probe_fake import FixtureDriver, served # noqa: E402 + +from agentenv_protocol.computer import ( # noqa: E402 + ExecResult, + Platform, + ReadyCheck, + capture, + click, + drag, + key_down, + run_in_session, + scroll, + type_text, +) +from agentenv_protocol.computer.conformance import CheckResult, ProbeReport, run_probe # noqa: E402 +from agentenv_protocol.computer.conformance import checks # noqa: E402 +from agentenv_protocol.computer.conformance.__main__ import main # noqa: E402 + + +@pytest.fixture(autouse=True) +def home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +async def _probe(driver, level="full", **kwargs) -> ProbeReport: + with served(driver) as url: + return await run_probe(url, level, **kwargs) + + +def _failed(report: ProbeReport) -> list[str]: + return [f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed"] + + +def _status(report: ProbeReport, name: str) -> str: + return next(c.status for c in report.checks if c.name == name) + + +@pytest.mark.asyncio +async def test_the_fake_driver_passes_every_check_at_full_and_cleans_up(home): + driver = FixtureDriver() + report = await _probe(driver) + assert report.passed, _failed(report) + assert report.driver == "FixtureDriver" and report.level == "full" + assert set(report.extensions.values()) == {"verified"} and len(report.extensions) == 11 + counts = report.counts() + assert counts["failed"] == counts["unverified"] == 0 and counts["not_applicable"] == 1 + assert {c.check for c in report.checks} == {f"P{n}" for n in range(10)} | {"X"} + assert not (home / ".agentenv-probe").exists() and ["kill", "4242"] in driver.commands + + +@pytest.mark.asyncio +async def test_basic_needs_no_fixture(home): + driver = FixtureDriver() + report = await _probe(driver, "basic") + assert report.passed, _failed(report) + assert {c.check for c in report.checks} == {"P0", "P1", "P2", "P5", "P8", "X"} + assert not driver.desk.started and not (home / ".agentenv-probe").exists() + + +@pytest.mark.asyncio +async def test_a_driver_without_exec_or_data_add_passes_basic_and_leaves_full_unverified(): + report = await _probe(MinimalDriver(FakeScreen(640, 400), platform=Platform(os="macos"))) + assert report.passed, _failed(report) + unverified = [c for c in report.checks if c.status == "unverified"] + assert {c.check for c in unverified} == {"P3", "P4", "P5", "P6", "P7", "P9"} + assert "the driver lacks ['urn:agentenv:computer-exec/v1', 'data/add']" in unverified[0].detail + assert "urn:agentenv:computer-exec/v1" not in report.extensions + assert report.extensions["urn:agentenv:computer/v1"] == "verified" + + +class NeverReady(FixtureDriver): + async def ready_checks(self): + return [ReadyCheck("session", False, "no desktop session")] + + +class ClicksOff(FixtureDriver): + @click + async def click_at(self, x, y, button, count, modifiers): + return await super().click_at(x + 60, y, button, count, modifiers) + + +class ShiftedCapture(FixtureDriver): + @capture + async def grab(self): + return ImageChops.offset(self.screen.image, 1, 0) + + +class AsciiOnly(FixtureDriver): + @type_text + async def type_in(self, text): + await super().type_in("".join(c for c in text if ord(c) < 128)) + + +class SwappedScroll(FixtureDriver): + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): + swapped = {"left": "right", "right": "left"}.get(direction, direction) + await super().scroll_at(x, y, swapped, amount, modifiers) + + +class StraightDrag(FixtureDriver): + @drag + async def drag_along(self, path, button, duration_ms, modifiers): + await super().drag_along([path[0], path[-1]], button, duration_ms, modifiers) + + +class NoHold(FixtureDriver): + @key_down + async def keys_down(self, keys): + self.screen.log.append(("key_down", tuple(keys))) + + +class ExitZero(FixtureDriver): + @run_in_session + async def run(self, argv, cwd, env, timeout_s): + result = await super().run(argv, cwd, env, timeout_s) + return ExecResult(0, result.stdout, result.stderr, result.timed_out, result.truncated) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("driver, broken", [ + (NeverReady, "computer-ready reports ready"), + (ClicksOff, "a click in frame space hits its cell, and the next frame shows it"), + (ShiftedCapture, "markers at odd and even pixels read back in frame pixels"), + (AsciiOnly, "Unicode and mixed-case text arrives whole"), + (SwappedScroll, "scrolls up, down, left and right each arrive that many times"), + (StraightDrag, "a drag starts and ends where it says, through its waypoints"), + (NoHold, "a key held by computer_keyboard_down stays down while text is typed"), + (ExitZero, "a command's exit code and output come back"), +]) +async def test_a_driver_broken_one_way_fails_the_check_for_it(driver, broken, monkeypatch): + monkeypatch.setattr(checks, "SETTLE_S", 0.5) + report = await _probe(driver(), ready_timeout_s=1) + failed = next(c for c in report.checks if c.name == broken) + assert failed.status == "failed" and not report.passed + assert failed.extensions and all(report.extensions[uri] == "failed" for uri in failed.extensions) + + +class NoFixture(FixtureDriver): + def _fixture(self, argv): + if argv[:1] == ["cat"] and argv[1].endswith("state.json"): + return ExecResult(0, json.dumps({"error": "the screen is 800x600; the fixture needs 1280x800"}).encode()) + if "python3" in " ".join(argv): + return ExecResult(0) + return super()._fixture(argv) + + +class SilentFixture(FixtureDriver): + def _fixture(self, argv): + if "python3" in " ".join(argv): + return ExecResult(0) + if argv[:1] == ["tail"]: + return ExecResult(0, b"ModuleNotFoundError: No module named 'tkinter'\n") + return super()._fixture(argv) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("driver, why", [ + (NoFixture, "the fixture didn't start: the screen is 800x600"), + (SilentFixture, "no layout after 0.6 s: ModuleNotFoundError: No module named 'tkinter'"), +]) +async def test_a_fixture_that_doesnt_start_leaves_full_unverified(driver, why, monkeypatch): + monkeypatch.setattr(checks, "FIXTURE_START_S", 0.6) + report = await _probe(driver()) + assert report.passed, _failed(report) + unverified = [c for c in report.checks if c.status == "unverified"] + assert len(unverified) == len(checks.FULL_CHECKS) and why in unverified[0].detail + + +def test_a_refusal_must_carry_the_code_and_applied_no(): + def result(error: bool, **data) -> CallToolResult: + return CallToolResult(content=[TextContent(type="text", text="x")], structuredContent=data, isError=error) + + assert "wasn't refused" in checks.Probe.refused(result(False, summary="Clicked"), "out_of_bounds") + assert "not 'out_of_bounds'" in checks.Probe.refused(result(True, code="internal"), "out_of_bounds") + assert "applied 'unknown'" in checks.Probe.refused(result(True, code="out_of_bounds", applied="unknown"), + "out_of_bounds") + assert checks.Probe.refused(result(True, code="out_of_bounds", applied="no"), "out_of_bounds") is None + + +def test_declared_leasing_is_unverified_until_the_probe_checks_leases(): + driver = SimpleNamespace(params=lambda uri: {"leasing": {}}, declares=lambda uri: True) + probe = checks.Probe(driver, None, 1) + probe._grants() + assert probe.results[0].status == "unverified" and probe.results[0].check == "P8" + + +def test_an_extension_is_failed_by_any_check_verified_by_a_pass_and_unverified_otherwise(): + results = [CheckResult(check="X", name="a", status="passed", extensions=["e1", "e2"]), + CheckResult(check="X", name="b", status="failed", extensions=["e2"]), + CheckResult(check="X", name="c", status="unverified", extensions=["e3"])] + report = ProbeReport.of("basic", "d", results, ["e1", "e2", "e3", "e4"]) + assert report.extensions == {"e1": "verified", "e2": "failed", "e3": "unverified", "e4": "unverified"} + assert not report.passed and report.counts() == {"passed": 1, "failed": 1, "unverified": 1, "not_applicable": 0} + + +def test_the_cli_prints_each_check_writes_the_report_and_exits_by_the_result(tmp_path, capsys): + out = tmp_path / "report.json" + with served(FixtureDriver()) as url: + passed = main([url, "--level", "basic", "--json", str(out)]) + with served(ExitZero()) as url: + failed = main([url, "--level", "basic"]) + unreachable = main(["http://127.0.0.1:9", "--timeout", "2"]) + printed = capsys.readouterr() + assert (passed, failed, unreachable) == (0, 1, 2) + assert "PASS P0 computer-ready reports ready" in printed.out and "FAIL X a command's exit code" in printed.out + assert "couldn't probe http://127.0.0.1:9" in printed.err + assert ProbeReport.model_validate_json(out.read_text()).passed diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_checks.py b/packages/agentenv-protocol/tests/test_computer_conformance_checks.py new file mode 100644 index 00000000..b2a142e0 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance_checks.py @@ -0,0 +1,283 @@ +"""Each probe check against stubs that answer the way a driver outside our base class might: the failure each way is +reported with what was seen, and checks for what a driver doesn't declare are left out.""" +from __future__ import annotations + +import base64 +import contextlib +import io +import json +from types import SimpleNamespace + +import pytest + +pytest.importorskip("mcp") +pytest.importorskip("PIL") + +from mcp.types import CallToolResult, ImageContent, TextContent, Tool # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol.client import ExtensionCallError # noqa: E402 +from agentenv_protocol.computer.card import ( # noqa: E402 + COMPUTER, + COMPUTER_EXEC, + COMPUTER_POINTER, + COMPUTER_READY, + COMPUTER_SPACES, + GROUP_TOOLS, +) +from agentenv_protocol.computer.conformance import checks # noqa: E402 +from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI # noqa: E402 +from agentenv_protocol.types import INTAKE_EXTENSION_URI # noqa: E402 + +SCHEMA = {"type": "object"} + + +def png(width: int, height: int, colour=(255, 255, 255)) -> str: + buffer = io.BytesIO() + PILImage.new("RGB", (width, height), colour).save(buffer, "PNG") + return base64.b64encode(buffer.getvalue()).decode() + + +def shot(width=100, height=80, frame="f1", parent=None, image=None, display=None) -> CallToolResult: + data = {"frame_id": frame, "width": width, "height": height, "parent_frame_id": parent, + "display": dict(zip(("width", "height", "scale"), display or (width, height, 1.0)))} + return CallToolResult(content=[TextContent(type="text", text="shot"), + ImageContent(type="image", data=image or png(width, height), mimeType="image/png")], + structuredContent=data) + + +def ok(**data) -> CallToolResult: + return CallToolResult(content=[TextContent(type="text", text="ok")], structuredContent=data) + + +def error(text: str, **data) -> CallToolResult: + return CallToolResult(content=[TextContent(type="text", text=text)], structuredContent=data, isError=True) + + +class Session: + def __init__(self, call=None, tools=None) -> None: + self.call = call or (lambda name, args: shot() if name == "computer_screenshot" else ok()) + self.tools = tools if tools is not None else [*checks.MINIMUM_TOOLS] + + async def list_tools(self): + return SimpleNamespace(tools=[Tool(name=n, inputSchema=SCHEMA, outputSchema=SCHEMA) for n in self.tools]) + + async def call_tool(self, name, args=None): + return self.call(name, args or {}) + + +class Driver: + def __init__(self, extensions=(), params=None, invoke=None, run=None, rpc=None, other=None) -> None: + self.extensions = list(extensions) + self._params = params or {} + self._invoke = invoke or (lambda uri, method, params: {}) + self._run = run or (lambda argv: {"exit_code": 0, "stdout": "", "stderr": ""}) + self._rpc = rpc or (lambda method, params: {"result": {}}) + self.other = other or Session() + self.card = {"capabilities": {"extensions": [{"uri": u, "params": self._params.get(u, {})} + for u in self.extensions]}} + + def declares(self, uri): + return uri in self.extensions + + def params(self, uri): + return self._params.get(uri, {}) + + async def invoke(self, uri, method, params=None): + return self._invoke(uri, method, params or {}) + + async def run(self, *argv, timeout_s=30): + return self._run(list(argv)) + + async def rpc(self, method, params=None): + return self._rpc(method, params or {}) + + async def add_file(self, path, data): + return self._rpc("data/add", {"path": path}) + + @contextlib.asynccontextmanager + async def session(self, role=None): + yield self.other + + +def probe(driver=None, session=None) -> checks.Probe: + return checks.Probe(driver or Driver([COMPUTER]), session or Session(), 0.1) + + +def details(p: checks.Probe) -> dict[str, tuple[str, str]]: + return {r.name: (r.status, r.detail) for r in p.results} + + +@pytest.mark.asyncio +async def test_a_card_without_readiness_or_tools_fails_p0_and_p1(): + p = probe(Driver([COMPUTER, COMPUTER_POINTER]), Session(tools=["computer_screenshot", "computer_pointer_move"])) + await p._ready() + await p._card() + got = details(p) + assert got["computer-ready reports ready"] == ("failed", "the card doesn't declare computer-ready/v1") + assert got["the seven minimum tools are listed with their schemas"][1].startswith("missing ['computer_click'") + assert "computer-pointer/v1 is declared without" in got["each group's tools come with its extension"][1] + + +@pytest.mark.asyncio +async def test_group_tools_without_their_extension_fail_p1(): + p = probe(session=Session(tools=[*checks.MINIMUM_TOOLS, *GROUP_TOOLS["keyboard"]])) + await p._card() + assert "are listed without urn:agentenv:computer-keyboard/v1" in details(p)["each group's tools come with its extension"][1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("answer, name, problem", [ + (lambda n, a: shot(display=(50, 40, 1.0)), "a screenshot is native: its image, frame and display agree", + "the display (50, 40)"), + (lambda n, a: error("out_of_bounds: no") if a.get("region") else shot(), "a region comes back as a native crop of its frame", + "out_of_bounds: no"), + (lambda n, a: shot(10, 10, frame="f2", parent="f1") if a.get("region") else shot(), + "a region comes back as a native crop of its frame", "region came back (10, 10)"), + (lambda n, a: shot(33, 26, frame="f2", parent="f9") if a.get("region") else shot(), + "a region comes back as a native crop of its frame", "its parent_frame_id is 'f9'"), + (lambda n, a: shot(33, 26, frame="f2", parent="f1", image=png(33, 26, (0, 0, 0))) if a.get("region") else shot(), + "a region comes back as a native crop of its frame", "aren't its frame's"), +]) +async def test_frames_that_arent_native_fail_p2(answer, name, problem): + p = probe(session=Session(call=answer)) + await p._frame() + status, detail = details(p)[name] + assert status == "failed" and problem in detail + + +@pytest.mark.asyncio +async def test_extensions_a_driver_doesnt_declare_get_no_checks(): + p = probe() + for section in (p._display, p._spaces, p._keyboard_settings, p._exec, p._data_get, p._tool_access, p._pointer, + p._pointer_input): + await section() + assert p.results == [] + + +@pytest.mark.asyncio +async def test_a_spaces_set_that_doesnt_take_fails(): + driver = Driver([COMPUTER, COMPUTER_SPACES], invoke=lambda uri, method, params: { + "default_space": "frame", "default_norm_scale": 1000}) + p = probe(driver) + await p._spaces() + assert "set norm, then get said" in next(iter(details(p).values()))[1] + + +@pytest.mark.asyncio +async def test_exec_that_runs_a_missing_program_fails_and_no_cap_means_no_cap_check(): + def invoke(uri, method, params): + if params["argv"] == ["agentenv-probe-no-such-program"]: + return {"exit_code": 127, "stdout": "", "stderr": "not found"} + raise ExtensionCallError(SimpleNamespace(status_code=400, json=lambda: {"error": {}}, text="", + request=None)) + + p = probe(Driver([COMPUTER, COMPUTER_EXEC], invoke=invoke)) + p.driver._run = lambda argv: invoke(None, None, {"argv": argv}) + await p._exec() + got = details(p) + assert got["a program that isn't there is refused with a code"] == ("failed", "it answered {'exit_code': 127, " + "'stdout': '', 'stderr': 'not found'}") + assert "output past max_output_bytes is cut there" not in got + + +@pytest.mark.asyncio +@pytest.mark.parametrize("listed, refused, problem", [ + (["computer_wait"], error("x", code="permission_denied"), "still listed"), + ([], ok(), "a disabled tool answered its role"), +]) +async def test_tool_access_that_doesnt_hide_or_refuse_fails(listed, refused, problem): + other = Session(call=lambda n, a: refused, tools=listed) + p = probe(Driver([COMPUTER, DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI], other=other)) + await p._tool_access() + assert problem in next(iter(details(p).values()))[1] + + +@pytest.mark.asyncio +async def test_a_fixture_data_add_refuses_is_unverified_and_a_root_user_fails_p9(): + refusing = Driver([COMPUTER, COMPUTER_EXEC, INTAKE_EXTENSION_URI], + rpc=lambda method, params: {"error": {"message": "read-only home"}}) + refusing.card["capabilities"]["extensions"][-1]["params"] = {"add": [{}]} + p = probe(refusing) + await p.full() + assert {r.detail for r in p.results} == {"the fixture didn't start: data/add refused it: read-only home"} + + root = probe(Driver([COMPUTER, COMPUTER_EXEC], run=lambda argv: {"stdout": "0\n" if argv == ["id", "-u"] else "0\n"})) + await root._security() + assert details(root)["commands and data/add run as one unprivileged user"] == ("failed", "commands run as root") + await root._unstage() + + +def fixture(state: dict, layout: dict | None = None): + """A driver and probe whose fixture is up with ``layout`` and answers ``state``.""" + run = lambda argv: {"exit_code": 0, "stdout": json.dumps(state)} # noqa: E731 + p = probe(Driver([COMPUTER, COMPUTER_EXEC], run=run)) + p.layout = layout or {"grid": [0, 0, 80, 80], "save": [0, 0, 10, 10], "subscribe": [0, 0, 10, 10], + "pad": [0, 0, 80, 60], "name": [0, 0, 10, 10], "markers": [[5, 5, "#ffffff"]], "cell": 10} + return p + + +@pytest.mark.asyncio +async def test_effects_without_spaces_right_or_double_clicks_leave_those_out_and_report_refusals(monkeypatch): + monkeypatch.setattr(checks, "SETTLE_S", 0.01) + p = fixture({"cells": {}, "right_clicks": 0, "double_clicks": 0, "saved_count": 0, "subscribed": False}) + p.session.call = lambda n, a: (shot() if n == "computer_screenshot" else error("busy") if a.get("x") else ok()) + p.features = {"buttons": ["left"], "max_click_count": 1} + await p._geometry() + await p._effects() + got = details(p) + assert "a click in norm space hits its cell" not in got and "a right click arrives as one" not in got + assert "a double click arrives as one" not in got and "norm points land on the same pixels" not in got + assert got["16 clicks sent at once all land"] == ("failed", "16 of 16 were refused") + assert got["a click in frame space hits its cell, and the next frame shows it"][1].startswith("AssertionError") + + +@pytest.mark.asyncio +async def test_crops_the_driver_refuses_fail_p3_and_p4(monkeypatch): + monkeypatch.setattr(checks, "SETTLE_S", 0.01) + p = fixture({"cells": {}}) + p.driver.extensions.append(COMPUTER_SPACES) + p.session.call = lambda n, a: error("no crops") if a.get("region") else shot() + await p._geometry() + await p._effects() + got = details(p) + assert got["norm points land on the same pixels"] == ("failed", "no crops") + assert got["a click in a crop's frame maps back to the screen"] == ("failed", "no crops") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drags, problem", [ + ([], "no drag reached the pad"), + ([{"start": [0, 0], "end": [300, 250], "points": 9, "path": []}], "the stroke was [0, 0] to [300, 250] in 9 points"), +]) +async def test_a_drag_that_doesnt_arrive_whole_fails_p7(drags, problem, monkeypatch): + monkeypatch.setattr(checks, "SETTLE_S", 0.01) + p = fixture({"drags": drags, "scroll_top": 0, "wheel": {}}) + await p._scroll_and_drag() + assert details(p)["a drag starts and ends where it says, through its waypoints"] == ("failed", problem) + + +@pytest.mark.asyncio +async def test_typing_without_the_keyboard_group_leaves_held_keys_out(monkeypatch): + monkeypatch.setattr(checks, "SETTLE_S", 0.01) + p = fixture({"name": "", "notes": "", "keys": [], "clipboard": "x"}, {"name": [0, 0, 10, 10], + "notes": [0, 0, 10, 10], "clipboard": "x"}) + await p._typing() + assert "a key held by computer_keyboard_down stays down while text is typed" not in details(p) + + +@pytest.mark.asyncio +async def test_a_data_add_the_app_cant_see_fails(): + p = fixture({"loaded": None}) + p.driver._rpc = lambda method, params: {"error": {"message": "no inbox"}} + await p._data_add() + assert details(p)["a file data/add writes is there for an app to read"] == ("failed", "data/add refused it: no inbox") + + +@pytest.mark.asyncio +async def test_a_section_that_raises_outside_its_checks_fails_as_one(): + p = probe(session=Session(call=lambda n, a: error("down"))) + p.session.list_tools = None + await p._guarded("P1", p._card) + assert details(p)["card could run"][0] == "failed" + assert COMPUTER_READY not in p.results[0].extensions From 722f65e2178e0c5b261108b5c27dd0eb6d305452 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:20:06 -0700 Subject: [PATCH 03/11] feat(env): add the computer_driver env type A computer_driver env is a ComputerDriverEnvironment subclass with its files: env.toml names its class as module = ":", put stores the folder's files as a file_artifact_universe and records what the class declares for each platform its supports names (the card's name, the extensions it provides, its tools and its computer/v1 features). The class is read in a process of its own by agentenv_protocol.computer._declare, where a module that isn't installed imports as an inert stub, so a driver's own dependencies needn't be installed where it is put. The bundle ledger hashes the folder's files, so a driver is written again only when one changes. Deploying one is refused until agent-env can create a computer to deploy it in. Core now requires agentenv-framework-protocol[computer], which put reads the class with. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/_declare.py | 95 +++++++++++++ pyproject.toml | 2 +- src/agent_env/bundle/ledger.py | 6 +- src/agent_env/bundle/plan.py | 10 +- src/agent_env/env/envs/computer_driver.py | 83 +++++++++++ src/agent_env/env/registry.py | 2 + tst/unit/bundle/computer_driver_env_test.py | 133 ++++++++++++++++++ .../golden/env/computer_driver.json | 53 +++++++ .../serialization/test_golden_wire_format.py | 13 ++ uv.lock | 4 +- 10 files changed, 395 insertions(+), 6 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py create mode 100644 src/agent_env/env/envs/computer_driver.py create mode 100644 tst/unit/bundle/computer_driver_env_test.py create mode 100644 tst/unit/serialization/golden/env/computer_driver.json diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py new file mode 100644 index 00000000..cb3fcef9 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py @@ -0,0 +1,95 @@ +"""Read a computer driver class where its own dependencies may not be installed, as a ``computer_driver`` env's put +does: ``python -m agentenv_protocol.computer._declare `` imports the file with every module +not installed here standing in as an inert stub, and prints what the class declares for each platform it supports as +JSON. Nothing runs beyond defining the module and its classes, and a class that can't be defined even so exits 1 +with why.""" +from __future__ import annotations + +import importlib.abc +import importlib.machinery +import importlib.util +import json +import sys +import types +from pathlib import Path + +from .card import COMPUTER +from .driver import ComputerDriverEnvironment +from .platform import supported + + +class _StubMeta(type): + def __getattr__(cls, name: str): + if name.startswith("__"): + raise AttributeError(name) + return cls + + def __call__(cls, *args, **kwargs): + return cls + + def __getitem__(cls, key): + return cls + + +class _Stub(metaclass=_StubMeta): + """Whatever a stubbed module holds: itself again on any attribute, call or index, and a class to derive from.""" + + +class _StubModule(types.ModuleType): + __path__: list = [] + + def __getattr__(self, name: str): + if name.startswith("__"): + raise AttributeError(name) + return _Stub + + +class _Stubs(importlib.abc.MetaPathFinder, importlib.abc.Loader): + """Last on ``sys.meta_path``: a module no other finder finds imports as a stub.""" + + def find_spec(self, fullname, path, target=None): + return importlib.machinery.ModuleSpec(fullname, self, is_package=True) + + def create_module(self, spec): + return _StubModule(spec.name) + + def exec_module(self, module) -> None: + return None + + +def declare(folder: Path, file: str, name: str) -> dict: + """What the class ``name`` in ``folder/file`` declares: its card's name, its ``supports``, and for each platform + those name, the extensions it provides, its tools and the computer/v1 features. Raises ValueError for a class + that isn't a driver or declares no platform.""" + sys.path.insert(0, str(folder)) + sys.meta_path.append(_Stubs()) + spec = importlib.util.spec_from_file_location("computer_driver_module", folder / file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + cls = getattr(module, name, None) + if not (isinstance(cls, type) and issubclass(cls, ComputerDriverEnvironment)): + raise ValueError(f"{file} has no ComputerDriverEnvironment subclass named {name}") + platforms = [platform for entry in cls.supports for platform in supported(entry)] + if not platforms: + raise ValueError(f"{name} declares no platform: name them with @computer_driver(supports=[...])") + cards = [(platform, cls.declaration(platform)) for platform in platforms] + return {"name": cards[0][1].name, "supports": list(cls.supports), "platforms": [ + {"platform": platform.as_dict(), "provides": [e.uri for e in card.capabilities.extensions], + "tools": [t.name for t in card.capabilities.tools], + "features": next(e.params for e in card.capabilities.extensions if e.uri == COMPUTER)} + for platform, card in cards]} + + +def main(argv: list[str]) -> int: + folder, file, name = argv + try: + declared = declare(Path(folder), file, name) + except Exception as e: + print(f"{type(e).__name__}: {e}", file=sys.stderr) + return 1 + print(json.dumps(declared)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/pyproject.toml b/pyproject.toml index 18e2a3b5..d0330d9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agentenv-framework-protocol>=0.1.0", + "agentenv-framework-protocol[computer]>=0.1.0", # a computer_driver env's put reads the driver class with it "certifi>=2024.2.2", # the public roots the local transfer CA's trust bundle carries "click>=8.0.0", "cryptography>=42", # the local transfer CA diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index a7405dd7..219c79d9 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -33,7 +33,7 @@ from .authoring import build_context_files from .parse import BundleKind -from .plan import Plan, Write, artifact_writer, env_writer, folder_files, unpinned_store_refs +from .plan import Plan, Write, artifact_writer, env_folder_writer, env_writer, folder_files, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage LEDGER_COLLECTION = "bundle_ledger" @@ -121,7 +121,9 @@ def digest(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Digest | config = _config(write) inputs = {"type": _type(write), "config": _sha256(_canonical(config)), "files": {}, "needs": {}, "store_refs": {}} - if write.kind in (BundleKind.ARTIFACT, BundleKind.SKILL) and not isinstance(write.source, ReferencedImage): + from_folder = write.kind in (BundleKind.ARTIFACT, BundleKind.SKILL) or ( + write.kind is BundleKind.ENV and env_folder_writer(get_env_registry().get(_type(write)))) + if from_folder and not isinstance(write.source, ReferencedImage): built = isinstance(write.source, BuiltImage) listing = build_context_files if built else folder_files for key, path in listing(self._plan.bundle_of(write.source.entry), write.source.entry).items(): diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 4ea3160a..b083bf12 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -34,6 +34,7 @@ from agent_env.entity.refs import EntityKind, RefRole, default_agent_refs, ref_sites from agent_env.env.env import Env from agent_env.env.envs._deployment import provider_refusal +from agent_env.env.envs.computer_driver import ComputerDriverEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.website import WebsiteEnv @@ -94,7 +95,9 @@ def _stored_env(id: str, version: int | None) -> Env: _FOLDER_HOOKS = {"written_besides": Artifact, "seeded_environments": SeedsEnvironments} # The env types whose from_toml reads only the toml and what it names, so a type built by one of them is written from # its env.toml and the ledger can list what it's made from. -_ENV_FROM_TOMLS = (Env, MCPServerEnv, WebsiteEnv, MultiEnv) +_ENV_FROM_TOMLS = (Env, MCPServerEnv, WebsiteEnv, MultiEnv, ComputerDriverEnv) +# The env types whose from_toml also reads every file in the env's folder, which the ledger hashes as an artifact's. +_ENV_FROM_FOLDERS = (ComputerDriverEnv,) _NOTHING_TO_RUN = "this bundle has no tasks or evals to run" @@ -1088,6 +1091,11 @@ def env_writer(cls: type | None) -> bool: return cls is not None and next(c for c in cls.__mro__ if "from_toml" in vars(c)) in _ENV_FROM_TOMLS +def env_folder_writer(cls: type | None) -> bool: + """Whether an env of type ``cls`` is written from every file in its folder, as well as its env.toml.""" + return cls is not None and next(c for c in cls.__mro__ if "from_toml" in vars(c)) in _ENV_FROM_FOLDERS + + def artifact_writer(cls: type | None) -> bool: """Whether an artifact of type ``cls`` is written from its bundle folder: by a ``from_toml`` of its own or one it inherits from a type that has one, not the base's, which writes a document and no folder.""" diff --git a/src/agent_env/env/envs/computer_driver.py b/src/agent_env/env/envs/computer_driver.py new file mode 100644 index 00000000..9334b15a --- /dev/null +++ b/src/agent_env/env/envs/computer_driver.py @@ -0,0 +1,83 @@ +"""Computer driver env: the code that serves a computer's tools, a ``ComputerDriverEnvironment`` subclass, with its +files and what it declares for each platform it supports, read from the class at put.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from typing import TYPE_CHECKING, Any, ClassVar, NoReturn, Optional + +from agent_env.artifact import Artifact +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.env.env import Env +from agent_env.store.ids import derive_id + +if TYPE_CHECKING: + from agent_env.bundle.authoring import AuthoringContext + from agent_env.env.env import DeployedEnv + +#: How long reading a driver's class at put may take. +DECLARE_TIMEOUT_S = 120 + + +class ComputerDriverEnv(Env): + type: ClassVar[str] = "computer_driver" + description = "A computer driver: the code that serves a computer's tools, and what it declares" + + def __init__(self, id: str, version: Optional[int], module: str, files: FileArtifactUniverse, declaration: dict, + metadata: Optional[dict[str, Any]] = None): + super().__init__(id, version, metadata=metadata) + self.module = module + self.files = files + #: The card's ``name``, the class's ``supports``, and per platform those name the extensions it ``provides``, + #: its ``tools`` and its computer/v1 ``features``. + self.declaration = declaration + + def to_dict(self) -> dict: + return {**super().to_dict(), "module": self.module, "declaration": self.declaration, + "files": {"id": self.files.id, "version": self.files.version, "type": self.files.type}} + + @classmethod + def from_dict(cls, data: dict) -> ComputerDriverEnv: + files = Artifact.get(data["files"]["id"], version=data["files"]["version"]) + return cls(id=data["id"], version=data.get("version"), module=data["module"], files=files, + declaration=data["declaration"], metadata=data.get("metadata") or {}) + + @classmethod + def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerDriverEnv: + """``module = ":"`` names the driver's class in its folder, which is read in a process of its own + (``agentenv_protocol.computer._declare``), so the driver's own dependencies need not be installed here.""" + fields = ctx.accept(data, module=str) + module = fields.get("module") + if module is None: + ctx.refuse([ctx.config_problem('module is required: the driver\'s file and class, such as ' + '"driver.py:ReferenceDriver"')]) + file, _, name = module.partition(":") + files = ctx.files() + if not name or file not in files: + ctx.refuse([ctx.config_problem(f'module {module!r} names no ":" in this folder; it has ' + f"{', '.join(sorted(files))}")]) + declaration = _declare(ctx, file, name) + universe = FileArtifactUniverse.put_bundled(derive_id(ctx.id, "files"), files=files) + return cls.put(id=ctx.id, module=module, files=universe, declaration=declaration) + + async def deploy(self, **kwargs) -> DeployedEnv: + raise NotImplementedError(f"{self.id} is a computer_driver env, which deploys inside a computer, and " + "agent-env can't create a computer yet; serve the driver from an image of its own " + "as an mcp_server env meanwhile") + + +def _declare(ctx: AuthoringContext, file: str, name: str) -> dict: + try: + done = subprocess.run([sys.executable, "-m", "agentenv_protocol.computer._declare", str(ctx.dir), file, name], + capture_output=True, text=True, timeout=DECLARE_TIMEOUT_S, check=False) + except subprocess.TimeoutExpired: + _refuse(ctx, f"reading {file}:{name} took over {DECLARE_TIMEOUT_S} s") + if done.returncode != 0: + _refuse(ctx, f"{file}:{name} can't be read: {(done.stderr.strip().splitlines() or ['no output'])[-1]}") + return json.loads(done.stdout) + + +def _refuse(ctx: AuthoringContext, problem: str) -> NoReturn: + ctx.refuse([ctx.config_problem(problem)]) diff --git a/src/agent_env/env/registry.py b/src/agent_env/env/registry.py index 7af41d3c..da666648 100644 --- a/src/agent_env/env/registry.py +++ b/src/agent_env/env/registry.py @@ -21,6 +21,7 @@ def _get_type(cls: type["Env"]) -> str: def _builtin_registry() -> dict[str, type["Env"]]: """The built-in env types, by type.""" + from agent_env.env.envs.computer_driver import ComputerDriverEnv from agent_env.env.envs.gateway_server import GatewayEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv @@ -28,6 +29,7 @@ def _builtin_registry() -> dict[str, type["Env"]]: from agent_env.env.envs.website import WebsiteEnv return { + _get_type(ComputerDriverEnv): ComputerDriverEnv, _get_type(GatewayEnv): GatewayEnv, _get_type(MCPServerEnv): MCPServerEnv, _get_type(MultiEnv): MultiEnv, diff --git a/tst/unit/bundle/computer_driver_env_test.py b/tst/unit/bundle/computer_driver_env_test.py new file mode 100644 index 00000000..91d42968 --- /dev/null +++ b/tst/unit/bundle/computer_driver_env_test.py @@ -0,0 +1,133 @@ +"""Computer driver envs written from a bundle's env folders: the driver's files stored with each version, its class read +where its own dependencies aren't installed, what it declares for each platform it supports, a new version when a file +changes, refusals before any write, and a deploy that waits for a computer.""" + +import json + +import pytest + +from agent_env.bundle import BundleError +from agent_env.bundle.materialize import materialize +from agent_env.env import Env +from agent_env.env.envs.computer_driver import ComputerDriverEnv +from tst.unit.bundle._support import layout, plan_of + +ROOT = "@local/~/desks" +DRIVER = '''import not_installed_here +from not_installed_here.input import Keyboard + +from agentenv_protocol.computer import (ComputerDriverEnvironment, Display, capture, click, computer_driver, display, + drag, press, scroll, type_text) + +SHIFT = Keyboard.keysym("Shift_L") + + +@computer_driver(name="desk", supports={supports}) +class Desk(ComputerDriverEnvironment): + buttons = {buttons} + + @display + def size(self): return Display(10, 10) + + @capture + def grab(self): return not_installed_here.grab() + + @click + def click_at(self, x, y, button, count, modifiers): pass + + @drag + def drag_along(self, path, button, duration_ms, modifiers): pass + + @scroll + def scroll_at(self, x, y, direction, amount, modifiers): pass + + @type_text + def type_in(self, text): pass + + @press + def press_keys(self, keys): pass +''' +LINUX = [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] +TOML = 'type = "computer_driver"\nmodule = "driver.py:Desk"\n' + + +@pytest.fixture +def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path / "desks" + + +def _bundle(root, toml=TOML, supports=LINUX, buttons=("left", "right"), **extra): + return layout(root, {"envs/desk/env.toml": toml, + "envs/desk/driver.py": DRIVER.format(supports=supports, buttons=buttons), **extra, + "tasks/t.json": json.dumps([{"id": "agent", "type": "deploy_agent", "env_ids": ["desk"]}])}) + + +def _run(root): + return {done.write.id: (done.version, done.reused, done.reasons) for done in materialize(plan_of(root)).writes} + + +def _problems(root): + with pytest.raises(BundleError) as caught: + _run(root) + return " ".join(caught.value.problems) + + +def test_a_driver_folder_is_written_with_its_files_and_what_its_class_declares(bundle_dir): + _bundle(bundle_dir) + + first = _run(bundle_dir) + env = Env.get(f"{ROOT}/desk") + + assert first[f"{ROOT}/desk"] == (1, False, ("new",)) + assert isinstance(env, ComputerDriverEnv) and env.module == "driver.py:Desk" + assert sorted(env.files.file_artifact_ids) == ["driver.py"] + declared = env.declaration + assert declared["name"] == "desk" and declared["supports"] == LINUX + assert [p["platform"] for p in declared["platforms"]] == [{"os": "linux", "arch": "amd64", "display": "x11"}, + {"os": "linux", "arch": "arm64", "display": "x11"}] + linux = declared["platforms"][0] + assert "urn:agentenv:computer-exec/v1" in linux["provides"] and "urn:agentenv:computer-pointer/v1" not in linux["provides"] + assert len(linux["tools"]) == 7 and linux["features"]["buttons"] == ["left", "right"] + assert ComputerDriverEnv.from_dict(env.to_dict()).to_dict() == env.to_dict() + + +def test_a_driver_is_written_again_only_when_one_of_its_files_changes(bundle_dir): + _bundle(bundle_dir) + _run(bundle_dir) + + assert _run(bundle_dir)[f"{ROOT}/desk"][1] is True + (bundle_dir / "envs/desk/driver.py").write_text(DRIVER.format(supports=LINUX, buttons=("left",))) + rewritten = _run(bundle_dir) + + assert rewritten[f"{ROOT}/desk"] == (2, False, ("files changed: driver.py",)) + assert Env.get(f"{ROOT}/desk").declaration["platforms"][0]["features"]["buttons"] == ["left"] + + +@pytest.mark.parametrize("toml, supports, buttons, problem", [ + ('type = "computer_driver"\n', LINUX, ("left",), "module is required"), + ('type = "computer_driver"\nmodule = "server.py:Desk"\n', LINUX, ("left",), + "module 'server.py:Desk' names no \":\" in this folder; it has driver.py"), + ('type = "computer_driver"\nmodule = "driver.py:Missing"\n', LINUX, ("left",), + "driver.py has no ComputerDriverEnvironment subclass named Missing"), + (TOML, LINUX, ("thumb",), "buttons include left"), + (TOML, [], ("left",), "Desk declares no platform: name them with @computer_driver(supports=[...])"), + ('type = "computer_driver"\nmodule = "driver.py:Desk"\nimage = "x"\n', LINUX, ("left",), "unknown key 'image'"), +]) +def test_a_driver_its_class_cant_be_read_from_is_refused_before_any_write(bundle_dir, toml, supports, buttons, + problem): + _bundle(bundle_dir, toml=toml, supports=supports, buttons=buttons) + + assert problem in _problems(bundle_dir) + with pytest.raises(Exception): + Env.get(f"{ROOT}/desk") + + +@pytest.mark.asyncio +async def test_a_driver_env_doesnt_deploy_until_there_is_a_computer_to_deploy_it_in(bundle_dir): + _bundle(bundle_dir) + _run(bundle_dir) + + with pytest.raises(NotImplementedError, match="deploys inside a computer, and agent-env can't create a computer"): + await Env.get(f"{ROOT}/desk").deploy() + diff --git a/tst/unit/serialization/golden/env/computer_driver.json b/tst/unit/serialization/golden/env/computer_driver.json new file mode 100644 index 00000000..d46cf903 --- /dev/null +++ b/tst/unit/serialization/golden/env/computer_driver.json @@ -0,0 +1,53 @@ +{ + "declaration": { + "name": "reference", + "platforms": [ + { + "features": { + "buttons": [ + "left" + ], + "max_click_count": 1 + }, + "platform": { + "arch": "amd64", + "display": "x11", + "os": "linux" + }, + "provides": [ + "urn:agentenv:computer/v1", + "urn:agentenv:computer-ready/v1" + ], + "tools": [ + "computer_screenshot", + "computer_click" + ] + } + ], + "supports": [ + { + "arch": [ + "amd64" + ], + "os": { + "display": [ + "x11" + ], + "type": "linux" + } + } + ] + }, + "files": { + "id": "tutorial-files", + "type": "file_artifact_universe", + "version": 4 + }, + "id": "reference-driver", + "metadata": { + "owner": "example-team" + }, + "module": "driver.py:ReferenceDriver", + "type": "computer_driver", + "version": 4 +} diff --git a/tst/unit/serialization/test_golden_wire_format.py b/tst/unit/serialization/test_golden_wire_format.py index c157bb7d..e606c7eb 100644 --- a/tst/unit/serialization/test_golden_wire_format.py +++ b/tst/unit/serialization/test_golden_wire_format.py @@ -37,6 +37,7 @@ from agent_env.artifact.artifacts.skill import SkillArtifact from agent_env.artifact.ref import ArtifactRef from agent_env.artifact.registry import get_artifact_registry +from agent_env.env.envs.computer_driver import ComputerDriverEnv from agent_env.env.envs.gateway_server import GatewayEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv @@ -142,6 +143,18 @@ def _website_env() -> WebsiteEnv: } ENV_FIXTURES = { + "computer_driver": lambda: ComputerDriverEnv( + id="reference-driver", + version=4, + module="driver.py:ReferenceDriver", + files=ARTIFACT_FIXTURES["file_artifact_universe"](), + declaration={"name": "reference", "supports": [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}], + "platforms": [{"platform": {"os": "linux", "arch": "amd64", "display": "x11"}, + "provides": ["urn:agentenv:computer/v1", "urn:agentenv:computer-ready/v1"], + "tools": ["computer_screenshot", "computer_click"], + "features": {"buttons": ["left"], "max_click_count": 1}}]}, + metadata={"owner": "example-team"}, + ), "gateway_server": lambda: GatewayEnv( id="gateway-env", version=11, diff --git a/uv.lock b/uv.lock index fdedaf38..eae55eba 100644 --- a/uv.lock +++ b/uv.lock @@ -48,7 +48,7 @@ version = "0.9.1361" source = { editable = "." } dependencies = [ { name = "a2a-sdk" }, - { name = "agentenv-framework-protocol" }, + { name = "agentenv-framework-protocol", extra = ["computer"] }, { name = "certifi" }, { name = "click" }, { name = "cryptography" }, @@ -137,7 +137,7 @@ vercel = [ requires-dist = [ { name = "a2a-sdk", specifier = "==0.3.26" }, { name = "agentenv-framework", extras = ["aws", "boxd", "daytona", "gcp", "sail", "tensorlake", "tenki", "vercel"], marker = "extra == 'dev'" }, - { name = "agentenv-framework-protocol", editable = "packages/agentenv-protocol" }, + { name = "agentenv-framework-protocol", extras = ["computer"], editable = "packages/agentenv-protocol" }, { name = "anyio", marker = "extra == 'vercel'", specifier = ">=4,<5" }, { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "botocore", marker = "extra == 'aws'", specifier = ">=1.35.0" }, From cc71fac7f1f2ead48dc358e7f696a2d3d0bd79c6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:20:51 -0700 Subject: [PATCH 04/11] feat(computer): ship the reference driver in a drivers bundle agent-env ships a drivers bundle with the reference driver, the X11 driver PR #336's example and end-to-end runs proved out, as the computer_driver env envs/reference, named reference by its aliases.toml. The example folder is removed: its fixture is the conformance probe's, and its offline test moves with the driver. A bundle that gives what it holds aliases and has nothing to run now passes the checks a run makes, as a library other bundles pull from; a run of it still says it has nothing to run. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 4 +- .../examples/computer_x11/Dockerfile | 19 -- .../examples/computer_x11/README.md | 60 ------ .../examples/computer_x11/fixture.py | 173 ------------------ .../examples/computer_x11/start.sh | 9 - pyproject.toml | 1 + src/agent_env/bundle/plan.py | 5 +- src/agent_env/drivers/README.md | 14 ++ src/agent_env/drivers/aliases.toml | 3 + .../drivers/envs/reference}/driver.py | 13 +- src/agent_env/drivers/envs/reference/env.toml | 2 + tst/unit/bundle/computer_driver_env_test.py | 18 ++ tst/unit/bundle/installed_test.py | 2 +- tst/unit/bundle/plan_test.py | 13 ++ tst/unit/bundle/shipped_bundles_test.py | 6 +- tst/unit/drivers/__init__.py | 0 .../unit/drivers/reference_driver_test.py | 39 ++-- tst/unit/test_clean_install_wheel.py | 2 +- 18 files changed, 95 insertions(+), 288 deletions(-) delete mode 100644 packages/agentenv-protocol/examples/computer_x11/Dockerfile delete mode 100644 packages/agentenv-protocol/examples/computer_x11/README.md delete mode 100644 packages/agentenv-protocol/examples/computer_x11/fixture.py delete mode 100644 packages/agentenv-protocol/examples/computer_x11/start.sh create mode 100644 src/agent_env/drivers/README.md create mode 100644 src/agent_env/drivers/aliases.toml rename {packages/agentenv-protocol/examples/computer_x11 => src/agent_env/drivers/envs/reference}/driver.py (96%) create mode 100644 src/agent_env/drivers/envs/reference/env.toml create mode 100644 tst/unit/drivers/__init__.py rename packages/agentenv-protocol/tests/test_computer_x11_example.py => tst/unit/drivers/reference_driver_test.py (90%) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 82f360b4..76ba0ec8 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -319,8 +319,8 @@ one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, `computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow). The driver marks the methods that capture the screen and send input; the base class serves the computer contract from them, so the tools, results and errors can't drift from it. A complete -driver for an X11 desktop, with an image to run it in, is in -[`examples/computer_x11/`](https://github.com/scaleapi/agentenv-framework/tree/main/packages/agentenv-protocol/examples/computer_x11). +driver for an X11 desktop is agent-env's reference driver, in its +[`drivers` bundle](https://github.com/scaleapi/agentenv-framework/tree/main/src/agent_env/drivers). ```python from agentenv_protocol.computer import ( diff --git a/packages/agentenv-protocol/examples/computer_x11/Dockerfile b/packages/agentenv-protocol/examples/computer_x11/Dockerfile deleted file mode 100644 index 02b4b7e3..00000000 --- a/packages/agentenv-protocol/examples/computer_x11/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# Built from packages/agentenv-protocol, so the driver runs on this checkout's SDK: -# docker build -f examples/computer_x11/Dockerfile -t x11-desktop . -FROM public.ecr.aws/docker/library/ubuntu:24.04 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3 python3-venv python3-tk xvfb x11-utils fonts-dejavu-core procps \ - libxcb1 libxcb-render0 libxcb-shm0 libxcb-randr0 libxcb-xfixes0 libxcb-xinerama0 \ - && rm -rf /var/lib/apt/lists/* -COPY pyproject.toml README.md LICENSE NOTICE THIRD_PARTY_NOTICES.md /opt/protocol/ -COPY src /opt/protocol/src -RUN python3 -m venv /opt/driver \ - && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer]" "mss>=9,<11" "python-xlib>=0.33,<0.34" -RUN useradd --create-home --uid 10001 user -COPY examples/computer_x11/start.sh examples/computer_x11/driver.py examples/computer_x11/fixture.py /opt/desktop/ -USER user -WORKDIR /home/user -ENV DISPLAY=:99 XDG_SESSION_TYPE=x11 -EXPOSE 18765 -CMD ["/bin/sh", "/opt/desktop/start.sh"] diff --git a/packages/agentenv-protocol/examples/computer_x11/README.md b/packages/agentenv-protocol/examples/computer_x11/README.md deleted file mode 100644 index c69a20f0..00000000 --- a/packages/agentenv-protocol/examples/computer_x11/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# An X11 computer driver - -[`driver.py`](driver.py) is a complete computer driver for an X11 desktop, written on -`ComputerDriverEnvironment` from `agentenv_protocol.computer`: `mss` captures the screen and XTEST, -through `python-xlib`, sends the input. It implements the seven required operations and the pointer -and keyboard groups, declares five buttons, triple clicks and 50-point drags, and adds a ready check -of its own for XTEST. The base class serves everything else from those: the `computer_*` tools, -frames and coordinate spaces, the input queue, the framework extensions and the card, and its Linux -defaults give the driver `computer-exec/v1` and `data/add`. - -The image runs the driver on Xvfb beside [`fixture.py`](fixture.py), the Fixture Desk: a Tk window -with a name field, a checkbox, a 100-item list, a grid of cells, a drawing pad and a Save button. It -records whatever reaches it in `~/fixture/state.json`, so what an agent did can be read back from -the app rather than from the driver's answers, and shows the text of `~/inbox/message.txt`, which -`data/add` can write. [`start.sh`](start.sh) restarts Xvfb and the app if either dies. - -## Run it - -```bash -cd packages/agentenv-protocol -docker build -f examples/computer_x11/Dockerfile -t x11-desktop . -docker run --rm -p 18765:18765 x11-desktop -``` - -The card is at `http://localhost:18765/.well-known/agent-env.json`, readiness at -`/agentenv/ext/computer-ready/status`, and the tools at the MCP endpoint `/mcp`: - -```python -import asyncio - -from mcp import ClientSession -from mcp.client.streamable_http import streamablehttp_client - - -async def main() -> None: - async with streamablehttp_client("http://localhost:18765/mcp") as (read, write, _): - async with ClientSession(read, write) as session: - await session.initialize() - shot = await session.call_tool("computer_screenshot", {}) - print(shot.structuredContent["summary"]) - await session.call_tool("computer_click", {"x": 300, "y": 52}) - typed = await session.call_tool("computer_type", {"text": "Ada Lovelace", "screenshot": True}) - print(typed.structuredContent["summary"]) - - -asyncio.run(main()) -``` - -`AGENTENV_COMPUTER_SETTINGS` takes the driver's settings as one JSON value, for example -`-e AGENTENV_COMPUTER_SETTINGS='{"default_space": "norm", "data_paths": ["fixture"]}'`. - -## In agent-env - -As a bundle env (`envs//Dockerfile`), the image is built from the env's own folder, so copy -the three files there and install `agentenv-framework-protocol[computer]` in place of this checkout. -Name the env with `environment_name` in its `env.toml`. A task then deploys it with `deploy_env` -and gives it to an agent with `deploy_agent`, as for any MCP env. - -`tests/test_computer_x11_example.py` runs the driver against fake `mss` and `Xlib` modules, so it -keeps up with the SDK without an X server. diff --git a/packages/agentenv-protocol/examples/computer_x11/fixture.py b/packages/agentenv-protocol/examples/computer_x11/fixture.py deleted file mode 100644 index 78cd2699..00000000 --- a/packages/agentenv-protocol/examples/computer_x11/fixture.py +++ /dev/null @@ -1,173 +0,0 @@ -"""The Fixture Desk: a Tk window whose every effect is recorded in ~/fixture/state.json, and whose widgets' screen -boxes are written to ~/fixture/layout.json, so a test can aim at them and read back what reached them.""" -import json -import os -import tkinter as tk -from pathlib import Path - -HOME = Path.home() / "fixture" -HOME.mkdir(exist_ok=True) -COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff") - -root = tk.Tk() -root.title("Fixture Desk") -root.geometry("1280x800+0+0") -root.configure(bg="#f0f0f0") -state = {"name": "", "saved_count": 0, "subscribed": False, "selected": None, "cells": {}, "drags": [], - "keys": [], "scroll_top": 0.0, "right_clicks": 0, "double_clicks": 0, "loaded": None} - - -def save_state(): - tmp = HOME / ".state.json.part" - tmp.write_text(json.dumps(state)) - os.replace(tmp, HOME / "state.json") - - -tk.Label(root, text="Name:", font=("DejaVu Sans", 16), bg="#f0f0f0").place(x=40, y=40) -name = tk.Entry(root, font=("DejaVu Sans", 16), width=30) -name.place(x=140, y=40) - - -def on_key(event): - state["name"] = name.get() - state["keys"] = (state["keys"] + [event.keysym])[-50:] - save_state() - - -name.bind("", on_key) - - -def on_save(): - state["name"] = name.get() - state["saved_count"] += 1 - save_state() - - -save = tk.Button(root, text="Save", font=("DejaVu Sans", 16), command=on_save, width=8) -save.place(x=560, y=34) - -subscribed = tk.BooleanVar(value=False) - - -def on_subscribe(): - state["subscribed"] = subscribed.get() - save_state() - - -subscribe = tk.Checkbutton(root, text="Subscribe", font=("DejaVu Sans", 16), variable=subscribed, - command=on_subscribe, bg="#f0f0f0") -subscribe.place(x=40, y=100) - -frame = tk.Frame(root) -frame.place(x=40, y=160, width=300, height=400) -listbox = tk.Listbox(frame, font=("DejaVu Sans", 14), exportselection=False) -listbox.pack(side="left", fill="both", expand=True) -bar = tk.Scrollbar(frame, command=listbox.yview) -bar.pack(side="right", fill="y") - - -def on_scroll(first, last): - bar.set(first, last) - state["scroll_top"] = float(first) - save_state() - - -listbox.configure(yscrollcommand=on_scroll) -for i in range(1, 101): - listbox.insert("end", f"Item {i}") - - -def on_select(event): - chosen = listbox.curselection() - state["selected"] = listbox.get(chosen[0]) if chosen else None - save_state() - - -listbox.bind("<>", on_select) - -grid = tk.Canvas(root, width=400, height=400, bg="#ffffff", highlightthickness=0) -grid.place(x=400, y=160) -CELL = 50 -for row in range(8): - for col in range(8): - grid.create_rectangle(col * CELL, row * CELL, (col + 1) * CELL, (row + 1) * CELL, fill=COLOURS[0], - outline="#999999", tags=(f"cell-{row}-{col}",)) - - -def on_cell(event, button): - row, col = event.y // CELL, event.x // CELL - key = f"{row},{col}" - count = (state["cells"].get(key, 0) + 1) % len(COLOURS) - state["cells"][key] = count - grid.itemconfigure(f"cell-{row}-{col}", fill=COLOURS[count]) - if button == 3: - state["right_clicks"] += 1 - save_state() - - -def on_double(event): - state["double_clicks"] += 1 - save_state() - - -grid.bind("", lambda e: on_cell(e, 1)) -grid.bind("", lambda e: on_cell(e, 3)) -grid.bind("", on_double) - -pad = tk.Canvas(root, width=400, height=400, bg="#fffbe6", highlightthickness=1) -pad.place(x=840, y=160) -stroke = [] - - -def on_press(event): - stroke.clear() - stroke.append((event.x, event.y)) - - -def on_motion(event): - if stroke: - pad.create_line(stroke[-1][0], stroke[-1][1], event.x, event.y, width=3) - stroke.append((event.x, event.y)) - - -def on_release(event): - if stroke: - stroke.append((event.x, event.y)) - state["drags"] = (state["drags"] + [{"start": stroke[0], "end": stroke[-1], "points": len(stroke)}])[-20:] - save_state() - stroke.clear() - - -pad.bind("", on_press) -pad.bind("", on_motion) -pad.bind("", on_release) - -loaded = tk.Label(root, text="", font=("DejaVu Sans", 14), bg="#f0f0f0") -loaded.place(x=40, y=600) - - -def poll_loaded(): - path = Path.home() / "inbox" / "message.txt" - if path.exists(): - text = path.read_text() - if text != state["loaded"]: - state["loaded"] = text - loaded.configure(text=f"Loaded: {text}") - save_state() - root.after(200, poll_loaded) - - -def write_layout(): - root.update_idletasks() - boxes = {} - for key, widget in (("name", name), ("save", save), ("subscribe", subscribe), ("list", listbox), - ("grid", grid), ("pad", pad)): - boxes[key] = [widget.winfo_rootx(), widget.winfo_rooty(), widget.winfo_width(), widget.winfo_height()] - boxes["list_row_height"] = listbox.bbox(0)[3] if listbox.bbox(0) else 0 - (HOME / "layout.json").write_text(json.dumps(boxes)) - save_state() - - -root.after(300, write_layout) -root.after(500, poll_loaded) -root.mainloop() diff --git a/packages/agentenv-protocol/examples/computer_x11/start.sh b/packages/agentenv-protocol/examples/computer_x11/start.sh deleted file mode 100644 index 31dcb19d..00000000 --- a/packages/agentenv-protocol/examples/computer_x11/start.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -# The desktop: Xvfb and the fixture each restart if they die, then the driver serves on MCP_PORT. -(while true; do Xvfb :99 -screen 0 1280x800x24 -nolisten tcp -ac >/tmp/xvfb.log 2>&1; sleep 0.5; done) & -for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done -(while true; do - for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done - python3 /opt/desktop/fixture.py >/tmp/fixture.log 2>&1; sleep 0.5 - done) & -exec /opt/driver/bin/python /opt/desktop/driver.py diff --git a/pyproject.toml b/pyproject.toml index d0330d9f..f37231d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,7 @@ agent-env = "agent_env.cli:cli" [project.entry-points."agent_env.bundles"] agents = "agent_env" +drivers = "agent_env" hello = "agent_env.examples" echo-mcp = "agent_env.examples" mail-mcp = "agent_env.examples" diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index b083bf12..2f17083e 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -531,9 +531,12 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr dict(self.origins), self._published(writes), aliases, override, dict(self.env_alias_uses)) def check(self) -> None: + """The checks a run makes before it reads a store. A bundle with nothing to run passes only when its + aliases.toml gives what it holds aliases: other bundles pull from it, as from a library.""" self._check_writers() self._check_loops() - if not any(entry.entry.kind in (BundleKind.TASK, BundleKind.EVAL) for entry in self.entries): + runs = any(entry.entry.kind in (BundleKind.TASK, BundleKind.EVAL) for entry in self.entries) + if not runs and not any(self.resolved.bundle.aliases.values()): self.problems.append(_NOTHING_TO_RUN) everything = set(self.nodes) self._check_store_refs(everything, stores=False) diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md new file mode 100644 index 00000000..b75176e1 --- /dev/null +++ b/src/agent_env/drivers/README.md @@ -0,0 +1,14 @@ +Computer drivers that come with agent-env, each a `computer_driver` env a computer env serves its tools through. + +`envs/reference/` is the reference driver, for an X11 desktop on Linux (`amd64` and `arm64`): `mss` captures the +screen and XTEST, through `python-xlib`, sends the input, foreground only. It serves the seven `computer_*` tools, the +pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer-spaces`, `computer-keyboard`, and +`computer-exec` and `data/add` through the base class's Linux defaults. Name it from another bundle as `reference`, the +alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/reference`. + +A run that names it writes it: its files, and what its class declares for each platform it supports, read without +`mss` or `python-xlib` installed. It deploys inside a computer, which agent-env can't create yet, so a deploy of it is +refused until then. `python -m agentenv_protocol.computer.conformance ` checks it, or any driver, where it runs. + +This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing +to run. diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml new file mode 100644 index 00000000..7d5edd69 --- /dev/null +++ b/src/agent_env/drivers/aliases.toml @@ -0,0 +1,3 @@ +# Ids a task, or --driver, can name these drivers by; a store's env of the id wins over an alias. +[envs] +reference = ["reference"] diff --git a/packages/agentenv-protocol/examples/computer_x11/driver.py b/src/agent_env/drivers/envs/reference/driver.py similarity index 96% rename from packages/agentenv-protocol/examples/computer_x11/driver.py rename to src/agent_env/drivers/envs/reference/driver.py index aa19b533..d9cae7d5 100644 --- a/packages/agentenv-protocol/examples/computer_x11/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -1,6 +1,7 @@ -"""An X11 computer driver: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off the -event loop, one at a time, on a connection checked before each call and reopened when it drops, so the driver outlives -a restarted X server.""" +"""The reference computer driver, for an X11 desktop: mss captures the screen and XTEST, through python-xlib, sends the +input. X calls run off the event loop, one at a time, on a connection checked before each call and reopened when it +drops, so the driver outlives a restarted X server. It needs ``agentenv-framework-protocol[computer]``, ``mss`` and +``python-xlib`` where it runs.""" from __future__ import annotations import asyncio @@ -59,9 +60,9 @@ DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) -@computer_driver(name="x11-desktop", +@computer_driver(name="reference", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) -class X11Driver(ComputerDriverEnvironment): +class ReferenceDriver(ComputerDriverEnvironment): buttons = ("left", "right", "middle", "back", "forward") max_click_count = 3 max_drag_points = 50 @@ -371,4 +372,4 @@ def _keysym(key: str) -> int: if __name__ == "__main__": - X11Driver().serve() + ReferenceDriver().serve() diff --git a/src/agent_env/drivers/envs/reference/env.toml b/src/agent_env/drivers/envs/reference/env.toml new file mode 100644 index 00000000..6ec1dff4 --- /dev/null +++ b/src/agent_env/drivers/envs/reference/env.toml @@ -0,0 +1,2 @@ +type = "computer_driver" +module = "driver.py:ReferenceDriver" diff --git a/tst/unit/bundle/computer_driver_env_test.py b/tst/unit/bundle/computer_driver_env_test.py index 91d42968..ec3d4956 100644 --- a/tst/unit/bundle/computer_driver_env_test.py +++ b/tst/unit/bundle/computer_driver_env_test.py @@ -7,7 +7,11 @@ import pytest from agent_env.bundle import BundleError +from agent_env.bundle.installed import installed_bundles from agent_env.bundle.materialize import materialize +from agent_env.bundle.plan import plan_bundle +from agent_env.bundle.resolve import resolve_bundle +from agent_env.bundle import parse_bundle from agent_env.env import Env from agent_env.env.envs.computer_driver import ComputerDriverEnv from tst.unit.bundle._support import layout, plan_of @@ -131,3 +135,17 @@ async def test_a_driver_env_doesnt_deploy_until_there_is_a_computer_to_deploy_it with pytest.raises(NotImplementedError, match="deploys inside a computer, and agent-env can't create a computer"): await Env.get(f"{ROOT}/desk").deploy() + +def test_a_run_naming_the_reference_driver_writes_the_shipped_one(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + layout(tmp_path / "uses", {"tasks/t.json": json.dumps([ + {"id": "agent", "type": "deploy_agent", "env_ids": ["reference"]}])}) + plan = plan_bundle(resolve_bundle(parse_bundle(tmp_path / "uses")), installed=installed_bundles) + + written = {done.write.id for done in materialize(plan).writes} + env = Env.get("@local/agentenv-framework/drivers/reference") + + assert "@local/agentenv-framework/drivers/reference" in written + assert env.declaration["name"] == "reference" and len(env.declaration["platforms"]) == 2 + assert {"urn:agentenv:computer-pointer/v1", "urn:agentenv:computer-keyboard/v1", + "urn:agentenv:computer-exec/v1"} <= set(env.declaration["platforms"][0]["provides"]) diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index e08889b4..151c08b1 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -127,7 +127,7 @@ def test_the_plugin_report_counts_the_bundles_agent_env_ships(monkeypatch, quiet assert result.exit_code == 0, result.output (core,) = json.loads(result.stdout)["plugins"] assert [(c["group"], c["name"], c["status"]) for c in core["contributions"]] == [ - (BUNDLES, "agents", "active"), (BUNDLES, "echo-mcp", "active"), (BUNDLES, "hello", "active"), + (BUNDLES, "agents", "active"), (BUNDLES, "drivers", "active"), (BUNDLES, "echo-mcp", "active"), (BUNDLES, "hello", "active"), (BUNDLES, "mail-mcp", "active")] assert "\n agent-env itself\n" in shown and "requires:" not in shown diff --git a/tst/unit/bundle/plan_test.py b/tst/unit/bundle/plan_test.py index c3bf560d..bce8ff7b 100644 --- a/tst/unit/bundle/plan_test.py +++ b/tst/unit/bundle/plan_test.py @@ -194,6 +194,19 @@ def test_a_bundle_with_nothing_to_run_is_refused(make): assert caught.value.problems == ("this bundle has no tasks or evals to run",) +def test_a_bundle_that_gives_aliases_and_has_nothing_to_run_passes_the_checks_as_a_library(tmp_path): + root = tmp_path / "library" + (root / "envs/mail").mkdir(parents=True) + (root / "envs/mail/Dockerfile").write_text("FROM scratch\n") + (root / "envs/mail/server.py").write_text('from agentenv_protocol import environment_card\n\n\n' + '@environment_card(name="mail")\nclass Server:\n pass\n') + (root / "aliases.toml").write_text('[envs]\nmail = ["mail"]\n') + resolved = resolve_bundle(parse_bundle(root)) + + check_bundle(resolved) + assert problems(resolved) == ("this bundle has no tasks or evals to run",) + + # Order diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index f918f524..44495505 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -26,7 +26,7 @@ def _shipped() -> dict: def test_agentenv_framework_ships_its_bundles(): - assert SHIPPED == ["agents", "echo-mcp", "hello", "mail-mcp"] + assert SHIPPED == ["agents", "drivers", "echo-mcp", "hello", "mail-mcp"] @pytest.mark.parametrize("name", SHIPPED) @@ -50,12 +50,12 @@ def test_the_shipped_aliases_are_agent_envs_alone_and_claude_code_names_the_defa -def test_the_shipped_env_alias_names_the_mail_mcp_examples_env_which_the_validator_deploys(): +def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_and_the_reference_driver(): table = installed_aliases(tuple(_shipped().values()), ENV_ALIASES) assert table.refused == {} and table.refused_claims == {} assert {alias: named.target_id for alias, named in table.aliases.items()} == { - "mail": "@local/agentenv-framework/mail-mcp/mail"} + "mail": "@local/agentenv-framework/mail-mcp/mail", "reference": "@local/agentenv-framework/drivers/reference"} assert A2AAgent.VALIDATION_ENV_ID == "mail" def test_a_task_naming_no_agent_plans_with_the_shipped_default(tmp_path, monkeypatch, local_stores, cli_routing): diff --git a/tst/unit/drivers/__init__.py b/tst/unit/drivers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/agentenv-protocol/tests/test_computer_x11_example.py b/tst/unit/drivers/reference_driver_test.py similarity index 90% rename from packages/agentenv-protocol/tests/test_computer_x11_example.py rename to tst/unit/drivers/reference_driver_test.py index 2700e276..42c63896 100644 --- a/packages/agentenv-protocol/tests/test_computer_x11_example.py +++ b/tst/unit/drivers/reference_driver_test.py @@ -1,24 +1,24 @@ -"""The X11 example driver against fake mss and Xlib modules: it passes the definition checks, serves the whole -computer contract, and turns tool calls into the XTEST events they mean, without resending input, leaving keys or -buttons down, or letting go of keys it didn't press.""" +"""The reference driver against fake mss and Xlib modules: it passes the definition checks, serves the whole computer +contract, and turns tool calls into the XTEST events they mean, without resending input, leaving keys or buttons down, +or letting go of keys it didn't press.""" from __future__ import annotations import asyncio +import contextlib import importlib.util import sys import types from pathlib import Path +import httpx import pytest +from mcp.shared.memory import create_connected_server_and_client_session -pytest.importorskip("mcp.server.fastmcp") -pytest.importorskip("PIL") +import agent_env +from agentenv_protocol.computer import COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_POINTER, Platform -from computer_fake import LINUX, http_client, mcp_session # noqa: E402 - -from agentenv_protocol.computer import COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_POINTER # noqa: E402 - -DRIVER = Path(__file__).resolve().parents[1] / "examples" / "computer_x11" / "driver.py" +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "reference" / "driver.py" +LINUX = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") WIDTH, HEIGHT = 640, 400 TOKYO = [0x01000000 | ord(ch) for ch in "東京"] #: The fake keymap puts the named keys, such as Shift_L, from this keycode up; printable ASCII is its keysym + 8. @@ -133,10 +133,23 @@ def module(name: str, **attrs) -> types.ModuleType: monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) - spec = importlib.util.spec_from_file_location("computer_x11_driver", DRIVER) + spec = importlib.util.spec_from_file_location("reference_driver", DRIVER) example = importlib.util.module_from_spec(spec) spec.loader.exec_module(example) - return example.X11Driver(), fake, lambda name: fake.keysym_to_keycode(XK.string_to_keysym(name)) + return example.ReferenceDriver(), fake, lambda name: fake.keysym_to_keycode(XK.string_to_keysym(name)) + + +@contextlib.asynccontextmanager +async def mcp_session(driver): + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +@contextlib.asynccontextmanager +async def http_client(driver): + app = (driver.mcp or driver.create_app()).streamable_http_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + yield client async def _events(fake: FakeX, session, tool: str, args: dict) -> list: @@ -154,7 +167,7 @@ async def test_the_example_serves_the_whole_computer_contract(x11): tools = {t.name for t in (await session.list_tools()).tools} ready = (await http.get("/agentenv/ext/computer-ready/status")).json() extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} - assert card["name"] == "x11-desktop" and len(tools) == 14 and "computer_keyboard_down" in tools + assert card["name"] == "reference" and len(tools) == 14 and "computer_keyboard_down" in tools assert {COMPUTER_POINTER, COMPUTER_KEYBOARD, COMPUTER_EXEC} <= set(extensions) assert extensions["urn:agentenv:computer/v1"]["params"]["max_click_count"] == 3 assert ready["ready"] and [c["name"] for c in ready["checks"]] == ["display", "capture", "xtest"] diff --git a/tst/unit/test_clean_install_wheel.py b/tst/unit/test_clean_install_wheel.py index 5427892a..26177b53 100644 --- a/tst/unit/test_clean_install_wheel.py +++ b/tst/unit/test_clean_install_wheel.py @@ -62,5 +62,5 @@ def test_bundle_entry_points_that_differ_from_pyproject_are_named(tmp_path, entr def test_the_gate_checks_the_folder_of_every_shipped_bundle(): roots = {name: root.relative_to(SCRIPT.parents[2]).as_posix() for name, root in clean_install.bundle_roots().items()} - assert roots == {"agents": "src/agent_env/agents", "echo-mcp": "src/agent_env/examples/echo-mcp", + assert roots == {"agents": "src/agent_env/agents", "drivers": "src/agent_env/drivers", "echo-mcp": "src/agent_env/examples/echo-mcp", "hello": "src/agent_env/examples/hello", "mail-mcp": "src/agent_env/examples/mail-mcp"} From 0100d5e9129e9b9ff42356c4d2395be474ddf9d4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:32:00 -0700 Subject: [PATCH 05/11] test(computer): run the reference driver on a scaffold desktop through agent-env, probed at full The scaffold is a hand-written Ubuntu 24.04 image (Xvfb, the probe's Fixture Desk, tini as PID 1) that serves the reference driver as an MCP server on the server provider, until agent-env composes computers. agent-env run deploys it and an env_outcome_verifier runs the conformance probe at full three times; a second task has claude-code drive the desk through the computer_* tools and checks the desk's own record of what reached it. Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/__init__.py | 0 .../scaffold/artifacts/check_desk/verify.py | 25 ++++++ .../scaffold/artifacts/check_probe/verify.py | 15 ++++ .../computer/scaffold/envs/desktop/Dockerfile | 18 +++++ .../computer/scaffold/envs/desktop/env.toml | 2 + .../computer/scaffold/envs/desktop/start.sh | 9 +++ .../computer/scaffold/tasks/agent.json | 8 ++ .../computer/scaffold/tasks/probe.json | 5 ++ tst/integration/computer/scaffold_test.py | 79 +++++++++++++++++++ 9 files changed, 161 insertions(+) create mode 100644 tst/integration/computer/__init__.py create mode 100644 tst/integration/computer/scaffold/artifacts/check_desk/verify.py create mode 100644 tst/integration/computer/scaffold/artifacts/check_probe/verify.py create mode 100644 tst/integration/computer/scaffold/envs/desktop/Dockerfile create mode 100644 tst/integration/computer/scaffold/envs/desktop/env.toml create mode 100644 tst/integration/computer/scaffold/envs/desktop/start.sh create mode 100644 tst/integration/computer/scaffold/tasks/agent.json create mode 100644 tst/integration/computer/scaffold/tasks/probe.json create mode 100644 tst/integration/computer/scaffold_test.py diff --git a/tst/integration/computer/__init__.py b/tst/integration/computer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/integration/computer/scaffold/artifacts/check_desk/verify.py b/tst/integration/computer/scaffold/artifacts/check_desk/verify.py new file mode 100644 index 00000000..414f26a8 --- /dev/null +++ b/tst/integration/computer/scaffold/artifacts/check_desk/verify.py @@ -0,0 +1,25 @@ +"""Whether the agent did the desk task through the computer: the Fixture Desk's own record of what reached it, read +through the driver's computer-exec.""" +import json +import uuid + +from agentenv_protocol import client as protocol + +EXEC = "urn:agentenv:computer-exec/v1" + + +async def verify(mcp_url): + base = mcp_url.rsplit("/mcp", 1)[0] + card = await protocol.get_card(base) + out = await protocol.invoke_extension(base, card, EXEC, {"argv": ["cat", "fixture/state.json"], + "request_id": uuid.uuid4().hex}, method="run") + state = json.loads(out["stdout"]) + return [ + {"id": "name", "result": state["name"] == "Ada Lovelace", "description": f"name {state['name']!r}"}, + {"id": "subscribed", "result": state["subscribed"] is True, "description": f"subscribed {state['subscribed']}"}, + {"id": "item", "result": state["selected"] == "Item 37", "description": f"selected {state['selected']!r}"}, + {"id": "saved", "result": state["saved_count"] >= 1, "description": f"saved {state['saved_count']} times"}, + {"id": "cell", "result": state["cells"].get("3,4", 0) >= 1, "description": f"cells {state['cells']}"}, + {"id": "typed-as-keys", "result": len(state["keys"]) >= len("Ada Lovelace"), + "description": f"{len(state['keys'])} key events reached the field"}, + ] diff --git a/tst/integration/computer/scaffold/artifacts/check_probe/verify.py b/tst/integration/computer/scaffold/artifacts/check_probe/verify.py new file mode 100644 index 00000000..3443b95c --- /dev/null +++ b/tst/integration/computer/scaffold/artifacts/check_probe/verify.py @@ -0,0 +1,15 @@ +"""The conformance probe at full against the desktop, RUNS times: a check passes only when it passed, or didn't +apply, in every run.""" +from agentenv_protocol.computer.conformance import run_probe + +RUNS = 3 + + +async def verify(mcp_url): + base, outcomes = mcp_url.rsplit("/mcp", 1)[0], {} + for _ in range(RUNS): + for check in (await run_probe(base, "full")).checks: + seen = outcomes.setdefault(f"{check.check} {check.name}", []) + seen.append(check.status if check.status in ("passed", "not_applicable") else f"{check.status}: {check.detail}") + return [{"id": name, "result": all(s in ("passed", "not_applicable") for s in seen), "description": "; ".join(seen)} + for name, seen in outcomes.items()] diff --git a/tst/integration/computer/scaffold/envs/desktop/Dockerfile b/tst/integration/computer/scaffold/envs/desktop/Dockerfile new file mode 100644 index 00000000..b2a3c914 --- /dev/null +++ b/tst/integration/computer/scaffold/envs/desktop/Dockerfile @@ -0,0 +1,18 @@ +# The scaffold: the reference driver on a hand-written Ubuntu 24.04 desktop, until agent-env composes computers. The +# test stages driver.py, fixture.py and the protocol package (protocol/) beside this file. +FROM public.ecr.aws/docker/library/ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-tk xvfb x11-utils fonts-dejavu-core procps tini \ + libxcb1 libxcb-render0 libxcb-shm0 libxcb-randr0 libxcb-xfixes0 libxcb-xinerama0 \ + && rm -rf /var/lib/apt/lists/* +COPY protocol /opt/protocol +RUN python3 -m venv /opt/driver \ + && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer]" "mss>=9,<11" "python-xlib>=0.33,<0.34" +RUN useradd --create-home --uid 10001 user +COPY start.sh driver.py fixture.py /opt/desktop/ +USER user +WORKDIR /home/user +ENV DISPLAY=:99 XDG_SESSION_TYPE=x11 +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/bin/sh", "/opt/desktop/start.sh"] diff --git a/tst/integration/computer/scaffold/envs/desktop/env.toml b/tst/integration/computer/scaffold/envs/desktop/env.toml new file mode 100644 index 00000000..c7adc72e --- /dev/null +++ b/tst/integration/computer/scaffold/envs/desktop/env.toml @@ -0,0 +1,2 @@ +environment_name = "desktop" +env_provider_type = "server" diff --git a/tst/integration/computer/scaffold/envs/desktop/start.sh b/tst/integration/computer/scaffold/envs/desktop/start.sh new file mode 100644 index 00000000..dce5b94a --- /dev/null +++ b/tst/integration/computer/scaffold/envs/desktop/start.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Xvfb and the Fixture Desk each restart if they die; the driver serves on MCP_PORT. +(while true; do Xvfb :99 -screen 0 1280x800x24 -nolisten tcp -ac >/tmp/xvfb.log 2>&1; sleep 0.5; done) & +for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done +(while true; do + for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done + python3 /opt/desktop/fixture.py "$HOME/fixture" >/tmp/fixture.log 2>&1; sleep 0.5 + done) & +exec /opt/driver/bin/python /opt/desktop/driver.py diff --git a/tst/integration/computer/scaffold/tasks/agent.json b/tst/integration/computer/scaffold/tasks/agent.json new file mode 100644 index 00000000..25cc9b49 --- /dev/null +++ b/tst/integration/computer/scaffold/tasks/agent.json @@ -0,0 +1,8 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop", "sandbox_type": "local"}, + {"id": "agent", "type": "deploy_agent", "a2a_agent_id": "claude-code", "env_ids": ["desktop"], "sandbox_type": "local"}, + {"id": "task", "type": "prompt_agent", "prompt_id": "task", "depends_on": ["env", "agent"], + "prompt": "You have a computer with a graphical desktop, which you use through the computer_* tools: take a screenshot to see it, then click, type, scroll and press keys. A window called Fixture Desk is open. In it: type the name Ada Lovelace into the Name field, tick the Subscribe checkbox, select Item 37 in the list on the left (scroll the list to find it), click the cell in the fourth row and fifth column of the white grid of squares, and finally click Save. Take a screenshot after each step to check it worked. Use only the computer_* tools, and reply DONE when everything is in place."}, + {"id": "check", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "check_desk", + "verifier_id": "desk", "depends_on": ["task"]} +] diff --git a/tst/integration/computer/scaffold/tasks/probe.json b/tst/integration/computer/scaffold/tasks/probe.json new file mode 100644 index 00000000..d71829cd --- /dev/null +++ b/tst/integration/computer/scaffold/tasks/probe.json @@ -0,0 +1,5 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop", "sandbox_type": "local"}, + {"id": "probe", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "check_probe", + "verifier_id": "probe", "depends_on": ["env"]} +] diff --git a/tst/integration/computer/scaffold_test.py b/tst/integration/computer/scaffold_test.py new file mode 100644 index 00000000..e87ba4e5 --- /dev/null +++ b/tst/integration/computer/scaffold_test.py @@ -0,0 +1,79 @@ +"""The reference driver on a desktop of its own before agent-env can compose a computer: the scaffold, a hand-written +Ubuntu 24.04 image under this folder, deployed by ``agent-env run`` as an MCP server on the server provider, passes +the conformance probe at full three runs in a row, and an agent drives its desktop through the computer_* tools. +Needs a Docker daemon and the local registry; the agent's task, a model endpoint too.""" + +import logging +import shutil +from pathlib import Path + +import pytest +from click.testing import CliRunner + +import agent_env +import agentenv_protocol +from agent_env.artifact.store import reset_artifact_store +from agent_env.cli import cli +from agent_env.config import configure, reset_config +from tst.util.capabilities import skip_without_model_endpoint + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +SCAFFOLD = Path(__file__).parent / "scaffold" +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "reference" / "driver.py" +FIXTURE = Path(agentenv_protocol.__file__).parent / "computer" / "conformance" / "fixture.py" +PROTOCOL = Path(agentenv_protocol.__file__).parents[2] +PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") + + +@pytest.fixture +def state(monkeypatch, tmp_path): + """Local stores under this test's folder. Whatever ran, no sandbox work folder may be left behind.""" + sandboxes = tmp_path / "sandboxes" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(sandboxes)) + configure() + reset_artifact_store() + logging.disable(logging.CRITICAL) + try: + yield tmp_path + assert not sandboxes.exists() or not any(sandboxes.iterdir()), "a run left a sandbox work folder" + finally: + logging.disable(logging.NOTSET) + reset_artifact_store() + reset_config() + + +def _scaffold(root: Path) -> Path: + """The scaffold bundle with the reference driver, the probe's fixture and this checkout's protocol beside its + Dockerfile.""" + shutil.copytree(SCAFFOLD, root, ignore=shutil.ignore_patterns("__pycache__")) + desktop = root / "envs" / "desktop" + shutil.copy(DRIVER, desktop / "driver.py") + shutil.copy(FIXTURE, desktop / "fixture.py") + shutil.copytree(PROTOCOL / "src", desktop / "protocol" / "src", ignore=shutil.ignore_patterns("__pycache__")) + for name in PROTOCOL_FILES: + shutil.copy(PROTOCOL / name, desktop / "protocol" / name) + return root + + +def _run(root: Path, task: str): + return CliRunner().invoke(cli, ["run", str(root), "--task", task]) + + +def test_the_reference_driver_passes_the_probe_at_full(state): + ran = _run(_scaffold(state / "scaffold"), "probe") + + assert ran.exit_code == 0, ran.output + assert "passed (probe: 1)" in ran.output, ran.output + + +@skip_without_model_endpoint() +def test_an_agent_drives_the_reference_drivers_desktop(state): + ran = _run(_scaffold(state / "scaffold"), "agent") + + assert ran.exit_code == 0, ran.output + assert "passed (check: 1)" in ran.output, ran.output From f042de5e0fe2f29cbee9375594d5fcbacba2812e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:35:08 -0700 Subject: [PATCH 06/11] docs(computer): document the conformance probe, the computer_driver env and the drivers bundle Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 ++- packages/agentenv-protocol/README.md | 35 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 65f1accb..0981e97b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,7 +105,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset` and `load_file_artifact_universe`. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset` and `load_file_artifact_universe`. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed; its deploy is refused until agent-env creates computers). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, and more). `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | @@ -120,6 +120,7 @@ and deserialized through a registry. | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux) aliased `reference`. It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. `tst/integration/computer/` serves it on a scaffold desktop image and runs the conformance probe and an agent against it. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ## Extension points diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 76ba0ec8..86ea4aa4 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -422,6 +422,41 @@ and `exec_timeout_s`. A driver reads them as `self.settings`, for example its ta Every caller is `default` until agents carry an identity of their own; `caller_identity(context)` names the caller whose latest frame and others' input a call uses. +`MyDriver.declaration(platform)` returns the card the class serves on a `Platform`, built without +running the driver, its operations or its dependencies. + +### Checking a driver: the conformance probe + +The probe checks a running driver against the contract through its card, tools and extensions. It +reports each check, and gives each extension the card declares a verdict: verified, unverified or +failed. + +```bash +python -m agentenv_protocol.computer.conformance http://localhost:18765 [--level basic|full] [--json report.json] +``` + +```python +from agentenv_protocol.computer.conformance import run_probe + +report = await run_probe("http://localhost:18765", "full") # a ProbeReport +failed = [c for c in report.checks if c.status == "failed"] +``` + +The CLI exits 0 when no check failed, 1 when one did and 2 when it can't reach the driver. + +| Level | Checks | Needs on the computer | +|---|---|---| +| `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add` | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` | + +At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into +`~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the +window's widgets report, reads back what reached them, and then removes the fixture. When the +fixture can't be staged, the `full` checks are unverified rather than failed. The probe leaves out +the checks for an extension the card doesn't declare. P8 is not applicable to a driver without +leasing, and unverified for one with it until the probe has lease checks. The probe calls as the +role `agentenv-probe`, so a driver's per-role tool access applies to it. + ## A2A agent framework The distribution exposes two unrelated decorators named `extension`: From 4b16037cbcfaefa210f1c6bd241e7655f9181d1e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:53:12 -0700 Subject: [PATCH 07/11] test(computer): read a driver class in-process in the protocol's own tests, and refuse one that takes too long to read Co-Authored-By: Claude Opus 5.5 --- .../tests/test_computer_declare.py | 113 ++++++++++++++++++ tst/unit/bundle/computer_driver_env_test.py | 8 ++ 2 files changed, 121 insertions(+) create mode 100644 packages/agentenv-protocol/tests/test_computer_declare.py diff --git a/packages/agentenv-protocol/tests/test_computer_declare.py b/packages/agentenv-protocol/tests/test_computer_declare.py new file mode 100644 index 00000000..80f5a1d4 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_declare.py @@ -0,0 +1,113 @@ +"""Reading a driver class where its own dependencies aren't installed: a module no finder finds imports as an inert +stub, the class is read without running it, and what it declares for each platform it supports comes back.""" +from __future__ import annotations + +import json +import re +import sys + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from agentenv_protocol.computer._declare import _Stub, _StubModule, declare, main # noqa: E402 + +DRIVER = '''import not_installed_here +from not_installed_here.input import Base, Keyboard + +from agentenv_protocol.computer import (ComputerDriverEnvironment, Display, capture, click, computer_driver, display, + drag, press, scroll, type_text) + +SHIFT = Keyboard.keysym("Shift_L")[0] + + +class Connection(Base): + pass + + +@computer_driver(name="desk", supports={supports}) +class Desk(ComputerDriverEnvironment): + buttons = {buttons} + + @display + def size(self): return Display(10, 10) + + @capture + def grab(self): return not_installed_here.grab() + + @click + def click_at(self, x, y, button, count, modifiers): pass + + @drag + def drag_along(self, path, button, duration_ms, modifiers): pass + + @scroll + def scroll_at(self, x, y, direction, amount, modifiers): pass + + @type_text + def type_in(self, text): pass + + @press + def press_keys(self, keys): pass +''' +LINUX = [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] + + +@pytest.fixture +def folder(tmp_path, monkeypatch): + """A driver's folder; the import path, the finders and the stub modules a declaration adds are taken back.""" + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.setattr(sys, "meta_path", list(sys.meta_path)) + yield tmp_path + for name in [name for name, module in sys.modules.items() if isinstance(module, _StubModule)]: + del sys.modules[name] + + +def _driver(folder, supports=LINUX, buttons=("left", "right")): + (folder / "driver.py").write_text(DRIVER.format(supports=supports, buttons=buttons)) + + +def test_a_class_whose_imports_arent_installed_declares_what_it_serves_on_each_platform(folder): + _driver(folder) + + declared = declare(folder, "driver.py", "Desk") + + assert declared["name"] == "desk" and declared["supports"] == LINUX + assert [p["platform"] for p in declared["platforms"]] == [{"os": "linux", "arch": "amd64", "display": "x11"}, + {"os": "linux", "arch": "arm64", "display": "x11"}] + linux = declared["platforms"][0] + assert {"urn:agentenv:computer/v1", "urn:agentenv:computer-exec/v1"} <= set(linux["provides"]) + assert len(linux["tools"]) == 7 and linux["features"]["buttons"] == ["left", "right"] + + +def test_a_stub_is_itself_again_except_for_dunders(folder): + _driver(folder) + declare(folder, "driver.py", "Desk") + stub = sys.modules["not_installed_here"] + + assert isinstance(stub, _StubModule) and stub.anything is _Stub + assert _Stub.a.b(1, k=2)[0] is _Stub and isinstance(_Stub, type) + assert not hasattr(stub, "__version__") and not hasattr(_Stub, "__wrapped__") + + +@pytest.mark.parametrize("name, supports, problem", [ + ("Connection", LINUX, "driver.py has no ComputerDriverEnvironment subclass named Connection"), + ("Missing", LINUX, "driver.py has no ComputerDriverEnvironment subclass named Missing"), + ("Desk", [], "Desk declares no platform: name them with @computer_driver(supports=[...])"), +]) +def test_a_name_that_isnt_a_driver_or_declares_no_platform_is_refused(folder, name, supports, problem): + _driver(folder, supports=supports) + + with pytest.raises(ValueError, match=re.escape(problem)): + declare(folder, "driver.py", name) + + +def test_main_prints_the_declaration_or_exits_1_with_why_it_cant(folder, capsys): + _driver(folder) + assert main([str(folder), "driver.py", "Desk"]) == 0 + assert json.loads(capsys.readouterr().out)["name"] == "desk" + + _driver(folder, buttons=("thumb",)) + assert main([str(folder), "driver.py", "Desk"]) == 1 + assert capsys.readouterr().err.startswith("TypeError: ") diff --git a/tst/unit/bundle/computer_driver_env_test.py b/tst/unit/bundle/computer_driver_env_test.py index ec3d4956..641fef60 100644 --- a/tst/unit/bundle/computer_driver_env_test.py +++ b/tst/unit/bundle/computer_driver_env_test.py @@ -13,6 +13,7 @@ from agent_env.bundle.resolve import resolve_bundle from agent_env.bundle import parse_bundle from agent_env.env import Env +from agent_env.env.envs import computer_driver from agent_env.env.envs.computer_driver import ComputerDriverEnv from tst.unit.bundle._support import layout, plan_of @@ -127,6 +128,13 @@ def test_a_driver_its_class_cant_be_read_from_is_refused_before_any_write(bundle Env.get(f"{ROOT}/desk") +def test_a_driver_whose_class_takes_too_long_to_read_is_refused(bundle_dir, monkeypatch): + monkeypatch.setattr(computer_driver, "DECLARE_TIMEOUT_S", 0.001) + _bundle(bundle_dir) + + assert "reading driver.py:Desk took over 0.001 s" in _problems(bundle_dir) + + @pytest.mark.asyncio async def test_a_driver_env_doesnt_deploy_until_there_is_a_computer_to_deploy_it_in(bundle_dir): _bundle(bundle_dir) From 8f0043f70235b80ce42765b6e5fa52c72203d26c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 18:30:52 -0700 Subject: [PATCH 08/11] fix(protocol): keep the probe's data/add check inside its stage The check wrote ~/inbox/message.txt for the Fixture Desk to pick up, so a probe left an inbox folder behind and would remove a user's own inbox/message.txt. The fixture now reads /inbox.txt, which goes with the stage. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/conformance/checks.py | 7 ++----- .../src/agentenv_protocol/computer/conformance/fixture.py | 8 ++++---- packages/agentenv-protocol/tests/probe_fake.py | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index c8e91d55..221026a6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -704,13 +704,10 @@ async def down_up() -> str | None: async def _data_add(self) -> None: async def picked_up() -> str | None: message = f"hello from data/add {time.time_ns()}" - added = await self.driver.add_file("inbox/message.txt", message.encode()) + added = await self.driver.add_file(f"{STAGE}/state/inbox.txt", message.encode()) if "error" in added: return f"data/add refused it: {added['error'].get('message')}" - try: - return await self.settled(lambda s: s["loaded"] == message, lambda s: s["loaded"]) - finally: - await self.driver.run("rm", "-f", "inbox/message.txt") + return await self.settled(lambda s: s["loaded"] == message, lambda s: s["loaded"]) await self.check("X", "a file data/add writes is there for an app to read", picked_up, *DATA_PLANE) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py index c4e7441f..1499e5e4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py @@ -1,7 +1,7 @@ """The Linux fixture the conformance probe stages with data/add and launches with computer-exec: a Tk window, the -Fixture Desk, whose widgets' screen boxes are written to /layout.json and every effect that reaches it to -/state.json, so the probe can aim at it and read back what arrived. Run as -``python3 fixture.py ``; it needs a 1280x800 screen or larger.""" +Fixture Desk, whose widgets' screen boxes are written to /layout.json and every effect that reaches it, +with whatever lands in /inbox.txt, to /state.json, so the probe can aim at it and read back +what arrived. Run as ``python3 fixture.py ``; it needs a 1280x800 screen or larger.""" from __future__ import annotations import json @@ -171,7 +171,7 @@ def wheeled(event, button: int) -> None: field.create_rectangle(x, y, x + 1, y + 1, fill=colour, outline="", width=0) def poll_inbox() -> None: - message = Path.home() / "inbox" / "message.txt" + message = state_dir / "inbox.txt" if message.exists() and (text := message.read_text()) != state["loaded"]: state["loaded"] = text save() diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index ef3d5ae0..c84c4751 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -65,7 +65,7 @@ def start(self) -> None: self.started = True def read(self) -> dict: - message = Path.home() / "inbox" / "message.txt" + message = Path.home() / STAGE / "state" / "inbox.txt" self.state["loaded"] = message.read_text() if message.exists() else self.state["loaded"] return self.state From aa0e449ef49ab29aeaac0676a49040871af3896d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 19:26:12 -0700 Subject: [PATCH 09/11] fix(protocol): probe in frame pixels whatever the default space, clean up a fixture that didn't start, and check tool access on a role of its own - Every point tool call names space "frame" unless it names another, so a driver whose default space is norm passes. - Staging runs inside the cleanup: a fixture that fails to start, or a staging step that raises, still has its folder removed, and the fixture is killed by the pid its launch reports, so one still starting after the timeout is stopped too. - The tool-access check hides a tool from a role new to each run, so it can't re-enable a tool a driver's deployment had disabled for agentenv-probe. The README says the probe calls as the default role. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 3 +- .../computer/conformance/checks.py | 32 ++++++++++------ .../computer/conformance/fixture.py | 1 - .../agentenv-protocol/tests/probe_fake.py | 5 +-- .../tests/test_computer_conformance.py | 38 +++++++++++++++++-- 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 86ea4aa4..57807604 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -455,7 +455,8 @@ window's widgets report, reads back what reached them, and then removes the fixt fixture can't be staged, the `full` checks are unverified rather than failed. The probe leaves out the checks for an extension the card doesn't declare. P8 is not applicable to a driver without leasing, and unverified for one with it until the probe has lease checks. The probe calls as the -role `agentenv-probe`, so a driver's per-role tool access applies to it. +default role; the tool-access check hides a tool from a role of its own, `agentenv-probe-`, +and gives it back. ## A2A agent framework diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 221026a6..f5c007d9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -7,6 +7,7 @@ import json import math import time +import uuid from collections.abc import Awaitable, Callable from importlib import resources @@ -36,7 +37,7 @@ DATA_PLANE = (INTAKE_EXTENSION_URI, DATA_OBJECTS_EXTENSION_URI) #: Where the fixture is staged and keeps its state, relative to the session user's home. STAGE = ".agentenv-probe" -#: The role the tool-access check hides a tool from. +#: What the tool-access check's role, new each run, and the probe's request ids start with. PROBE_ROLE = "agentenv-probe" #: How long the probe waits for an effect to reach the fixture, and how often it looks. SETTLE_S = 5.0 @@ -63,6 +64,7 @@ def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: floa self.results: list[CheckResult] = [] self.features = driver.params(COMPUTER) self.layout: dict = {} + self.fixture_pid: str | None = None async def basic(self) -> None: for check, section in (("P0", self._ready), ("P1", self._card), ("P2", self._frame), ("P5", self._refusals), @@ -78,10 +80,13 @@ async def full(self) -> None: missing.append("data/add") if missing: return self._skip_full(f"the fixture is staged with data/add and computer-exec; the driver lacks {missing}") - problem = await self._stage() - if problem: - return self._skip_full(f"the fixture didn't start: {problem}") try: + try: + problem = await self._stage() + except Exception as e: + problem = f"{type(e).__name__}: {e}" + if problem: + return self._skip_full(f"the fixture didn't start: {problem}") for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), ("X", self._pointer_input), ("X", self._data_add)): @@ -118,7 +123,9 @@ def _skip_full(self, why: str) -> None: # Talking to the driver async def tool(self, name: str, args: dict | None = None) -> CallToolResult: - return await self.session.call_tool(name, args or {}) + """Call ``name``, its points in frame pixels unless ``args`` names a space, whatever the driver's default.""" + args = args or {} + return await self.session.call_tool(name, {"space": "frame", **args} if name in POINT_TOOLS else args) async def shot(self, **args) -> tuple[CallToolResult, object]: result = await self.tool("computer_screenshot", args) @@ -378,16 +385,17 @@ async def _tool_access(self) -> None: return async def per_role() -> str | None: - hidden = {"role": PROBE_ROLE, "tools": ["computer_wait"]} + role = f"{PROBE_ROLE}-{uuid.uuid4().hex[:8]}" + hidden = {"role": role, "tools": ["computer_wait"]} await self.driver.invoke(DISABLE_TOOL_EXTENSION_URI, "disable", hidden) try: - async with self.driver.session(PROBE_ROLE) as other: + async with self.driver.session(role) as other: listed = {t.name for t in (await other.list_tools()).tools} refused = await other.call_tool("computer_wait", {"ms": 0}) allowed = await self.tool("computer_wait", {"ms": 0}) finally: await self.driver.invoke(ENABLE_TOOL_EXTENSION_URI, "enable", hidden) - async with self.driver.session(PROBE_ROLE) as other: + async with self.driver.session(role) as other: back = await other.call_tool("computer_wait", {"ms": 0}) if "computer_wait" in listed: return "a disabled tool is still listed for its role" @@ -436,7 +444,9 @@ async def _stage(self) -> str | None: if "error" in added: return f"data/add refused it: {added['error'].get('message')}" await self.driver.run("sh", "-c", f"rm -rf {STAGE}/state") - await self.driver.run("sh", "-c", f"python3 {STAGE}/fixture.py {STAGE}/state > {STAGE}/fixture.log 2>&1 &") + started = await self.driver.run( + "sh", "-c", f"python3 {STAGE}/fixture.py {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") + self.fixture_pid = started["stdout"].strip() or None deadline = time.monotonic() + FIXTURE_START_S while time.monotonic() < deadline: layout = await self.driver.run("cat", f"{STAGE}/state/layout.json") @@ -451,8 +461,8 @@ async def _stage(self) -> str | None: return f"no layout after {FIXTURE_START_S:g} s: {log['stdout'].strip() or log['stderr'].strip()}" async def _unstage(self) -> None: - if pid := self.layout.get("pid"): - await self.driver.run("kill", str(pid)) + if self.fixture_pid: + await self.driver.run("kill", self.fixture_pid) await self.driver.run("rm", "-rf", STAGE) async def _base(self) -> tuple[str, float, object]: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py index 1499e5e4..cd4b5bee 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py @@ -186,7 +186,6 @@ def write_layout() -> None: boxes["markers"] = [[field.winfo_rootx() + x, field.winfo_rooty() + y, colour] for x, y, colour in MARKERS] boxes["cell"] = CELL boxes["clipboard"] = CLIPBOARD - boxes["pid"] = os.getpid() save() (state_dir / "layout.json").write_text(json.dumps(boxes)) diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index c84c4751..cd00615b 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -37,8 +37,7 @@ LAYOUT = {"name": [140, 40, 420, 34], "save": [560, 34, 120, 44], "subscribe": [40, 100, 160, 34], "notes": [760, 40, 380, 90], "list": [40, 160, 280, 400], "grid": [400, 160, 400, 400], "pad": [840, 160, 400, 300], "wheel": [840, 500, 200, 120], "field": [*FIELD, 48, 48], "list_row_height": 24, - "markers": [[FIELD[0] + x, FIELD[1] + y, c] for x, y, c in MARKERS], "cell": 50, "clipboard": CLIPBOARD, - "pid": 4242} + "markers": [[FIELD[0] + x, FIELD[1] + y, c] for x, y, c in MARKERS], "cell": 50, "clipboard": CLIPBOARD} KEYSYMS = {" ": "space", ",": "comma", "!": "exclam", "\t": "Tab", "\n": "Return"} @@ -185,7 +184,7 @@ def _fixture(self, argv: list[str]) -> ExecResult | None: return ExecResult(0, json.dumps(self.desk.read()).encode()) if self.desk.started else ExecResult(1) if f"python3 {STAGE}/fixture.py" in text: self.desk.start() - return ExecResult(0) + return ExecResult(0, b"4242\n") if argv in (["id", "-u"], ["stat", "-c", "%u", f"{STAGE}/fixture.py"]): return ExecResult(0, b"1000\n") if argv[:1] in (["kill"], ["tail"]) or f"rm -rf {STAGE}/state" in text: diff --git a/packages/agentenv-protocol/tests/test_computer_conformance.py b/packages/agentenv-protocol/tests/test_computer_conformance.py index aa329edf..3cded207 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance.py @@ -17,6 +17,7 @@ from probe_fake import FixtureDriver, served # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 + ComputerSettings, ExecResult, Platform, ReadyCheck, @@ -30,7 +31,9 @@ ) from agentenv_protocol.computer.conformance import CheckResult, ProbeReport, run_probe # noqa: E402 from agentenv_protocol.computer.conformance import checks # noqa: E402 +from agentenv_protocol.computer.conformance.client import Driver # noqa: E402 from agentenv_protocol.computer.conformance.__main__ import main # noqa: E402 +from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI # noqa: E402 @pytest.fixture(autouse=True) @@ -65,6 +68,24 @@ async def test_the_fake_driver_passes_every_check_at_full_and_cleans_up(home): assert not (home / ".agentenv-probe").exists() and ["kill", "4242"] in driver.commands +@pytest.mark.asyncio +async def test_a_driver_whose_default_space_is_norm_passes_at_full(): + report = await _probe(FixtureDriver(settings=ComputerSettings(default_space="norm"))) + assert report.passed, _failed(report) + + +@pytest.mark.asyncio +async def test_a_tool_already_hidden_from_the_probes_role_stays_hidden(): + with served(FixtureDriver()) as url: + driver = await Driver.connect(url, 10) + await driver.invoke(DISABLE_TOOL_EXTENSION_URI, "disable", {"role": checks.PROBE_ROLE, "tools": ["computer_wait"]}) + report = await run_probe(url, "basic") + async with driver.session(checks.PROBE_ROLE) as session: + refused = await session.call_tool("computer_wait", {"ms": 0}) + assert report.passed, _failed(report) + assert refused.isError and refused.structuredContent["code"] == "permission_denied" + + @pytest.mark.asyncio async def test_basic_needs_no_fixture(home): driver = FixtureDriver() @@ -165,23 +186,34 @@ def _fixture(self, argv): class SilentFixture(FixtureDriver): def _fixture(self, argv): if "python3" in " ".join(argv): - return ExecResult(0) + return ExecResult(0, b"4243\n") if argv[:1] == ["tail"]: return ExecResult(0, b"ModuleNotFoundError: No module named 'tkinter'\n") return super()._fixture(argv) +class ExecBreaks(FixtureDriver): + def _fixture(self, argv): + if f"rm -rf {checks.STAGE}/state" in " ".join(argv): + raise RuntimeError("exec broke") + return super()._fixture(argv) + + @pytest.mark.asyncio @pytest.mark.parametrize("driver, why", [ (NoFixture, "the fixture didn't start: the screen is 800x600"), (SilentFixture, "no layout after 0.6 s: ModuleNotFoundError: No module named 'tkinter'"), + (ExecBreaks, "the fixture didn't start: ExtensionCallError"), ]) -async def test_a_fixture_that_doesnt_start_leaves_full_unverified(driver, why, monkeypatch): +async def test_a_fixture_that_doesnt_start_leaves_full_unverified_and_is_cleaned_up(driver, why, monkeypatch, home): monkeypatch.setattr(checks, "FIXTURE_START_S", 0.6) - report = await _probe(driver()) + probed = driver() + report = await _probe(probed) assert report.passed, _failed(report) unverified = [c for c in report.checks if c.status == "unverified"] assert len(unverified) == len(checks.FULL_CHECKS) and why in unverified[0].detail + assert not (home / checks.STAGE).exists() + assert (["kill", "4243"] in probed.commands) is (driver is SilentFixture) def test_a_refusal_must_carry_the_code_and_applied_no(): From 4b2da9c7dc7a27c3d7caab4bdee9cd91c2d259f9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 19:26:13 -0700 Subject: [PATCH 10/11] fix(computer): keep a driver's import output out of its declaration, and check a computer_driver env.toml in a dry run - _declare sends what the driver module prints while it loads to stderr, so stdout carries only the JSON put reads. - ComputerDriverEnv.accept_toml checks the keys and that module names a file of the folder, so a dry run and the bundle check refuse it before anything is written; from_toml reuses it. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/_declare.py | 5 ++++- .../tests/test_computer_declare.py | 9 ++++++--- src/agent_env/env/envs/computer_driver.py | 18 +++++++++++++----- tst/unit/bundle/computer_driver_env_test.py | 11 +++++++++++ 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py index cb3fcef9..bf6e1fff 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py @@ -5,6 +5,7 @@ with why.""" from __future__ import annotations +import contextlib import importlib.abc import importlib.machinery import importlib.util @@ -81,9 +82,11 @@ def declare(folder: Path, file: str, name: str) -> dict: def main(argv: list[str]) -> int: + """Print the declaration as the only output; what the module prints while it loads goes to stderr.""" folder, file, name = argv try: - declared = declare(Path(folder), file, name) + with contextlib.redirect_stdout(sys.stderr): + declared = declare(Path(folder), file, name) except Exception as e: print(f"{type(e).__name__}: {e}", file=sys.stderr) return 1 diff --git a/packages/agentenv-protocol/tests/test_computer_declare.py b/packages/agentenv-protocol/tests/test_computer_declare.py index 80f5a1d4..ac4ac2b0 100644 --- a/packages/agentenv-protocol/tests/test_computer_declare.py +++ b/packages/agentenv-protocol/tests/test_computer_declare.py @@ -20,6 +20,7 @@ drag, press, scroll, type_text) SHIFT = Keyboard.keysym("Shift_L")[0] +print("loading the desk driver") class Connection(Base): @@ -103,11 +104,13 @@ def test_a_name_that_isnt_a_driver_or_declares_no_platform_is_refused(folder, na declare(folder, "driver.py", name) -def test_main_prints_the_declaration_or_exits_1_with_why_it_cant(folder, capsys): +def test_main_prints_only_the_declaration_or_exits_1_with_why_it_cant(folder, capsys): _driver(folder) assert main([str(folder), "driver.py", "Desk"]) == 0 - assert json.loads(capsys.readouterr().out)["name"] == "desk" + printed = capsys.readouterr() + assert json.loads(printed.out)["name"] == "desk" and printed.err == "loading the desk driver\n" _driver(folder, buttons=("thumb",)) assert main([str(folder), "driver.py", "Desk"]) == 1 - assert capsys.readouterr().err.startswith("TypeError: ") + printed = capsys.readouterr() + assert printed.out == "" and printed.err.splitlines()[-1].startswith("TypeError: Desk: buttons include left") diff --git a/src/agent_env/env/envs/computer_driver.py b/src/agent_env/env/envs/computer_driver.py index 9334b15a..03430972 100644 --- a/src/agent_env/env/envs/computer_driver.py +++ b/src/agent_env/env/envs/computer_driver.py @@ -46,8 +46,18 @@ def from_dict(cls, data: dict) -> ComputerDriverEnv: @classmethod def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerDriverEnv: - """``module = ":"`` names the driver's class in its folder, which is read in a process of its own - (``agentenv_protocol.computer._declare``), so the driver's own dependencies need not be installed here.""" + """The driver's class is read in a process of its own (``agentenv_protocol.computer._declare``), so the + driver's own dependencies need not be installed here.""" + module = cls.accept_toml(data, ctx)["module"] + file, _, name = module.partition(":") + declaration = _declare(ctx, file, name) + universe = FileArtifactUniverse.put_bundled(derive_id(ctx.id, "files"), files=ctx.files()) + return cls.put(id=ctx.id, module=module, files=universe, declaration=declaration) + + @classmethod + def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: + """The keys of ``data``, an env.toml, a computer driver env takes: ``module = ":"``, naming a file + of its folder. Raises BundleError listing every problem.""" fields = ctx.accept(data, module=str) module = fields.get("module") if module is None: @@ -58,9 +68,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerDriverEnv: if not name or file not in files: ctx.refuse([ctx.config_problem(f'module {module!r} names no ":" in this folder; it has ' f"{', '.join(sorted(files))}")]) - declaration = _declare(ctx, file, name) - universe = FileArtifactUniverse.put_bundled(derive_id(ctx.id, "files"), files=files) - return cls.put(id=ctx.id, module=module, files=universe, declaration=declaration) + return fields async def deploy(self, **kwargs) -> DeployedEnv: raise NotImplementedError(f"{self.id} is a computer_driver env, which deploys inside a computer, and " diff --git a/tst/unit/bundle/computer_driver_env_test.py b/tst/unit/bundle/computer_driver_env_test.py index 641fef60..19a8f507 100644 --- a/tst/unit/bundle/computer_driver_env_test.py +++ b/tst/unit/bundle/computer_driver_env_test.py @@ -11,6 +11,7 @@ from agent_env.bundle.materialize import materialize from agent_env.bundle.plan import plan_bundle from agent_env.bundle.resolve import resolve_bundle +from agent_env.bundle.run import dry_run_bundle from agent_env.bundle import parse_bundle from agent_env.env import Env from agent_env.env.envs import computer_driver @@ -25,6 +26,7 @@ drag, press, scroll, type_text) SHIFT = Keyboard.keysym("Shift_L") +print("loading the desk driver") @computer_driver(name="desk", supports={supports}) @@ -128,6 +130,15 @@ def test_a_driver_its_class_cant_be_read_from_is_refused_before_any_write(bundle Env.get(f"{ROOT}/desk") +def test_a_dry_run_refuses_a_driver_folder_its_env_toml_doesnt_name_a_class_of(bundle_dir): + _bundle(bundle_dir, toml='type = "computer_driver"\nmodule = "server.py:Desk"\n') + + with pytest.raises(BundleError) as caught: + dry_run_bundle(bundle_dir) + + assert "module 'server.py:Desk' names no" in " ".join(caught.value.problems) + + def test_a_driver_whose_class_takes_too_long_to_read_is_refused(bundle_dir, monkeypatch): monkeypatch.setattr(computer_driver, "DECLARE_TIMEOUT_S", 0.001) _bundle(bundle_dir) From 2e38c6c4ae3630c635d3ee53e810ffa3f8a03d17 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 19:26:13 -0700 Subject: [PATCH 11/11] test(installer): agent-env ships five bundles now, with drivers Co-Authored-By: Claude Opus 5.5 --- tst/installer/test_container_journey.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tst/installer/test_container_journey.py b/tst/installer/test_container_journey.py index 02aa69ad..57972171 100644 --- a/tst/installer/test_container_journey.py +++ b/tst/installer/test_container_journey.py @@ -133,7 +133,7 @@ def test_a_new_user_installs_agent_env_adds_plugins_runs_a_task_and_removes_them container.ok("python", "-m", "pip", "install", "-q", "--no-index", "--find-links", "/wheels", "uv") container.ok("uv", "tool", "install", "agentenv-framework") rows = container.ok("agent-env", "plugin", "list").stdout.split("STATUS\n", 1)[1] - assert re.fullmatch(r"agentenv-framework +\S+ +4 bundles +ok\n", rows), rows + assert re.fullmatch(r"agentenv-framework +\S+ +5 bundles +ok\n", rows), rows added = container.ok("agent-env", "plugin", "add", browser, grader, "--yes") assert "toy_navigate active" in added.stdout and "toy_grade active" in added.stdout