From 9114b552e3323b3a47bb592cc388aa461fa2cf57 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 11:05:05 -0700 Subject: [PATCH 01/18] feat(protocol): structured errors on tools and extensions AgentEnvError(code, message, retryable, applied, hint, status). Raised in an extension it answers with its status and the fields in the error body; raised in a tool, on mcp's FastMCP and on fastmcp alike, it answers isError with ": " as text and the fields as structured content. Any other extension failure stays extension_failed / 500, now with applied "unknown". invoke_extension raises ExtensionCallError, an httpx.HTTPStatusError carrying the parsed fields, so callers that catch HTTPStatusError keep working. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/__init__.py | 5 + .../agent_env_environment.py | 67 ++++++-- .../src/agentenv_protocol/client.py | 28 +++- .../src/agentenv_protocol/errors.py | 29 ++++ .../src/agentenv_protocol/types.py | 6 +- .../agentenv-protocol/tests/test_client.py | 27 ++- .../agentenv-protocol/tests/test_errors.py | 158 ++++++++++++++++++ 7 files changed, 299 insertions(+), 21 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/errors.py create mode 100644 packages/agentenv-protocol/tests/test_errors.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py index c68ef7f2..fd209caf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py @@ -1,6 +1,8 @@ """Open data-plane protocol and server SDK for agent environments.""" from . import client +from .client import ExtensionCallError +from .errors import Applied, AgentEnvError from .preflight import intake_fit_check from .tool_client import ToolResult, ToolSession, ToolSessionError from .agent_env_environment import ( @@ -144,6 +146,9 @@ "AddDataResponse", "GetDataResponse", "error_body", + "AgentEnvError", + "Applied", + "ExtensionCallError", "PROTOCOL_VERSION", "RPC_PATH", "WELL_KNOWN_PATH", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index c1e9b9c6..134ceb04 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -21,6 +21,7 @@ from starlette.responses import Response from starlette.routing import Route +from .errors import AgentEnvError from .tool_context import EMPTY, ToolContext, bound, context_from_http, context_from_mcp from .transfers import WriteNamespaceGrant from .types import DATA_OBJECTS_EXTENSION_URI, MCP_TRANSPORT, METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH, AddDataRequest, AddDataResponse, EnvironmentCapabilities, EnvironmentCard, EnvironmentExtension, EnvironmentInterface, EnvironmentTool, GetDataResponse, ResetDataResponse, error_body @@ -308,12 +309,18 @@ async def call_tool(name: str, arguments: dict, *args: Any, **kwargs: Any) -> An return await inner(name, arguments, *args, **kwargs) context = kwargs.get("context", args[0] if args else None) with bound(context_from_mcp(context, name, arguments)) as tool_context: - if hook is not None: - answer = await _maybe_await(hook(tool_context)) - if answer is not None: - _check_call_tool_result(answer) - return answer - return await inner(name, arguments, *args, **kwargs) + try: + if hook is not None: + answer = await _maybe_await(hook(tool_context)) + if answer is not None: + _check_call_tool_result(answer) + return answer + return await inner(name, arguments, *args, **kwargs) + except Exception as e: + error = _agentenv_error(e) + if error is None: + raise + return _error_result(error) functools.update_wrapper(call_tool, inner) setattr(call_tool, _TOOL_CONTEXT_BOUND_ATTR, True) @@ -620,12 +627,18 @@ async def on_call_tool(self, context: Any, call_next: Callable) -> Any: message = context.message arguments = dict(getattr(message, "arguments", None) or {}) with bound(context_from_mcp(context.fastmcp_context, message.name, arguments)) as tool_context: - if hook is not None: - answer = await _maybe_await(hook(tool_context)) - if answer is not None: - _check_call_tool_result(answer) - return _fastmcp_tool_result(answer) - return await call_next(context) + try: + if hook is not None: + answer = await _maybe_await(hook(tool_context)) + if answer is not None: + _check_call_tool_result(answer) + return _fastmcp_tool_result(answer) + return await call_next(context) + except Exception as e: + error = _agentenv_error(e) + if error is None: + raise + return _fastmcp_tool_result(_error_result(error)) middleware = ToolContextMiddleware() setattr(middleware, _TOOL_CONTEXT_BOUND_ATTR, True) @@ -646,6 +659,24 @@ def _fastmcp_tool_result(answer: Any) -> Any: return ToolResult(content=list(answer.content), structured_content=structured, is_error=bool(is_error)) +def _agentenv_error(exc: BaseException) -> AgentEnvError | None: + """The AgentEnvError a tool raised, found under the ToolError both FastMCPs wrap it in.""" + while exc is not None: + if isinstance(exc, AgentEnvError): + return exc + exc = exc.__cause__ + return None + + +def _error_result(error: AgentEnvError) -> Any: + from mcp.types import CallToolResult, TextContent + return CallToolResult( + isError=True, + content=[TextContent(type="text", text=f"{error.code}: {error.message}")], + structuredContent=error.fields(), + ) + + def _schema_from_signature(fn: Callable) -> dict: """Derive a JSON-Schema object for a handler's call params from its signature (Annotated[Field] descriptions included). Injected parameters (``ToolContext``, FastMCP ``Context``) are not wire parameters and are left out.""" @@ -961,22 +992,28 @@ async def handler(request: Request) -> Response: raise ValueError("request body must be a JSON object") params = _coerce_params(fn, params) except Exception as e: - return _json_response(error_body("invalid_params", str(e)), 400) + return _error_response(AgentEnvError("invalid_params", str(e))) try: signature.bind(**params) except TypeError as e: - return _json_response(error_body("invalid_params", str(e)), 400) + return _error_response(AgentEnvError("invalid_params", str(e))) with bound(context_from_http(request, op, params)) as tool_context: call_kwargs = {**params, slot: tool_context} if slot else params try: result = await _maybe_await(fn(**call_kwargs)) + except AgentEnvError as e: + return _error_response(e) except Exception as e: logger.exception("extension invocation failed") - return _json_response(error_body("extension_failed", str(e)), 500) + return _error_response(AgentEnvError("extension_failed", str(e), applied="unknown", status=500)) return _json_response(result if result is not None else {}) return handler +def _error_response(error: AgentEnvError) -> Response: + return _json_response(error_body(**error.fields()), error.status) + + class _Op(NamedTuple): name: str rpc_method: str diff --git a/packages/agentenv-protocol/src/agentenv_protocol/client.py b/packages/agentenv-protocol/src/agentenv_protocol/client.py index 9d3be78a..e934b772 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/client.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/client.py @@ -12,6 +12,29 @@ logger = logging.getLogger(__name__) +class ExtensionCallError(httpx.HTTPStatusError): + """A non-2xx answer from an extension method, with the error fields the server sent. A body without them + (a proxy's own error page, an older server) leaves ``code`` None and ``applied`` ``"unknown"``.""" + + def __init__(self, response: httpx.Response) -> None: + error = _error_fields(response) + self.code: str | None = error.get("code") + self.detail: str = error.get("message") or response.reason_phrase + self.retryable: bool = bool(error.get("retryable", False)) + self.applied: str = error.get("applied", "unknown") + self.hint: str | None = error.get("hint") + super().__init__(f"{response.status_code} {self.code or 'error'}: {self.detail}", request=response.request, response=response) + + +def _error_fields(response: httpx.Response) -> dict: + try: + body = response.json() + except ValueError: + return {} + error = body.get("error") if isinstance(body, dict) else None + return error if isinstance(error, dict) else {} + + async def _rpc(base_url: str, method: str, params: dict, timeout: int, verify: bool) -> dict: request = {"jsonrpc": "2.0", "id": 1, "method": method, "params": params} async with httpx.AsyncClient(verify=verify) as client: @@ -168,7 +191,7 @@ async def invoke_extension(base_url: str, card: dict, uri: str, params: dict | N Reads the endpoint and HTTP verb of the extension method named `method` from the card, so the card alone is enough to invoke; without `method`, the first method listed. Returns the parsed - JSON result; raises on a non-2xx response. + JSON result; a non-2xx response raises `ExtensionCallError`. """ ext = find_extension(card, uri) if ext is None: @@ -187,5 +210,6 @@ async def invoke_extension(base_url: str, card: dict, uri: str, params: dict | N response = await client.get(url, params=params or {}, timeout=timeout) else: response = await client.request(op["method"], url, json=params or {}, timeout=timeout) - response.raise_for_status() + if response.status_code >= 400: + raise ExtensionCallError(response) return response.json() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/errors.py new file mode 100644 index 00000000..9ba47e4d --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/errors.py @@ -0,0 +1,29 @@ +"""Errors a tool or extension raises to answer its caller with a code, not a crash.""" +from __future__ import annotations + +from typing import Literal + +#: Whether the call's effect happened: ``unknown`` when it may have been delivered before something failed. +Applied = Literal["yes", "no", "unknown"] + + +class AgentEnvError(Exception): + """Raised in an extension, answers with ``status`` and these fields as the error body; raised in a tool, + answers ``isError`` with ``": "`` as text and the fields as structured content. Callers retry + on their own only when ``retryable`` and ``applied == "no"``.""" + + def __init__(self, code: str, message: str, *, retryable: bool = False, applied: Applied = "no", + hint: str | None = None, status: int = 400) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retryable = retryable + self.applied = applied + self.hint = hint + self.status = status + + def fields(self) -> dict: + fields = {"code": self.code, "message": self.message, "retryable": self.retryable, "applied": self.applied} + if self.hint is not None: + fields["hint"] = self.hint + return fields diff --git a/packages/agentenv-protocol/src/agentenv_protocol/types.py b/packages/agentenv-protocol/src/agentenv_protocol/types.py index 1f95e52c..56a860ea 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/types.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/types.py @@ -1,15 +1,15 @@ """Wire types for the v1 data-plane protocol.""" from __future__ import annotations -from typing import Annotated, Literal, Optional, Union +from typing import Annotated, Any, Literal, Optional, Union from pydantic import BaseModel, Field, TypeAdapter, ValidationError from .transfers import RelativePath -def error_body(code: str, message: str) -> dict: - return {"ok": False, "error": {"code": code, "message": message}} +def error_body(code: str, message: str, **fields: Any) -> dict: + return {"ok": False, "error": {"code": code, "message": message, **fields}} PROTOCOL_VERSION = "1.0" diff --git a/packages/agentenv-protocol/tests/test_client.py b/packages/agentenv-protocol/tests/test_client.py index 844119a6..13088bf0 100644 --- a/packages/agentenv-protocol/tests/test_client.py +++ b/packages/agentenv-protocol/tests/test_client.py @@ -8,7 +8,7 @@ import httpx import pytest -from agentenv_protocol import DataPart, client as protocol_v1 +from agentenv_protocol import DataPart, ExtensionCallError, client as protocol_v1 _BASE = "http://gw/svc/mcp-items" @@ -223,6 +223,31 @@ async def test_invoke_extension_get_uses_query_params(): assert client.get.call_args.args[0] == f"{_BASE}/agentenv/ext/ping" +def _error_response(status: int, **kwargs) -> httpx.Response: + return httpx.Response(status, request=httpx.Request("POST", f"{_BASE}/agentenv/ext/set_errors"), **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response, fields", [ + (_error_response(429, json={"ok": False, "error": { + "code": "limit_exceeded", "message": "3 leases held", "retryable": True, "applied": "no", "hint": "release one"}}), + ("limit_exceeded", "3 leases held", True, "no", "release one")), + # A server from before the structured fields: its code, and nothing claimed about the effect. + (_error_response(500, json={"ok": False, "error": {"code": "extension_failed", "message": "kaboom"}}), + ("extension_failed", "kaboom", False, "unknown", None)), + # A proxy's own error page. + (_error_response(502, text="Bad Gateway"), (None, "Bad Gateway", False, "unknown", None)), + (_error_response(404, json=["not", "an", "object"]), (None, "Not Found", False, "unknown", None)), +]) +async def test_invoke_extension_raises_the_error_fields_it_was_sent(response, fields): + with _patch(_client(response)), pytest.raises(ExtensionCallError) as raised: + await protocol_v1.invoke_extension(_BASE, _CARD_SET_ERRORS, "urn:agentenv:set-errors/v1", {}) + error = raised.value + assert (error.code, error.detail, error.retryable, error.applied, error.hint) == fields + assert isinstance(error, httpx.HTTPStatusError) and error.response is response + assert str(error).startswith(f"{response.status_code} {fields[0] or 'error'}: ") + + @pytest.mark.asyncio async def test_invoke_extension_unknown_uri_raises(): with pytest.raises(ValueError, match="not advertised"): diff --git a/packages/agentenv-protocol/tests/test_errors.py b/packages/agentenv-protocol/tests/test_errors.py new file mode 100644 index 00000000..ec8123bf --- /dev/null +++ b/packages/agentenv-protocol/tests/test_errors.py @@ -0,0 +1,158 @@ +"""Structured errors: an AgentEnvError raised in an extension answers with its status and fields, raised in a tool +it answers isError with the fields as structured content on both FastMCPs, and the client parses them back.""" +from __future__ import annotations + +from unittest.mock import patch + +import anyio +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +fastmcp = pytest.importorskip("fastmcp") + +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 + +from agentenv_protocol import ( # noqa: E402 + AgentEnvEnvironment, + AgentEnvError, + ExtensionCallError, + ToolContext, + client, + environment_card, + extension, + tool, +) + +_URI = "urn:agentenv:frames/v1" + + +@environment_card(name="frames") +class _Env(AgentEnvEnvironment): + def __init__(self) -> None: + self.refuse_hook = False + + def on_tool_call(self, context: ToolContext): + if self.refuse_hook: + raise AgentEnvError("device_unavailable", "not ready", retryable=True, status=503) + return None + + @tool() + def click(self, x: int) -> str: + if x > 100: + raise AgentEnvError("out_of_bounds", f"({x}, 0) is outside frame f1 (100×100)", hint="x < 100") + return "ok" + + @tool() + def crash(self) -> str: + raise RuntimeError("kaboom") + + @extension(_URI) + async def stale(self, frame: str) -> dict: + raise AgentEnvError("stale_frame", f"frame {frame} is stale", retryable=True, status=409) + + @extension("urn:agentenv:partial/v1") + async def partial(self) -> dict: + raise AgentEnvError("delivery_failed", "typed 3 of 5 characters", applied="unknown", status=502) + + @extension("urn:agentenv:broken/v1") + async def broken(self) -> dict: + raise RuntimeError("kaboom") + + +def test_fields_carry_the_defaults_and_a_hint_only_when_given(): + assert AgentEnvError("invalid_argument", "bad").fields() == { + "code": "invalid_argument", "message": "bad", "retryable": False, "applied": "no"} + error = AgentEnvError("invalid_argument", "unknown key", hint="did you mean Return?") + assert error.fields()["hint"] == "did you mean Return?" and error.status == 400 and str(error) == "unknown key" + + +def _http(env: _Env) -> httpx.AsyncClient: + app = env.create_app().streamable_http_app() + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") + + +@pytest.mark.asyncio +async def test_an_extension_answers_with_the_errors_status_and_fields(): + async with _http(_Env()) as http: + stale = await http.post("/agentenv/ext/stale", json={"frame": "f3"}) + partial = await http.post("/agentenv/ext/partial", json={}) + assert stale.status_code == 409 + assert stale.json() == {"ok": False, "error": { + "code": "stale_frame", "message": "frame f3 is stale", "retryable": True, "applied": "no"}} + assert partial.status_code == 502 and partial.json()["error"]["applied"] == "unknown" + + +@pytest.mark.asyncio +async def test_any_other_extension_failure_is_extension_failed_with_applied_unknown(): + async with _http(_Env()) as http: + r = await http.post("/agentenv/ext/broken", json={}) + assert r.status_code == 500 + assert r.json()["error"] == {"code": "extension_failed", "message": "kaboom", "retryable": False, "applied": "unknown"} + + +@pytest.mark.asyncio +async def test_invalid_params_never_ran_so_applied_is_no(): + async with _http(_Env()) as http: + r = await http.post("/agentenv/ext/stale", json={"frame": "f1", "extra": 1}) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "invalid_params" and r.json()["error"]["applied"] == "no" + + +@pytest.mark.asyncio +async def test_a_tool_error_is_an_error_result_with_structured_fields_on_mcps_fastmcp(): + env = _Env() + with anyio.fail_after(30): + async with create_connected_server_and_client_session(env.create_app()) as session: + refused = await session.call_tool("click", {"x": 140}) + served = await session.call_tool("click", {"x": 4}) + crashed = await session.call_tool("crash", {}) + env.refuse_hook = True + hooked = await session.call_tool("click", {"x": 4}) + assert refused.isError is True + assert refused.content[0].text == "out_of_bounds: (140, 0) is outside frame f1 (100×100)" + assert refused.structuredContent == {"code": "out_of_bounds", "message": "(140, 0) is outside frame f1 (100×100)", + "retryable": False, "applied": "no", "hint": "x < 100"} + assert served.isError is False and served.content[0].text == "ok" + assert crashed.isError is True and "kaboom" in crashed.content[0].text and crashed.structuredContent is None + assert hooked.isError is True and hooked.structuredContent["code"] == "device_unavailable" + assert hooked.structuredContent["retryable"] is True + + +@pytest.mark.asyncio +async def test_a_tool_error_is_an_error_result_with_structured_fields_on_fastmcp(): + app = fastmcp.FastMCP("frames") + env = _Env() + env.mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as session: + refused = await session.call_tool("click", {"x": 140}, raise_on_error=False) + crashed = await session.call_tool("crash", {}, raise_on_error=False) + env.refuse_hook = True + hooked = await session.call_tool("click", {"x": 4}, raise_on_error=False) + assert refused.is_error is True + assert refused.content[0].text == "out_of_bounds: (140, 0) is outside frame f1 (100×100)" + assert refused.structured_content["code"] == "out_of_bounds" and refused.structured_content["hint"] == "x < 100" + assert crashed.is_error is True and "kaboom" in crashed.content[0].text and not crashed.structured_content + assert hooked.is_error is True and hooked.structured_content["code"] == "device_unavailable" + + +@pytest.mark.asyncio +async def test_the_client_raises_the_servers_fields_as_an_extension_call_error(): + env = _Env() + app = env.create_app().streamable_http_app() + card = await _card(app) + asgi = httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") + with patch("agentenv_protocol.client.httpx.AsyncClient", return_value=asgi): + with pytest.raises(ExtensionCallError) as raised: + await client.invoke_extension("http://t", card, _URI, {"frame": "f9"}) + error = raised.value + assert isinstance(error, httpx.HTTPStatusError) and error.response.status_code == 409 + assert (error.code, error.detail, error.retryable, error.applied, error.hint) == ( + "stale_frame", "frame f9 is stale", True, "no", None) + assert str(error) == "409 stale_frame: frame f9 is stale" + + +async def _card(app) -> dict: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as http: + return (await http.get("/.well-known/agent-env.json")).json() From c0d1d57b7d28b60c05b04f975e0170f392670614 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 11:12:33 -0700 Subject: [PATCH 02/18] feat(protocol): several methods per extension @extension(uri, op=...) makes a handler one method of an extension: handlers sharing the URI merge into one card entry, each method under methods. with its own verb and endpoint, by default /agentenv/ext//. The environment's extension_params(uri) adds extension-level params beside the methods, and declared_extensions() advertises declaration-only extensions. Two handlers for one method, a handler without op on a URI another uses, and methods disagreeing on description or required are refused at mount. Without op, routes and cards are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../agent_env_environment.py | 96 +++++++-- .../tests/test_extension_methods.py | 195 ++++++++++++++++++ 2 files changed, 278 insertions(+), 13 deletions(-) create mode 100644 packages/agentenv-protocol/tests/test_extension_methods.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index 134ceb04..ffa079a7 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -31,6 +31,7 @@ _OP_ATTR = "_aee_op" _EXT_ATTR = "_aee_ext" _EXT_ROUTE_ATTR = "_aee_ext_route" +_EXT_OP_ATTR = "_aee_ext_op" _TOOL_ATTR = "_aee_tool" _HIDDEN_BY_DEFAULT_ATTR = "_aee_hidden_by_default" _CARD_CONFIG_ATTR = "_aee_card_config" @@ -69,7 +70,7 @@ def get_data(fn: Callable) -> Callable: return fn -def extension(uri: str, *, description: str | None = None, params: dict | None = None, +def extension(uri: str, *, op: str | None = None, description: str | None = None, params: dict | None = None, required: bool | None = None, method: str = "POST", path: str | None = None) -> Callable: """Mark a handler method as the invocation handler for a card extension. @@ -87,7 +88,15 @@ def extension(uri: str, *, description: str | None = None, params: dict | None = where the request JSON-Schema is derived from the signature. Pass an explicit ``params`` to override the advertisement. A handler that wants the raw params dict can declare ``**params``. + + With ``op``, the handler is one method of an extension that has several: handlers sharing the URI + merge into one card entry, each method under ``methods.`` with its own verb and ``endpoint``, + by default ``/ext//`` where ```` is the URN's name without + ``urn:agentenv:`` and the version. Extension-level params come from the environment's + ``extension_params(uri)``, merged beside ``methods``, so ``params`` is refused with ``op``. """ + if op is not None and params is not None: + raise TypeError("an extension method's advertisement comes from extension_params(uri), not params=") # Normalize the HTTP verb once, here, so the advertised card method, the # route registration, and the handler's request-binding (which checks for # the exact string "GET") all agree. Without this, ``@extension(method="get")`` @@ -96,19 +105,33 @@ def extension(uri: str, *, description: str | None = None, params: dict | None = http_method = method.upper() def deco(fn: Callable) -> Callable: - op = fn.__name__ or "invoke" - route_path = path or f"{RPC_PATH}/ext/{op}" - advertised = params if params is not None else { - "endpoint": route_path, - "methods": {op: {"method": http_method, "request": _schema_from_signature(fn)}}, - } + if op is None: + name = fn.__name__ or "invoke" + route_path = path or f"{RPC_PATH}/ext/{name}" + advertised = params if params is not None else { + "endpoint": route_path, + "methods": {name: {"method": http_method, "request": _schema_from_signature(fn)}}, + } + else: + route_path = path or f"{RPC_PATH}/ext/{_extension_slug(uri)}/{op}" + advertised = {"methods": {op: {"method": http_method, "endpoint": route_path, "request": _schema_from_signature(fn)}}} setattr(fn, _EXT_ATTR, EnvironmentExtension(uri=uri, description=description, params=advertised, required=required)) setattr(fn, _EXT_ROUTE_ATTR, (route_path, http_method)) + setattr(fn, _EXT_OP_ATTR, op) return fn return deco +def _extension_slug(uri: str) -> str: + """``urn:agentenv:computer-display/v1`` -> ``computer-display``; another namespace keeps its name as a path.""" + name = uri.removeprefix("urn:").removeprefix("agentenv:") + head, _, version = name.rpartition("/") + if head and version[:1] == "v" and version[1:].isdigit(): + name = head + return name.replace(":", "/") + + def tool(name: str | None = None, *, description: str | None = None, hidden_by_default: bool = False) -> Callable: """Mark a handler method as an MCP tool. @@ -178,7 +201,7 @@ class AgentEnvApplication(ABC): def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: self.handler = handler ext_handlers = _discover_extensions(handler) - environment_card = _merge_extensions(environment_card, ext_handlers) + environment_card = _merge_extensions(environment_card, ext_handlers + _declared_extensions(handler, ext_handlers)) tool_handlers = _discover_tools(handler, environment_card.name) if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") @@ -388,6 +411,14 @@ def serve(self, transport: str = "streamable-http") -> None: self.mount(self.mcp) self.mcp.run(transport=transport) + def extension_params(self, uri: str) -> dict: + """Extension-level params (features, limits) advertised beside the methods of an extension served with ``op``.""" + return {} + + def declared_extensions(self) -> list[EnvironmentExtension]: + """Declaration-only extensions to advertise, like ``intake/v1``: a promise with params and no handler.""" + return [] + def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -728,20 +759,59 @@ def _discover_methods(handler: Any) -> dict: def _discover_extensions(handler: Any) -> list: - """Collect (descriptor, bound_method, route_path, http_method) for every @extension method.""" - seen_uris: set = set() + """Collect (descriptor, bound_method, route_path, http_method) for every @extension method. The methods of an + extension served with ``op`` share one descriptor: their ``methods`` merged beside ``extension_params(uri)``.""" + ops_by_uri: dict = {} seen_paths: set = set() ordered: list = [] for attr, descriptor in _marked_members(handler, _EXT_ATTR): route_path, http_method = getattr(attr, _EXT_ROUTE_ATTR) - if descriptor.uri in seen_uris: + op = getattr(attr, _EXT_OP_ATTR, None) + ops = ops_by_uri.setdefault(descriptor.uri, set()) + if None in ops or (op is None and ops): raise ValueError(f"Multiple handlers for extension {descriptor.uri}") + if op in ops: + raise ValueError(f"Multiple handlers for extension {descriptor.uri} method {op}") if route_path in seen_paths: raise ValueError(f"Multiple handlers for extension path {route_path}") - seen_uris.add(descriptor.uri) + ops.add(op) seen_paths.add(route_path) ordered.append((descriptor, attr, route_path, http_method)) - return ordered + merged = { + uri: _merged_extension(handler, [d for d, *_ in ordered if d.uri == uri]) + for uri, ops in ops_by_uri.items() if None not in ops + } + return [(merged.get(d.uri, d), attr, path, method) for d, attr, path, method in ordered] + + +def _merged_extension(handler: Any, descriptors: list) -> EnvironmentExtension: + uri = descriptors[0].uri + methods = {name: entry for d in descriptors for name, entry in d.params["methods"].items()} + hook = getattr(handler, "extension_params", None) + params = {**(hook(uri) if hook is not None else {}), "methods": methods} + return EnvironmentExtension( + uri=uri, + description=_one_value(uri, "description", [d.description for d in descriptors]), + params=params, + required=_one_value(uri, "required", [d.required for d in descriptors]), + ) + + +def _one_value(uri: str, field: str, values: list) -> Any: + given = {v for v in values if v is not None} + if len(given) > 1: + raise ValueError(f"the methods of extension {uri} declare different {field} values: {sorted(map(str, given))}") + return given.pop() if given else None + + +def _declared_extensions(handler: Any, ext_handlers: list) -> list: + hook = getattr(handler, "declared_extensions", None) + declared = list(hook()) if hook is not None else [] + served = {descriptor.uri for descriptor, *_ in ext_handlers} + clash = sorted(d.uri for d in declared if d.uri in served) + if clash: + raise ValueError(f"extensions both declared and served by a handler: {clash}") + return [(d,) for d in declared] def _merge_extensions(card: EnvironmentCard, ext_handlers: list) -> EnvironmentCard: diff --git a/packages/agentenv-protocol/tests/test_extension_methods.py b/packages/agentenv-protocol/tests/test_extension_methods.py new file mode 100644 index 00000000..c2940f15 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_extension_methods.py @@ -0,0 +1,195 @@ +"""Several methods per extension: handlers given ``op`` merge into one card entry with an endpoint per method, with +the environment's extension_params beside them; declaration-only extensions; the refusals at mount.""" +from __future__ import annotations + +from unittest.mock import patch + +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") + +from agentenv_protocol import ( # noqa: E402 + AgentEnvEnvironment, + AgentEnvFastMCPApplication, + EnvironmentCapabilities, + EnvironmentCard, + EnvironmentExtension, + client, + environment_card, + extension, +) +from agentenv_protocol.agent_env_environment import _extension_slug # noqa: E402 + +_DISPLAY = "urn:agentenv:computer-display/v1" +_POINTER = "urn:agentenv:computer-pointer/v1" + + +@environment_card(name="display") +class _Display(AgentEnvEnvironment): + def __init__(self) -> None: + self.size = (1280, 800, 1.0) + + def extension_params(self, uri: str) -> dict: + return {"modes": [[1280, 800], [1920, 1080]]} if uri == _DISPLAY else {} + + def declared_extensions(self) -> list[EnvironmentExtension]: + return [EnvironmentExtension(uri=_POINTER, description="Pointer tools.", params={"buttons": ["left"]})] + + @extension(_DISPLAY, op="get", method="GET", description="Display resolution and scale.") + async def display_get(self) -> dict: + width, height, scale = self.size + return {"width": width, "height": height, "scale": scale} + + @extension(_DISPLAY, op="set") + async def display_set(self, width: int, height: int, scale: float = 1.0) -> dict: + self.size = (width, height, scale) + return await self.display_get() + + @extension("urn:agentenv:legacy/v1") + async def legacy(self) -> dict: + return {"legacy": True} + + +def _card(handler, card: EnvironmentCard | None = None) -> EnvironmentCard: + return AgentEnvFastMCPApplication(card or EnvironmentCard(name="t"), handler).environment_card + + +def _extensions(card: EnvironmentCard) -> dict: + return {e.uri: e for e in card.capabilities.extensions} + + +def _http(env: AgentEnvEnvironment) -> httpx.AsyncClient: + return _client(env.create_app().streamable_http_app()) + + +def _client(app) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") + + +@pytest.mark.parametrize("uri, slug", [ + ("urn:agentenv:computer-display/v1", "computer-display"), + ("urn:agentenv:computer-display/v12", "computer-display"), + ("urn:trycua:windows/v1", "trycua/windows"), + ("urn:agentenv:unversioned", "unversioned"), + ("urn:agentenv:mail/vnext", "mail/vnext"), +]) +def test_extension_slug_drops_the_agentenv_namespace_and_the_version(uri, slug): + assert _extension_slug(uri) == slug + + +def test_methods_sharing_a_uri_merge_into_one_card_entry_with_their_own_endpoints(): + extensions = _extensions(_card(_Display())) + display = extensions[_DISPLAY] + assert display.description == "Display resolution and scale." + assert display.params["modes"] == [[1280, 800], [1920, 1080]] and "endpoint" not in display.params + methods = display.params["methods"] + assert set(methods) == {"get", "set"} + assert methods["get"]["method"] == "GET" and methods["get"]["endpoint"] == "/agentenv/ext/computer-display/get" + assert methods["set"]["method"] == "POST" and methods["set"]["endpoint"] == "/agentenv/ext/computer-display/set" + assert methods["set"]["request"]["required"] == ["width", "height"] + assert extensions[_POINTER] == EnvironmentExtension(uri=_POINTER, description="Pointer tools.", params={"buttons": ["left"]}) + assert extensions["urn:agentenv:legacy/v1"].params["endpoint"] == "/agentenv/ext/legacy" + + +@pytest.mark.asyncio +async def test_each_method_is_served_at_its_endpoint_and_the_client_reaches_it_by_name(): + app = _Display().create_app().streamable_http_app() + async with _client(app) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + with patch("agentenv_protocol.client.httpx.AsyncClient", return_value=_client(app)): + changed = await client.invoke_extension("http://t", card, _DISPLAY, {"width": 1920, "height": 1080}, method="set") + got = await http.get("/agentenv/ext/computer-display/get") + legacy = await http.post("/agentenv/ext/legacy", json={}) + assert changed == {"width": 1920, "height": 1080, "scale": 1.0} + assert got.json() == {"width": 1920, "height": 1080, "scale": 1.0} and legacy.json() == {"legacy": True} + + +@pytest.mark.asyncio +async def test_an_explicit_path_is_the_methods_endpoint(): + class _Paths(AgentEnvEnvironment): + @extension(_DISPLAY, op="get", method="GET", path="/display") + async def get(self) -> dict: + return {"ok": True} + + env = _Paths() + assert _extensions(_card(env))[_DISPLAY].params["methods"]["get"]["endpoint"] == "/display" + async with _http(env) as http: + assert (await http.get("/display")).json() == {"ok": True} + + +def test_a_card_declared_extension_wins_over_the_merged_one(): + card = EnvironmentCard(name="d", capabilities=EnvironmentCapabilities(extensions=[EnvironmentExtension(uri=_DISPLAY)])) + assert [e for e in _card(_Display(), card).capabilities.extensions if e.uri == _DISPLAY] == [EnvironmentExtension(uri=_DISPLAY)] + + +def test_a_plain_handler_without_the_hooks_serves_methods_with_no_extra_params(): + class _Plain: + @extension(_DISPLAY, op="get") + async def get(self) -> dict: + return {} + + assert _extensions(_card(_Plain()))[_DISPLAY].params == { + "methods": {"get": {"method": "POST", "endpoint": "/agentenv/ext/computer-display/get", + "request": {"type": "object", "properties": {}}}}} + + +def test_params_are_refused_with_op(): + with pytest.raises(TypeError, match="extension_params"): + extension(_DISPLAY, op="get", params={"methods": {}}) + + +def test_two_handlers_for_one_method_are_refused(): + class _Dup: + @extension(_DISPLAY, op="get", path="/a") + async def a(self) -> dict: + return {} + + @extension(_DISPLAY, op="get", path="/b") + async def b(self) -> dict: + return {} + + with pytest.raises(ValueError, match="Multiple handlers for extension urn:agentenv:computer-display/v1 method get"): + _card(_Dup()) + + +@pytest.mark.parametrize("first, second", [("get", None), (None, "get"), (None, None)]) +def test_a_handler_without_op_shares_its_uri_with_no_other(first, second): + class _Mixed: + @extension(_DISPLAY, op=first, path="/a") + async def a(self) -> dict: + return {} + + @extension(_DISPLAY, op=second, path="/b") + async def b(self) -> dict: + return {} + + with pytest.raises(ValueError, match="Multiple handlers for extension urn:agentenv:computer-display/v1$"): + _card(_Mixed()) + + +def test_methods_disagreeing_on_description_are_refused(): + class _Described: + @extension(_DISPLAY, op="get", description="one") + async def a(self) -> dict: + return {} + + @extension(_DISPLAY, op="set", description="two") + async def b(self) -> dict: + return {} + + with pytest.raises(ValueError, match="declare different description values"): + _card(_Described()) + + +def test_an_extension_both_declared_and_served_is_refused(): + class _Both(AgentEnvEnvironment): + def declared_extensions(self) -> list[EnvironmentExtension]: + return [EnvironmentExtension(uri=_DISPLAY)] + + @extension(_DISPLAY, op="get") + async def a(self) -> dict: + return {} + + with pytest.raises(ValueError, match="both declared and served"): + _card(_Both()) From 0acb5084c10bebcde929b57b4804103c969a7e7c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 11:16:13 -0700 Subject: [PATCH 03/18] feat(protocol): @tool annotations, structured output and _meta; served_tools @tool takes annotations (MCP ToolAnnotations fields), structured_output and meta, passed through to mcp's FastMCP and to fastmcp, with the hidden-by-default marker merged into meta. A tool returning Annotated[CallToolResult, Model] is sent as is on either FastMCP and listed with Model's schema. served_tools(tools) lets an environment choose at mount which @tool methods it serves, and with which descriptions. Co-Authored-By: Claude Opus 5.5 --- .../agent_env_environment.py | 94 +++++++++++- .../tests/test_tool_annotations.py | 143 ++++++++++++++++++ 2 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 packages/agentenv-protocol/tests/test_tool_annotations.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index ffa079a7..0bde81d5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from typing import Annotated, Any, Callable, ClassVar, Literal, NamedTuple, Union, get_args, get_origin, get_type_hints -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from pydantic.fields import FieldInfo from starlette.requests import Request from starlette.responses import Response @@ -34,6 +34,7 @@ _EXT_OP_ATTR = "_aee_ext_op" _TOOL_ATTR = "_aee_tool" _HIDDEN_BY_DEFAULT_ATTR = "_aee_hidden_by_default" +_TOOL_OPTIONS_ATTR = "_aee_tool_options" _CARD_CONFIG_ATTR = "_aee_card_config" #: Set on the tool dispatch installed by ``_bind_tool_calls``; a guard downstream checks it at boot. _TOOL_CONTEXT_BOUND_ATTR = "__agentenv_tool_context__" @@ -132,7 +133,8 @@ def _extension_slug(uri: str) -> str: return name.replace(":", "/") -def tool(name: str | None = None, *, description: str | None = None, hidden_by_default: bool = False) -> Callable: +def tool(name: str | None = None, *, description: str | None = None, hidden_by_default: bool = False, + annotations: dict | None = None, structured_output: bool | None = None, meta: dict | None = None) -> Callable: """Mark a handler method as an MCP tool. At mount the bound method is registered as a real MCP tool on the server app and advertised @@ -155,6 +157,12 @@ def tool(name: str | None = None, *, description: str | None = None, hidden_by_d ``{HIDDEN_BY_DEFAULT_META_KEY: true}``, and a gateway hides it from every role until a tool-access step names it. Steps for every tool (``tools="*"``) never show it, though they still clear earlier named steps as they do for any tool. + + ``annotations`` (MCP ``ToolAnnotations`` fields, such as ``{"readOnlyHint": True}``) and ``meta`` are listed with + the tool. ``structured_output`` follows mcp's FastMCP: None derives an output schema from the return annotation + where it can, True requires one, False lists none; on a fastmcp app it is fastmcp's own inference, off with + False. A tool returning ``Annotated[CallToolResult, Model]`` sends its result as is on either, listed with + ``Model``'s schema. """ if callable(name): raise TypeError("use @tool(...) with parentheses, not bare @tool") @@ -167,11 +175,18 @@ def deco(fn: Callable) -> Callable: )) if hidden_by_default: setattr(fn, _HIDDEN_BY_DEFAULT_ATTR, True) + setattr(fn, _TOOL_OPTIONS_ATTR, _ToolOptions(annotations, structured_output, meta)) return fn return deco +class _ToolOptions(NamedTuple): + annotations: dict | None + structured_output: bool | None + meta: dict | None + + def environment_card(_cls: type | None = None, **config: Any) -> Callable: """Configure the class's EnvironmentCard; kwargs are EnvironmentCard fields. @@ -202,7 +217,7 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: self.handler = handler ext_handlers = _discover_extensions(handler) environment_card = _merge_extensions(environment_card, ext_handlers + _declared_extensions(handler, ext_handlers)) - tool_handlers = _discover_tools(handler, environment_card.name) + tool_handlers = _served_tools(handler, _discover_tools(handler, environment_card.name)) if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") environment_card = _merge_tools(environment_card, tool_handlers) @@ -262,9 +277,23 @@ def _add_route(self, app: Any, path: str, methods: list, handler: Callable) -> N app.custom_route(path, methods=methods)(handler) def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> None: - # passed only when set, so a FastMCP-shaped target without the parameter still mounts unflagged tools - extra = {"meta": {HIDDEN_BY_DEFAULT_META_KEY: True}} if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False) else {} - app.tool(name=descriptor.name, description=descriptor.description, **extra)(injecting(fn)) + # passed only when set, so a FastMCP-shaped target without the parameters still mounts plain tools + options = getattr(fn, _TOOL_OPTIONS_ATTR, None) or _ToolOptions(None, None, None) + fastmcp = self._binding == "middleware" + extra: dict = {} + meta = {**(options.meta or {}), **({HIDDEN_BY_DEFAULT_META_KEY: True} if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False) else {})} + if meta: + extra["meta"] = meta + if options.annotations is not None: + extra["annotations"] = options.annotations if fastmcp else _mcp_tool_annotations(options.annotations) + if options.structured_output is not None and not fastmcp: + extra["structured_output"] = options.structured_output + elif options.structured_output is False: + extra["output_schema"] = None + registered = injecting(fn) + if fastmcp and _returns_call_tool_result(fn): + registered, extra["output_schema"] = _fastmcp_returning(registered) + app.tool(name=descriptor.name, description=descriptor.description, **extra)(registered) def _check_tool_binding(self, app: Any) -> None: """Decide, before any route or tool is added, how every tools/call on ``app`` gets its ToolContext, and @@ -419,6 +448,10 @@ def declared_extensions(self) -> list[EnvironmentExtension]: """Declaration-only extensions to advertise, like ``intake/v1``: a promise with params and no handler.""" return [] + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + """The ``@tool`` methods to serve, chosen at mount; a returned descriptor's description replaces the marked one.""" + return tools + def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -690,6 +723,39 @@ def _fastmcp_tool_result(answer: Any) -> Any: return ToolResult(content=list(answer.content), structured_content=structured, is_error=bool(is_error)) +def _mcp_tool_annotations(annotations: Any) -> Any: + from mcp.types import ToolAnnotations + return ToolAnnotations.model_validate(annotations) + + +def _returns_call_tool_result(fn: Callable) -> bool: + returns = _resolved_hints(fn).get("return") + if get_origin(returns) is Annotated: + returns = get_args(returns)[0] + result_type = getattr(sys.modules.get("mcp.types"), "CallToolResult", None) + return result_type is not None and isinstance(returns, type) and issubclass(returns, result_type) + + +def _fastmcp_returning(fn: Callable) -> tuple: + """A tool returning ``CallToolResult`` as fastmcp sends it as is: a twin returning fastmcp's ``ToolResult``, and the + output schema of ``Annotated[CallToolResult, Model]``'s model (None for a bare ``CallToolResult``).""" + from fastmcp.tools import ToolResult + signature, annotations = _reduced_signature(fn, "") + returns = annotations.get("return") + schema = TypeAdapter(get_args(returns)[1]).json_schema() if get_origin(returns) is Annotated else None + + if inspect.iscoroutinefunction(fn): + async def wrapper(*args: Any, **kwargs: Any) -> Any: + return _fastmcp_tool_result(await fn(*args, **kwargs)) + else: + def wrapper(*args: Any, **kwargs: Any) -> Any: + return _fastmcp_tool_result(fn(*args, **kwargs)) + functools.update_wrapper(wrapper, fn) + wrapper.__signature__ = signature.replace(return_annotation=ToolResult) + wrapper.__annotations__ = {**annotations, "return": ToolResult} + return wrapper, schema + + def _agentenv_error(exc: BaseException) -> AgentEnvError | None: """The AgentEnvError a tool raised, found under the ToolError both FastMCPs wrap it in.""" while exc is not None: @@ -852,6 +918,22 @@ def _discover_tools(handler: Any, environment_name: str) -> list: return ordered +def _served_tools(handler: Any, tool_handlers: list) -> list: + """The tools the handler's ``served_tools`` keeps, in its order and with its descriptors; all of them without one.""" + hook = getattr(handler, "served_tools", None) + if hook is None: + return tool_handlers + chosen = list(hook([descriptor for descriptor, _fn in tool_handlers])) + by_name = {descriptor.name: fn for descriptor, fn in tool_handlers} + names = [descriptor.name for descriptor in chosen] + unknown = sorted(set(names) - set(by_name)) + if unknown: + raise ValueError(f"served_tools returned tools with no @tool method: {unknown}") + if len(set(names)) != len(names): + raise ValueError(f"served_tools returned a tool twice: {sorted({n for n in names if names.count(n) > 1})}") + return [(descriptor, by_name[descriptor.name]) for descriptor in chosen] + + def _merge_tools(card: EnvironmentCard, tool_handlers: list) -> EnvironmentCard: """Advertise decorator-declared tools under card.capabilities; card-declared names win.""" if not tool_handlers: diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py new file mode 100644 index 00000000..23a0ab28 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -0,0 +1,143 @@ +"""@tool annotations, structured output and _meta on both FastMCPs, and served_tools choosing what a handler serves.""" +from __future__ import annotations + +from typing import Annotated + +import anyio +import pytest +from pydantic import BaseModel + +pytest.importorskip("mcp.server.fastmcp") +fastmcp = pytest.importorskip("fastmcp") + +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 +from mcp.types import CallToolResult, ImageContent, TextContent # noqa: E402 + +from agentenv_protocol import ( # noqa: E402 + HIDDEN_BY_DEFAULT_META_KEY, + AgentEnvEnvironment, + AgentEnvFastMCPApplication, + EnvironmentCard, + EnvironmentTool, + environment_card, + tool, +) + + +class _Frame(BaseModel): + frame_id: str + width: int + + +@environment_card(name="tools") +class _Env(AgentEnvEnvironment): + @tool(annotations={"readOnlyHint": True, "title": "Look"}, meta={"agentenv.io/group": "minimum"}) + def look(self) -> _Frame: + return _Frame(frame_id="f1", width=1280) + + @tool(structured_output=False) + def plain(self) -> dict: + return {"a": 1} + + @tool(hidden_by_default=True, meta={"agentenv.io/group": "admin"}) + def secret(self) -> str: + return "s" + + @tool() + def shot(self) -> Annotated[CallToolResult, _Frame]: + return CallToolResult( + content=[TextContent(type="text", text="Captured f2"), ImageContent(type="image", data="aGk=", mimeType="image/png")], + structuredContent={"frame_id": "f2", "width": 640}, + ) + + +async def _mcp_tools(env: AgentEnvEnvironment) -> dict: + with anyio.fail_after(30): + async with create_connected_server_and_client_session(env.create_app()) as session: + return {t.name: t for t in (await session.list_tools()).tools} + + +@pytest.mark.asyncio +async def test_annotations_meta_and_output_schemas_are_listed_on_mcps_fastmcp(): + tools = await _mcp_tools(_Env()) + look = tools["look"] + assert look.annotations.readOnlyHint is True and look.annotations.title == "Look" + assert look.meta == {"agentenv.io/group": "minimum"} + assert look.outputSchema["properties"]["frame_id"] == {"title": "Frame Id", "type": "string"} + assert tools["plain"].outputSchema is None + assert tools["secret"].meta == {"agentenv.io/group": "admin", HIDDEN_BY_DEFAULT_META_KEY: True} + assert tools["shot"].outputSchema["required"] == ["frame_id", "width"] + + +@pytest.mark.asyncio +async def test_structured_results_reach_the_caller_with_their_content_blocks(): + with anyio.fail_after(30): + async with create_connected_server_and_client_session(_Env().create_app()) as session: + look = await session.call_tool("look", {}) + shot = await session.call_tool("shot", {}) + assert look.structuredContent == {"frame_id": "f1", "width": 1280} + assert [c.type for c in shot.content] == ["text", "image"] and shot.structuredContent == {"frame_id": "f2", "width": 640} + + +@pytest.mark.asyncio +async def test_annotations_meta_and_output_schemas_are_listed_on_fastmcp(): + app = fastmcp.FastMCP("tools") + _Env().mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + tools = {t.name: t for t in await client.list_tools()} + look = await client.call_tool("look", {}) + shot = await client.call_tool("shot", {}) + assert tools["look"].annotations.readOnlyHint is True + assert tools["look"].meta["agentenv.io/group"] == "minimum" + assert tools["look"].outputSchema["properties"]["frame_id"]["type"] == "string" + assert tools["plain"].outputSchema is None + assert tools["secret"].meta[HIDDEN_BY_DEFAULT_META_KEY] is True + assert tools["shot"].outputSchema["required"] == ["frame_id", "width"] + assert look.structured_content == {"frame_id": "f1", "width": 1280} + assert [c.type for c in shot.content] == ["text", "image"] and shot.structured_content == {"frame_id": "f2", "width": 640} + + +@pytest.mark.asyncio +async def test_a_bare_call_tool_result_on_fastmcp_lists_no_output_schema(): + @environment_card(name="bare") + class _Bare(AgentEnvEnvironment): + @tool() + async def bare(self) -> CallToolResult: + return CallToolResult(content=[TextContent(type="text", text="done")]) + + app = fastmcp.FastMCP("bare") + _Bare().mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + listed = (await client.list_tools())[0] + result = await client.call_tool("bare", {}) + assert listed.outputSchema is None and result.content[0].text == "done" + + +class _Chooser(_Env): + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + return [t.model_copy(update={"description": "Take a look."}) if t.name == "look" else t + for t in tools if t.name != "secret"] + + +@pytest.mark.asyncio +async def test_served_tools_chooses_what_is_listed_and_advertised(): + env = _Chooser() + tools = await _mcp_tools(env) + assert set(tools) == {"look", "plain", "shot"} and tools["look"].description == "Take a look." + advertised = {t.name: t for t in AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Chooser()).environment_card.capabilities.tools} + assert set(advertised) == {"look", "plain", "shot"} and advertised["look"].description == "Take a look." + + +@pytest.mark.parametrize("chosen, error", [ + (lambda tools: [*tools, EnvironmentTool(name="ghost")], r"no @tool method: \['ghost'\]"), + (lambda tools: [*tools, tools[0]], "a tool twice"), +]) +def test_served_tools_naming_an_unmarked_tool_or_one_twice_is_refused(chosen, error): + class _Bad(_Env): + def served_tools(self, tools): + return chosen(tools) + + with pytest.raises(ValueError, match=error): + AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Bad()) From 605f1f72cb36a484f559c29d2974c5a89d08895c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 11:22:48 -0700 Subject: [PATCH 04/18] feat(protocol): per-role tool access without a gateway An environment with tool_access = True filters tools/list and refuses tools/call per caller role (AgentEnv-Role, or a gateway-stamped _meta role) on both FastMCPs, and serves disable-tool/v1 and enable-tool/v1 at the gateway's paths and bodies, so modify_env_tool_access works on an env with no gateway in front. agentenv_protocol.tool_access keeps the gateway's rule semantics, hidden-by-default tools included; a refused call answers permission_denied. The gateway keeps its own code, since its image installs no protocol, and tst/unit/env/gateway/tool_access_parity_test.py runs the same rule cases against both. Co-Authored-By: Claude Opus 5.5 --- .../agent_env_environment.py | 71 ++++++- .../src/agentenv_protocol/tool_access.py | 98 +++++++++ .../tests/test_tool_access.py | 195 ++++++++++++++++++ .../env/gateway/tool_access_parity_test.py | 71 +++++++ 4 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/tool_access.py create mode 100644 packages/agentenv-protocol/tests/test_tool_access.py create mode 100644 tst/unit/env/gateway/tool_access_parity_test.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index 0bde81d5..e328e2a1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -22,6 +22,8 @@ from starlette.routing import Route from .errors import AgentEnvError +from .tool_access import DISABLE as TOOL_DISABLE, ENABLE as TOOL_ENABLE, PATHS as TOOL_ACCESS_PATHS, ToolAccess, caller_role +from .tool_access import extensions as tool_access_extensions from .tool_context import EMPTY, ToolContext, bound, context_from_http, context_from_mcp from .transfers import WriteNamespaceGrant from .types import DATA_OBJECTS_EXTENSION_URI, MCP_TRANSPORT, METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH, AddDataRequest, AddDataResponse, EnvironmentCapabilities, EnvironmentCard, EnvironmentExtension, EnvironmentInterface, EnvironmentTool, GetDataResponse, ResetDataResponse, error_body @@ -221,12 +223,21 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") environment_card = _merge_tools(environment_card, tool_handlers) + self._tool_access = ( + ToolAccess(descriptor.name for descriptor, fn in tool_handlers if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False)) + if self.supports_tools and getattr(handler, "tool_access", False) else None + ) + if self._tool_access is not None: + environment_card = _merge_extensions(environment_card, [(e,) for e in tool_access_extensions()]) methods = _discover_methods(handler) environment_card = _merge_operations(environment_card, methods) environment_card = _merge_data_objects(environment_card, methods) self.environment_card = environment_card self._dispatch = _jsonrpc_handler(methods) self._ext_routes = [(path, method, _extension_handler(fn, method)) for _descriptor, fn, path, method in ext_handlers] + if self._tool_access is not None: + self._ext_routes += [(TOOL_ACCESS_PATHS[action], "POST", _tool_access_handler(self._tool_access, action)) + for action in (TOOL_DISABLE, TOOL_ENABLE)] # Register with the advertised descriptor (card-declared wins in _merge_tools) so # tools/list cannot drift from the card; falls back to the discovered descriptor. caps_tools = (environment_card.capabilities.tools if environment_card.capabilities else None) or [] @@ -311,7 +322,7 @@ def _check_tool_binding(self, app: Any) -> None: if isinstance(middleware, list) and callable(getattr(app, "add_middleware", None)): if any(getattr(entry, _TOOL_CONTEXT_BOUND_ATTR, False) for entry in middleware): raise RuntimeError("this fastmcp app already carries an AgentEnv tool-context middleware; mount one handler per app") - self._middleware = _tool_context_middleware(self._hook) + self._middleware = _tool_context_middleware(self._hook, self._tool_access) self._binding = "middleware" return manager = getattr(app, "_tool_manager", None) @@ -321,10 +332,12 @@ def _check_tool_binding(self, app: Any) -> None: self._binding = "dispatch" return readers = [descriptor.name for descriptor, fn in self._tools if _tool_context_slot(fn)] - if readers or self._hook is not None: + readers += ["on_tool_call"] if self._hook is not None else [] + readers += ["tool_access"] if self._tool_access is not None else [] + if readers: raise RuntimeError( "this app has no tool dispatch to bind a ToolContext around (mcp>=1.25,<2 keeps one at " - f"FastMCP._tool_manager; fastmcp exposes a middleware list), and {', '.join(readers) or 'on_tool_call'} " + f"FastMCP._tool_manager; fastmcp exposes a middleware list), and {', '.join(readers)} " "would read an empty caller" ) self._binding = None @@ -355,6 +368,7 @@ def _bind_tool_dispatch(self, app: Any) -> None: manager = app._tool_manager inner = manager.call_tool hook = self._hook + access = self._tool_access async def call_tool(name: str, arguments: dict, *args: Any, **kwargs: Any) -> Any: if manager.get_tool(name) is None: @@ -362,6 +376,8 @@ async def call_tool(name: str, arguments: dict, *args: Any, **kwargs: Any) -> An context = kwargs.get("context", args[0] if args else None) with bound(context_from_mcp(context, name, arguments)) as tool_context: try: + if access is not None: + access.check_call(tool_context) if hook is not None: answer = await _maybe_await(hook(tool_context)) if answer is not None: @@ -377,6 +393,20 @@ async def call_tool(name: str, arguments: dict, *args: Any, **kwargs: Any) -> An functools.update_wrapper(call_tool, inner) setattr(call_tool, _TOOL_CONTEXT_BOUND_ATTR, True) manager.call_tool = call_tool + if access is not None: + self._filter_tool_list(app, manager, access) + + @staticmethod + def _filter_tool_list(app: Any, manager: Any, access: ToolAccess) -> None: + """tools/list under the live request's caller, read through ``FastMCP.get_context``, as tools/call reads it.""" + inner = manager.list_tools + + def list_tools(*args: Any, **kwargs: Any) -> Any: + role = caller_role(context_from_mcp(app.get_context(), "", {})) + return [t for t in inner(*args, **kwargs) if not access.is_disabled(role, t.name)] + + functools.update_wrapper(list_tools, inner) + manager.list_tools = list_tools def _card_for(self, app: Any) -> EnvironmentCard: """Declare the MCP endpoint only when the app says where it serves it; FastMCP-shaped targets may not.""" @@ -411,6 +441,9 @@ class AgentEnvEnvironment: """ mcp: Any = None + #: Filter tools/list and refuse tools/call per caller role, and serve disable-tool/v1 and enable-tool/v1 at the + #: gateway's paths and with its rules, for an env agents reach with no gateway in front. + tool_access: ClassVar[bool] = False _mounted: bool = False def _build_card(self) -> EnvironmentCard: @@ -676,22 +709,32 @@ def _check_call_tool_result(value: Any) -> None: raise TypeError(f"on_tool_call must return None or mcp.types.CallToolResult, got {type(value).__name__}") -def _tool_context_middleware(hook: Callable | None) -> Any: +def _tool_context_middleware(hook: Callable | None, access: ToolAccess | None = None) -> Any: """A fastmcp ``Middleware`` that runs every tools/call under its ToolContext, the handler's ``on_tool_call`` - first. A ``Middleware`` subclass rather than a bare ``(context, call_next)`` callable because fastmcp runs the - typed ``on_call_tool`` exactly once per call, where a bare callable runs once per dispatch phase. fastmcp is - not a dependency of this package: it is imported here, where an app built with it has just been handed over.""" + first, and with ``access`` filters tools/list and refuses tools/call per caller role. A ``Middleware`` subclass + rather than a bare ``(context, call_next)`` callable because fastmcp runs the typed hooks exactly once per call, + where a bare callable runs once per dispatch phase. fastmcp is not a dependency of this package: it is imported + here, where an app built with it has just been handed over.""" try: from fastmcp.server.middleware import Middleware except ImportError as e: raise RuntimeError("this app carries a fastmcp middleware list but fastmcp cannot be imported") from e class ToolContextMiddleware(Middleware): + async def on_list_tools(self, context: Any, call_next: Callable) -> Any: + tools = await call_next(context) + if access is None: + return tools + role = caller_role(context_from_mcp(context.fastmcp_context, "", {})) + return [t for t in tools if not access.is_disabled(role, t.name)] + async def on_call_tool(self, context: Any, call_next: Callable) -> Any: message = context.message arguments = dict(getattr(message, "arguments", None) or {}) with bound(context_from_mcp(context.fastmcp_context, message.name, arguments)) as tool_context: try: + if access is not None: + access.check_call(tool_context) if hook is not None: answer = await _maybe_await(hook(tool_context)) if answer is not None: @@ -1166,6 +1209,20 @@ def _error_response(error: AgentEnvError) -> Response: return _json_response(error_body(**error.fields()), error.status) +def _tool_access_handler(access: ToolAccess, action: str) -> Callable: + async def handler(request: Request) -> Response: + raw = await request.body() + try: + body = json.loads(raw) if raw else None + except ValueError: + return _error_response(AgentEnvError("invalid_params", "request body must be JSON")) + try: + return _json_response(access.change(action, body)) + except AgentEnvError as e: + return _error_response(e) + return handler + + class _Op(NamedTuple): name: str rpc_method: str diff --git a/packages/agentenv-protocol/src/agentenv_protocol/tool_access.py b/packages/agentenv-protocol/src/agentenv_protocol/tool_access.py new file mode 100644 index 00000000..5391465d --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/tool_access.py @@ -0,0 +1,98 @@ +"""Per-role tool visibility for an environment agents reach with no gateway in front, under the gateway's rules and +at its paths, so ``modify_env_tool_access`` and an env's ``[tools]`` behave the same with or without one.""" +from __future__ import annotations + +from typing import Iterable + +from .errors import AgentEnvError +from .tool_context import DEFAULT_ROLE, ToolContext +from .types import EnvironmentExtension + +WILDCARD = "*" +DISABLE_TOOL_EXTENSION_URI = "urn:agentenv:disable-tool/v1" +ENABLE_TOOL_EXTENSION_URI = "urn:agentenv:enable-tool/v1" +DISABLE = "disable" +ENABLE = "enable" +PATHS = {DISABLE: "/tools/disable", ENABLE: "/tools/enable"} + +_REQUEST = { + "type": "object", + "properties": { + "role": {"type": "string"}, + "tools": {"anyOf": [{"type": "array", "items": {"type": "string"}}, {"const": WILDCARD}]}, + }, + "required": ["role", "tools"], +} + + +class ToolAccess: + """Rules per role, each a tool name or ``*`` mapped to disabled or not. A call's role is looked up first, then + ``*``; the first rule naming the tool decides, then a ``*`` rule, except that a tool hidden by default skips + ``*`` rules and stays hidden until a rule names it.""" + + def __init__(self, hidden_by_default: Iterable[str] = ()) -> None: + self.rules: dict[str, dict[str, bool]] = {} + self.hidden_by_default = set(hidden_by_default) + + def is_disabled(self, role: str, name: str) -> bool: + hidden = name in self.hidden_by_default + for r in (role, WILDCARD): + rules = self.rules.get(r, {}) + if name in rules: + return rules[name] + if WILDCARD in rules and not hidden: + return rules[WILDCARD] + return hidden + + def apply(self, role: str, tool: str, disabled: bool) -> None: + if role == WILDCARD and tool == WILDCARD: + self.rules.clear() + self.rules[WILDCARD] = {WILDCARD: disabled} + elif role == WILDCARD: + for rules in self.rules.values(): + rules[tool] = disabled + self.rules.setdefault(WILDCARD, {})[tool] = disabled + elif tool == WILDCARD: + self.rules[role] = {WILDCARD: disabled} + else: + self.rules.setdefault(role, {})[tool] = disabled + + def change(self, action: str, body: object) -> dict: + """A ``disable-tool/v1`` or ``enable-tool/v1`` request applied: ``{role, tools: [...] | "*"}`` to + ``{role, disabled, allowed}``, the role's own rules after the change.""" + role = body.get("role") if isinstance(body, dict) else None + if not isinstance(role, str) or not role: + raise AgentEnvError("invalid_params", "missing or invalid 'role' field") + tools = body.get("tools") + if tools == WILDCARD: + names = [WILDCARD] + elif isinstance(tools, list) and all(isinstance(t, str) for t in tools): + names = tools + else: + raise AgentEnvError("invalid_params", "'tools' must be a list of strings or the string '*'") + for name in names: + self.apply(role, name, action == DISABLE) + rules = self.rules.get(role, {}) + return {"role": role, "disabled": sorted(k for k, v in rules.items() if v), + "allowed": sorted(k for k, v in rules.items() if not v)} + + def check_call(self, context: ToolContext) -> None: + role = caller_role(context) + if self.is_disabled(role, context.tool): + raise AgentEnvError("permission_denied", f"Tool '{context.tool}' is disabled for role '{role}'", status=403) + + +def caller_role(context: ToolContext) -> str: + """The role a rule is keyed by: the caller's, or the gateway's unforwarded ``default``.""" + return context.caller.role or DEFAULT_ROLE + + +def extensions() -> list[EnvironmentExtension]: + return [ + EnvironmentExtension(uri=uri, description=description, required=False, params={ + "endpoint": PATHS[action], "methods": {action: {"method": "POST", "request": _REQUEST}}}) + for uri, action, description in ( + (DISABLE_TOOL_EXTENSION_URI, DISABLE, "Disable tools for a role."), + (ENABLE_TOOL_EXTENSION_URI, ENABLE, "Enable tools for a role."), + ) + ] diff --git a/packages/agentenv-protocol/tests/test_tool_access.py b/packages/agentenv-protocol/tests/test_tool_access.py new file mode 100644 index 00000000..ac04104a --- /dev/null +++ b/packages/agentenv-protocol/tests/test_tool_access.py @@ -0,0 +1,195 @@ +"""Per-role tool access with no gateway: tools/list filtered and tools/call refused per caller role on both FastMCPs, +disable-tool/v1 and enable-tool/v1 at the gateway's paths and bodies, and the rules' edge cases.""" +from __future__ import annotations + +import contextlib + +import anyio +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +fastmcp = pytest.importorskip("fastmcp") + +from mcp import ClientSession # noqa: E402 +from mcp.client import streamable_http as _streamable_http # noqa: E402 +from mcp.server.fastmcp import FastMCP # noqa: E402 + +from agentenv_protocol import ( # noqa: E402 + ROLE_HEADER, + ROLE_META_KEY, + AgentEnvEnvironment, + AgentEnvFastMCPApplication, + EnvironmentCard, + environment_card, + tool, +) +from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI, ToolAccess # noqa: E402 + +pytestmark = pytest.mark.skipif(not hasattr(_streamable_http, "streamable_http_client"), + reason="mcp without streamable_http_client") + +_BASE = "http://127.0.0.1:8000" + + +@environment_card(name="desk") +class _Env(AgentEnvEnvironment): + tool_access = True + + @tool() + def read(self) -> str: + return "read" + + @tool() + def write(self) -> str: + return "write" + + @tool(hidden_by_default=True) + def grade(self) -> str: + return "grade" + + +@contextlib.asynccontextmanager +async def _served(): + """The env on mcp's FastMCP over streamable HTTP, and a way to open a session as a role.""" + app = FastMCP("desk", json_response=True) + _Env().mount(app) + asgi = app.streamable_http_app() + + @contextlib.asynccontextmanager + async def session(role: str | None = None): + headers = {ROLE_HEADER: role} if role else {} + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE, headers=headers) as http: + async with _streamable_http.streamable_http_client(f"{_BASE}/mcp", http_client=http) as (read, write, _): + async with ClientSession(read, write) as s: + await s.initialize() + yield s + + async with app.session_manager.run(): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE) as rest: + yield rest, session + + +async def _names(session) -> list: + return sorted(t.name for t in (await session.list_tools()).tools) + + +def test_the_card_advertises_both_extensions_at_the_gateways_paths(): + card = AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Env()).environment_card + extensions = {e.uri: e for e in card.capabilities.extensions} + assert extensions[DISABLE_TOOL_EXTENSION_URI].params["endpoint"] == "/tools/disable" + assert extensions[DISABLE_TOOL_EXTENSION_URI].params["methods"]["disable"]["method"] == "POST" + assert extensions[ENABLE_TOOL_EXTENSION_URI].params["methods"]["enable"]["request"]["required"] == ["role", "tools"] + + +def test_an_env_without_tool_access_advertises_neither(): + class _Plain(_Env): + tool_access = False + + card = AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Plain()).environment_card + assert not [e for e in (card.capabilities.extensions or []) if "tool/v1" in e.uri] + + +@pytest.mark.asyncio +async def test_roles_see_and_call_only_what_their_rules_allow(): + with anyio.fail_after(30): + async with _served() as (rest, session): + async with session() as anyone: + assert await _names(anyone) == ["read", "write"] + hidden = await anyone.call_tool("grade", {}) + changed = await rest.post("/tools/disable", json={"role": "agent", "tools": ["write"]}) + shown = await rest.post("/tools/enable", json={"role": "grader", "tools": ["grade"]}) + async with session("agent") as agent: + agent_tools = await _names(agent) + refused = await agent.call_tool("write", {}) + allowed = await agent.call_tool("read", {}) + forwarded = await agent.call_tool("grade", {}, meta={ROLE_META_KEY: "grader"}) + async with session("grader") as grader: + grader_tools = await _names(grader) + async with session("default") as default: + default_tools = await _names(default) + assert hidden.isError and hidden.structuredContent["code"] == "permission_denied" + assert changed.json() == {"role": "agent", "disabled": ["write"], "allowed": []} + assert shown.json() == {"role": "grader", "disabled": [], "allowed": ["grade"]} + assert agent_tools == ["read"] + assert refused.isError and refused.content[0].text == "permission_denied: Tool 'write' is disabled for role 'agent'" + assert refused.structuredContent["applied"] == "no" and not allowed.isError + assert not forwarded.isError and forwarded.content[0].text == "grade", "a gateway-stamped role wins over the header" + assert grader_tools == ["grade", "read", "write"] and default_tools == ["read", "write"] + + +@pytest.mark.asyncio +async def test_everything_off_then_one_tool_on_for_one_role(): + with anyio.fail_after(30): + async with _served() as (rest, session): + await rest.post("/tools/disable", json={"role": "*", "tools": "*"}) + await rest.post("/tools/enable", json={"role": "grader", "tools": ["read"]}) + async with session("agent") as agent: + agent_tools = await _names(agent) + async with session("grader") as grader: + grader_tools = await _names(grader) + assert agent_tools == [] and grader_tools == ["read"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"tools": ["read"]}, + {"role": "", "tools": ["read"]}, + {"role": "agent", "tools": "read"}, + {"role": "agent", "tools": ["read", 3]}, + ["not", "an", "object"], +]) +async def test_a_malformed_request_is_refused_and_changes_nothing(body): + with anyio.fail_after(30): + async with _served() as (rest, session): + r = await rest.post("/tools/disable", json=body) + garbled = await rest.post("/tools/disable", content=b"{not json") + async with session("agent") as agent: + tools = await _names(agent) + assert r.status_code == 400 and r.json()["error"]["code"] == "invalid_params" + assert garbled.status_code == 400 and tools == ["read", "write"] + + +@pytest.mark.asyncio +async def test_roles_are_honoured_on_fastmcp_too(): + app = fastmcp.FastMCP("desk") + _Env().mount(app) + asgi = app.http_app(path="/mcp", json_response=True) + with anyio.fail_after(30): + async with asgi.router.lifespan_context(asgi): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE) as rest: + await rest.post("/tools/disable", json={"role": "agent", "tools": ["write"]}) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE, + headers={ROLE_HEADER: "agent"}) as http: + async with _streamable_http.streamable_http_client(f"{_BASE}/mcp", http_client=http) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + tools = await _names(session) + refused = await session.call_tool("write", {}) + assert tools == ["read"] + assert refused.isError and refused.structuredContent["code"] == "permission_denied" + + +def test_tool_access_needs_a_dispatch_to_read_the_caller_from(): + class _Shapeless: + def tool(self, name=None, description=None, **kwargs): + return lambda fn: fn + + def custom_route(self, path, methods): + return lambda fn: fn + + with pytest.raises(RuntimeError, match="tool_access would read an empty caller"): + AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Env()).add_routes_to_app(_Shapeless()) + + +def test_a_rule_for_every_role_reaches_roles_that_already_have_rules(): + access = ToolAccess(hidden_by_default=["grade"]) + access.apply("agent", "read", True) + access.apply("*", "write", True) + access.apply("*", "grade", False) + assert access.rules == {"agent": {"read": True, "write": True, "grade": False}, "*": {"write": True, "grade": False}} + assert access.is_disabled("agent", "grade") is False and access.is_disabled("other", "write") is True + access.apply("agent", "*", False) + assert access.rules["agent"] == {"*": False}, "a rule for every tool of a role replaces that role's named rules" + assert access.is_disabled("agent", "write") is False + assert access.is_disabled("agent", "grade") is False, "a hidden tool skips the role's '*' rule; the every-role rule naming it decides" diff --git a/tst/unit/env/gateway/tool_access_parity_test.py b/tst/unit/env/gateway/tool_access_parity_test.py new file mode 100644 index 00000000..f8131b61 --- /dev/null +++ b/tst/unit/env/gateway/tool_access_parity_test.py @@ -0,0 +1,71 @@ +"""The gateway and an env serving tool access itself (agentenv_protocol.tool_access) keep separate code, since the +gateway's image installs no protocol; the same rule cases run against both so a task's tool-access steps mean the +same with or without a gateway in front.""" +from __future__ import annotations + +import random + +import pytest +from agentenv_protocol.tool_access import ToolAccess + +from agent_env.env.gateway.gateway import Gateway + +ROLES = ("default", "grader", "sim") +TOOLS = ("h", "n") +HIDDEN = {"h"} + + +def _both() -> tuple[Gateway, ToolAccess]: + gateway = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[]) + gateway._hidden_by_default = set(HIDDEN) + return gateway, ToolAccess(hidden_by_default=HIDDEN) + + +def _assert_same(gateway: Gateway, access: ToolAccess, steps: list) -> None: + assert access.rules == gateway._role_rules, steps + for role in ROLES + ("*", "never-mentioned"): + for tool in TOOLS + ("unlisted",): + assert access.is_disabled(role, tool) is gateway._is_disabled(role, tool), (steps, role, tool) + + +def test_random_step_sequences_leave_both_with_the_same_rules_and_answers(): + rng = random.Random(20261026) + for _ in range(5000): + gateway, access = _both() + steps = [(rng.choice(ROLES + ("*",)), rng.choice(TOOLS + ("*",)), rng.random() < 0.5) + for _ in range(rng.randint(0, 10))] + for step in steps: + gateway._apply_rule(*step) + access.apply(*step) + _assert_same(gateway, access, steps) + + +@pytest.mark.parametrize("steps", [ + [], + [("*", "*", False)], + [("*", "*", True), ("grader", "n", False)], + [("sim", "*", True), ("sim", "h", False)], + [("sim", "h", False), ("sim", "*", True)], + [("*", "h", False), ("default", "h", True)], + [("default", "n", True), ("*", "n", False)], + [("grader", "*", True), ("*", "*", False), ("grader", "h", False)], +]) +def test_the_steps_a_task_writes(steps): + gateway, access = _both() + for step in steps: + gateway._apply_rule(*step) + access.apply(*step) + _assert_same(gateway, access, steps) + + +@pytest.mark.parametrize("action, body", [ + ("disable", {"role": "sim", "tools": ["n", "h"]}), + ("enable", {"role": "sim", "tools": "*"}), + ("disable", {"role": "*", "tools": ["n"]}), +]) +def test_a_change_answers_with_the_gateways_body(action, body): + gateway, access = _both() + for tool in body["tools"] if isinstance(body["tools"], list) else ["*"]: + gateway._apply_rule(body["role"], tool, action == "disable") + expected = {"role": body["role"], **gateway._format_role_rules(gateway._role_rules.get(body["role"], {}))} + assert access.change(action, body) == expected From 23ff361469825393b22ee2a19f511198539ff974 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 11:49:29 -0700 Subject: [PATCH 05/18] feat(protocol): ComputerDriverEnvironment, the computer driver base class agentenv_protocol.computer: a driver subclasses ComputerDriverEnvironment and marks the methods that capture and send input (@display, @capture, @click, @drag, @scroll, @type_text, @press; the pointer and keyboard groups; @set_display; @run_in_session and @write_files, with Linux defaults). The class is checked when defined: every required operation, whole groups, one method per operation per platform, and features that match the decorators. Predicates (os, distribution, version, arch, display) choose a method per platform. The base class serves the contract from those operations: - the seven computer_* tools and the pointer and keyboard group tools, with structured results, a one-line summary as text, output schemas and the errors table's codes and applied; - frames: f ids, native region crops that map back exactly, the last 64 kept, stale when the display geometry changes, each caller's latest; - frame and norm coordinate spaces, refused rather than clamped outside the frame or range, with the default written into the tool descriptions; - the input queue (one call at a time, input_busy past the wait) and others_since_frame; - computer-ready (real checks; tools refuse device_unavailable until they pass once), computer-display, computer-spaces, computer-keyboard, computer-exec (idempotent by request_id, output capped, timed out, a background app's open pipes drained rather than waited on); - data/add into the session user's home from inline bytes, https URLs and read grants, written atomically with path checks; data/get of the data paths, inline or under a write_namespace grant; - the generated card, and per-role tool access. Settings come from AGENTENV_COMPUTER_SETTINGS. The new computer extra installs mcp, uvicorn and Pillow. Tests drive the whole base class through an in-memory screen in tests/computer_fake.py, which doesn't ship. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/pyproject.toml | 7 + .../agentenv_protocol/computer/__init__.py | 67 +++ .../agentenv_protocol/computer/dataplane.py | 109 ++++ .../src/agentenv_protocol/computer/driver.py | 356 +++++++++++++ .../src/agentenv_protocol/computer/errors.py | 71 +++ .../agentenv_protocol/computer/extensions.py | 132 +++++ .../src/agentenv_protocol/computer/frames.py | 148 ++++++ .../src/agentenv_protocol/computer/keys.py | 66 +++ .../src/agentenv_protocol/computer/linux.py | 96 ++++ .../agentenv_protocol/computer/operations.py | 183 +++++++ .../agentenv_protocol/computer/platform.py | 76 +++ .../src/agentenv_protocol/computer/queue.py | 45 ++ .../agentenv_protocol/computer/settings.py | 43 ++ .../src/agentenv_protocol/computer/spaces.py | 49 ++ .../src/agentenv_protocol/computer/tools.py | 469 ++++++++++++++++++ .../agentenv-protocol/tests/computer_fake.py | 218 ++++++++ .../tests/test_computer_dataplane.py | 153 ++++++ .../tests/test_computer_driver.py | 370 ++++++++++++++ .../tests/test_computer_extensions.py | 179 +++++++ .../tests/test_computer_frames.py | 157 ++++++ .../tests/test_computer_queue.py | 360 ++++++++++++++ .../tests/test_computer_spaces.py | 85 ++++ pyproject.toml | 1 + uv.lock | 104 +++- 24 files changed, 3543 insertions(+), 1 deletion(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py create mode 100644 packages/agentenv-protocol/tests/computer_fake.py create mode 100644 packages/agentenv-protocol/tests/test_computer_dataplane.py create mode 100644 packages/agentenv-protocol/tests/test_computer_driver.py create mode 100644 packages/agentenv-protocol/tests/test_computer_extensions.py create mode 100644 packages/agentenv-protocol/tests/test_computer_frames.py create mode 100644 packages/agentenv-protocol/tests/test_computer_queue.py create mode 100644 packages/agentenv-protocol/tests/test_computer_spaces.py diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 8057ee21..5733bf01 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -35,6 +35,12 @@ agent = [ "regex>=2023.12.25", "uvicorn>=0.30", ] +# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP and crops and encodes frames. +computer = [ + "mcp>=1.25,<2", + "Pillow>=10.4", + "uvicorn>=0.30", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -43,6 +49,7 @@ dev = [ "pytest-asyncio>=0.23.0", "mcp>=1.25,<2", "fastmcp>=3.4,<4", # the standalone-fastmcp mount path + "Pillow>=10.4", # the computer extra's frame crops and encodings ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py new file mode 100644 index 00000000..7d014c4e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -0,0 +1,67 @@ +"""The computer driver SDK: subclass ComputerDriverEnvironment, mark the methods that capture and send input, and +serve() it. Needs the ``computer`` extra: ``pip install 'agentenv-framework-protocol[computer]'``.""" + +from .driver import DEFAULT_IDENTITY, ComputerDriverEnvironment, computer_driver +from .errors import ( + BackgroundUnavailable, + ComputerError, + DeliveryFailed, + DeviceUnavailable, + InputBusy, + Internal, + InvalidArgument, + LimitExceeded, + OutOfBounds, + PermissionDenied, + StaleFrame, + TargetUnavailable, + Timeout, + Unsupported, +) +from .extensions import ( + COMPUTER, + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_POINTER, + COMPUTER_READY, + COMPUTER_SPACES, +) +from .operations import ( + Display, + Effect, + ExecResult, + Image, + ReadyCheck, + capture, + click, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + run_in_session, + scroll, + set_display, + type_text, + write_files, +) +from .platform import Platform +from .settings import SETTINGS_ENV, ComputerSettings + +__all__ = [ + "ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", + "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", + "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", + "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", + "run_in_session", "write_files", + "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", + "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed", + "Internal", + "COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD", + "COMPUTER_EXEC", +] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py new file mode 100644 index 00000000..daf6ccb0 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py @@ -0,0 +1,109 @@ +"""The data plane on a computer: ``data/add`` writes files under the session user's home, ``data/get`` returns the +env's ``data_paths``. Each file part names its path, relative to the home, under the ``data-objects/v1`` marker.""" +from __future__ import annotations + +import base64 +import binascii +import mimetypes +import tempfile +from pathlib import Path +from typing import Optional + +import httpx +from pydantic import ValidationError + +from ..agent_env_environment import add_data, get_data +from ..transfers import NamespaceUploader, ReadObject, WriteNamespaceGrant, download +from ..types import DATA_OBJECTS_EXTENSION_URI, FilePart, FileWithBytes, FileWithUri, uploaded_file_part, uploaded_object_path + +#: The most an https URL part may fetch; a read grant carries its own limit. +MAX_URL_BYTES = 1 << 30 + + +class DataPlane: + """The data plane handlers, mixed into ComputerDriverEnvironment.""" + + @add_data + async def _data_add(self, parts: list) -> None: + files = {} + for part in parts: + try: + path = uploaded_object_path(part) if isinstance(part, FilePart) else None + except ValueError: + marker = part.metadata[DATA_OBJECTS_EXTENSION_URI] + raise ValueError(f"the part's path {marker.get('path')!r} is not a normalized path relative to " + "the home") from None + if path is None: + raise ValueError(f"data/add on a computer takes file parts naming their path under " + f"{DATA_OBJECTS_EXTENSION_URI}, got a {part.kind} part") + files[path] = await _fetch(part) + await self._call("write_files", files) + + @get_data + async def _data_get(self, write_namespace: Optional[WriteNamespaceGrant] = None) -> list: + files = _data_files(Path.home(), self.settings.data_paths) + if write_namespace is not None: + uploader = NamespaceUploader(write_namespace) + parts = [] + for relative, source in files: + await uploader.upload(relative, source) + parts.append(uploaded_file_part(relative, name=source.name, mime_type=_mime(source))) + return parts + return [ + FilePart(file=FileWithBytes(bytes=base64.b64encode(source.read_bytes()).decode("ascii"), + name=source.name, mimeType=_mime(source)), + metadata={DATA_OBJECTS_EXTENSION_URI: {"path": relative}}) + for relative, source in files + ] + + +async def _fetch(part: FilePart) -> bytes: + """A part's bytes: inline, from a read grant under the marker, or from an https URL.""" + marker = part.metadata[DATA_OBJECTS_EXTENSION_URI] + if "read" in marker: + try: + grant = ReadObject.model_validate(marker["read"]) + except ValidationError: + raise ValueError("the part's read grant is not a valid ReadObject") from None + with tempfile.TemporaryDirectory() as scratch: + destination = Path(scratch) / "object" + await download(grant, destination) + return destination.read_bytes() + if isinstance(part.file, FileWithBytes): + try: + return base64.b64decode(part.file.bytes, validate=True) + except (binascii.Error, ValueError): + raise ValueError("the part's bytes are not base64") from None + return await _get_url(part.file) + + +async def _get_url(file: FileWithUri) -> bytes: + if not file.uri.startswith("https://"): + raise ValueError("a file part's uri is an https URL") + data = bytearray() + async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: + async with client.stream("GET", file.uri) as response: + response.raise_for_status() + async for chunk in response.aiter_bytes(): + data += chunk + if len(data) > MAX_URL_BYTES: + raise ValueError(f"the file at the part's uri is over {MAX_URL_BYTES} bytes") + return bytes(data) + + +def _data_files(home: Path, data_paths: list[str]) -> list[tuple[str, Path]]: + """Every file under the data paths, by its path relative to the home; a path not there yet is skipped.""" + root = home.resolve() + found: dict[str, Path] = {} + for relative in data_paths: + target = home / relative + if not target.exists() or not target.resolve().is_relative_to(root): + continue + for path in sorted(target.rglob("*")) if target.is_dir() else [target]: + if path.is_file() and path.resolve().is_relative_to(root): + found[path.relative_to(home).as_posix()] = path + return list(found.items()) + + +def _mime(path: Path) -> str: + return mimetypes.guess_type(path.name)[0] or "application/octet-stream" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py new file mode 100644 index 00000000..0d1fa6c4 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -0,0 +1,356 @@ +"""ComputerDriverEnvironment: a driver writes screen operations, and the base class serves the computer contract +from them: the tools agents call, the framework extensions agentenv calls, the data plane and the card.""" +from __future__ import annotations + +import asyncio +from collections import OrderedDict +from typing import Any, Callable, ClassVar + +from ..agent_env_environment import AgentEnvEnvironment, environment_card +from ..errors import AgentEnvError +from ..tool_context import ToolContext +from ..types import EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension +from . import linux +from .dataplane import DataPlane +from .errors import DeliveryFailed, DeviceUnavailable, Internal +from .extensions import ( + COMPUTER, + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_POINTER, + COMPUTER_SPACES, + KEYBOARD_RANGES, + MAX_EXEC_TIMEOUT_S, + Extensions, +) +from .frames import Frame, FrameStore, to_pil +from .keys import canonical +from .operations import GROUPS, INPUT, Display, ReadyCheck, check_class, declared, default_for, resolve +from .platform import Platform, detect +from .queue import InputQueue +from .settings import ComputerSettings, load_settings +from .tools import GROUP_TOOLS, POINT_OPTIONAL_WITH_POINTER, POINT_TOOLS, Tools, _result, effect_data + +#: The caller every call comes from until agents carry an identity of their own. +DEFAULT_IDENTITY = "default" +_BUTTONS = ("left", "right", "middle", "back", "forward") +#: The handlers each optional operation or group brings; without it they are withdrawn. +_HANDLERS = { + "set_display": ("_display_set",), + "keyboard": ("_keyboard_get", "_keyboard_set"), + "run_in_session": ("_exec_run",), + "write_files": ("_data_add",), +} + + +class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironment): + """Subclass it and mark the methods that capture and send input (see ``operations``); the class is checked when + it is defined. Features are class attributes, on the card. A subclass's ``__init__`` calls ``super().__init__()``. + """ + + buttons: ClassVar[tuple[str, ...]] = ("left",) + max_click_count: ClassVar[int] = 1 + #: ``"all"`` for every W3C key, or the key values this computer can press. + keys: ClassVar[Any] = "all" + max_text_length: ClassVar[int] = 10_000 + max_wait_ms: ClassVar[int] = 60_000 + max_drag_points: ClassVar[int] = 2 + max_duration_ms: ClassVar[int] = 10_000 + max_key_repeat: ClassVar[int] = 100 + max_scroll_amount: ClassVar[int] = 100 + #: The (width, height) modes computer-display/v1 set takes; with ``@set_display`` only. + display_modes: ClassVar[tuple] = () + layouts: ClassVar[tuple[str, ...]] = ("us",) + #: The platforms the driver supports, as the driver env declares them: ``[{"os": {...}, "arch": [...]}]``. + supports: ClassVar[list] = [] + tool_access = True + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + check_class(cls) + _check_features(cls) + + def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None: + self.settings = settings if settings is not None else load_settings() + self.platform = platform if platform is not None else detect() + self._operations = resolve(type(self), self.platform) + self._frames = FrameStore() + self._queue = InputQueue() + self._ready = False + self._ready_lock = asyncio.Lock() + self._exec_runs: OrderedDict = OrderedDict() + self._held: dict[str, list[str]] = {} + self._releases: dict[str, asyncio.Task] = {} + self._served: dict[str, EnvironmentTool] = {} + for capability, handlers in _HANDLERS.items(): + if not self.has(capability): + for name in handlers: + setattr(self, name, None) + + def has(self, capability: str) -> bool: + """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" + if capability in GROUPS: + return all(op in self._operations for op in GROUPS[capability]) + return capability in self._operations + + def caller_identity(self, context: ToolContext) -> str: + """Who a call comes from, for each caller's latest frame and others' input since a frame.""" + return DEFAULT_IDENTITY + + async def ready_checks(self) -> list[ReadyCheck]: + """Checks of the driver's own, run with the base class's by computer-ready/v1; never sends input.""" + return [] + + # The card. + + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + unserved = {name for group, names in GROUP_TOOLS.items() if not self.has(group) for name in names} + self._served = {t.name: t.model_copy(deep=True) for t in tools if t.name not in unserved} + if not self.has("pointer"): + for name in POINT_OPTIONAL_WITH_POINTER: + schema = self._served[name].inputSchema + schema["required"] = [*schema.get("required", []), "x", "y"] + self._describe_tools() + return list(self._served.values()) + + def create_app(self) -> Any: + self._require_init() + app = super().create_app() + self._describe_tools() + return app + + def mount(self, app: Any) -> Any: + self._require_init() + mounted = super().mount(app) + self._describe_tools() + return mounted + + def _describe_tools(self) -> None: + """Write the default coordinate space into the descriptions of the tools that take a point, and carry the + served descriptions and input schemas over to mcp's FastMCP's tools/list, which reads the tool functions.""" + space, scale = self.settings.default_space, self.settings.default_norm_scale + note = (" Points are in frame space by default: pixels of the frame's image." if space == "frame" else + f" Points are in norm space by default: 0 to {scale} over the frame, so send x and y on that grid.") + for name in POINT_TOOLS: + descriptor = self._served.get(name) + if descriptor is not None: + descriptor.description = " ".join((getattr(type(self), name).__doc__ or "").split()) + note + manager = getattr(self.mcp, "_tool_manager", None) + for name, descriptor in self._served.items(): + registered = manager.get_tool(name) if manager is not None else None + if registered is not None: + registered.description = descriptor.description + registered.parameters["required"] = sorted( + {*registered.parameters.get("required", []), *(descriptor.inputSchema or {}).get("required", [])}) + + def extension_params(self, uri: str) -> dict: + if uri == COMPUTER_DISPLAY: + return {"modes": [list(m) for m in self.display_modes]} + if uri == COMPUTER_SPACES: + return {"spaces": ["frame", "norm"]} + if uri == COMPUTER_KEYBOARD: + return {**{name: {"min": low, "max": high} for name, (low, high) in KEYBOARD_RANGES.items()}, + "layouts": list(self.layouts)} + if uri == COMPUTER_EXEC: + return {"max_output_bytes": self.settings.exec_output_bytes, "max_timeout_s": MAX_EXEC_TIMEOUT_S} + return {} + + def declared_extensions(self) -> list[EnvironmentExtension]: + features = { + "buttons": list(self.buttons), "max_click_count": self.max_click_count, + "keys": self.keys if self.keys == "all" else list(self.keys), "max_text_length": self.max_text_length, + "max_wait_ms": self.max_wait_ms, "max_drag_points": self.max_drag_points, + "max_duration_ms": self.max_duration_ms, "max_key_repeat": self.max_key_repeat, + "max_scroll_amount": self.max_scroll_amount, + } + declared = [EnvironmentExtension( + uri=COMPUTER, description="The seven computer_* tools: screenshot, click, drag, scroll, type, key, wait.", + params={**features, "delivery": "foreground", "platform": self.platform.as_dict(), + "supports": list(self.supports)})] + if self.has("pointer"): + declared.append(EnvironmentExtension( + uri=COMPUTER_POINTER, description="The computer_pointer_* tools, and click and scroll at the pointer.", + params={"buttons": list(self.buttons), "kinds": ["foreground"]})) + intake = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")]) + if self.has("write_files"): + intake.add = [IntakeFormat(part="file", format="file-tree", load="additive")] + declared.append(intake_extension(intake, "data/add writes files under the session user's home; data/get " + "returns the env's data paths.")) + return declared + + # What the tools and extensions call. + + async def _call(self, operation: str, *args: Any) -> Any: + """Run one of the driver's operations: an AgentEnvError passes through, anything else is an input that may + have been delivered (delivery_failed) or an internal failure.""" + method = getattr(self, self._operations[operation]) + try: + result = method(*args) + return await result if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future) else result + except AgentEnvError: + raise + except Exception as e: + error = DeliveryFailed if operation in INPUT else Internal + raise error(f"@{operation} failed: {type(e).__name__}: {e}") from e + + async def _display_now(self) -> Display: + display = await self._call("display") + if not isinstance(display, Display) or display.width <= 0 or display.height <= 0 or display.scale <= 0: + raise Internal(f"@display returned {display!r}, not a Display with a positive size and scale") + return display + + async def _capture(self, identity: str) -> Frame: + display = await self._display_now() + image = to_pil(await self._call("capture"), display) + return self._frames.add(image, display, self._queue.snapshot(), identity=identity) + + async def _pointer_screen(self) -> tuple[int, int]: + where = await self._call("pointer_position") + try: + x, y = where + return int(x), int(y) + except (TypeError, ValueError): + raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None + + async def _act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, + summary: str, repeat: int = 1, hold: tuple | None = None, move_to: tuple | None = None): + """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result + with others' input since ``frame`` and, when asked, a fresh frame.""" + identity = self.caller_identity(ctx) + sent = False + async with self._queue.turn(self.settings.queue_wait_s): + others = self._queue.others_since(frame.inputs, identity) if frame is not None else 0 + effect = None + try: + if move_to is not None: + await self._call("pointer_move", *move_to) + sent = True + for _ in range(repeat): + effect = await self._call(operation, *args) + sent = True + if hold is not None: + await asyncio.sleep(hold[0] / 1000) + await self._call("key_up", list(reversed(hold[1]))) + except AgentEnvError as e: + if sent: + e.applied = "unknown" + raise + finally: + if sent: + self._queue.delivered(identity) + data = {"applied": "yes", "delivery": "foreground", "others_since_frame": others, "effect": effect_data(effect)} + if not screenshot: + return _result(summary, data) + new = await self._capture(identity) + return _result(f"{summary}, then captured {new.id}", {**data, "frame": new.data()}, new.encode()) + + async def _ensure_ready(self) -> None: + """Refuse with device_unavailable until the ready checks have passed once; run them if none has yet.""" + if self._ready: + return + async with self._ready_lock: + if self._ready: + return + failed = [c for c in await self._run_ready_checks() if not c.ok] + if failed: + raise DeviceUnavailable("the computer isn't ready: " + "; ".join(f"{c.name}: {c.detail}" for c in failed)) + + async def _run_ready_checks(self) -> list[ReadyCheck]: + checks = [] + try: + display = await self._display_now() + checks.append(ReadyCheck("display", True, f"{display.width}×{display.height} at scale {display.scale:g}")) + except AgentEnvError as e: + display = None + checks.append(ReadyCheck("display", False, e.message)) + if display is None: + checks.append(ReadyCheck("capture", False, "the display didn't answer")) + else: + try: + image = to_pil(await self._call("capture"), display) + checks.append(ReadyCheck("capture", True, f"{image.size[0]}×{image.size[1]}")) + except AgentEnvError as e: + checks.append(ReadyCheck("capture", False, e.message)) + try: + checks += list(await self.ready_checks()) + except Exception as e: + checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) + if all(c.ok for c in checks): + self._ready = True + return checks + + def _hold(self, identity: str, keys: list[str]) -> None: + held = self._held.setdefault(identity, []) + held += [k for k in keys if k not in held] + previous = self._releases.pop(identity, None) + if previous is not None: + previous.cancel() + self._releases[identity] = asyncio.ensure_future(self._release_after(identity, self.settings.max_hold_ms)) + + def _release(self, identity: str, keys: list[str]) -> None: + held = [k for k in self._held.get(identity, []) if k not in keys] + if held: + self._held[identity] = held + return + self._held.pop(identity, None) + release = self._releases.pop(identity, None) + if release is not None and release is not asyncio.current_task(): + release.cancel() + + async def _release_after(self, identity: str, ms: int) -> None: + """Release what ``identity`` still holds once ``max_hold_ms`` has passed.""" + await asyncio.sleep(ms / 1000) + keys = list(reversed(self._held.get(identity, []))) + if keys: + async with self._queue.turn(self.settings.queue_wait_s): + await self._call("key_up", keys) + self._release(identity, keys) + + def _require_init(self) -> None: + if not hasattr(self, "_operations"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + + # The Linux defaults for the OS operations. + + @default_for("run_in_session", os="linux") + async def _linux_run_in_session(self, argv: list, cwd: str | None, env: dict | None, timeout_s: float): + return await linux.run_in_session(argv, cwd, env, timeout_s, output_bytes=self.settings.exec_output_bytes) + + @default_for("write_files", os="linux") + async def _linux_write_files(self, files: dict) -> None: + await asyncio.to_thread(linux.write_files, files) + + +def computer_driver(name: str | None = None, *, supports: list | None = None) -> Callable: + """Name a driver, its card's name unless ENVIRONMENT_NAME gives one, and declare the platforms it supports.""" + def deco(cls: type) -> type: + if not (isinstance(cls, type) and issubclass(cls, ComputerDriverEnvironment)): + raise TypeError("@computer_driver decorates a ComputerDriverEnvironment subclass") + if supports is not None: + cls.supports = list(supports) + return environment_card(name=name)(cls) if name else cls + return deco + + +def _check_features(cls: type) -> None: + problems = [] + unknown = [b for b in cls.buttons if b not in _BUTTONS] + if "left" not in cls.buttons or unknown: + problems.append(f"buttons include left and come from {list(_BUTTONS)}, got {list(cls.buttons)}") + for name, low in (("max_click_count", 1), ("max_text_length", 1), ("max_wait_ms", 60_000), ("max_drag_points", 2), + ("max_duration_ms", 0), ("max_key_repeat", 1), ("max_scroll_amount", 1)): + value = getattr(cls, name) + if not isinstance(value, int) or value < low: + problems.append(f"{name} is an integer of at least {low}, got {value!r}") + if cls.keys != "all": + try: + if [canonical(k) for k in cls.keys] != list(cls.keys): + problems.append("keys are W3C key values in their W3C case") + except AgentEnvError as e: + problems.append(f"keys: {e.message}") + if ("set_display" in declared(cls)) != bool(cls.display_modes): + problems.append("display_modes and @set_display come together") + if problems: + raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) + diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py new file mode 100644 index 00000000..854b992e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py @@ -0,0 +1,71 @@ +"""The computer contract's error codes as exceptions a driver raises: ``raise OutOfBounds("...")``.""" +from __future__ import annotations + +from typing import ClassVar + +from ..errors import AgentEnvError, Applied + + +class ComputerError(AgentEnvError): + """One row of the errors table; a subclass fixes its code, status, whether it may be retried and whether the + action may have happened.""" + + CODE: ClassVar[str] = "internal" + STATUS: ClassVar[int] = 500 + RETRYABLE: ClassVar[bool] = False + APPLIED: ClassVar[Applied] = "unknown" + + def __init__(self, message: str, *, hint: str | None = None) -> None: + super().__init__(self.CODE, message, retryable=self.RETRYABLE, applied=self.APPLIED, hint=hint, status=self.STATUS) + + +class InvalidArgument(ComputerError): + CODE, STATUS, APPLIED = "invalid_argument", 400, "no" + + +class OutOfBounds(ComputerError): + CODE, STATUS, APPLIED = "out_of_bounds", 400, "no" + + +class StaleFrame(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "stale_frame", 409, "no", True + + +class InputBusy(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "input_busy", 409, "no", True + + +class PermissionDenied(ComputerError): + CODE, STATUS, APPLIED = "permission_denied", 403, "no" + + +class Unsupported(ComputerError): + CODE, STATUS, APPLIED = "unsupported", 501, "no" + + +class BackgroundUnavailable(ComputerError): + CODE, STATUS, APPLIED = "background_unavailable", 501, "no" + + +class TargetUnavailable(ComputerError): + CODE, STATUS, APPLIED = "target_unavailable", 404, "no" + + +class DeviceUnavailable(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "device_unavailable", 503, "no", True + + +class LimitExceeded(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "limit_exceeded", 429, "no", True + + +class Timeout(ComputerError): + CODE, STATUS = "timeout", 504 + + +class DeliveryFailed(ComputerError): + CODE, STATUS = "delivery_failed", 502 + + +class Internal(ComputerError): + pass diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py new file mode 100644 index 00000000..249ae895 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -0,0 +1,132 @@ +"""The framework extensions agentenv calls, never agents: readiness, display, coordinate spaces, keyboard settings +and commands in the session.""" +from __future__ import annotations + +import asyncio +from collections import OrderedDict +from typing import Optional + +from ..agent_env_environment import extension +from .errors import Internal, InvalidArgument, Unsupported +from .operations import Display, ExecResult +from .spaces import check_space + +COMPUTER = "urn:agentenv:computer/v1" +COMPUTER_READY = "urn:agentenv:computer-ready/v1" +COMPUTER_DISPLAY = "urn:agentenv:computer-display/v1" +COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1" +COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1" +COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1" +COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" + +MAX_EXEC_TIMEOUT_S = 3600 +#: How many command results are kept for a repeated request_id. +KEEP_EXEC_RESULTS = 256 +KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)} + + +class Extensions: + """The extension handlers, mixed into ComputerDriverEnvironment; one whose operation the driver lacks is + withdrawn when the driver is built.""" + + @extension(COMPUTER_READY, op="status", method="GET", + description="Whether the computer is usable now, by real checks; never sends input.") + async def _ready_status(self) -> dict: + checks = await self._run_ready_checks() + return {"ready": all(c.ok for c in checks), + "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]} + + @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.") + async def _display_get(self) -> dict: + return self._display_data(await self._display_now()) + + @extension(COMPUTER_DISPLAY, op="set") + async def _display_set(self, width: int, height: int, scale: Optional[float] = None) -> dict: + if [width, height] not in [list(m) for m in self.display_modes]: + modes = ", ".join(f"{w}×{h}" for w, h in self.display_modes) + raise InvalidArgument(f"{width}×{height} is not one of this computer's display modes: {modes}") + now = await self._display_now() + changed = await self._call("set_display", width, height, now.scale if scale is None else scale) + if not isinstance(changed, Display): + changed = await self._display_now() + return self._display_data(changed) + + @extension(COMPUTER_SPACES, op="get", method="GET", + description="Get or set the default coordinate space, written into the tool descriptions.") + async def _spaces_get(self) -> dict: + return {"default_space": self.settings.default_space, "default_norm_scale": self.settings.default_norm_scale} + + @extension(COMPUTER_SPACES, op="set") + async def _spaces_set(self, default_space: str, default_norm_scale: Optional[int] = None) -> dict: + space, scale = check_space(default_space, default_norm_scale, self.settings.default_space, + self.settings.default_norm_scale) + self.settings = self.settings.model_copy(update={"default_space": space, "default_norm_scale": scale}) + self._describe_tools() + return await self._spaces_get() + + @extension(COMPUTER_KEYBOARD, op="get", method="GET", + description="Get or set the tap hold, typing delay, layout and maximum hold.") + async def _keyboard_get(self) -> dict: + s = self.settings + return {"tap_hold_ms": s.tap_hold_ms, "type_delay_ms": s.type_delay_ms, "layout": s.layout, + "max_hold_ms": s.max_hold_ms} + + @extension(COMPUTER_KEYBOARD, op="set") + async def _keyboard_set(self, tap_hold_ms: Optional[int] = None, type_delay_ms: Optional[int] = None, + layout: Optional[str] = None, max_hold_ms: Optional[int] = None) -> dict: + given = {"tap_hold_ms": tap_hold_ms, "type_delay_ms": type_delay_ms, "max_hold_ms": max_hold_ms} + for name, value in given.items(): + low, high = KEYBOARD_RANGES[name] + if value is not None and not low <= value <= high: + raise InvalidArgument(f"{name} is from {low} to {high}, got {value}") + if layout is not None and layout not in self.layouts: + raise Unsupported(f"this computer types in the layouts {list(self.layouts)}, not {layout!r}") + update = {k: v for k, v in {**given, "layout": layout}.items() if v is not None} + self.settings = self.settings.model_copy(update=update) + return await self._keyboard_get() + + @extension(COMPUTER_EXEC, op="run", + description="Run a command inside the desktop session as the session user; a repeated request_id " + "returns the first result.") + async def _exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None, env: Optional[dict] = None, + timeout_s: Optional[float] = None) -> dict: + if not (isinstance(argv, list) and argv and all(isinstance(a, str) for a in argv)): + raise InvalidArgument("argv is a non-empty list of strings; name a shell to use one: ['bash', '-lc', '...']") + if not (isinstance(request_id, str) and request_id): + raise InvalidArgument("request_id is a non-empty string") + if env is not None and not all(isinstance(k, str) and isinstance(v, str) for k, v in env.items()): + raise InvalidArgument("env maps names to string values") + timeout = self.settings.exec_timeout_s if timeout_s is None else timeout_s + if not 0 < timeout <= MAX_EXEC_TIMEOUT_S: + raise InvalidArgument(f"timeout_s is above 0 and at most {MAX_EXEC_TIMEOUT_S}, got {timeout}") + runs: OrderedDict = self._exec_runs + task = runs.get(request_id) + if task is None: + task = asyncio.ensure_future(self._run_command(argv, cwd, env, timeout)) + runs[request_id] = task + + def forget_a_failure(done: asyncio.Task) -> None: + if done.cancelled() or done.exception() is not None: + runs.pop(request_id, None) + + task.add_done_callback(forget_a_failure) + while len(runs) > KEEP_EXEC_RESULTS: + runs.popitem(last=False) + return await asyncio.shield(task) + + async def _run_command(self, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: + result = await self._call("run_in_session", argv, cwd, env, timeout) + if not isinstance(result, ExecResult): + raise Internal(f"@run_in_session returned {type(result).__name__}, not an ExecResult") + cap = self.settings.exec_output_bytes + return { + "exit_code": result.exit_code, + "stdout": result.stdout[:cap].decode("utf-8", errors="replace"), + "stderr": result.stderr[:cap].decode("utf-8", errors="replace"), + "truncated": result.truncated or len(result.stdout) > cap or len(result.stderr) > cap, + "timed_out": result.timed_out, + } + + def _display_data(self, display: Display) -> dict: + return {"width": display.width, "height": display.height, "scale": display.scale, + "modes": [list(m) for m in self.display_modes]} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py new file mode 100644 index 00000000..bf88fac4 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -0,0 +1,148 @@ +"""Frames: every screenshot, kept by id so any caller can point into it later, and its mapping to screen pixels.""" +from __future__ import annotations + +import base64 +import io +import math +from collections import OrderedDict +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from PIL import Image as PILImage + +from .errors import Internal, StaleFrame +from .operations import Display, Image + +KEEP = 64 + + +@dataclass(frozen=True) +class Frame: + """A capture, or a crop of one: ``origin`` is where this frame's pixels start in the root capture, and the root + capture's ``root_size`` image covers the whole ``display``.""" + + id: str + image: PILImage.Image = field(repr=False) + display: Display + origin: tuple[int, int] + root_size: tuple[int, int] + parent_id: str | None + captured_at: datetime + inputs: tuple = field(repr=False) + + @property + def width(self) -> int: + return self.image.size[0] + + @property + def height(self) -> int: + return self.image.size[1] + + def to_screen(self, fx: float, fy: float) -> tuple[int, int]: + """The screen pixel under frame pixel (fx, fy); exact at identity scale.""" + rx, ry = self.origin[0] + fx, self.origin[1] + fy + return (int(math.floor(rx * self.display.width / self.root_size[0])), + int(math.floor(ry * self.display.height / self.root_size[1]))) + + def from_screen(self, sx: float, sy: float) -> tuple[int, int]: + rx = sx * self.root_size[0] / self.display.width + ry = sy * self.root_size[1] / self.display.height + return int(math.floor(rx - self.origin[0])), int(math.floor(ry - self.origin[1])) + + def contains(self, fx: float, fy: float) -> bool: + return 0 <= fx < self.width and 0 <= fy < self.height + + def data(self) -> dict: + return { + "frame_id": self.id, + "width": self.width, + "height": self.height, + "display": {"width": self.display.width, "height": self.display.height, "scale": self.display.scale}, + "parent_frame_id": self.parent_id, + "captured_at": self.captured_at.isoformat().replace("+00:00", "Z"), + } + + def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str]: + """The frame as base64 and its MIME type.""" + buffer = io.BytesIO() + if fmt == "jpeg": + self.image.convert("RGB").save(buffer, format="JPEG", quality=quality or 85) + mime = "image/jpeg" + else: + self.image.save(buffer, format="PNG") + mime = "image/png" + return base64.b64encode(buffer.getvalue()).decode("ascii"), mime + + +class FrameStore: + """The last ``KEEP`` frames by id, ``f``, and each caller's latest; a frame no longer kept is unknown.""" + + def __init__(self, keep: int = KEEP) -> None: + self._keep = keep + self._frames: OrderedDict[str, Frame] = OrderedDict() + self._next = 1 + self._latest: dict[str, str] = {} + + def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identity: str, + parent: Frame | None = None, origin: tuple[int, int] = (0, 0)) -> Frame: + frame = Frame( + id=f"f{self._next}", + image=image, + display=parent.display if parent is not None else display, + origin=(parent.origin[0] + origin[0], parent.origin[1] + origin[1]) if parent is not None else (0, 0), + root_size=parent.root_size if parent is not None else image.size, + parent_id=parent.id if parent is not None else None, + captured_at=parent.captured_at if parent is not None else datetime.now(timezone.utc), + inputs=parent.inputs if parent is not None else inputs, + ) + self._next += 1 + self._frames[frame.id] = frame + while len(self._frames) > self._keep: + self._frames.popitem(last=False) + self._latest[identity] = frame.id + return frame + + def get(self, frame_id: str | None, identity: str) -> Frame: + """The named frame, or the caller's latest when none is named.""" + if frame_id is None: + frame_id = self._latest.get(identity) + if frame_id is None: + raise StaleFrame("there is no frame yet: take a screenshot first") + frame = self._frames.get(frame_id) + if frame is None: + raise StaleFrame(f"frame {frame_id} is unknown or no longer kept: take a new screenshot") + return frame + + def latest(self, identity: str) -> Frame | None: + return self._frames.get(self._latest.get(identity, "")) + + def check_current(self, frame: Frame, now: Display) -> None: + if (frame.display.width, frame.display.height, frame.display.scale) != (now.width, now.height, now.scale): + raise StaleFrame( + f"frame {frame.id} was captured at {_size(frame.display)} and the display is now {_size(now)}: " + "take a new screenshot") + + +def to_pil(capture: object, display: Display) -> PILImage.Image: + """A capture as a Pillow image, checked to be the display at native resolution.""" + if isinstance(capture, PILImage.Image): + image = capture + elif isinstance(capture, Image): + if capture.mode in ("png", "jpeg"): + image = PILImage.open(io.BytesIO(capture.data)) + image.load() + elif capture.mode in ("BGRA", "BGRX"): + image = PILImage.frombytes("RGB", (capture.width, capture.height), capture.data, "raw", "BGRX") + else: + image = PILImage.frombytes(capture.mode, (capture.width, capture.height), capture.data) + else: + raise Internal(f"@capture returned {type(capture).__name__}, not an Image") + expected = (round(display.width * display.scale), round(display.height * display.scale)) + if image.size != expected: + raise Internal(f"@capture returned {image.size[0]}×{image.size[1]}, but the display at scale " + f"{display.scale:g} is {expected[0]}×{expected[1]}") + return image + + +def _size(display: Display) -> str: + return f"{display.width}×{display.height} at scale {display.scale:g}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py new file mode 100644 index 00000000..b8bab2aa --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py @@ -0,0 +1,66 @@ +"""W3C UI Events key values: a single character, or a named key, case-insensitive and passed on in its W3C case. +Model spellings (ctrl, Return, ctrl+s) are an adapter's to map; a refused name gets a hint, never a remapping.""" +from __future__ import annotations + +import difflib + +from .errors import InvalidArgument, Unsupported + +NAMED = ( + "Alt", "AltGraph", "CapsLock", "Control", "Fn", "FnLock", "Hyper", "Meta", "NumLock", "ScrollLock", "Shift", + "Super", "Symbol", "SymbolLock", + "Enter", "Tab", + "ArrowDown", "ArrowLeft", "ArrowRight", "ArrowUp", "End", "Home", "PageDown", "PageUp", + "Backspace", "Clear", "Copy", "CrSel", "Cut", "Delete", "EraseEof", "ExSel", "Insert", "Paste", "Redo", "Undo", + "Accept", "Again", "Attn", "Cancel", "ContextMenu", "Escape", "Execute", "Find", "Help", "Pause", "Play", + "Props", "Select", "ZoomIn", "ZoomOut", + "BrightnessDown", "BrightnessUp", "Eject", "LogOff", "Power", "PowerOff", "PrintScreen", "Hibernate", "Standby", + "WakeUp", + *(f"F{n}" for n in range(1, 25)), + "AudioVolumeDown", "AudioVolumeMute", "AudioVolumeUp", "MediaPlayPause", "MediaStop", "MediaTrackNext", + "MediaTrackPrevious", + "BrowserBack", "BrowserFavorites", "BrowserForward", "BrowserHome", "BrowserRefresh", "BrowserSearch", + "BrowserStop", +) +_BY_LOWER = {name.lower(): name for name in NAMED} +_SPELLINGS = { + "ctrl": "Control", "control_l": "Control", "cmd": "Meta", "command": "Meta", "win": "Meta", "windows": "Meta", + "super_l": "Meta", "option": "Alt", "opt": "Alt", "return": "Enter", "esc": "Escape", "del": "Delete", + "space": "' '", "spacebar": "' '", "up": "ArrowUp", "down": "ArrowDown", + "left": "ArrowLeft", "right": "ArrowRight", "pgup": "PageUp", "pgdn": "PageDown", "ins": "Insert", + "menu": "ContextMenu", "print": "PrintScreen", +} + + +def canonical(key: object) -> str: + """``key`` as its W3C value, or InvalidArgument with a hint naming the closest valid one.""" + if not isinstance(key, str) or not key: + raise InvalidArgument(f"a key is a W3C key value, got {key!r}") + if len(key) == 1: + return key + name = _BY_LOWER.get(key.lower()) + if name is not None: + return name + raise InvalidArgument(f"{key!r} is not a W3C key value", hint=_hint(key)) + + +def chord(keys: object, supported: object) -> list[str]: + """A chord's keys in W3C case, refused whole when one isn't a key or isn't one ``supported`` names.""" + if not isinstance(keys, list) or not keys: + raise InvalidArgument("keys is a non-empty list of W3C key values, such as ['Control', 's']") + names = [canonical(k) for k in keys] + if supported != "all": + lacking = [n for n in names if n not in supported] + if lacking: + raise Unsupported(f"this computer doesn't support the key(s) {lacking}") + return names + + +def _hint(key: str) -> str: + lower = key.lower() + if "+" in key or " " in key.strip(): + return "one key per entry: keys is a list, such as ['Control', 's']" + if lower in _SPELLINGS: + return f"did you mean {_SPELLINGS[lower]}?" + close = difflib.get_close_matches(lower, list(_BY_LOWER), n=1, cutoff=0.6) + return f"did you mean {_BY_LOWER[close[0]]}?" if close else "named keys are W3C UI Events key values, such as Enter" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py new file mode 100644 index 00000000..303e42bc --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py @@ -0,0 +1,96 @@ +"""The base class's Linux defaults for the OS operations. The driver runs as the session user inside the desktop +session, so both act directly: a command inherits the session's environment, a file lands as that user.""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +import tempfile +from pathlib import Path + +from .errors import InvalidArgument +from .operations import ExecResult + +#: How long output is still kept after the command exits. An app it started in the background may hold its stdout +#: or stderr open; that stream is then drained, and discarded, until the app closes it, so the app never blocks on +#: a full pipe or dies writing to a closed one. +DRAIN_S = 1.0 +_draining: set = set() + + +async def run_in_session(argv: list[str], cwd: str | None, env: dict[str, str] | None, timeout_s: float, *, + output_bytes: int) -> ExecResult: + """Run ``argv`` with the driver's own environment plus ``env``, in ``cwd`` (relative to the home), keeping at + most ``output_bytes`` of each stream; past ``timeout_s`` its process group is killed.""" + home = Path.home() + workdir = home / cwd if cwd else home + try: + process = await asyncio.create_subprocess_exec( + *argv, cwd=workdir, env={**os.environ, **(env or {})}, stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, start_new_session=True) + except (FileNotFoundError, PermissionError, NotADirectoryError) as e: + raise InvalidArgument(f"cannot run {argv[0]!r}: {e.strerror or e}") from None + outputs = [_Capped(output_bytes), _Capped(output_bytes)] + readers = [asyncio.ensure_future(out.read(stream)) for out, stream in zip(outputs, (process.stdout, process.stderr))] + timed_out = not await _exited(process, timeout_s) + if timed_out: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + await _exited(process, DRAIN_S) + _, pending = await asyncio.wait(readers, timeout=DRAIN_S) + for out in outputs: + out.returned = True + for reader in pending: + _draining.add(reader) + reader.add_done_callback(_draining.discard) + stdout, stderr = outputs + return ExecResult(exit_code=None if timed_out else process.returncode, stdout=bytes(stdout.kept), + stderr=bytes(stderr.kept), timed_out=timed_out, truncated=stdout.cut or stderr.cut) + + +async def _exited(process: asyncio.subprocess.Process, timeout_s: float) -> bool: + """Whether the process exits within ``timeout_s``. Read from ``returncode``, which is set when the process exits: + ``Process.wait()`` also waits for the pipes to close, which a background child can hold open.""" + deadline = asyncio.get_running_loop().time() + timeout_s + while process.returncode is None: + if asyncio.get_running_loop().time() >= deadline: + return False + await asyncio.sleep(0.02) + return True + + +class _Capped: + def __init__(self, limit: int) -> None: + self.limit = limit + self.kept = bytearray() + self.cut = False + self.returned = False + + async def read(self, stream: asyncio.StreamReader) -> None: + while chunk := await stream.read(65536): + if self.returned: + continue + room = self.limit - len(self.kept) + if len(chunk) > room: + self.cut = True + self.kept += chunk[:max(room, 0)] + + +def write_files(files: dict[str, bytes]) -> None: + """Write each file under the home atomically: a temporary file beside it, then a rename.""" + home = Path.home().resolve() + for relative, data in files.items(): + destination = home / relative + if not destination.resolve().is_relative_to(home): + raise InvalidArgument(f"{relative!r} is outside the home") + destination.parent.mkdir(parents=True, exist_ok=True) + handle, temporary = tempfile.mkstemp(dir=destination.parent, prefix=f".{destination.name}.", suffix=".part") + try: + with os.fdopen(handle, "wb") as out: + out.write(data) + os.replace(temporary, destination) + except BaseException: + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) + raise diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py new file mode 100644 index 00000000..0f7419da --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -0,0 +1,183 @@ +"""The operations a driver implements, marked with decorators; the base class builds the contract from them. + +Each operation is called with screen pixels, positionally, in the order below, and may be sync or async: + + display() -> Display set_display(width, height, scale) -> Display + capture() -> Image | PIL.Image.Image the whole display, at native resolution + click(x, y, button, count, modifiers) -> Effect | None + drag(path, button, duration_ms, modifiers) -> Effect | None path: [(x, y), ...] + scroll(x, y, direction, amount, modifiers) -> Effect | None + type_text(text) -> Effect | None press(keys) -> Effect | None, one chord + pointer_move(x, y), pointer_down(button), pointer_up(button), pointer_position() -> (x, y) + key_down(keys), key_up(keys) + run_in_session(argv, cwd, env, timeout_s) -> ExecResult write_files({path: bytes}) +""" +from __future__ import annotations + +import inspect +from dataclasses import dataclass +from typing import Any, Callable + +from .platform import Platform, matches, normalize, overlap + +REQUIRED = ("display", "capture", "click", "drag", "scroll", "type_text", "press") +GROUPS = { + "pointer": ("pointer_move", "pointer_down", "pointer_up", "pointer_position"), + "keyboard": ("key_down", "key_up"), +} +OPTIONAL = ("set_display", "run_in_session", "write_files") +INPUT = frozenset({"click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", + "key_down", "key_up"}) + +_OP_ATTR = "_aec_operation" + + +@dataclass(frozen=True) +class Display: + """The display: its resolution in screen pixels, and how many image pixels a capture has per screen pixel.""" + width: int + height: int + scale: float = 1.0 + + +@dataclass(frozen=True) +class Image: + """A capture: encoded ``png`` or ``jpeg`` bytes, or raw pixels in a Pillow raw mode (``RGB``, ``RGBA``, ``BGRA``, + ``BGRX``).""" + width: int + height: int + data: bytes + mode: str = "png" + + +@dataclass(frozen=True) +class Effect: + """What a driver can confirm an input did; unknown fields stay None.""" + focus_changed: bool | None = None + pointer_moved: bool | None = None + clipboard_used: bool | None = None + detail: str | None = None + + +@dataclass(frozen=True) +class ExecResult: + exit_code: int | None + stdout: bytes = b"" + stderr: bytes = b"" + timed_out: bool = False + truncated: bool = False + + +@dataclass(frozen=True) +class ReadyCheck: + name: str + ok: bool + detail: str = "" + + +@dataclass(frozen=True) +class _Mark: + operation: str + predicates: dict + default: bool = False + + +def _marker(operation: str) -> Callable: + def decorate(fn: Callable | None = None, /, **predicates: Any) -> Callable: + mark = _Mark(operation, normalize(predicates)) + + def deco(f: Callable) -> Callable: + setattr(f, _OP_ATTR, mark) + return f + + return deco(fn) if fn is not None else deco + + decorate.__name__ = decorate.__qualname__ = operation + decorate.__doc__ = (f"Mark the method that performs ``{operation}``. Use bare, or with predicates " + f"(os, distribution, version, arch, display) to choose it per platform.") + return decorate + + +display = _marker("display") +set_display = _marker("set_display") +capture = _marker("capture") +click = _marker("click") +drag = _marker("drag") +scroll = _marker("scroll") +type_text = _marker("type_text") +press = _marker("press") +pointer_move = _marker("pointer_move") +pointer_down = _marker("pointer_down") +pointer_up = _marker("pointer_up") +pointer_position = _marker("pointer_position") +key_down = _marker("key_down") +key_up = _marker("key_up") +run_in_session = _marker("run_in_session") +write_files = _marker("write_files") + + +def default_for(operation: str, **predicates: Any) -> Callable: + """Mark one of the base class's per-OS defaults, which a driver's own method for the operation replaces.""" + def deco(f: Callable) -> Callable: + setattr(f, _OP_ATTR, _Mark(operation, normalize(predicates), default=True)) + return f + return deco + + +def declared(cls: type) -> dict[str, list[tuple[str, dict]]]: + """Every operation the class marks, as (method name, predicates), with the base class's defaults dropped for + an operation the driver implements itself.""" + found: dict[str, list[tuple[str, _Mark]]] = {} + for name in dir(cls): + if name.startswith("__"): + continue + mark = getattr(inspect.getattr_static(cls, name, None), _OP_ATTR, None) + if isinstance(mark, _Mark): + found.setdefault(mark.operation, []).append((name, mark)) + out = {} + for operation, marks in found.items(): + own = [(name, mark) for name, mark in marks if not mark.default] + out[operation] = [(name, mark.predicates) for name, mark in (own or marks)] + return out + + +def check_class(cls: type) -> None: + """The class-definition checks: every required operation, whole groups, one method per operation per + platform. Raises TypeError naming what is wrong.""" + ops = declared(cls) + problems = [] + missing = [op for op in REQUIRED if op not in ops] + if missing: + problems.append(f"missing required operation(s): {', '.join('@' + op for op in missing)}") + for group, members in GROUPS.items(): + have = [op for op in members if op in ops] + if have and len(have) != len(members): + lacking = [op for op in members if op not in ops] + problems.append(f"the {group} group needs all of {', '.join('@' + op for op in members)}; " + f"missing {', '.join('@' + op for op in lacking)}") + for op, methods in ops.items(): + for i, (a, pa) in enumerate(methods): + for b, pb in methods[i + 1:]: + if overlap(pa, pb): + problems.append(f"@{op} is marked on both {a} and {b}, and some platform matches both") + if problems: + raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) + + +def resolve(cls: type, running: Platform) -> dict[str, str]: + """The method each operation uses on ``running``, by name. A required operation with no method for it raises + RuntimeError; a group is kept only when all of it matches.""" + chosen = {} + for op, methods in declared(cls).items(): + hits = [name for name, predicates in methods if matches(predicates, running)] + if hits: + chosen[op] = hits[0] + missing = [op for op in REQUIRED if op not in chosen] + if missing: + raise RuntimeError(f"{cls.__qualname__} has no method for {', '.join('@' + op for op in missing)} " + f"on this platform ({running.as_dict()})") + for members in GROUPS.values(): + if not all(op in chosen for op in members): + for op in members: + chosen.pop(op, None) + return chosen diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py new file mode 100644 index 00000000..949d17e0 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py @@ -0,0 +1,76 @@ +"""The platform a driver runs on, and the predicates (``os``, ``distribution``, ``version``, ``arch``, ``display``) +that choose one method per operation for it.""" +from __future__ import annotations + +import os +import platform as _platform +import sys +from dataclasses import dataclass +from pathlib import Path + +PREDICATE_KEYS = ("os", "distribution", "version", "arch", "display") + +_ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"} +_OS = {"linux": "linux", "darwin": "macos", "win32": "windows"} + + +@dataclass(frozen=True) +class Platform: + os: str | None = None + distribution: str | None = None + version: str | None = None + arch: str | None = None + display: str | None = None + + def as_dict(self) -> dict: + return {k: getattr(self, k) for k in PREDICATE_KEYS if getattr(self, k) is not None} + + +def detect(environ: dict | None = None, os_release: Path = Path("/etc/os-release")) -> Platform: + """The running platform: the OS, ``/etc/os-release``'s ID and VERSION_ID on Linux, the machine, and the + display from ``XDG_SESSION_TYPE``, else ``DISPLAY`` (x11) or ``WAYLAND_DISPLAY`` (wayland).""" + environ = os.environ if environ is None else environ + name = _OS.get(sys.platform, sys.platform) + release = _os_release(os_release) if name == "linux" else {} + display = None + if name == "linux": + session = (environ.get("XDG_SESSION_TYPE") or "").strip().lower() + if session in ("x11", "wayland"): + display = session + elif environ.get("DISPLAY"): + display = "x11" + elif environ.get("WAYLAND_DISPLAY"): + display = "wayland" + machine = _platform.machine().lower() + return Platform(os=name, distribution=release.get("ID"), version=release.get("VERSION_ID"), + arch=_ARCH.get(machine, machine or None), display=display) + + +def _os_release(path: Path) -> dict: + try: + text = path.read_text() + except OSError: + return {} + fields = {} + for line in text.splitlines(): + key, sep, value = line.partition("=") + if sep and key.strip(): + fields[key.strip()] = value.strip().strip("\"'") + return fields + + +def normalize(predicates: dict) -> dict[str, frozenset]: + """Predicates as sets of allowed values; an unknown key is refused.""" + unknown = sorted(set(predicates) - set(PREDICATE_KEYS)) + if unknown: + raise TypeError(f"unknown predicate(s) {unknown}; predicates are {list(PREDICATE_KEYS)}") + return {k: frozenset([v] if isinstance(v, str) else v) for k, v in predicates.items()} + + +def matches(predicates: dict[str, frozenset], running: Platform) -> bool: + return all(getattr(running, key) in allowed for key, allowed in predicates.items()) + + +def overlap(a: dict[str, frozenset], b: dict[str, frozenset]) -> bool: + """Whether some platform could satisfy both: no key both constrain to disjoint values.""" + return all(a[k] & b[k] for k in set(a) & set(b)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py new file mode 100644 index 00000000..1f890335 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py @@ -0,0 +1,45 @@ +"""The input queue: input calls run one at a time in arrival order, and delivered inputs are counted per caller so +a result can say how many others landed since the frame it named.""" +from __future__ import annotations + +import asyncio +import contextlib +from collections import Counter +from typing import AsyncIterator + +from .errors import InputBusy + + +class InputQueue: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._total = 0 + self._by_caller: Counter = Counter() + + @contextlib.asynccontextmanager + async def turn(self, wait_s: float) -> AsyncIterator[None]: + """Hold the queue for one call, or refuse with input_busy after ``wait_s``. Waiting is first come, first + served; a waiter woken as its time runs out still takes its turn.""" + waiter = asyncio.ensure_future(self._lock.acquire()) + done, _ = await asyncio.wait({waiter}, timeout=wait_s) + if not done: + waiter.cancel() + try: + await waiter + except asyncio.CancelledError: + raise InputBusy(f"waited {wait_s:g} s for earlier input to finish") from None + try: + yield + finally: + self._lock.release() + + def delivered(self, caller: str) -> None: + self._total += 1 + self._by_caller[caller] += 1 + + def snapshot(self) -> tuple: + return self._total, dict(self._by_caller) + + def others_since(self, snapshot: tuple, caller: str) -> int: + total, by_caller = snapshot + return (self._total - total) - (self._by_caller[caller] - by_caller.get(caller, 0)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py new file mode 100644 index 00000000..320ed333 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py @@ -0,0 +1,43 @@ +"""The settings a computer driver reads at start, from one JSON value in ``AGENTENV_COMPUTER_SETTINGS``.""" +from __future__ import annotations + +import json +import os +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from ..transfers import RelativePath + +SETTINGS_ENV = "AGENTENV_COMPUTER_SETTINGS" + +Space = Literal["frame", "norm"] + + +class ComputerSettings(BaseModel): + """``data_paths`` are relative to the session user's home and are what ``data/get`` returns; the rest are the + defaults the card reports and the framework extensions change.""" + + model_config = ConfigDict(extra="forbid", validate_assignment=True) + + data_paths: list[RelativePath] = Field(default_factory=list) + default_space: Space = "frame" + default_norm_scale: int = Field(default=1000, ge=1, le=1_000_000) + queue_wait_s: float = Field(default=30.0, gt=0, le=3600) + tap_hold_ms: int = Field(default=50, ge=0, le=1000) + type_delay_ms: int = Field(default=10, ge=0, le=1000) + max_hold_ms: int = Field(default=10_000, ge=0, le=600_000) + layout: str = "us" + drag_duration_ms: int = Field(default=500, ge=0) + exec_output_bytes: int = Field(default=1 << 20, gt=0) + exec_timeout_s: float = Field(default=60.0, gt=0, le=3600) + + +def load_settings(environ: dict | None = None) -> ComputerSettings: + raw = (os.environ if environ is None else environ).get(SETTINGS_ENV) + if not raw: + return ComputerSettings() + try: + return ComputerSettings.model_validate(json.loads(raw)) + except (ValueError, ValidationError) as e: + raise ValueError(f"{SETTINGS_ENV} is not valid computer settings: {e}") from None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py new file mode 100644 index 00000000..d3465112 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py @@ -0,0 +1,49 @@ +"""Coordinate spaces: points in a frame's pixels (``frame``) or on a 0..norm_scale grid over it (``norm``), mapped +to the frame's pixels and refused, never clamped, when outside it.""" +from __future__ import annotations + +import math + +from .errors import InvalidArgument, OutOfBounds +from .frames import Frame +from .settings import Space + +SPACES = ("frame", "norm") + + +def frame_point(frame: Frame, x: float, y: float, space: Space, norm_scale: int) -> tuple[float, float]: + if space == "norm": + if not (0 <= x <= norm_scale and 0 <= y <= norm_scale): + raise OutOfBounds(f"({x:g}, {y:g}) is outside the norm range 0..{norm_scale} of frame {frame.id}") + fx, fy = min(x * frame.width / norm_scale, frame.width - 1), min(y * frame.height / norm_scale, frame.height - 1) + return fx, fy + if not frame.contains(x, y): + raise OutOfBounds(f"({x:g}, {y:g}) is outside frame {frame.id} ({frame.width}×{frame.height})") + return x, y + + +def frame_region(frame: Frame, region: list, space: Space, norm_scale: int) -> tuple[int, int, int, int]: + """``[x0, y0, x1, y1]`` as a pixel box within the frame, end exclusive.""" + if len(region) != 4: + raise InvalidArgument(f"region is [x0, y0, x1, y1], got {len(region)} numbers") + x0, y0, x1, y1 = region + if space == "norm": + if not all(0 <= v <= norm_scale for v in region): + raise OutOfBounds(f"region {list(region)} is outside the norm range 0..{norm_scale} of frame {frame.id}") + x0, x1 = x0 * frame.width / norm_scale, x1 * frame.width / norm_scale + y0, y1 = y0 * frame.height / norm_scale, y1 * frame.height / norm_scale + box = (int(math.floor(x0)), int(math.floor(y0)), int(math.ceil(x1)), int(math.ceil(y1))) + if not (0 <= box[0] < box[2] <= frame.width and 0 <= box[1] < box[3] <= frame.height): + raise OutOfBounds(f"region {list(region)} is not a non-empty box within frame {frame.id} " + f"({frame.width}×{frame.height})") + return box + + +def check_space(space: str | None, norm_scale: int | None, default_space: Space, default_scale: int) -> tuple: + space = space or default_space + if space not in SPACES: + raise InvalidArgument(f"space is 'frame' or 'norm', got {space!r}") + scale = default_scale if norm_scale is None else norm_scale + if not (isinstance(scale, int) and 1 <= scale <= 1_000_000): + raise InvalidArgument(f"norm_scale is an integer from 1 to 1000000, got {norm_scale!r}") + return space, scale diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py new file mode 100644 index 00000000..9297efee --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -0,0 +1,469 @@ +"""The seven computer_* tools and the pointer and keyboard groups' tools: fixed names, parameters and results, built +from the driver's operations. Every result is structured, with a one-line summary repeated as its text block.""" +from __future__ import annotations + +import asyncio +import unicodedata +from typing import Annotated, Literal, Optional + +from mcp.types import CallToolResult, ImageContent, TextContent +from pydantic import BaseModel, Field + +from ..agent_env_environment import tool +from ..tool_context import ToolContext +from .errors import InvalidArgument, Unsupported +from .frames import Frame +from .keys import chord +from .operations import Effect +from .spaces import check_space, frame_point, frame_region + +_READ = {"readOnlyHint": True, "openWorldHint": False} +_INPUT = {"readOnlyHint": False, "destructiveHint": False, "idempotentHint": False, "openWorldHint": False} + +Number = Annotated[float, Field(description="A coordinate in the chosen space")] +FrameId = Annotated[Optional[str], Field(description="The frame the point refers to; defaults to your latest")] +SpaceArg = Annotated[Optional[Literal["frame", "norm"]], Field( + description="frame: pixels of the frame's image; norm: 0 to norm_scale over it")] +NormScale = Annotated[Optional[int], Field(description="The norm grid's size, for space=norm")] +Modifiers = Annotated[Optional[list[str]], Field(description="W3C key names held for this action, such as ['Shift']")] +Screenshot = Annotated[bool, Field(description="Return a fresh frame after acting")] +Button = Annotated[Optional[str], Field(description="Omitted, the primary press; others only where declared")] + + +class DisplayData(BaseModel): + width: int + height: int + scale: float + + +class FrameData(BaseModel): + frame_id: str + width: int + height: int + display: DisplayData + parent_frame_id: Optional[str] = None + captured_at: str + + +class ScreenshotResult(FrameData): + summary: str + + +class EffectData(BaseModel): + focus_changed: Optional[bool] = None + pointer_moved: Optional[bool] = None + clipboard_used: Optional[bool] = None + detail: Optional[str] = None + + +class InputResult(BaseModel): + summary: str + applied: Literal["yes", "no", "unknown"] + delivery: Literal["foreground", "background"] + others_since_frame: int + effect: Optional[EffectData] = None + frame: Optional[FrameData] = None + + +class WaitResult(BaseModel): + summary: str + waited_ms: int + frame: Optional[FrameData] = None + + +class Point(BaseModel): + x: int + y: int + + +class PointerPositionResult(BaseModel): + summary: str + screen: Point + frame_id: Optional[str] = None + x: Optional[int] = None + y: Optional[int] = None + inside_frame: Optional[bool] = None + + +class PointerData(BaseModel): + id: str + kind: Literal["foreground", "background"] + label: str + default: bool + + +class PointerListResult(BaseModel): + summary: str + pointers: list[PointerData] + + +#: The tools a group serves, and the tools whose description names the default coordinate space. +GROUP_TOOLS = { + "pointer": ("computer_pointer_move", "computer_pointer_down", "computer_pointer_up", "computer_pointer_position", + "computer_pointer_list"), + "keyboard": ("computer_keyboard_down", "computer_keyboard_up"), +} +POINT_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_pointer_move", + "computer_pointer_down", "computer_pointer_up") +POINT_OPTIONAL_WITH_POINTER = ("computer_click", "computer_scroll") + + +def _result(summary: str, data: dict, image: tuple[str, str] | None = None) -> CallToolResult: + content: list = [TextContent(type="text", text=summary)] + if image is not None: + content.append(ImageContent(type="image", data=image[0], mimeType=image[1])) + return CallToolResult(content=content, structuredContent={"summary": summary, **data}) + + +def _fmt(x: float, y: float) -> str: + return f"({x:g}, {y:g})" + + +class Tools: + """The tools, mixed into ComputerDriverEnvironment.""" + + @tool(annotations=_READ) + async def computer_screenshot( + self, + ctx: ToolContext, + region: Annotated[Optional[list[float]], Field( + description="[x0, y0, x1, y1] within a frame: return that part at native resolution as a new frame")] = None, + frame_id: Annotated[Optional[str], Field(description="The frame region refers to; defaults to your latest")] = None, + format: Annotated[Literal["png", "jpeg"], Field(description="The image encoding")] = "png", + quality: Annotated[Optional[int], Field(description="JPEG quality, 1 to 95")] = None, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, ScreenshotResult]: + """Capture the screen as a frame: an image plus a frame_id you can point into later, the image size, and the + display resolution and scale. With region, crop an existing frame at native resolution (zoom).""" + await self._ensure_ready() + identity = self.caller_identity(ctx) + if quality is not None and not (format == "jpeg" and 1 <= quality <= 95): + raise InvalidArgument("quality is for format=jpeg, from 1 to 95") + if region is None: + if frame_id is not None: + raise InvalidArgument("frame_id names the frame a region crops; give a region too") + frame = await self._capture(identity) + summary = f"Captured {frame.id}: {frame.width}×{frame.height} of the {_display(frame)} display" + else: + parent = self._frames.get(frame_id, identity) + self._frames.check_current(parent, await self._display_now()) + space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) + box = frame_region(parent, region, space, scale) + frame = self._frames.add(parent.image.crop(box), parent.display, parent.inputs, identity=identity, + parent=parent, origin=(box[0], box[1])) + summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})" + return _result(summary, frame.data(), frame.encode(format, quality)) + + @tool(annotations=_INPUT) + async def computer_click( + self, + ctx: ToolContext, + x: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, + y: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, + frame_id: FrameId = None, + button: Button = None, + count: Annotated[int, Field(description="1 by default; double and triple clicks where declared")] = 1, + modifiers: Modifiers = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Click at a point of a frame.""" + await self._ensure_ready() + button = self._button(button) + if not 1 <= count <= self.max_click_count: + raise InvalidArgument(f"count is from 1 to {self.max_click_count} on this computer, got {count}") + held = self._modifiers(modifiers) + frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) + times = f" ×{count}" if count > 1 else "" + return await self._act(ctx, "click", (sx, sy, button, count, held), frame=frame, screenshot=screenshot, + summary=f"Clicked {button}{times} {where}") + + @tool(annotations=_INPUT) + async def computer_drag( + self, + ctx: ToolContext, + path: Annotated[list[list[float]], Field(description="Points [[x, y], ...]: at least a start and an end")], + frame_id: FrameId = None, + button: Button = None, + duration_ms: Annotated[Optional[int], Field(description="How long the move takes")] = None, + modifiers: Modifiers = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Press at the first point, move through the path, and release at the last.""" + await self._ensure_ready() + button = self._button(button) + if not isinstance(path, list) or not 2 <= len(path) <= self.max_drag_points: + raise InvalidArgument(f"path has from 2 to {self.max_drag_points} points on this computer") + duration = self.settings.drag_duration_ms if duration_ms is None else duration_ms + if not 0 <= duration <= self.max_duration_ms: + raise InvalidArgument(f"duration_ms is from 0 to {self.max_duration_ms} on this computer, got {duration}") + held = self._modifiers(modifiers) + identity = self.caller_identity(ctx) + frame = self._frames.get(frame_id, identity) + self._frames.check_current(frame, await self._display_now()) + space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) + screen = [] + for point in path: + if not (isinstance(point, list) and len(point) == 2): + raise InvalidArgument("each path point is [x, y]") + screen.append(frame.to_screen(*frame_point(frame, point[0], point[1], space, scale))) + return await self._act(ctx, "drag", (screen, button, duration, held), frame=frame, screenshot=screenshot, + summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") + + @tool(annotations=_INPUT) + async def computer_scroll( + self, + ctx: ToolContext, + direction: Annotated[Literal["up", "down", "left", "right"], Field( + description="Where the view moves through the content: down shows what is further down")], + x: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, + y: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, + frame_id: FrameId = None, + amount: Annotated[int, Field(description="How many steps (wheel notches)")] = 3, + modifiers: Modifiers = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Scroll the content under a point.""" + await self._ensure_ready() + if not 1 <= amount <= self.max_scroll_amount: + raise InvalidArgument(f"amount is from 1 to {self.max_scroll_amount} on this computer, got {amount}") + held = self._modifiers(modifiers) + frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) + return await self._act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, screenshot=screenshot, + summary=f"Scrolled {direction} {amount} {where}") + + @tool(annotations=_INPUT) + async def computer_type( + self, + ctx: ToolContext, + text: Annotated[str, Field(description="Any Unicode; \\n types Enter and \\t types Tab")], + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Type text into whatever has keyboard focus. Text that can't be typed whole is refused before anything is + typed; the clipboard is left as it was.""" + await self._ensure_ready() + if not text: + raise InvalidArgument("text is empty") + if len(text) > self.max_text_length: + raise InvalidArgument(f"text is at most {self.max_text_length} characters on this computer, got {len(text)}") + control = sorted({f"U+{ord(c):04X}" for c in text if c not in "\n\t" and unicodedata.category(c) == "Cc"}) + if control: + raise InvalidArgument(f"text has control characters {control}; press keys with computer_key") + identity = self.caller_identity(ctx) + return await self._act(ctx, "type_text", (text,), frame=self._frames.latest(identity), screenshot=screenshot, + summary=f"Typed {len(text)} characters") + + @tool(annotations=_INPUT) + async def computer_key( + self, + ctx: ToolContext, + keys: Annotated[list[str], Field( + description="One chord, W3C key names pressed in order and released in reverse: ['Control', 's'] or " + "['Enter']")], + repeat: Annotated[int, Field(description="How many times to press the chord")] = 1, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Press one key chord.""" + await self._ensure_ready() + names = chord(keys, self.keys) + if not 1 <= repeat <= self.max_key_repeat: + raise InvalidArgument(f"repeat is from 1 to {self.max_key_repeat} on this computer, got {repeat}") + identity = self.caller_identity(ctx) + times = f" ×{repeat}" if repeat > 1 else "" + return await self._act(ctx, "press", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") + + @tool(annotations=_READ) + async def computer_wait( + self, + ctx: ToolContext, + ms: Annotated[int, Field(description="How long to wait, in milliseconds")] = 1000, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, WaitResult]: + """Wait, then optionally return a fresh frame.""" + await self._ensure_ready() + if not 0 <= ms <= self.max_wait_ms: + raise InvalidArgument(f"ms is from 0 to {self.max_wait_ms} on this computer, got {ms}") + await asyncio.sleep(ms / 1000) + summary = f"Waited {ms} ms" + if not screenshot: + return _result(summary, {"waited_ms": ms}) + frame = await self._capture(self.caller_identity(ctx)) + return _result(f"{summary}, then captured {frame.id}", {"waited_ms": ms, "frame": frame.data()}, frame.encode()) + + @tool(annotations=_INPUT) + async def computer_pointer_move( + self, + ctx: ToolContext, + x: Number, + y: Number, + frame_id: FrameId = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Move the pointer to a point; hover effects happen.""" + await self._ensure_ready() + frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) + return await self._act(ctx, "pointer_move", (sx, sy), frame=frame, screenshot=screenshot, + summary=f"Moved the pointer {where}") + + @tool(annotations=_INPUT) + async def computer_pointer_down( + self, + ctx: ToolContext, + button: Button = None, + x: Annotated[Optional[float], Field(description="Move here first")] = None, + y: Annotated[Optional[float], Field(description="Move here first")] = None, + frame_id: FrameId = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Press a button, optionally after moving to a point.""" + return await self._button_action(ctx, "pointer_down", "Pressed", button, x, y, frame_id, screenshot, space, + norm_scale) + + @tool(annotations=_INPUT) + async def computer_pointer_up( + self, + ctx: ToolContext, + button: Button = None, + x: Annotated[Optional[float], Field(description="Move here first")] = None, + y: Annotated[Optional[float], Field(description="Move here first")] = None, + frame_id: FrameId = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Release a button, optionally after moving to a point.""" + return await self._button_action(ctx, "pointer_up", "Released", button, x, y, frame_id, screenshot, space, + norm_scale) + + @tool(annotations=_READ) + async def computer_pointer_position( + self, + ctx: ToolContext, + frame_id: Annotated[Optional[str], Field(description="The frame to report in; defaults to your latest")] = None, + ) -> Annotated[CallToolResult, PointerPositionResult]: + """Where the pointer is, in a frame's pixels and in screen units, and whether it is inside that frame.""" + await self._ensure_ready() + sx, sy = await self._pointer_screen() + identity = self.caller_identity(ctx) + frame = self._frames.get(frame_id, identity) if frame_id is not None else self._frames.latest(identity) + data: dict = {"screen": {"x": sx, "y": sy}} + if frame is None: + return _result(f"The pointer is at screen ({sx}, {sy})", data) + fx, fy = frame.from_screen(sx, sy) + inside = frame.contains(fx, fy) + data.update(frame_id=frame.id, x=fx, y=fy, inside_frame=inside) + where = f"({fx}, {fy}) in {frame.id}" if inside else f"outside {frame.id}, at ({fx}, {fy}) in its pixels" + return _result(f"The pointer is {where}, screen ({sx}, {sy})", data) + + @tool(annotations=_READ) + async def computer_pointer_list(self) -> Annotated[CallToolResult, PointerListResult]: + """The pointers you may use, and which one is your default.""" + pointers = [{"id": "default", "kind": "foreground", "label": "The computer's pointer", "default": True}] + return _result("1 pointer: default (foreground)", {"pointers": pointers}) + + @tool(annotations=_INPUT) + async def computer_keyboard_down( + self, + ctx: ToolContext, + keys: Annotated[list[str], Field(description="W3C key names, pressed in the order given")], + hold_ms: Annotated[Optional[int], Field( + description="Hold this long, then release in reverse order; without it they stay down until " + "computer_keyboard_up")] = None, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Press keys and keep them down.""" + await self._ensure_ready() + names = chord(keys, self.keys) + if hold_ms is not None and not 0 <= hold_ms <= self.settings.max_hold_ms: + raise InvalidArgument(f"hold_ms is from 0 to {self.settings.max_hold_ms}, got {hold_ms}") + identity = self.caller_identity(ctx) + if hold_ms is not None: + return await self._act(ctx, "key_down", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + hold=(hold_ms, names), summary=f"Held {'+'.join(names)} for {hold_ms} ms") + result = await self._act(ctx, "key_down", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + summary=f"Pressed and held {'+'.join(names)}") + self._hold(identity, names) + return result + + @tool(annotations=_INPUT) + async def computer_keyboard_up( + self, + ctx: ToolContext, + keys: Annotated[Optional[list[str]], Field( + description="The keys to release, in order; without it, every key you are holding")] = None, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Release held keys.""" + await self._ensure_ready() + identity = self.caller_identity(ctx) + names = chord(keys, self.keys) if keys is not None else list(self._held.get(identity, [])) + if not names: + raise InvalidArgument("you are holding no keys") + result = await self._act(ctx, "key_up", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + summary=f"Released {'+'.join(names)}") + self._release(identity, names) + return result + + async def _button_action(self, ctx, op, verb, button, x, y, frame_id, screenshot, space, norm_scale): + await self._ensure_ready() + button = self._button(button) + if (x is None) != (y is None): + raise InvalidArgument("give both x and y, or neither") + if x is None: + return await self._act(ctx, op, (button,), frame=self._frames.latest(self.caller_identity(ctx)), + screenshot=screenshot, summary=f"{verb} {button}") + frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) + return await self._act(ctx, op, (button,), frame=frame, screenshot=screenshot, move_to=(sx, sy), + summary=f"{verb} {button} {where}") + + async def _point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple: + """The frame, the screen pixel and a phrase for a point; with no point and the pointer group, the pointer.""" + identity = self.caller_identity(ctx) + if x is None and y is None and self.has("pointer") and frame_id is None: + sx, sy = await self._pointer_screen() + return self._frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})" + if x is None or y is None: + raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if self.has("pointer") else "")) + frame = self._frames.get(frame_id, identity) + self._frames.check_current(frame, await self._display_now()) + space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) + fx, fy = frame_point(frame, x, y, space, scale) + in_space = f" ({space} {scale})" if space == "norm" else "" + return frame, frame.to_screen(fx, fy), f"at {_fmt(x, y)}{in_space} in {frame.id}" + + def _button(self, button: str | None) -> str: + button = button or "left" + if button not in self.buttons: + raise Unsupported(f"this computer has no {button} button; it has {', '.join(self.buttons)}") + return button + + def _modifiers(self, modifiers: list | None) -> list[str]: + if not modifiers: + return [] + if not isinstance(modifiers, list): + raise InvalidArgument("modifiers is a list of W3C key names") + return chord(modifiers, self.keys) + + +def _display(frame: Frame) -> str: + d = frame.display + return f"{d.width}×{d.height}" + (f" at scale {d.scale:g}" if d.scale != 1 else "") + + +def effect_data(effect: object) -> dict | None: + if not isinstance(effect, Effect): + return None + data = {k: v for k, v in vars(effect).items() if v is not None} + return data or None + diff --git a/packages/agentenv-protocol/tests/computer_fake.py b/packages/agentenv-protocol/tests/computer_fake.py new file mode 100644 index 00000000..6133104c --- /dev/null +++ b/packages/agentenv-protocol/tests/computer_fake.py @@ -0,0 +1,218 @@ +"""An in-memory screen and a driver on it that implements every operation, so the tests exercise the whole base +class with no display. A click paints its button's colour at its point; everything that reaches the screen is +logged; ``fail`` makes the next call of an operation raise.""" +from __future__ import annotations + +import asyncio +import contextlib + +import httpx +from mcp.shared.memory import create_connected_server_and_client_session +from PIL import Image as PILImage + +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + ComputerSettings, + Display, + Effect, + Platform, + capture, + click, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + scroll, + set_display, + type_text, +) + +LINUX = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") +BACKGROUND = (255, 255, 255) +COLOURS = {"left": (255, 0, 0), "right": (0, 0, 255), "middle": (0, 255, 0)} + + +class FakeScreen: + def __init__(self, width: int = 1280, height: int = 800, scale: float = 1.0) -> None: + self.log: list = [] + self.pointer = (0, 0) + self.held: list = [] + self.scrolled = [0, 0] + self.resize(width, height, scale) + + def resize(self, width: int, height: int, scale: float) -> None: + self.display = Display(width, height, scale) + self.image = PILImage.new("RGB", (round(width * scale), round(height * scale)), BACKGROUND) + + def paint(self, x: int, y: int, colour: tuple) -> None: + """Colour the image pixels under screen pixel (x, y).""" + s = self.display.scale + for px in range(int(x * s), max(int((x + 1) * s), int(x * s) + 1)): + for py in range(int(y * s), max(int((y + 1) * s), int(y * s) + 1)): + self.image.putpixel((px, py), colour) + + def colour_at(self, x: int, y: int) -> tuple: + s = self.display.scale + return self.image.getpixel((int(x * s), int(y * s))) + + +class FakeDriver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle") + max_click_count = 3 + max_drag_points = 10 + display_modes = ((1280, 800), (1920, 1080), (640, 400)) + + def __init__(self, screen: FakeScreen | None = None, settings: ComputerSettings | None = None, + platform: Platform = LINUX) -> None: + super().__init__(settings=settings or ComputerSettings(), platform=platform) + self.screen = screen or FakeScreen() + self.fail: dict = {} + self.slow: dict = {} + + async def _step(self, op: str) -> None: + if op in self.slow: + await asyncio.sleep(self.slow[op]) + error = self.fail.pop(op, None) + if error is not None: + raise error + + @display + async def screen_size(self) -> Display: + await self._step("display") + return self.screen.display + + @set_display + async def resize(self, width: int, height: int, scale: float) -> Display: + await self._step("set_display") + self.screen.resize(width, height, scale) + return self.screen.display + + @capture + async def grab(self): + await self._step("capture") + return self.screen.image.copy() + + @click + async def click_at(self, x, y, button, count, modifiers): + await self._step("click") + self.screen.log.append(("click", x, y, button, count, tuple(modifiers))) + self.screen.pointer = (x, y) + self.screen.paint(x, y, COLOURS[button]) + return Effect(pointer_moved=True) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers): + await self._step("drag") + self.screen.log.append(("drag", tuple(map(tuple, path)), button, duration_ms, tuple(modifiers))) + self.screen.pointer = tuple(path[-1]) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): + await self._step("scroll") + self.screen.log.append(("scroll", x, y, direction, amount, tuple(modifiers))) + dx, dy = {"up": (0, -1), "down": (0, 1), "left": (-1, 0), "right": (1, 0)}[direction] + self.screen.scrolled = [self.screen.scrolled[0] + dx * amount, self.screen.scrolled[1] + dy * amount] + + @type_text + async def type_in(self, text): + await self._step("type_text") + self.screen.log.append(("type", text)) + + @press + async def press_keys(self, keys): + await self._step("press") + self.screen.log.append(("press", tuple(keys), self.settings.tap_hold_ms)) + + @pointer_move + async def move(self, x, y): + await self._step("pointer_move") + self.screen.log.append(("move", x, y)) + self.screen.pointer = (x, y) + + @pointer_down + async def down(self, button): + await self._step("pointer_down") + self.screen.log.append(("down", button)) + + @pointer_up + async def up(self, button): + await self._step("pointer_up") + self.screen.log.append(("up", button)) + + @pointer_position + async def where(self): + await self._step("pointer_position") + return self.screen.pointer + + @key_down + async def keys_down(self, keys): + await self._step("key_down") + self.screen.log.append(("key_down", tuple(keys))) + self.screen.held += [k for k in keys if k not in self.screen.held] + + @key_up + async def keys_up(self, keys): + await self._step("key_up") + self.screen.log.append(("key_up", tuple(keys))) + self.screen.held = [k for k in self.screen.held if k not in keys] + + +class MinimalDriver(ComputerDriverEnvironment): + """Only the required operations, synchronous ones: no pointer or keyboard group, no display modes.""" + + def __init__(self, screen: FakeScreen | None = None, platform: Platform = LINUX) -> None: + super().__init__(settings=ComputerSettings(), platform=platform) + self.screen = screen or FakeScreen(640, 400) + + @display + def screen_size(self) -> Display: + return self.screen.display + + @capture + def grab(self): + return self.screen.image.copy() + + @click + def click_at(self, x, y, button, count, modifiers): + self.screen.log.append(("click", x, y, button, count, tuple(modifiers))) + self.screen.paint(x, y, COLOURS[button]) + + @drag + def drag_along(self, path, button, duration_ms, modifiers): + self.screen.log.append(("drag", tuple(map(tuple, path)))) + + @scroll + def scroll_at(self, x, y, direction, amount, modifiers): + self.screen.log.append(("scroll", x, y, direction, amount)) + + @type_text + def type_in(self, text): + self.screen.log.append(("type", text)) + + @press + def press_keys(self, keys): + self.screen.log.append(("press", tuple(keys))) + + +@contextlib.asynccontextmanager +async def mcp_session(driver: ComputerDriverEnvironment): + """An MCP client session on the driver's app, in memory.""" + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +@contextlib.asynccontextmanager +async def http_client(driver: ComputerDriverEnvironment): + """An HTTP client on the driver's app, for its card, extensions and data plane.""" + app = (driver.mcp or driver.create_app()).streamable_http_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + yield client + + +def rpc(method: str, params: dict | None = None) -> dict: + return {"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}} diff --git a/packages/agentenv-protocol/tests/test_computer_dataplane.py b/packages/agentenv-protocol/tests/test_computer_dataplane.py new file mode 100644 index 00000000..212d9912 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_dataplane.py @@ -0,0 +1,153 @@ +"""The data plane on a computer: data/add writes file parts atomically under the session user's home, from inline +bytes, an https URL or a read grant, with paths checked; data/get returns the data paths, inline or uploaded.""" +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, http_client, rpc # noqa: E402 + +from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI # noqa: E402 +from agentenv_protocol.computer import ComputerSettings # noqa: E402 + + +@pytest.fixture +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +def _part(path: str, data: bytes = b"hello", **marker) -> dict: + return {"kind": "file", "file": {"bytes": base64.b64encode(data).decode()}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": path, **marker}}} + + +async def _add(driver: FakeDriver, *parts: dict) -> dict: + async with http_client(driver) as http: + return (await http.post("/agentenv", json=rpc("data/add", {"parts": list(parts)}))).json() + + +@pytest.mark.asyncio +async def test_file_parts_land_under_the_home_atomically(home): + (home / "notes").mkdir() + (home / "notes" / "a.txt").write_text("old") + answer = await _add(FakeDriver(), _part("notes/a.txt", b"new"), _part("deep/er/b.bin", b"\x00\x01")) + assert answer["result"] == {} + assert (home / "notes" / "a.txt").read_text() == "new" and (home / "deep" / "er" / "b.bin").read_bytes() == b"\x00\x01" + assert sorted(p.name for p in (home / "notes").iterdir()) == ["a.txt"], "no temporary file is left behind" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("part, error", [ + (_part("../outside"), "is not a normalized path relative to the home"), + (_part("/etc/passwd"), "is not a normalized path relative to the home"), + ({"kind": "text", "text": "hi"}, "takes file parts naming their path"), + ({"kind": "file", "file": {"bytes": "aGk="}}, "takes file parts naming their path"), + ({"kind": "file", "file": {"bytes": "not base64!"}, "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "a"}}}, + "the part's bytes are not base64"), + ({"kind": "file", "file": {"uri": "http://plain/x"}, "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "a"}}}, + "a file part's uri is an https URL"), + (_part("a", read={"not": "a grant"}), "the part's read grant is not a valid ReadObject"), +]) +async def test_a_bad_part_is_refused_and_nothing_is_written(home, part, error): + answer = await _add(FakeDriver(), part) + assert answer["error"]["data"]["code"] == "add_failed" and error in answer["error"]["message"] + assert list(home.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_a_symlink_out_of_the_home_is_refused(home, tmp_path_factory): + outside = tmp_path_factory.mktemp("outside") + (home / "link").symlink_to(outside) + answer = await _add(FakeDriver(), _part("link/escaped.txt")) + assert "'link/escaped.txt' is outside the home" in answer["error"]["message"] + assert list(outside.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_an_https_url_part_is_fetched_by_the_driver(home): + transport = httpx.MockTransport(lambda request: httpx.Response(200, content=b"from the web")) + real = httpx.AsyncClient + with patch("agentenv_protocol.computer.dataplane.httpx.AsyncClient", + lambda **kwargs: real(**{"transport": transport, **kwargs})): + answer = await _add(FakeDriver(), {"kind": "file", "file": {"uri": "https://example.test/f"}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "web.txt"}}}) + assert answer["result"] == {} and (home / "web.txt").read_bytes() == b"from the web" + + +@pytest.mark.asyncio +async def test_an_https_url_over_the_limit_is_refused(home): + transport = httpx.MockTransport(lambda request: httpx.Response(200, content=b"x" * 64)) + real = httpx.AsyncClient + with patch("agentenv_protocol.computer.dataplane.httpx.AsyncClient", + lambda **kwargs: real(**{"transport": transport, **kwargs})), \ + patch("agentenv_protocol.computer.dataplane.MAX_URL_BYTES", 10): + answer = await _add(FakeDriver(), {"kind": "file", "file": {"uri": "https://example.test/f"}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "big"}}}) + assert "is over 10 bytes" in answer["error"]["message"] and not (home / "big").exists() + + +@pytest.mark.asyncio +async def test_a_read_grant_part_is_downloaded_through_the_transfers_helper(home): + grant = {"media_type": "text/plain", "max_bytes": 100, + "read": {"kind": "http-get", "url": "https://store.test/o", + "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat()}} + + async def fake_download(source, destination): + assert source.read.url == "https://store.test/o" + destination.write_bytes(b"granted") + + with patch("agentenv_protocol.computer.dataplane.download", fake_download): + answer = await _add(FakeDriver(), _part("granted.txt", b"ignored", read=grant)) + assert answer["result"] == {} and (home / "granted.txt").read_bytes() == b"granted" + + +@pytest.mark.asyncio +async def test_data_get_returns_the_data_paths_inline(home, tmp_path_factory): + (home / "app" / "db").mkdir(parents=True) + (home / "app" / "db" / "state.json").write_text('{"n": 1}') + (home / "app" / "db" / "blob.bin").write_bytes(b"\x01") + (home / "notes.md").write_text("# notes") + outside = tmp_path_factory.mktemp("outside") + (outside / "secret").write_text("no") + (home / "app" / "db" / "escape").symlink_to(outside / "secret") + driver = FakeDriver(settings=ComputerSettings(data_paths=["app", "notes.md", "missing", "app/db"])) + async with http_client(driver) as http: + parts = (await http.post("/agentenv", json=rpc("data/get"))).json()["result"]["parts"] + by_path = {p["metadata"][DATA_OBJECTS_EXTENSION_URI]["path"]: p["file"] for p in parts} + assert sorted(by_path) == ["app/db/blob.bin", "app/db/state.json", "notes.md"] + assert base64.b64decode(by_path["app/db/state.json"]["bytes"]) == b'{"n": 1}' + assert by_path["app/db/state.json"]["mimeType"] == "application/json" and by_path["notes.md"]["name"] == "notes.md" + assert by_path["app/db/blob.bin"]["mimeType"] == "application/octet-stream" + + +@pytest.mark.asyncio +async def test_data_get_uploads_under_a_write_namespace_grant(home): + (home / "notes.md").write_text("# notes") + grant = {"root_path": "runs/1", "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + "max_objects": 5, "max_object_bytes": 100, "max_total_bytes": 500, + "write": {"kind": "http-post-policy", "url": "https://store.test/", "fields": {}, + "path_field": "key", "file_field": "file"}} + upload = AsyncMock() + with patch("agentenv_protocol.computer.dataplane.NamespaceUploader") as uploader: + uploader.return_value.upload = upload + driver = FakeDriver(settings=ComputerSettings(data_paths=["notes.md"])) + async with http_client(driver) as http: + parts = (await http.post("/agentenv", json=rpc("data/get", {"write_namespace": grant}))).json()["result"]["parts"] + assert upload.await_args.args[0] == "notes.md" and upload.await_args.args[1] == home / "notes.md" + assert parts == [{"kind": "file", "file": {"uri": "notes.md", "mimeType": "text/markdown", "name": "notes.md"}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "notes.md"}}}] + + +@pytest.mark.asyncio +async def test_a_failed_write_leaves_no_temporary_file(home): + with patch("agentenv_protocol.computer.linux.os.replace", side_effect=OSError("disk full")): + answer = await _add(FakeDriver(), _part("notes/a.txt")) + assert "disk full" in answer["error"]["message"] and list((home / "notes").iterdir()) == [] diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py new file mode 100644 index 00000000..1c25c62c --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -0,0 +1,370 @@ +"""ComputerDriverEnvironment as a class: the checks at class definition, operations chosen per platform, settings, +the generated card, readiness and how driver failures are reported.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +import asyncio # noqa: E402 +import base64 # noqa: E402 +import io # noqa: E402 + +from computer_fake import LINUX, FakeDriver, FakeScreen, MinimalDriver, http_client, mcp_session # noqa: E402 +from mcp.server.fastmcp import FastMCP # noqa: E402 +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol.computer import ( # noqa: E402 + SETTINGS_ENV, + ComputerDriverEnvironment, + ComputerSettings, + Display, + Image, + InvalidArgument, + OutOfBounds, + Platform, + ReadyCheck, + capture, + click, + computer_driver, + display, + drag, + pointer_move, + press, + run_in_session, + scroll, + set_display, + type_text, +) +from agentenv_protocol.computer.keys import canonical # noqa: E402 +from agentenv_protocol.computer.operations import declared, resolve # noqa: E402 +from agentenv_protocol.computer.platform import detect, normalize # noqa: E402 +from agentenv_protocol.computer.settings import load_settings # noqa: E402 + +MACOS = Platform(os="macos", arch="arm64") + + +def _required(namespace: dict) -> dict: + """The required operations as no-op methods, with ``namespace`` added on top.""" + ops = { + "screen": display(lambda self: Display(10, 10)), + "grab": capture(lambda self: PILImage.new("RGB", (10, 10))), + "click_at": click(lambda self, *a: None), + "drag_along": drag(lambda self, *a: None), + "scroll_at": scroll(lambda self, *a: None), + "type_in": type_text(lambda self, *a: None), + "press_keys": press(lambda self, *a: None), + } + return {**ops, **namespace} + + +def _define(namespace: dict, **features) -> type: + return type("Driver", (ComputerDriverEnvironment,), {**_required(namespace), **features}) + + +def test_a_driver_missing_required_operations_is_refused_when_defined(): + with pytest.raises(TypeError, match=r"missing required operation\(s\): @drag, @press"): + type("Driver", (ComputerDriverEnvironment,), + {k: v for k, v in _required({}).items() if k not in ("drag_along", "press_keys")}) + + +def test_half_a_group_is_refused(): + with pytest.raises(TypeError, match="the pointer group needs all of .*missing @pointer_down, @pointer_up, " + "@pointer_position"): + _define({"move": pointer_move(lambda self, x, y: None)}) + + +def test_two_methods_one_platform_could_both_match_are_refused(): + with pytest.raises(TypeError, match="@capture is marked on both"): + _define({"grab_x11": capture(display="x11")(lambda self: None)}) + + +def test_predicates_choose_one_method_per_platform(): + Driver = _define({ + "grab": capture(display="x11")(lambda self: "x11"), + "grab_wayland": capture(display="wayland")(lambda self: "wayland"), + }) + assert resolve(Driver, LINUX)["capture"] == "grab" + assert resolve(Driver, Platform(os="linux", display="wayland"))["capture"] == "grab_wayland" + with pytest.raises(RuntimeError, match=r"no method for @capture on this platform \(\{'os': 'macos'"): + Driver(platform=MACOS) + + +def test_a_group_with_a_member_that_doesnt_match_the_platform_is_dropped(): + assert FakeDriver(platform=LINUX).has("pointer") + Driver = type("Driver", (FakeDriver,), {"move": pointer_move(os="windows")(lambda self, x, y: None)}) + driver = Driver() + assert not driver.has("pointer") and "pointer_down" not in driver._operations + + +def test_unknown_predicates_are_refused(): + with pytest.raises(TypeError, match=r"unknown predicate\(s\) \['kernel'\]"): + capture(kernel="6.8") + assert normalize({"arch": ["amd64", "arm64"], "os": "linux"}) == {"arch": frozenset({"amd64", "arm64"}), + "os": frozenset({"linux"})} + + +@pytest.mark.parametrize("features, error", [ + ({"buttons": ("right",)}, "buttons include left"), + ({"buttons": ("left", "thumb")}, "buttons include left"), + ({"max_click_count": 0}, "max_click_count is an integer of at least 1"), + ({"max_wait_ms": 1000}, "max_wait_ms is an integer of at least 60000"), + ({"keys": ("enter",)}, "keys are W3C key values in their W3C case"), + ({"keys": ("Return",)}, "keys: 'Return' is not a W3C key value"), + ({"display_modes": ((10, 10),)}, "display_modes and @set_display come together"), +]) +def test_features_are_checked_when_the_class_is_defined(features, error): + with pytest.raises(TypeError, match=error): + _define({}, **features) + + +def test_set_display_without_modes_is_refused(): + with pytest.raises(TypeError, match="display_modes and @set_display come together"): + _define({"resize": set_display(lambda self, w, h, s: None)}) + + +def test_a_driver_method_replaces_the_linux_default(): + Driver = _define({"run": run_in_session(lambda self, *a: None)}) + assert declared(Driver)["run_in_session"] == [("run", {})] + assert declared(MinimalDriver)["run_in_session"] == [("_linux_run_in_session", {"os": frozenset({"linux"})})] + + +def test_the_platform_is_detected(tmp_path, monkeypatch): + release = tmp_path / "os-release" + release.write_text('NAME="Ubuntu"\nID=ubuntu\nVERSION_ID="24.04"\n# comment\nbare\n') + monkeypatch.setattr("agentenv_protocol.computer.platform.sys.platform", "linux") + monkeypatch.setattr("agentenv_protocol.computer.platform._platform.machine", lambda: "aarch64") + assert detect({"DISPLAY": ":0"}, release) == Platform("linux", "ubuntu", "24.04", "arm64", "x11") + assert detect({"XDG_SESSION_TYPE": "Wayland", "DISPLAY": ":0"}, release).display == "wayland" + assert detect({"WAYLAND_DISPLAY": "wayland-0"}, release).display == "wayland" + assert detect({}, tmp_path / "missing").as_dict() == {"os": "linux", "arch": "arm64"} + monkeypatch.setattr("agentenv_protocol.computer.platform.sys.platform", "darwin") + assert detect({"DISPLAY": ":0"}, release) == Platform("macos", None, None, "arm64", None) + + +def test_settings_come_from_one_json_value(): + assert load_settings({}) == ComputerSettings() + settings = load_settings({SETTINGS_ENV: '{"data_paths": ["notes"], "default_space": "norm", "tap_hold_ms": 20}'}) + assert settings.data_paths == ["notes"] and settings.default_space == "norm" and settings.tap_hold_ms == 20 + for raw in ('{"data_paths": ["../out"]}', '{"tap_hold_ms": -1}', '{"unknown": 1}', "not json"): + with pytest.raises(ValueError, match=SETTINGS_ENV): + load_settings({SETTINGS_ENV: raw}) + + +def test_a_driver_reads_its_settings_and_platform_from_the_environment(monkeypatch): + monkeypatch.setenv(SETTINGS_ENV, '{"queue_wait_s": 2}') + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) + driver = _define({})() + assert driver.settings.queue_wait_s == 2 and driver.platform == LINUX + + +@pytest.mark.asyncio +async def test_the_card_declares_the_contract_the_driver_implements(): + async with http_client(FakeDriver()) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + computer = extensions["urn:agentenv:computer/v1"]["params"] + assert computer["buttons"] == ["left", "right", "middle"] and computer["max_click_count"] == 3 + assert computer["keys"] == "all" and computer["platform"] == LINUX.as_dict() and computer["delivery"] == "foreground" + assert extensions["urn:agentenv:computer-pointer/v1"]["params"]["buttons"] == ["left", "right", "middle"] + assert set(extensions["urn:agentenv:computer-keyboard/v1"]["params"]["methods"]) == {"get", "set"} + assert set(extensions["urn:agentenv:computer-display/v1"]["params"]["methods"]) == {"get", "set"} + assert extensions["urn:agentenv:computer-exec/v1"]["params"]["max_timeout_s"] == 3600 + for uri in ("urn:agentenv:computer-ready/v1", "urn:agentenv:computer-spaces/v1", "urn:agentenv:intake/v1", + "urn:agentenv:data-objects/v1", "urn:agentenv:disable-tool/v1", "urn:agentenv:enable-tool/v1"): + assert uri in extensions + assert card["capabilities"]["operations"] == ["data/add", "data/get"] + assert {t["name"] for t in card["capabilities"]["tools"]} >= {"computer_click", "computer_pointer_list"} + assert card["additionalInterfaces"] == [{"url": "/mcp", "transport": "mcp"}] + + +@pytest.mark.asyncio +async def test_a_minimal_driver_declares_only_what_it_has(): + async with http_client(MinimalDriver(platform=MACOS)) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + display_set = await http.post("/agentenv/ext/computer-display/set", json={"width": 1, "height": 1}) + exec_run = await http.post("/agentenv/ext/computer-exec/run", json={"argv": ["true"], "request_id": "r"}) + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + assert "urn:agentenv:computer-pointer/v1" not in extensions and "urn:agentenv:computer-keyboard/v1" not in extensions + assert "urn:agentenv:computer-exec/v1" not in extensions + assert set(extensions["urn:agentenv:computer-display/v1"]["params"]["methods"]) == {"get"} + assert extensions["urn:agentenv:intake/v1"]["params"] == {"get": [{"part": "file", "format": "file-tree"}]} + assert card["capabilities"]["operations"] == ["data/get"] + assert {t["name"] for t in card["capabilities"]["tools"]} == { + "computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", "computer_key", + "computer_wait"} + assert display_set.status_code == 404 and exec_run.status_code == 404 + + +def test_computer_driver_names_the_driver_and_declares_its_platforms(): + @computer_driver(name="reference", supports=[{"os": {"type": "linux"}, "arch": ["amd64"]}]) + class Reference(MinimalDriver): + pass + + driver = Reference() + card = driver._build_card() + assert card.name == "reference" and Reference.supports[0]["arch"] == ["amd64"] + assert computer_driver()(Reference) is Reference + with pytest.raises(TypeError, match="decorates a ComputerDriverEnvironment subclass"): + computer_driver(name="x")(object) + + +def test_a_driver_that_skips_super_init_is_told_so(): + class Forgetful(MinimalDriver): + def __init__(self) -> None: + pass + + with pytest.raises(TypeError, match=r"Forgetful.__init__ must call super\(\).__init__\(\)"): + Forgetful().create_app() + + +@pytest.mark.asyncio +async def test_tools_refuse_until_the_ready_checks_pass_once(): + driver = FakeDriver() + driver.fail["display"] = RuntimeError("no X server") + async with mcp_session(driver) as session: + refused = await session.call_tool("computer_screenshot", {}) + served = await session.call_tool("computer_screenshot", {}) + assert refused.isError and refused.structuredContent["code"] == "device_unavailable" + assert refused.structuredContent["retryable"] is True + assert "display: @display failed: RuntimeError: no X server" in refused.structuredContent["message"] + assert not served.isError and driver._ready + + +@pytest.mark.asyncio +async def test_the_drivers_own_ready_checks_count(): + class Checked(FakeDriver): + healthy = False + + async def ready_checks(self): + return [ReadyCheck("session", self.healthy, "logged in" if self.healthy else "no session")] + + class Raising(FakeDriver): + async def ready_checks(self): + raise RuntimeError("probe crashed") + + driver = Checked() + async with http_client(driver) as http: + before = (await http.get("/agentenv/ext/computer-ready/status")).json() + driver.healthy = True + after = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert before["ready"] is False and before["checks"][-1] == {"name": "session", "ok": False, "detail": "no session"} + assert after["ready"] is True + async with http_client(Raising()) as http: + raised = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert raised["checks"][-1] == {"name": "driver", "ok": False, "detail": "RuntimeError: probe crashed"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("captured, error", [ + (lambda screen: "pixels", "@capture returned str, not an Image"), + (lambda screen: PILImage.new("RGB", (5, 5)), "@capture returned 5×5, but the display at scale 1 is 1280×800"), +]) +async def test_a_bad_capture_is_an_internal_error(captured, error): + class Bad(FakeDriver): + @capture + async def grab(self): + return captured(self.screen) + + async with http_client(Bad()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert status["ready"] is False and error in status["checks"][1]["detail"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result", [None, Display(0, 10), Display(10, 10, 0)]) +async def test_a_bad_display_is_an_internal_error(result): + class Bad(FakeDriver): + @display + async def screen_size(self): + return result + + async with http_client(Bad()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert status["checks"][0]["ok"] is False and "@display returned" in status["checks"][0]["detail"] + assert status["checks"][1] == {"name": "capture", "ok": False, "detail": "the display didn't answer"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["png", "jpeg", "RGB", "BGRA"]) +async def test_a_capture_may_be_encoded_or_raw_pixels(mode): + screen = FakeScreen(4, 2) + screen.image.putpixel((1, 0), (10, 20, 30)) + + class Raw(FakeDriver): + @capture + async def grab(self): + if mode in ("png", "jpeg"): + buffer = io.BytesIO() + self.screen.image.save(buffer, format=mode.upper()) + return Image(4, 2, buffer.getvalue(), mode) + if mode == "BGRA": + r, g, b = self.screen.image.split() + return Image(4, 2, PILImage.merge("RGBA", (b, g, r, r)).tobytes(), "BGRA") + return Image(4, 2, self.screen.image.tobytes(), "RGB") + + async with mcp_session(Raw(screen)) as session: + shot = await session.call_tool("computer_screenshot", {}) + assert not shot.isError and shot.structuredContent["width"] == 4 + decoded = PILImage.open(io.BytesIO(base64.b64decode(shot.content[1].data))).convert("RGB") + assert decoded.size == (4, 2) + if mode != "jpeg": + assert decoded.getpixel((1, 0)) == (10, 20, 30) + + +@pytest.mark.asyncio +async def test_a_failing_input_operation_may_have_been_delivered_and_others_were_not(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + driver.fail["click"] = RuntimeError("XTEST gone") + crashed = await session.call_tool("computer_click", {"x": 1, "y": 1}) + driver.fail["click"] = OutOfBounds("the window moved") + refused = await session.call_tool("computer_click", {"x": 1, "y": 1}) + driver.fail["capture"] = RuntimeError("mss failed") + broken = await session.call_tool("computer_screenshot", {}) + assert crashed.structuredContent == {"code": "delivery_failed", "message": "@click failed: RuntimeError: XTEST gone", + "retryable": False, "applied": "unknown"} + assert refused.structuredContent["code"] == "out_of_bounds" and refused.structuredContent["applied"] == "no" + assert broken.structuredContent["code"] == "internal" and broken.structuredContent["applied"] == "unknown" + + +@pytest.mark.asyncio +async def test_a_failure_after_part_of_a_call_was_delivered_is_applied_unknown(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + driver.fail["pointer_down"] = OutOfBounds("the window moved") + moved_then_refused = await session.call_tool("computer_pointer_down", {"x": 3, "y": 4}) + assert moved_then_refused.structuredContent["code"] == "out_of_bounds" + assert moved_then_refused.structuredContent["applied"] == "unknown" + assert driver.screen.log[-1] == ("move", 3, 4) + + +@pytest.mark.asyncio +async def test_a_driver_mounted_on_an_app_serves_the_same_tools(): + driver = MinimalDriver() + app = driver.mount(FastMCP("mounted")) + async with create_connected_server_and_client_session(app) as session: + tools = {t.name: t for t in (await session.list_tools()).tools} + assert tools["computer_click"].inputSchema["required"] == ["x", "y"] + assert "frame space by default" in tools["computer_scroll"].description + + +@pytest.mark.asyncio +async def test_concurrent_first_calls_run_the_ready_checks_once(): + driver = FakeDriver() + driver.slow["display"] = 0.05 + async with mcp_session(driver) as session: + results = await asyncio.gather(*(session.call_tool("computer_wait", {"ms": 0}) for _ in range(3))) + assert all(not r.isError for r in results) and driver._ready + + +def test_keys_and_modifiers_that_arent_lists_of_names_are_refused(): + for key in (3, ""): + with pytest.raises(InvalidArgument, match="a key is a W3C key value"): + canonical(key) + with pytest.raises(InvalidArgument, match="modifiers is a list"): + FakeDriver()._modifiers("Shift") diff --git a/packages/agentenv-protocol/tests/test_computer_extensions.py b/packages/agentenv-protocol/tests/test_computer_extensions.py new file mode 100644 index 00000000..2793bdca --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_extensions.py @@ -0,0 +1,179 @@ +"""The framework extensions: computer-display, computer-keyboard and computer-exec with the Linux default, which runs +commands in the session, caps their output, times them out and answers a repeated request_id with the first result.""" +from __future__ import annotations + +import asyncio +import sys +import time + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, MinimalDriver, http_client # noqa: E402 + +from agentenv_protocol.computer import ComputerSettings, ExecResult, linux, run_in_session, set_display # noqa: E402 + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the Linux defaults need a POSIX host") + + +def _run(**body) -> dict: + return {"argv": ["sh", "-c", "true"], "request_id": "r", **body} + + +@pytest.mark.asyncio +async def test_display_get_and_set_within_the_declared_modes(): + driver = FakeDriver() + async with http_client(driver) as http: + got = (await http.get("/agentenv/ext/computer-display/get")).json() + changed = await http.post("/agentenv/ext/computer-display/set", json={"width": 640, "height": 400, "scale": 2}) + refused = await http.post("/agentenv/ext/computer-display/set", json={"width": 800, "height": 600}) + assert got == {"width": 1280, "height": 800, "scale": 1.0, "modes": [[1280, 800], [1920, 1080], [640, 400]]} + assert changed.json()["width"] == 640 and changed.json()["scale"] == 2.0 + assert driver.screen.image.size == (1280, 800) + assert refused.status_code == 400 and "800×600 is not one of this computer's display modes" in refused.json()["error"]["message"] + + +@pytest.mark.asyncio +async def test_a_set_display_that_returns_nothing_answers_with_the_display_it_reads(): + class Quiet(FakeDriver): + @set_display + async def resize(self, width, height, scale): + self.screen.resize(width, height, scale) + + async with http_client(Quiet()) as http: + changed = (await http.post("/agentenv/ext/computer-display/set", json={"width": 1920, "height": 1080})).json() + assert (changed["width"], changed["height"]) == (1920, 1080) + + +@pytest.mark.asyncio +async def test_keyboard_settings_reach_the_driver_within_their_ranges(): + driver = FakeDriver() + async with http_client(driver) as http: + before = (await http.get("/agentenv/ext/computer-keyboard/get")).json() + after = (await http.post("/agentenv/ext/computer-keyboard/set", json={"tap_hold_ms": 80, "layout": "us"})).json() + too_long = await http.post("/agentenv/ext/computer-keyboard/set", json={"max_hold_ms": 600_001}) + layout = await http.post("/agentenv/ext/computer-keyboard/set", json={"layout": "de"}) + assert before == {"tap_hold_ms": 50, "type_delay_ms": 10, "layout": "us", "max_hold_ms": 10_000} + assert after["tap_hold_ms"] == 80 and driver.settings.tap_hold_ms == 80 + assert too_long.status_code == 400 and too_long.json()["error"]["code"] == "invalid_argument" + assert layout.status_code == 501 and layout.json()["error"]["code"] == "unsupported" + + +@pytest.mark.asyncio +async def test_a_command_runs_in_the_session_with_its_env_and_cwd(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + (tmp_path / "work").mkdir() + async with http_client(FakeDriver()) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", 'echo "$GREETING from $(pwd)"; echo oops >&2; exit 3'], cwd="work", + env={"GREETING": "hi"})) + assert r.json() == {"exit_code": 3, "stdout": f"hi from {(tmp_path / 'work').resolve()}\n", "stderr": "oops\n", + "truncated": False, "timed_out": False} + + +@pytest.mark.asyncio +async def test_a_command_past_its_timeout_is_killed_with_its_children(): + async with http_client(FakeDriver()) as http: + started = time.monotonic() + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", "sleep 30 & sleep 30; echo never"], timeout_s=0.3)) + assert r.json()["timed_out"] is True and r.json()["exit_code"] is None and r.json()["stdout"] == "" + assert time.monotonic() - started < 5 + + +@pytest.mark.asyncio +async def test_output_is_capped_per_stream(): + driver = FakeDriver(settings=ComputerSettings(exec_output_bytes=10)) + async with http_client(driver) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["sh", "-c", "yes | head -c 100000"])) + assert r.json()["stdout"] == "y\ny\ny\ny\ny\n" and r.json()["truncated"] is True + + +@pytest.mark.asyncio +async def test_a_background_app_holding_the_output_open_doesnt_hold_the_call(): + async with http_client(FakeDriver()) as http: + started = time.monotonic() + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", "echo started; (sleep 3; echo late) &"])) + elapsed = time.monotonic() - started + assert r.json()["exit_code"] == 0 and r.json()["stdout"] == "started\n" and elapsed < 2.5 + + +@pytest.mark.asyncio +async def test_a_repeated_request_id_returns_the_first_result_without_running_again(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + count = ["sh", "-c", "echo x >> runs; wc -l < runs"] + driver = FakeDriver() + async with http_client(driver) as http: + first = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id="a"))).json() + again = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id="a"))).json() + both = await asyncio.gather(*(http.post("/agentenv/ext/computer-exec/run", + json=_run(argv=["sh", "-c", "sleep 0.2; " + count[2]], request_id="b")) + for _ in range(3))) + other = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id="c"))).json() + assert first == again and first["stdout"].strip() == "1" + assert len({r.json()["stdout"] for r in both}) == 1 and both[0].json()["stdout"].strip() == "2" + assert other["stdout"].strip() == "3" + + +@pytest.mark.asyncio +async def test_a_failed_command_is_not_remembered(): + async with http_client(FakeDriver()) as http: + missing = await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["no-such-binary-xyz"])) + again = await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["no-such-binary-xyz"])) + assert missing.status_code == again.status_code == 400 + assert missing.json()["error"]["code"] == "invalid_argument" and "no-such-binary-xyz" in missing.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"argv": []}, {"argv": "ls -l"}, {"argv": ["ls", 1]}, {"request_id": ""}, {"env": {"A": 1}}, + {"timeout_s": 0}, {"timeout_s": 3601}, +]) +async def test_a_bad_command_request_is_refused(body): + async with http_client(FakeDriver()) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run(**body)) + assert r.status_code == 400 and r.json()["error"]["code"] in ("invalid_argument", "invalid_params") + + +@pytest.mark.asyncio +async def test_a_drivers_own_run_in_session_is_capped_and_checked(): + class Own(MinimalDriver): + output = ExecResult(0, b"a" * 50, b"", False) + + @run_in_session + async def run(self, argv, cwd, env, timeout_s): + return self.output + + driver = Own() + driver.settings = ComputerSettings(exec_output_bytes=8) + async with http_client(driver) as http: + capped = (await http.post("/agentenv/ext/computer-exec/run", json=_run())).json() + driver.output = "not a result" + broken = await http.post("/agentenv/ext/computer-exec/run", json=_run(request_id="other")) + assert capped == {"exit_code": 0, "stdout": "aaaaaaaa", "stderr": "", "truncated": True, "timed_out": False} + assert broken.status_code == 500 and broken.json()["error"]["code"] == "internal" + + +@pytest.mark.asyncio +async def test_a_background_apps_output_is_drained_after_the_call_returns(monkeypatch): + monkeypatch.setattr(linux, "DRAIN_S", 0.1) + async with http_client(FakeDriver()) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", "echo now; (sleep 0.4; echo later) &"])) + draining = len(linux._draining) + await asyncio.sleep(1.0) + assert r.json()["stdout"] == "now\n" and draining >= 1 and not linux._draining + + +@pytest.mark.asyncio +async def test_only_the_latest_command_results_are_kept(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr("agentenv_protocol.computer.extensions.KEEP_EXEC_RESULTS", 1) + count = ["sh", "-c", "echo x >> runs; wc -l < runs"] + async with http_client(FakeDriver()) as http: + for request_id in ("a", "b", "a"): + last = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id=request_id))).json() + assert last["stdout"].strip() == "3", "a's first result was dropped, so a ran again" diff --git a/packages/agentenv-protocol/tests/test_computer_frames.py b/packages/agentenv-protocol/tests/test_computer_frames.py new file mode 100644 index 00000000..58beea5b --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_frames.py @@ -0,0 +1,157 @@ +"""Frames: screenshots as frames by id, native crops that map back exactly, encodings, retention, staleness after a +display change, HiDPI mapping, and each caller's latest frame.""" +from __future__ import annotations + +import base64 +import io + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, FakeScreen, http_client, mcp_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol import ROLE_META_KEY # noqa: E402 +from agentenv_protocol.computer.frames import KEEP # noqa: E402 + + +def _image(result) -> PILImage.Image: + return PILImage.open(io.BytesIO(base64.b64decode(result.content[1].data))) + + +@pytest.mark.asyncio +async def test_a_screenshot_is_a_native_frame_with_its_geometry(): + async with mcp_session(FakeDriver(FakeScreen(1280, 800))) as session: + first = await session.call_tool("computer_screenshot", {}) + second = await session.call_tool("computer_screenshot", {"format": "jpeg", "quality": 40}) + data = first.structuredContent + assert data["frame_id"] == "f1" and (data["width"], data["height"]) == (1280, 800) + assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and data["parent_frame_id"] is None + assert data["captured_at"].endswith("Z") and data["summary"] == "Captured f1: 1280×800 of the 1280×800 display" + assert first.content[0].text == data["summary"] and first.content[1].mimeType == "image/png" + assert _image(first).size == (1280, 800) + assert second.structuredContent["frame_id"] == "f2" and second.content[1].mimeType == "image/jpeg" + assert _image(second).format == "JPEG" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("args", [{"quality": 50}, {"format": "jpeg", "quality": 0}, {"format": "jpeg", "quality": 96}]) +async def test_quality_is_for_jpeg_within_range(args): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_screenshot", args) + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_a_region_is_a_native_crop_whose_points_map_back_exactly(): + driver = FakeDriver(FakeScreen(1280, 800)) + driver.screen.paint(700, 450, (1, 2, 3)) + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + crop = await session.call_tool("computer_screenshot", {"region": [600, 400, 900, 500]}) + nested = await session.call_tool("computer_screenshot", {"region": [100, 50, 200, 100], "frame_id": "f2"}) + clicked = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f3"}) + assert crop.structuredContent["parent_frame_id"] == "f1" and (crop.structuredContent["width"], + crop.structuredContent["height"]) == (300, 100) + assert crop.structuredContent["summary"] == "Cropped f2: 300×100 of f1 at (600, 400)" + assert crop.structuredContent["captured_at"] == (await _captured_at(driver, "f1")) + assert _image(crop).getpixel((100, 50))[:3] == (1, 2, 3) + assert nested.structuredContent["parent_frame_id"] == "f2" and _image(nested).getpixel((0, 0))[:3] == (1, 2, 3) + assert not clicked.isError and driver.screen.log[-1][1:3] == (701, 451) + + +async def _captured_at(driver: FakeDriver, frame_id: str) -> str: + return driver._frames.get(frame_id, "default").data()["captured_at"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("args, code", [ + ({"frame_id": "f1"}, "invalid_argument"), + ({"region": [0, 0, 10]}, "invalid_argument"), + ({"region": [10, 10, 10, 20]}, "out_of_bounds"), + ({"region": [0, 0, 1281, 10]}, "out_of_bounds"), + ({"region": [-1, 0, 10, 10]}, "out_of_bounds"), + ({"region": [0, 0, 10, 10], "frame_id": "f9"}, "stale_frame"), + ({"region": [0, 0, 1001, 10], "space": "norm"}, "out_of_bounds"), +]) +async def test_a_bad_region_is_refused(args, code): + async with mcp_session(FakeDriver()) as session: + await session.call_tool("computer_screenshot", {}) + refused = await session.call_tool("computer_screenshot", args) + assert refused.isError and refused.structuredContent["code"] == code + + +@pytest.mark.asyncio +async def test_a_region_in_norm_space_crops_the_same_box(): + async with mcp_session(FakeDriver(FakeScreen(1000, 500))) as session: + await session.call_tool("computer_screenshot", {}) + crop = await session.call_tool("computer_screenshot", {"region": [500, 500, 1000, 1000], "space": "norm"}) + assert (crop.structuredContent["width"], crop.structuredContent["height"]) == (500, 250) + + +@pytest.mark.asyncio +async def test_only_the_last_frames_are_kept(): + async with mcp_session(FakeDriver(FakeScreen(8, 8))) as session: + for _ in range(KEEP + 1): + await session.call_tool("computer_screenshot", {}) + evicted = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f1"}) + kept = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f2"}) + assert evicted.structuredContent["code"] == "stale_frame" and "no longer kept" in evicted.structuredContent["message"] + assert evicted.structuredContent["retryable"] is True and not kept.isError + + +@pytest.mark.asyncio +async def test_a_frame_goes_stale_when_the_display_changes(): + driver = FakeDriver() + async with mcp_session(driver) as session, http_client(driver) as http: + await session.call_tool("computer_screenshot", {}) + await http.post("/agentenv/ext/computer-display/set", json={"width": 1920, "height": 1080}) + stale = await session.call_tool("computer_click", {"x": 10, "y": 10}) + crop = await session.call_tool("computer_screenshot", {"region": [0, 0, 10, 10]}) + await session.call_tool("computer_screenshot", {}) + fresh = await session.call_tool("computer_click", {"x": 1900, "y": 10}) + assert stale.structuredContent["code"] == "stale_frame" + assert "captured at 1280×800 at scale 1 and the display is now 1920×1080 at scale 1" in stale.structuredContent["message"] + assert crop.structuredContent["code"] == "stale_frame" and not fresh.isError + + +@pytest.mark.asyncio +async def test_with_no_frame_yet_a_point_cannot_be_placed(): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_click", {"x": 1, "y": 1}) + assert refused.structuredContent["code"] == "stale_frame" + assert refused.structuredContent["message"] == "there is no frame yet: take a screenshot first" + + +@pytest.mark.asyncio +async def test_hidpi_frame_pixels_map_to_screen_pixels(): + driver = FakeDriver(FakeScreen(640, 400, scale=2.0)) + async with mcp_session(driver) as session: + shot = await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_click", {"x": 21, "y": 41}) + await session.call_tool("computer_click", {"x": 1279, "y": 799}) + position = await session.call_tool("computer_pointer_position", {}) + assert (shot.structuredContent["width"], shot.structuredContent["height"]) == (1280, 800) + assert shot.structuredContent["summary"] == "Captured f1: 1280×800 of the 640×400 at scale 2 display" + assert driver.screen.log[-2][1:3] == (10, 20) and driver.screen.log[-1][1:3] == (639, 399) + assert position.structuredContent["x"] == 1278 and position.structuredContent["screen"] == {"x": 639, "y": 399} + + +@pytest.mark.asyncio +async def test_each_caller_has_its_own_latest_frame(): + class ByRole(FakeDriver): + def caller_identity(self, context): + return context.caller.role or "default" + + driver = ByRole() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}, meta={ROLE_META_KEY: "alice"}) + await session.call_tool("computer_screenshot", {"region": [0, 0, 10, 10]}, meta={ROLE_META_KEY: "alice"}) + await session.call_tool("computer_screenshot", {}, meta={ROLE_META_KEY: "bob"}) + alice = await session.call_tool("computer_click", {"x": 9, "y": 9}, meta={ROLE_META_KEY: "alice"}) + bob = await session.call_tool("computer_click", {"x": 900, "y": 9}, meta={ROLE_META_KEY: "bob"}) + carol = await session.call_tool("computer_click", {"x": 1, "y": 1}, meta={ROLE_META_KEY: "carol"}) + assert alice.structuredContent["summary"].endswith("in f2") and bob.structuredContent["summary"].endswith("in f3") + assert carol.structuredContent["code"] == "stale_frame" diff --git a/packages/agentenv-protocol/tests/test_computer_queue.py b/packages/agentenv-protocol/tests/test_computer_queue.py new file mode 100644 index 00000000..e176f0c2 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_queue.py @@ -0,0 +1,360 @@ +"""Input: the input tools' parameters and refusals, the queue that runs input one call at a time in arrival order, +others' input since a frame, screenshots after acting, and the pointer and keyboard groups.""" +from __future__ import annotations + +import asyncio +import time + +import anyio +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import COLOURS, FakeDriver, FakeScreen, MinimalDriver, mcp_session # noqa: E402 + +from agentenv_protocol import ROLE_META_KEY # noqa: E402 +from agentenv_protocol.computer import ComputerSettings, pointer_position # noqa: E402 + + +class ByRole(FakeDriver): + def caller_identity(self, context): + return context.caller.role or "default" + + +async def _shot(session, **meta) -> dict: + return (await session.call_tool("computer_screenshot", {}, meta=meta or None)).structuredContent + + +@pytest.mark.asyncio +async def test_a_click_reaches_the_screen_with_its_button_count_and_modifiers(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + result = await session.call_tool("computer_click", {"x": 5, "y": 6, "button": "right", "count": 2, + "modifiers": ["shift", "Control"]}) + assert result.structuredContent == { + "summary": "Clicked right ×2 at (5, 6) in f1", "applied": "yes", "delivery": "foreground", + "others_since_frame": 0, "effect": {"pointer_moved": True}} + assert driver.screen.log[-1] == ("click", 5, 6, "right", 2, ("Shift", "Control")) + assert driver.screen.colour_at(5, 6) == COLOURS["right"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("args, code, message", [ + ({"button": "back"}, "unsupported", "this computer has no back button; it has left, right, middle"), + ({"count": 4}, "invalid_argument", "count is from 1 to 3 on this computer, got 4"), + ({"count": 0}, "invalid_argument", "count is from 1 to 3"), + ({"modifiers": ["Hyperdrive"]}, "invalid_argument", "'Hyperdrive' is not a W3C key value"), + ({"x": 5, "y": None}, "invalid_argument", "give x and y, or neither to act at the pointer"), + ({"x": None, "y": None, "frame_id": "f1"}, "invalid_argument", "give x and y"), +]) +async def test_a_bad_click_is_refused_and_nothing_is_sent(args, code, message): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + refused = await session.call_tool("computer_click", {"x": 5, "y": 6, **args}) + assert refused.structuredContent["code"] == code and message in refused.structuredContent["message"] + assert refused.structuredContent["applied"] == "no" and driver.screen.log == [] + + +@pytest.mark.asyncio +async def test_without_the_pointer_group_a_click_needs_its_point(): + async with mcp_session(MinimalDriver()) as session: + await _shot(session) + refused = await session.call_tool("computer_click", {"x": 5}) + assert refused.isError and "give x and y" in refused.content[0].text + + +@pytest.mark.asyncio +async def test_a_drag_checks_every_point_before_sending_anything(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + dragged = await session.call_tool("computer_drag", {"path": [[1, 1], [50, 60], [99, 98]], "duration_ms": 300}) + out = await session.call_tool("computer_drag", {"path": [[1, 1], [1280, 5]]}) + short = await session.call_tool("computer_drag", {"path": [[1, 1]]}) + long = await session.call_tool("computer_drag", {"path": [[1, 1]] * 11}) + slow = await session.call_tool("computer_drag", {"path": [[1, 1], [2, 2]], "duration_ms": 10_001}) + bad = await session.call_tool("computer_drag", {"path": [[1, 1], [2, 2, 3]]}) + middle = await session.call_tool("computer_drag", {"path": [[1, 1], [2, 2]], "button": "middle"}) + assert dragged.structuredContent["summary"] == "Dragged left through 3 points of f1 in 300 ms" + assert driver.screen.log[0] == ("drag", ((1, 1), (50, 60), (99, 98)), "left", 300, ()) + assert out.structuredContent["code"] == "out_of_bounds" + assert short.structuredContent["code"] == long.structuredContent["code"] == "invalid_argument" + assert slow.structuredContent["code"] == bad.structuredContent["code"] == "invalid_argument" + assert not middle.isError and driver.screen.log[1] == ("drag", ((1, 1), (2, 2)), "middle", 500, ()) + assert len(driver.screen.log) == 2 + + +@pytest.mark.asyncio +async def test_scroll_moves_the_content_and_can_scroll_at_the_pointer(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + await session.call_tool("computer_scroll", {"x": 10, "y": 10, "direction": "down"}) + await session.call_tool("computer_pointer_move", {"x": 30, "y": 40}) + at_pointer = await session.call_tool("computer_scroll", {"direction": "left", "amount": 2}) + refused = await session.call_tool("computer_scroll", {"x": 1, "y": 1, "direction": "up", "amount": 101}) + assert driver.screen.scrolled == [-2, 3] + assert at_pointer.structuredContent["summary"] == "Scrolled left 2 at the pointer, screen (30, 40)" + assert driver.screen.log[-1] == ("scroll", 30, 40, "left", 2, ()) + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_a_click_without_a_point_clicks_at_the_pointer(): + driver = FakeDriver() + driver.screen.pointer = (70, 80) + async with mcp_session(driver) as session: + clicked = await session.call_tool("computer_click", {}) + assert clicked.structuredContent["others_since_frame"] == 0 and driver.screen.log[-1][1:3] == (70, 80) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text, code", [("", "invalid_argument"), ("a" * 10_001, "invalid_argument"), + ("bell\x07", "invalid_argument"), ("del\x7f", "invalid_argument")]) +async def test_text_that_cant_be_typed_whole_is_refused(text, code): + driver = FakeDriver() + async with mcp_session(driver) as session: + refused = await session.call_tool("computer_type", {"text": text}) + assert refused.structuredContent["code"] == code and driver.screen.log == [] + + +@pytest.mark.asyncio +async def test_unicode_newlines_and_tabs_are_typed(): + driver = FakeDriver() + async with mcp_session(driver) as session: + typed = await session.call_tool("computer_type", {"text": "Grüße 東京\n\tok", "screenshot": True}) + assert driver.screen.log == [("type", "Grüße 東京\n\tok")] + assert typed.structuredContent["summary"] == "Typed 12 characters, then captured f1" + assert typed.structuredContent["frame"]["frame_id"] == "f1" and typed.content[1].type == "image" + + +@pytest.mark.asyncio +async def test_a_key_chord_is_pressed_in_w3c_case_and_repeated_by_the_base_class(): + driver = FakeDriver(settings=ComputerSettings(tap_hold_ms=25)) + async with mcp_session(driver) as session: + pressed = await session.call_tool("computer_key", {"keys": ["control", "S"], "repeat": 3}) + single = await session.call_tool("computer_key", {"keys": [" "]}) + assert pressed.structuredContent["summary"] == "Pressed Control+S ×3" + assert driver.screen.log == [("press", ("Control", "S"), 25)] * 3 + [("press", (" ",), 25)] + assert single.structuredContent["applied"] == "yes" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("keys, repeat, code, hint", [ + (["ctrl", "s"], 1, "invalid_argument", "did you mean Control?"), + (["Control+s"], 1, "invalid_argument", "one key per entry: keys is a list, such as ['Control', 's']"), + (["Entr"], 1, "invalid_argument", "did you mean Enter?"), + (["Qwertyuiop"], 1, "invalid_argument", "named keys are W3C UI Events key values, such as Enter"), + ([], 1, "invalid_argument", None), + (["Enter"], 101, "invalid_argument", None), +]) +async def test_a_refused_key_says_what_was_meant(keys, repeat, code, hint): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_key", {"keys": keys, "repeat": repeat}) + assert refused.structuredContent["code"] == code and refused.structuredContent.get("hint") == hint + + +@pytest.mark.asyncio +async def test_a_key_that_isnt_a_string_is_refused_by_the_tools_schema(): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_key", {"keys": [3]}) + assert refused.isError and refused.structuredContent is None + + +@pytest.mark.asyncio +async def test_a_driver_that_declares_its_keys_refuses_the_others(): + class Few(FakeDriver): + keys = ("Enter", "a", "Control") + + driver = Few() + async with mcp_session(driver) as session: + refused = await session.call_tool("computer_key", {"keys": ["Control", "Tab"]}) + allowed = await session.call_tool("computer_key", {"keys": ["control", "a"]}) + assert refused.structuredContent["code"] == "unsupported" and "['Tab']" in refused.structuredContent["message"] + assert not allowed.isError + + +@pytest.mark.asyncio +async def test_wait_waits_within_range_and_can_return_a_frame(): + async with mcp_session(FakeDriver()) as session: + started = time.monotonic() + waited = await session.call_tool("computer_wait", {"ms": 120}) + elapsed = time.monotonic() - started + shot = await session.call_tool("computer_wait", {"ms": 0, "screenshot": True}) + refused = await session.call_tool("computer_wait", {"ms": 60_001}) + assert waited.structuredContent == {"summary": "Waited 120 ms", "waited_ms": 120} + assert elapsed >= 0.1 + assert shot.structuredContent["frame"]["frame_id"] == "f1" and shot.content[1].type == "image" + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_input_runs_one_call_at_a_time_in_arrival_order(): + driver = FakeDriver() + driver.slow["click"] = 0.05 + async with mcp_session(driver) as session: + await _shot(session) + results = await asyncio.gather(*(session.call_tool("computer_click", {"x": i, "y": i}) for i in range(6))) + assert all(not r.isError for r in results) + assert [entry[1] for entry in driver.screen.log] == list(range(6)) + + +@pytest.mark.asyncio +async def test_a_call_waiting_past_the_limit_is_refused_input_busy(): + driver = FakeDriver(settings=ComputerSettings(queue_wait_s=0.1)) + driver.slow["click"] = 0.4 + async with mcp_session(driver) as session: + await _shot(session) + first, second = await asyncio.gather(session.call_tool("computer_click", {"x": 1, "y": 1}), + session.call_tool("computer_key", {"keys": ["a"]})) + assert not first.isError + assert second.structuredContent == {"code": "input_busy", "message": "waited 0.1 s for earlier input to finish", + "retryable": True, "applied": "no"} + + +@pytest.mark.asyncio +async def test_screenshots_and_waits_never_queue_behind_input(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + driver.slow["click"] = 0.5 + click = asyncio.ensure_future(session.call_tool("computer_click", {"x": 1, "y": 1})) + await asyncio.sleep(0.05) + started = time.monotonic() + await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_wait", {"ms": 0}) + observed = time.monotonic() - started + await click + assert observed < 0.4 + + +@pytest.mark.asyncio +async def test_others_input_since_a_frame_is_counted_and_ones_own_is_not(): + driver = ByRole() + async with mcp_session(driver) as session: + alice_frame = await _shot(session, **{ROLE_META_KEY: "alice"}) + await session.call_tool("computer_click", {"x": 1, "y": 1}, meta={ROLE_META_KEY: "alice"}) + await _shot(session, **{ROLE_META_KEY: "bob"}) + for _ in range(2): + await session.call_tool("computer_key", {"keys": ["a"]}, meta={ROLE_META_KEY: "bob"}) + alice = await session.call_tool("computer_click", {"x": 2, "y": 2, "frame_id": alice_frame["frame_id"]}, + meta={ROLE_META_KEY: "alice"}) + bob = await session.call_tool("computer_key", {"keys": ["b"]}, meta={ROLE_META_KEY: "bob"}) + assert alice.structuredContent["others_since_frame"] == 2 + assert bob.structuredContent["others_since_frame"] == 1 + + +@pytest.mark.asyncio +async def test_an_input_can_return_a_fresh_frame_that_shows_it(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + clicked = await session.call_tool("computer_click", {"x": 3, "y": 3, "screenshot": True}) + assert clicked.structuredContent["frame"]["frame_id"] == "f2" + assert clicked.structuredContent["summary"] == "Clicked left at (3, 3) in f1, then captured f2" + + +@pytest.mark.asyncio +async def test_the_pointer_tools(): + driver = FakeDriver() + async with mcp_session(driver) as session: + before = await session.call_tool("computer_pointer_position", {}) + await _shot(session) + moved = await session.call_tool("computer_pointer_move", {"x": 11, "y": 12}) + down = await session.call_tool("computer_pointer_down", {"button": "right", "x": 20, "y": 21}) + up = await session.call_tool("computer_pointer_up", {"button": "right"}) + inside = await session.call_tool("computer_pointer_position", {}) + await session.call_tool("computer_screenshot", {"region": [0, 0, 10, 10]}) + outside = await session.call_tool("computer_pointer_position", {}) + named = await session.call_tool("computer_pointer_position", {"frame_id": "f1"}) + half = await session.call_tool("computer_pointer_down", {"x": 1}) + listed = await session.call_tool("computer_pointer_list", {}) + assert before.structuredContent == {"summary": "The pointer is at screen (0, 0)", "screen": {"x": 0, "y": 0}} + assert moved.structuredContent["summary"] == "Moved the pointer at (11, 12) in f1" + assert down.structuredContent["summary"] == "Pressed right at (20, 21) in f1" + assert up.structuredContent["summary"] == "Released right" + assert driver.screen.log == [("move", 11, 12), ("move", 20, 21), ("down", "right"), ("up", "right")] + assert inside.structuredContent["inside_frame"] is True and inside.structuredContent["x"] == 20 + assert outside.structuredContent["inside_frame"] is False and "outside f2" in outside.structuredContent["summary"] + assert named.structuredContent["frame_id"] == "f1" + assert half.structuredContent["code"] == "invalid_argument" + assert listed.structuredContent["pointers"] == [ + {"id": "default", "kind": "foreground", "label": "The computer's pointer", "default": True}] + + +@pytest.mark.asyncio +async def test_a_bad_pointer_position_is_an_internal_error(): + class Lost(FakeDriver): + @pointer_position + async def where(self): + return "somewhere" + + async with mcp_session(Lost()) as session: + broken = await session.call_tool("computer_pointer_position", {}) + assert broken.structuredContent["code"] == "internal" + + +@pytest.mark.asyncio +async def test_keys_held_down_stay_down_until_released(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_keyboard_down", {"keys": ["shift"]}) + await session.call_tool("computer_keyboard_down", {"keys": ["Control"]}) + held = list(driver.screen.held) + partly = await session.call_tool("computer_keyboard_up", {"keys": ["Shift"]}) + rest = await session.call_tool("computer_keyboard_up", {}) + nothing = await session.call_tool("computer_keyboard_up", {}) + assert held == ["Shift", "Control"] and driver.screen.held == [] + assert partly.structuredContent["summary"] == "Released Shift" and rest.structuredContent["summary"] == "Released Control" + assert nothing.structuredContent["code"] == "invalid_argument" and driver._held == {} + + +@pytest.mark.asyncio +async def test_a_hold_with_hold_ms_releases_in_reverse_order(): + driver = FakeDriver() + async with mcp_session(driver) as session: + held = await session.call_tool("computer_keyboard_down", {"keys": ["Shift", "a"], "hold_ms": 30}) + refused = await session.call_tool("computer_keyboard_down", {"keys": ["a"], "hold_ms": 10_001}) + assert held.structuredContent["summary"] == "Held Shift+a for 30 ms" + assert driver.screen.log == [("key_down", ("Shift", "a")), ("key_up", ("a", "Shift"))] + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_keys_held_past_max_hold_ms_are_released(): + driver = FakeDriver(settings=ComputerSettings(max_hold_ms=50)) + async with mcp_session(driver) as session: + await session.call_tool("computer_keyboard_down", {"keys": ["Alt"]}) + with anyio.fail_after(5): + while driver.screen.held: + await asyncio.sleep(0.01) + assert driver.screen.log[-1] == ("key_up", ("Alt",)) and driver._held == {} + + +@pytest.mark.asyncio +async def test_a_second_hold_restarts_the_release_timer(): + driver = FakeDriver(settings=ComputerSettings(max_hold_ms=150)) + async with mcp_session(driver) as session: + await session.call_tool("computer_keyboard_down", {"keys": ["Alt"]}) + await asyncio.sleep(0.1) + await session.call_tool("computer_keyboard_down", {"keys": ["Shift"]}) + await asyncio.sleep(0.1) + still = list(driver.screen.held) + with anyio.fail_after(5): + while driver.screen.held: + await asyncio.sleep(0.01) + assert still == ["Alt", "Shift"] and driver.screen.log[-1] == ("key_up", ("Shift", "Alt")) + + +@pytest.mark.asyncio +async def test_the_minimal_driver_serves_no_group_tools(): + driver = MinimalDriver(FakeScreen(64, 40)) + async with mcp_session(driver) as session: + await _shot(session) + clicked = await session.call_tool("computer_click", {"x": 3, "y": 4}) + missing = await session.call_tool("computer_pointer_move", {"x": 1, "y": 1}) + assert not clicked.isError and driver.screen.log == [("click", 3, 4, "left", 1, ())] + assert missing.isError and "Unknown tool" in missing.content[0].text diff --git a/packages/agentenv-protocol/tests/test_computer_spaces.py b/packages/agentenv-protocol/tests/test_computer_spaces.py new file mode 100644 index 00000000..ebafee3f --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_spaces.py @@ -0,0 +1,85 @@ +"""Coordinate spaces: frame pixels and a 0..norm_scale grid, one mapping to the frame's pixels, refused and never +clamped outside the frame or the range, with the default space set per deploy.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, FakeScreen, http_client, mcp_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol.computer import ComputerSettings, InvalidArgument, OutOfBounds # noqa: E402 +from agentenv_protocol.computer.frames import FrameStore # noqa: E402 +from agentenv_protocol.computer.operations import Display # noqa: E402 +from agentenv_protocol.computer.spaces import check_space, frame_point # noqa: E402 + + +def _frame(width: int, height: int): + return FrameStore().add(PILImage.new("RGB", (width, height)), Display(width, height), (0, {}), identity="default") + + +@pytest.mark.parametrize("width, height, x, y, expected", [ + (1280, 800, 0, 0, (0, 0)), + (1280, 800, 1000, 1000, (1279, 799)), + (1280, 800, 500, 500, (640, 400)), + (1279, 799, 500, 500, (639.5, 399.5)), + (3, 3, 999, 0, (2, 0)), + (3, 3, 666, 334, (1.998, 1.002)), +]) +def test_norm_points_land_on_the_frames_pixels(width, height, x, y, expected): + fx, fy = frame_point(_frame(width, height), x, y, "norm", 1000) + assert (round(fx, 3), round(fy, 3)) == expected + + +def test_frame_points_are_its_pixels_and_odd_pixels_stay_exact(): + frame = _frame(1279, 799) + assert frame_point(frame, 1278, 798, "frame", 1000) == (1278, 798) + assert frame.to_screen(1278.9, 0.5) == (1278, 0) + + +@pytest.mark.parametrize("x, y, space", [(1280, 0, "frame"), (0, -1, "frame"), (1000.5, 0, "norm"), (-0.1, 5, "norm")]) +def test_points_outside_the_frame_or_range_are_refused(x, y, space): + with pytest.raises(OutOfBounds): + frame_point(_frame(1280, 800), x, y, space, 1000) + + +@pytest.mark.parametrize("space, scale", [("grid", None), (None, 0), (None, 2_000_000), (None, 1.5)]) +def test_a_bad_space_or_scale_is_refused(space, scale): + with pytest.raises(InvalidArgument): + check_space(space, scale, "frame", 1000) + + +@pytest.mark.asyncio +async def test_the_default_space_and_scale_come_from_the_settings_and_a_call_can_override_them(): + driver = FakeDriver(FakeScreen(1280, 800), settings=ComputerSettings(default_space="norm", default_norm_scale=999)) + async with mcp_session(driver) as session: + tools = {t.name: t for t in (await session.list_tools()).tools} + await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_click", {"x": 999, "y": 0}) + await session.call_tool("computer_click", {"x": 7, "y": 9, "space": "frame"}) + await session.call_tool("computer_click", {"x": 1, "y": 1, "norm_scale": 1}) + assert "0 to 999 over the frame" in tools["computer_click"].description + assert "0 to 999 over the frame" in tools["computer_drag"].description + assert "default space" not in tools["computer_type"].description + assert [entry[1:3] for entry in driver.screen.log] == [(1279, 0), (7, 9), (1279, 799)] + + +@pytest.mark.asyncio +async def test_setting_the_default_space_rewrites_the_descriptions(): + driver = FakeDriver() + async with http_client(driver) as http, mcp_session(driver) as session: + before = {t.name: t.description for t in (await session.list_tools()).tools} + applied = (await http.post("/agentenv/ext/computer-spaces/set", + json={"default_space": "norm", "default_norm_scale": 1000})).json() + after = {t.name: t.description for t in (await session.list_tools()).tools} + card = (await http.get("/.well-known/agent-env.json")).json() + current = (await http.get("/agentenv/ext/computer-spaces/get")).json() + refused = await http.post("/agentenv/ext/computer-spaces/set", json={"default_space": "grid"}) + assert "frame space by default" in before["computer_click"] + assert applied == current == {"default_space": "norm", "default_norm_scale": 1000} + assert "norm space by default: 0 to 1000" in after["computer_click"] + card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click") + assert card_click["description"] == after["computer_click"] + assert refused.status_code == 400 and refused.json()["error"]["code"] == "invalid_argument" diff --git a/pyproject.toml b/pyproject.toml index 786625db..f3ccfdbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ daytona = [ dev = [ "agentenv-framework[aws,boxd,daytona,explorer,gcp,sail,tensorlake,tenki,vercel]", # exercise portable optional providers and stores in CI "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x + "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs "griffelib==2.3.0", # the plugin API check; exact, since what it reports as a break changes between releases "tomli-w==1.2.0", # the Codex agent's config writer, which only its image installs; its tests import it "moto>=5.0.0", diff --git a/uv.lock b/uv.lock index 7610893b..12e9a0a9 100644 --- a/uv.lock +++ b/uv.lock @@ -90,6 +90,7 @@ dev = [ { name = "griffelib" }, { name = "httpx2" }, { name = "moto" }, + { name = "pillow" }, { name = "psycopg2-binary" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -163,6 +164,7 @@ requires-dist = [ { name = "modal", specifier = ">=1.5.2.dev22" }, { name = "moto", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "packaging", specifier = ">=22" }, + { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, { name = "psycopg2-binary", marker = "extra == 'dev'", specifier = ">=2.9.0" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pymongo", specifier = ">=4.7" }, @@ -206,10 +208,16 @@ agent = [ { name = "regex" }, { name = "uvicorn" }, ] +computer = [ + { name = "mcp" }, + { name = "pillow" }, + { name = "uvicorn" }, +] dev = [ { name = "a2a-sdk", extra = ["http-server"] }, { name = "fastmcp" }, { name = "mcp" }, + { name = "pillow" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "python-multipart" }, @@ -223,7 +231,10 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'dev'", specifier = ">=3.4,<4" }, { name = "httpx", specifier = ">=0.27" }, { name = "mcp", marker = "extra == 'agent'", specifier = ">=1.25,<2" }, + { name = "mcp", marker = "extra == 'computer'", specifier = ">=1.25,<2" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.25,<2" }, + { name = "pillow", marker = "extra == 'computer'", specifier = ">=10.4" }, + { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, { name = "pydantic", specifier = ">=2.11" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, @@ -233,8 +244,9 @@ requires-dist = [ { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, { name = "starlette", specifier = ">=0.37" }, { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, + { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "dev"] +provides-extras = ["agent", "computer", "dev"] [[package]] name = "aiofile" @@ -2718,6 +2730,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/e8/6d75ffd9784bce2e93d1ae4415649427e39a53bb172d4672b2b59c6f0a7b/pathable-0.6.0-py3-none-any.whl", hash = "sha256:82c4ca6c98c502ad12e0d4e9779b6210afee93c38990988c8c5d1b49bdcdf566", size = 18983, upload-time = "2026-05-19T18:15:10.728Z" }, ] +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/d7/af6af1b8cc55e07cb0489d5b0950bf5a9bb1461eeba825700b6d8d0e26ea/pillow-13.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb", size = 5592324, upload-time = "2026-10-16T22:15:24.672Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d2/2c12cb4fbb5d95f38d1001ff3d95fafcb830077018cba40e56a8f932dfbc/pillow-13.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061", size = 4856298, upload-time = "2026-10-16T22:15:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ee/229b27472483a4801ea94990815e710391836afdf987b4b722b62bd75db6/pillow-13.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0", size = 6517304, upload-time = "2026-10-16T22:15:29.43Z" }, + { url = "https://files.pythonhosted.org/packages/36/46/f509b74787f40ce0627501089d630b2c741b63b244c1f40b640b3b01e181/pillow-13.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c", size = 7140667, upload-time = "2026-10-16T22:15:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/5a398b8ec2c37145397d6461c0b551c23ec03072aa64f357aed25e582b00/pillow-13.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f", size = 6610583, upload-time = "2026-10-16T22:15:34.025Z" }, + { url = "https://files.pythonhosted.org/packages/4f/7b/f2f1109b230fd4e2d57ab2305de31adc37ea9b4ff8719436c623cb484415/pillow-13.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806", size = 7218241, upload-time = "2026-10-16T22:15:36.214Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d3/069edb9083b267746baf59756282cacef904d3475c5288c7216fb5195a98/pillow-13.0.0-cp311-cp311-win32.whl", hash = "sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5", size = 6515278, upload-time = "2026-10-16T22:15:38.129Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f6/6e2d96413e4c9dbec12ea584d9e646d602877d4cce3ea669099f959deac6/pillow-13.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3", size = 7278471, upload-time = "2026-10-16T22:15:40.286Z" }, + { url = "https://files.pythonhosted.org/packages/70/e7/baff21f8cb60496c8c250609501b88919eeedb4088d3da106f6c62f3f3c0/pillow-13.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13", size = 2698981, upload-time = "2026-10-16T22:15:42.637Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/68/c0/fb7b1caf25ed5ac5d46116f09b8b0706b9559833c65c667cecb1a53153f2/pillow-13.0.0-pp311-pypy311_pp80-macosx_10_15_x86_64.whl", hash = "sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca", size = 5521630, upload-time = "2026-10-16T22:17:50.008Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e6/4d11d5b1137f4fc080605cce410cfaae4f303ad400bdec75d27d3083fa29/pillow-13.0.0-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039", size = 4806023, upload-time = "2026-10-16T22:17:51.932Z" }, + { url = "https://files.pythonhosted.org/packages/f1/45/0923202a658e8ff6aafd72c1a4395ebadcdebaa182c437d8595d3dd3db32/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad", size = 5464831, upload-time = "2026-10-16T22:17:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/07/52/d3e38f0cad1dadea2a658b9a7d6ba8a04048f5ae14f011b806e5c3bd6bcb/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2", size = 6043133, upload-time = "2026-10-16T22:17:55.625Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/2ed7685980c1d64f8131a7d87a19c369424747577152622b73ddeff163b7/pillow-13.0.0-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c", size = 7365208, upload-time = "2026-10-16T22:17:57.509Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + [[package]] name = "platformdirs" version = "4.13.0" From 431895543bb0c55d9502466d7d61a428febd1b6e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 11:51:05 -0700 Subject: [PATCH 06/18] docs(protocol): structured errors, extension methods, tool access and computer drivers Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 153 +++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 8f9cf600..a94f18f4 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -90,10 +90,32 @@ step names it. Steps that name it show and hide it as they do any tool. Steps fo that role's earlier named steps, and a full reset (`role: "*", tools: "*"`) clears all of them, so grant a hidden tool after such steps. Trigger tool actions call it as usual. +`@tool(..., annotations={"readOnlyHint": True}, structured_output=True, meta={...})` lists the tool +with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and on fastmcp alike. +`structured_output` follows mcp's FastMCP (None derives an output schema from the return annotation +where it can, False lists none); a tool returning `Annotated[CallToolResult, Model]` sends its +result as is, image blocks included, and is listed with `Model`'s schema. An environment chooses +at mount which of its `@tool` methods it serves, and with which descriptions, by overriding +`served_tools(tools)`. + `AgentEnvStarletteApplication` mounts the same handler onto a Starlette/FastAPI app instead of FastMCP; since those apps have no MCP tool registry, constructing one with `@tool` methods raises. +### Errors + +Raise `AgentEnvError(code, message, *, retryable=False, applied="no", hint=None, status=400)` to +answer a caller with a code rather than a crash. From an `@extension` method it answers with +`status` and the body `{"ok": false, "error": {code, message, retryable, applied, hint?}}`; from a +`@tool` it answers `isError` with `": "` as the text and the same fields as the +structured content. `applied` says whether the call's effect happened (`yes`, `no`, or `unknown` +when it may have been delivered before something failed), and callers retry on their own only when +`retryable` is true and `applied` is `no`. Any other exception in an extension still answers +`extension_failed` with status 500, now with `applied: "unknown"`. On the client side, +`invoke_extension` raises `ExtensionCallError`, an `httpx.HTTPStatusError` carrying `code`, +`detail`, `retryable`, `applied` and `hint`; a body without those fields, such as a proxy's error +page, leaves `code` None and `applied` `unknown`. + ### Who is calling A tool that needs the caller declares a parameter annotated `ToolContext` (any name) and receives @@ -180,6 +202,34 @@ extension's, and `client.invoke_extension(base_url, card, uri, params, method=.. its HTTP verb. Without `method`, `invoke_extension` calls the first method listed; an extension with several methods, such as a gateway's `urn:agentenv:clock/v1`, should always be called by name. +An extension with several methods is served by giving each handler an `op`. Handlers sharing a URI +merge into one card entry, each method under `params.methods.` with its own verb and endpoint, +by default `/agentenv/ext//`, where `` is the URN's name without +`urn:agentenv:` and the version. Extension-level params, such as features and limits, come from the +environment's `extension_params(uri)`, merged beside `methods`. A declaration-only extension, a +promise with params and no handler like `intake/v1`, is returned from `declared_extensions()`. +Two handlers for one method, or a handler without `op` on a URI another handler also serves, are +refused at mount. + +```python +@extension("urn:agentenv:computer-display/v1", op="get", method="GET") +async def display_get(self) -> dict: ... + +@extension("urn:agentenv:computer-display/v1", op="set") +async def display_set(self, width: int, height: int) -> dict: ... +``` + +### Tool access without a gateway + +An environment agents reach directly, with no gateway in front, can enforce per-role tool access +itself: set `tool_access = True` on the class. It then filters `tools/list` and refuses `tools/call` +(`permission_denied`) by the caller's role, read from `AgentEnv-Role` or a gateway-stamped `_meta` +role, and serves `urn:agentenv:disable-tool/v1` and `urn:agentenv:enable-tool/v1` at the gateway's +paths (`POST /tools/disable`, `POST /tools/enable`) with its bodies (`{role, tools: [...] | "*"}` to +`{role, disabled, allowed}`) and its rules, hidden-by-default tools included. So agent-env's +`modify_env_tool_access` step works on such an env unchanged. A call with no role is `default`, as +at the gateway. + Where the binding lives depends on the app. On mcp's own FastMCP it wraps the tool manager's dispatch; on the standalone `fastmcp` package it joins the app's middleware chain as the outermost layer, so a handler's own middleware runs under it too. An app with neither seam mounts unbound, with a warning, unless a tool declares a `ToolContext` parameter or the handler defines `on_tool_call`: then the mount is refused rather than serving an empty caller. ### Uploading an export through a grant @@ -254,6 +304,109 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session: - **Tools:** `list_tools()` returns the tools the role may use as `EnvironmentTool`, the type a card's `capabilities.tools` holds. Each `inputSchema` is the JSON Schema a model API takes as the tool's parameters. +## Computer drivers + +A computer driver is what runs inside a computer an agent uses through screenshots and input. Write +one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, which needs the +`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow). +The driver marks the methods that capture the screen and send input; the base class serves the +computer contract from them, so the tools, results and errors can't drift from it. + +```python +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, Display, computer_driver, + capture, click, display, drag, press, scroll, type_text, +) + + +@computer_driver(name="my-driver", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}]) +class MyDriver(ComputerDriverEnvironment): + buttons = ("left", "right") # features, on the card + max_click_count = 2 + + @display + async def size(self) -> Display: ... # width, height, scale + @capture(display="x11") + async def grab(self): ... # the whole display, native resolution + @click + async def click_at(self, x, y, button, count, modifiers): ... + @drag + async def drag_along(self, path, button, duration_ms, modifiers): ... + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): ... + @type_text + async def type_in(self, text): ... + @press + async def press_keys(self, keys): ... + + +if __name__ == "__main__": + MyDriver().serve() +``` + +**Operations.** Operations get points in screen pixels, after the base class has mapped, checked +and queued the call, and may be sync or async. `@capture` returns a Pillow image or an +`Image(width, height, data, mode)` (PNG or JPEG bytes, or raw `RGB`, `RGBA`, `BGRA` or `BGRX` +pixels), at the display's size times its scale. Input operations may return an `Effect`. + +| Decorator | Required | Gives | +|---|---|---| +| `@display`, `@capture`, `@click`, `@drag`, `@scroll`, `@type_text`, `@press` | yes | the seven `computer_*` tools and `computer/v1` | +| `@set_display` | no, with `display_modes` | `computer-display/v1` `set` (`get` is always served) | +| `@pointer_move`, `@pointer_down`, `@pointer_up`, `@pointer_position` | all or none | `computer-pointer/v1` and its five tools; click and scroll at the pointer | +| `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | +| `@run_in_session` | no; a Linux default | `computer-exec/v1` | +| `@write_files` | no; a Linux default | `data/add` | + +The class is checked when it is defined: a missing required operation, half a group, two methods +for one operation that some platform could both match, or features that don't match the +decorators raise `TypeError`. Predicates (`os`, `distribution`, `version`, `arch`, `display`) +choose a method per platform, read from `/etc/os-release`, the machine and `XDG_SESSION_TYPE` or +`DISPLAY`: `@capture(display="x11")` and `@capture(display="wayland")` can sit side by side. A +driver's own `@run_in_session` or `@write_files` replaces the base class's Linux default, which runs +as the driver's user inside the desktop session. + +**What the base class serves:** + +- **Tools.** `computer_screenshot` (a frame: `frame_id`, image size, display resolution and scale; + `region` crops a frame at native resolution), `computer_click`, `computer_drag`, + `computer_scroll`, `computer_type`, `computer_key` (W3C key names) and `computer_wait`, plus the + group tools. Every result is structured, with a one-line summary as its text block; input results + carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame + after acting. +- **Frames and spaces.** Points are pixels of a named frame (your latest by default), or a + 0..`norm_scale` grid over it with `space: "norm"`. A point outside the frame or the range is + refused (`out_of_bounds`), never clamped. The last 64 frames are kept, and a frame goes stale + (`stale_frame`) when the display geometry it was captured at changes. +- **The input queue.** Input calls run one at a time in arrival order; one that waits past + `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. +- **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a + capture, plus the driver's own `ready_checks()`; tools refuse `device_unavailable` until they + pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool + descriptions), `computer-keyboard/v1` and `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, + request_id}`: output capped per stream, a repeated `request_id` answered with the first result). +- **The data plane.** `data/add` takes file parts that name their path, relative to the session + user's home, under the `data-objects/v1` marker (`{"path": ...}`), as inline bytes, an https URL, + or a `transfers.ReadObject` read grant under the marker's `read`, and writes them atomically. + `data/get` returns the files under the settings' `data_paths`, inline or uploaded under a + `write_namespace` grant. +- **The card** (the name from `ENVIRONMENT_NAME`, then `@computer_driver`'s), and per-role tool + access (`tool_access` is on). + +**Errors.** A driver raises the errors table's exceptions, all `AgentEnvError`s: +`InvalidArgument`, `OutOfBounds`, `StaleFrame`, `InputBusy`, `PermissionDenied`, `Unsupported`, +`BackgroundUnavailable`, `TargetUnavailable`, `DeviceUnavailable` and `LimitExceeded` mean nothing +was sent (`applied: "no"`); `Timeout`, `DeliveryFailed` and `Internal` mean it may have been. Any +other exception from an input operation is reported `delivery_failed` with `applied: "unknown"`. + +**Settings** reach the driver as one JSON value in `AGENTENV_COMPUTER_SETTINGS` +(`ComputerSettings`): `data_paths`, `default_space`, `default_norm_scale`, `queue_wait_s`, +`tap_hold_ms`, `type_delay_ms`, `max_hold_ms`, `layout`, `drag_duration_ms`, `exec_output_bytes` +and `exec_timeout_s`. A driver reads them as `self.settings`, for example its tap hold in `@press`. + +Every caller is `default` until agents carry an identity of their own; `caller_identity(context)` +names the caller whose latest frame and others' input a call uses. + ## A2A agent framework The distribution exposes two unrelated decorators named `extension`: From 441ac12d6a7aec456dae6d774a5989f36fe9cf43 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 12:08:07 -0700 Subject: [PATCH 07/18] fix(protocol): keep the computer base class's members out of a driver's way Found end to end: an X11 driver's own `_button` helper replaced the base class's `_button`, so every click failed with a TypeError. The base class's private members are now all `_computer_*`, and a driver that redefines one of them, another private member of the base class, or a computer_* tool is refused when it is defined. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 4 +- .../agentenv_protocol/computer/dataplane.py | 6 +- .../src/agentenv_protocol/computer/driver.py | 154 ++++++++++-------- .../agentenv_protocol/computer/extensions.py | 44 ++--- .../src/agentenv_protocol/computer/tools.py | 110 ++++++------- .../tests/test_computer_driver.py | 22 ++- .../tests/test_computer_frames.py | 2 +- .../tests/test_computer_queue.py | 4 +- 8 files changed, 186 insertions(+), 160 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index a94f18f4..fc395d54 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -364,7 +364,9 @@ decorators raise `TypeError`. Predicates (`os`, `distribution`, `version`, `arch choose a method per platform, read from `/etc/os-release`, the machine and `XDG_SESSION_TYPE` or `DISPLAY`: `@capture(display="x11")` and `@capture(display="wayland")` can sit side by side. A driver's own `@run_in_session` or `@write_files` replaces the base class's Linux default, which runs -as the driver's user inside the desktop session. +as the driver's user inside the desktop session. The base class's own members are named `_computer_*` +and `computer_*`; a driver that redefines one of them, or another private member of the base class, +is refused when it is defined, so a helper of the driver's can't silently replace one. **What the base class serves:** diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py index daf6ccb0..ad1f7bd6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py @@ -24,7 +24,7 @@ class DataPlane: """The data plane handlers, mixed into ComputerDriverEnvironment.""" @add_data - async def _data_add(self, parts: list) -> None: + async def _computer_data_add(self, parts: list) -> None: files = {} for part in parts: try: @@ -37,10 +37,10 @@ async def _data_add(self, parts: list) -> None: raise ValueError(f"data/add on a computer takes file parts naming their path under " f"{DATA_OBJECTS_EXTENSION_URI}, got a {part.kind} part") files[path] = await _fetch(part) - await self._call("write_files", files) + await self._computer_call("write_files", files) @get_data - async def _data_get(self, write_namespace: Optional[WriteNamespaceGrant] = None) -> list: + async def _computer_data_get(self, write_namespace: Optional[WriteNamespaceGrant] = None) -> list: files = _data_files(Path.home(), self.settings.data_paths) if write_namespace is not None: uploader = NamespaceUploader(write_namespace) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 0d1fa6c4..4f003dc1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -37,10 +37,10 @@ _BUTTONS = ("left", "right", "middle", "back", "forward") #: The handlers each optional operation or group brings; without it they are withdrawn. _HANDLERS = { - "set_display": ("_display_set",), - "keyboard": ("_keyboard_get", "_keyboard_set"), - "run_in_session": ("_exec_run",), - "write_files": ("_data_add",), + "set_display": ("_computer_display_set",), + "keyboard": ("_computer_keyboard_get", "_computer_keyboard_set"), + "run_in_session": ("_computer_exec_run",), + "write_files": ("_computer_data_add",), } @@ -68,21 +68,22 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) + _check_names(cls) check_class(cls) _check_features(cls) def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None: self.settings = settings if settings is not None else load_settings() self.platform = platform if platform is not None else detect() - self._operations = resolve(type(self), self.platform) - self._frames = FrameStore() - self._queue = InputQueue() - self._ready = False - self._ready_lock = asyncio.Lock() - self._exec_runs: OrderedDict = OrderedDict() - self._held: dict[str, list[str]] = {} - self._releases: dict[str, asyncio.Task] = {} - self._served: dict[str, EnvironmentTool] = {} + self._computer_operations = resolve(type(self), self.platform) + self._computer_frames = FrameStore() + self._computer_queue = InputQueue() + self._computer_ready = False + self._computer_ready_lock = asyncio.Lock() + self._computer_exec_runs: OrderedDict = OrderedDict() + self._computer_held: dict[str, list[str]] = {} + self._computer_releases: dict[str, asyncio.Task] = {} + self._computer_served: dict[str, EnvironmentTool] = {} for capability, handlers in _HANDLERS.items(): if not self.has(capability): for name in handlers: @@ -91,8 +92,8 @@ def __init__(self, settings: ComputerSettings | None = None, platform: Platform def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" if capability in GROUPS: - return all(op in self._operations for op in GROUPS[capability]) - return capability in self._operations + return all(op in self._computer_operations for op in GROUPS[capability]) + return capability in self._computer_operations def caller_identity(self, context: ToolContext) -> str: """Who a call comes from, for each caller's latest frame and others' input since a frame.""" @@ -106,38 +107,38 @@ async def ready_checks(self) -> list[ReadyCheck]: def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: unserved = {name for group, names in GROUP_TOOLS.items() if not self.has(group) for name in names} - self._served = {t.name: t.model_copy(deep=True) for t in tools if t.name not in unserved} + self._computer_served = {t.name: t.model_copy(deep=True) for t in tools if t.name not in unserved} if not self.has("pointer"): for name in POINT_OPTIONAL_WITH_POINTER: - schema = self._served[name].inputSchema + schema = self._computer_served[name].inputSchema schema["required"] = [*schema.get("required", []), "x", "y"] - self._describe_tools() - return list(self._served.values()) + self._computer_describe_tools() + return list(self._computer_served.values()) def create_app(self) -> Any: - self._require_init() + self._computer_require_init() app = super().create_app() - self._describe_tools() + self._computer_describe_tools() return app def mount(self, app: Any) -> Any: - self._require_init() + self._computer_require_init() mounted = super().mount(app) - self._describe_tools() + self._computer_describe_tools() return mounted - def _describe_tools(self) -> None: + def _computer_describe_tools(self) -> None: """Write the default coordinate space into the descriptions of the tools that take a point, and carry the served descriptions and input schemas over to mcp's FastMCP's tools/list, which reads the tool functions.""" space, scale = self.settings.default_space, self.settings.default_norm_scale note = (" Points are in frame space by default: pixels of the frame's image." if space == "frame" else f" Points are in norm space by default: 0 to {scale} over the frame, so send x and y on that grid.") for name in POINT_TOOLS: - descriptor = self._served.get(name) + descriptor = self._computer_served.get(name) if descriptor is not None: descriptor.description = " ".join((getattr(type(self), name).__doc__ or "").split()) + note manager = getattr(self.mcp, "_tool_manager", None) - for name, descriptor in self._served.items(): + for name, descriptor in self._computer_served.items(): registered = manager.get_tool(name) if manager is not None else None if registered is not None: registered.description = descriptor.description @@ -181,10 +182,10 @@ def declared_extensions(self) -> list[EnvironmentExtension]: # What the tools and extensions call. - async def _call(self, operation: str, *args: Any) -> Any: + async def _computer_call(self, operation: str, *args: Any) -> Any: """Run one of the driver's operations: an AgentEnvError passes through, anything else is an input that may have been delivered (delivery_failed) or an internal failure.""" - method = getattr(self, self._operations[operation]) + method = getattr(self, self._computer_operations[operation]) try: result = method(*args) return await result if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future) else result @@ -194,72 +195,72 @@ async def _call(self, operation: str, *args: Any) -> Any: error = DeliveryFailed if operation in INPUT else Internal raise error(f"@{operation} failed: {type(e).__name__}: {e}") from e - async def _display_now(self) -> Display: - display = await self._call("display") + async def _computer_display_now(self) -> Display: + display = await self._computer_call("display") if not isinstance(display, Display) or display.width <= 0 or display.height <= 0 or display.scale <= 0: raise Internal(f"@display returned {display!r}, not a Display with a positive size and scale") return display - async def _capture(self, identity: str) -> Frame: - display = await self._display_now() - image = to_pil(await self._call("capture"), display) - return self._frames.add(image, display, self._queue.snapshot(), identity=identity) + async def _computer_capture(self, identity: str) -> Frame: + display = await self._computer_display_now() + image = to_pil(await self._computer_call("capture"), display) + return self._computer_frames.add(image, display, self._computer_queue.snapshot(), identity=identity) - async def _pointer_screen(self) -> tuple[int, int]: - where = await self._call("pointer_position") + async def _computer_pointer(self) -> tuple[int, int]: + where = await self._computer_call("pointer_position") try: x, y = where return int(x), int(y) except (TypeError, ValueError): raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None - async def _act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, + async def _computer_act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, summary: str, repeat: int = 1, hold: tuple | None = None, move_to: tuple | None = None): """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result with others' input since ``frame`` and, when asked, a fresh frame.""" identity = self.caller_identity(ctx) sent = False - async with self._queue.turn(self.settings.queue_wait_s): - others = self._queue.others_since(frame.inputs, identity) if frame is not None else 0 + async with self._computer_queue.turn(self.settings.queue_wait_s): + others = self._computer_queue.others_since(frame.inputs, identity) if frame is not None else 0 effect = None try: if move_to is not None: - await self._call("pointer_move", *move_to) + await self._computer_call("pointer_move", *move_to) sent = True for _ in range(repeat): - effect = await self._call(operation, *args) + effect = await self._computer_call(operation, *args) sent = True if hold is not None: await asyncio.sleep(hold[0] / 1000) - await self._call("key_up", list(reversed(hold[1]))) + await self._computer_call("key_up", list(reversed(hold[1]))) except AgentEnvError as e: if sent: e.applied = "unknown" raise finally: if sent: - self._queue.delivered(identity) + self._computer_queue.delivered(identity) data = {"applied": "yes", "delivery": "foreground", "others_since_frame": others, "effect": effect_data(effect)} if not screenshot: return _result(summary, data) - new = await self._capture(identity) + new = await self._computer_capture(identity) return _result(f"{summary}, then captured {new.id}", {**data, "frame": new.data()}, new.encode()) - async def _ensure_ready(self) -> None: + async def _computer_ensure_ready(self) -> None: """Refuse with device_unavailable until the ready checks have passed once; run them if none has yet.""" - if self._ready: + if self._computer_ready: return - async with self._ready_lock: - if self._ready: + async with self._computer_ready_lock: + if self._computer_ready: return - failed = [c for c in await self._run_ready_checks() if not c.ok] + failed = [c for c in await self._computer_ready_checks() if not c.ok] if failed: raise DeviceUnavailable("the computer isn't ready: " + "; ".join(f"{c.name}: {c.detail}" for c in failed)) - async def _run_ready_checks(self) -> list[ReadyCheck]: + async def _computer_ready_checks(self) -> list[ReadyCheck]: checks = [] try: - display = await self._display_now() + display = await self._computer_display_now() checks.append(ReadyCheck("display", True, f"{display.width}×{display.height} at scale {display.scale:g}")) except AgentEnvError as e: display = None @@ -268,7 +269,7 @@ async def _run_ready_checks(self) -> list[ReadyCheck]: checks.append(ReadyCheck("capture", False, "the display didn't answer")) else: try: - image = to_pil(await self._call("capture"), display) + image = to_pil(await self._computer_call("capture"), display) checks.append(ReadyCheck("capture", True, f"{image.size[0]}×{image.size[1]}")) except AgentEnvError as e: checks.append(ReadyCheck("capture", False, e.message)) @@ -277,48 +278,48 @@ async def _run_ready_checks(self) -> list[ReadyCheck]: except Exception as e: checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) if all(c.ok for c in checks): - self._ready = True + self._computer_ready = True return checks - def _hold(self, identity: str, keys: list[str]) -> None: - held = self._held.setdefault(identity, []) + def _computer_hold(self, identity: str, keys: list[str]) -> None: + held = self._computer_held.setdefault(identity, []) held += [k for k in keys if k not in held] - previous = self._releases.pop(identity, None) + previous = self._computer_releases.pop(identity, None) if previous is not None: previous.cancel() - self._releases[identity] = asyncio.ensure_future(self._release_after(identity, self.settings.max_hold_ms)) + self._computer_releases[identity] = asyncio.ensure_future(self._computer_release_after(identity, self.settings.max_hold_ms)) - def _release(self, identity: str, keys: list[str]) -> None: - held = [k for k in self._held.get(identity, []) if k not in keys] + def _computer_release(self, identity: str, keys: list[str]) -> None: + held = [k for k in self._computer_held.get(identity, []) if k not in keys] if held: - self._held[identity] = held + self._computer_held[identity] = held return - self._held.pop(identity, None) - release = self._releases.pop(identity, None) + self._computer_held.pop(identity, None) + release = self._computer_releases.pop(identity, None) if release is not None and release is not asyncio.current_task(): release.cancel() - async def _release_after(self, identity: str, ms: int) -> None: + async def _computer_release_after(self, identity: str, ms: int) -> None: """Release what ``identity`` still holds once ``max_hold_ms`` has passed.""" await asyncio.sleep(ms / 1000) - keys = list(reversed(self._held.get(identity, []))) + keys = list(reversed(self._computer_held.get(identity, []))) if keys: - async with self._queue.turn(self.settings.queue_wait_s): - await self._call("key_up", keys) - self._release(identity, keys) + async with self._computer_queue.turn(self.settings.queue_wait_s): + await self._computer_call("key_up", keys) + self._computer_release(identity, keys) - def _require_init(self) -> None: - if not hasattr(self, "_operations"): + def _computer_require_init(self) -> None: + if not hasattr(self, "_computer_operations"): raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") # The Linux defaults for the OS operations. @default_for("run_in_session", os="linux") - async def _linux_run_in_session(self, argv: list, cwd: str | None, env: dict | None, timeout_s: float): + async def _computer_linux_run_in_session(self, argv: list, cwd: str | None, env: dict | None, timeout_s: float): return await linux.run_in_session(argv, cwd, env, timeout_s, output_bytes=self.settings.exec_output_bytes) @default_for("write_files", os="linux") - async def _linux_write_files(self, files: dict) -> None: + async def _computer_linux_write_files(self, files: dict) -> None: await asyncio.to_thread(linux.write_files, files) @@ -333,6 +334,17 @@ def deco(cls: type) -> type: return deco +def _check_names(cls: type) -> None: + """Refuse a driver member that would replace the base class's own: its private members and the tools, which + define the contract. The hooks (``caller_identity``, ``ready_checks``, ...) and the features are the driver's.""" + reserved = {name for name in dir(ComputerDriverEnvironment) + if (name.startswith("_") and not name.startswith("__")) or name.startswith("computer_")} + clash = sorted(name for name in vars(cls) if name in reserved or name.startswith(("_computer_", "computer_"))) + if clash: + raise TypeError(f"{cls.__qualname__} redefines {', '.join(clash)}, which ComputerDriverEnvironment uses " + "itself; give the driver's own members other names") + + def _check_features(cls: type) -> None: problems = [] unknown = [b for b in cls.buttons if b not in _BUTTONS] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 249ae895..0f2cbe23 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -31,48 +31,48 @@ class Extensions: @extension(COMPUTER_READY, op="status", method="GET", description="Whether the computer is usable now, by real checks; never sends input.") - async def _ready_status(self) -> dict: - checks = await self._run_ready_checks() + async def _computer_ready_status(self) -> dict: + checks = await self._computer_ready_checks() return {"ready": all(c.ok for c in checks), "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]} @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.") - async def _display_get(self) -> dict: - return self._display_data(await self._display_now()) + async def _computer_display_get(self) -> dict: + return self._computer_display_data(await self._computer_display_now()) @extension(COMPUTER_DISPLAY, op="set") - async def _display_set(self, width: int, height: int, scale: Optional[float] = None) -> dict: + async def _computer_display_set(self, width: int, height: int, scale: Optional[float] = None) -> dict: if [width, height] not in [list(m) for m in self.display_modes]: modes = ", ".join(f"{w}×{h}" for w, h in self.display_modes) raise InvalidArgument(f"{width}×{height} is not one of this computer's display modes: {modes}") - now = await self._display_now() - changed = await self._call("set_display", width, height, now.scale if scale is None else scale) + now = await self._computer_display_now() + changed = await self._computer_call("set_display", width, height, now.scale if scale is None else scale) if not isinstance(changed, Display): - changed = await self._display_now() - return self._display_data(changed) + changed = await self._computer_display_now() + return self._computer_display_data(changed) @extension(COMPUTER_SPACES, op="get", method="GET", description="Get or set the default coordinate space, written into the tool descriptions.") - async def _spaces_get(self) -> dict: + async def _computer_spaces_get(self) -> dict: return {"default_space": self.settings.default_space, "default_norm_scale": self.settings.default_norm_scale} @extension(COMPUTER_SPACES, op="set") - async def _spaces_set(self, default_space: str, default_norm_scale: Optional[int] = None) -> dict: + async def _computer_spaces_set(self, default_space: str, default_norm_scale: Optional[int] = None) -> dict: space, scale = check_space(default_space, default_norm_scale, self.settings.default_space, self.settings.default_norm_scale) self.settings = self.settings.model_copy(update={"default_space": space, "default_norm_scale": scale}) - self._describe_tools() - return await self._spaces_get() + self._computer_describe_tools() + return await self._computer_spaces_get() @extension(COMPUTER_KEYBOARD, op="get", method="GET", description="Get or set the tap hold, typing delay, layout and maximum hold.") - async def _keyboard_get(self) -> dict: + async def _computer_keyboard_get(self) -> dict: s = self.settings return {"tap_hold_ms": s.tap_hold_ms, "type_delay_ms": s.type_delay_ms, "layout": s.layout, "max_hold_ms": s.max_hold_ms} @extension(COMPUTER_KEYBOARD, op="set") - async def _keyboard_set(self, tap_hold_ms: Optional[int] = None, type_delay_ms: Optional[int] = None, + async def _computer_keyboard_set(self, tap_hold_ms: Optional[int] = None, type_delay_ms: Optional[int] = None, layout: Optional[str] = None, max_hold_ms: Optional[int] = None) -> dict: given = {"tap_hold_ms": tap_hold_ms, "type_delay_ms": type_delay_ms, "max_hold_ms": max_hold_ms} for name, value in given.items(): @@ -83,12 +83,12 @@ async def _keyboard_set(self, tap_hold_ms: Optional[int] = None, type_delay_ms: raise Unsupported(f"this computer types in the layouts {list(self.layouts)}, not {layout!r}") update = {k: v for k, v in {**given, "layout": layout}.items() if v is not None} self.settings = self.settings.model_copy(update=update) - return await self._keyboard_get() + return await self._computer_keyboard_get() @extension(COMPUTER_EXEC, op="run", description="Run a command inside the desktop session as the session user; a repeated request_id " "returns the first result.") - async def _exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None, env: Optional[dict] = None, + async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None, env: Optional[dict] = None, timeout_s: Optional[float] = None) -> dict: if not (isinstance(argv, list) and argv and all(isinstance(a, str) for a in argv)): raise InvalidArgument("argv is a non-empty list of strings; name a shell to use one: ['bash', '-lc', '...']") @@ -99,10 +99,10 @@ async def _exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None timeout = self.settings.exec_timeout_s if timeout_s is None else timeout_s if not 0 < timeout <= MAX_EXEC_TIMEOUT_S: raise InvalidArgument(f"timeout_s is above 0 and at most {MAX_EXEC_TIMEOUT_S}, got {timeout}") - runs: OrderedDict = self._exec_runs + runs: OrderedDict = self._computer_exec_runs task = runs.get(request_id) if task is None: - task = asyncio.ensure_future(self._run_command(argv, cwd, env, timeout)) + task = asyncio.ensure_future(self._computer_run_command(argv, cwd, env, timeout)) runs[request_id] = task def forget_a_failure(done: asyncio.Task) -> None: @@ -114,8 +114,8 @@ def forget_a_failure(done: asyncio.Task) -> None: runs.popitem(last=False) return await asyncio.shield(task) - async def _run_command(self, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: - result = await self._call("run_in_session", argv, cwd, env, timeout) + async def _computer_run_command(self, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: + result = await self._computer_call("run_in_session", argv, cwd, env, timeout) if not isinstance(result, ExecResult): raise Internal(f"@run_in_session returned {type(result).__name__}, not an ExecResult") cap = self.settings.exec_output_bytes @@ -127,6 +127,6 @@ async def _run_command(self, argv: list, cwd: Optional[str], env: Optional[dict] "timed_out": result.timed_out, } - def _display_data(self, display: Display) -> dict: + def _computer_display_data(self, display: Display) -> dict: return {"width": display.width, "height": display.height, "scale": display.scale, "modes": [list(m) for m in self.display_modes]} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index 9297efee..8b9c6bc8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -136,21 +136,21 @@ async def computer_screenshot( ) -> Annotated[CallToolResult, ScreenshotResult]: """Capture the screen as a frame: an image plus a frame_id you can point into later, the image size, and the display resolution and scale. With region, crop an existing frame at native resolution (zoom).""" - await self._ensure_ready() + await self._computer_ensure_ready() identity = self.caller_identity(ctx) if quality is not None and not (format == "jpeg" and 1 <= quality <= 95): raise InvalidArgument("quality is for format=jpeg, from 1 to 95") if region is None: if frame_id is not None: raise InvalidArgument("frame_id names the frame a region crops; give a region too") - frame = await self._capture(identity) + frame = await self._computer_capture(identity) summary = f"Captured {frame.id}: {frame.width}×{frame.height} of the {_display(frame)} display" else: - parent = self._frames.get(frame_id, identity) - self._frames.check_current(parent, await self._display_now()) + parent = self._computer_frames.get(frame_id, identity) + self._computer_frames.check_current(parent, await self._computer_display_now()) space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) box = frame_region(parent, region, space, scale) - frame = self._frames.add(parent.image.crop(box), parent.display, parent.inputs, identity=identity, + frame = self._computer_frames.add(parent.image.crop(box), parent.display, parent.inputs, identity=identity, parent=parent, origin=(box[0], box[1])) summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})" return _result(summary, frame.data(), frame.encode(format, quality)) @@ -170,14 +170,14 @@ async def computer_click( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Click at a point of a frame.""" - await self._ensure_ready() - button = self._button(button) + await self._computer_ensure_ready() + button = self._computer_button(button) if not 1 <= count <= self.max_click_count: raise InvalidArgument(f"count is from 1 to {self.max_click_count} on this computer, got {count}") - held = self._modifiers(modifiers) - frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) + held = self._computer_modifiers(modifiers) + frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) times = f" ×{count}" if count > 1 else "" - return await self._act(ctx, "click", (sx, sy, button, count, held), frame=frame, screenshot=screenshot, + return await self._computer_act(ctx, "click", (sx, sy, button, count, held), frame=frame, screenshot=screenshot, summary=f"Clicked {button}{times} {where}") @tool(annotations=_INPUT) @@ -194,24 +194,24 @@ async def computer_drag( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Press at the first point, move through the path, and release at the last.""" - await self._ensure_ready() - button = self._button(button) + await self._computer_ensure_ready() + button = self._computer_button(button) if not isinstance(path, list) or not 2 <= len(path) <= self.max_drag_points: raise InvalidArgument(f"path has from 2 to {self.max_drag_points} points on this computer") duration = self.settings.drag_duration_ms if duration_ms is None else duration_ms if not 0 <= duration <= self.max_duration_ms: raise InvalidArgument(f"duration_ms is from 0 to {self.max_duration_ms} on this computer, got {duration}") - held = self._modifiers(modifiers) + held = self._computer_modifiers(modifiers) identity = self.caller_identity(ctx) - frame = self._frames.get(frame_id, identity) - self._frames.check_current(frame, await self._display_now()) + frame = self._computer_frames.get(frame_id, identity) + self._computer_frames.check_current(frame, await self._computer_display_now()) space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) screen = [] for point in path: if not (isinstance(point, list) and len(point) == 2): raise InvalidArgument("each path point is [x, y]") screen.append(frame.to_screen(*frame_point(frame, point[0], point[1], space, scale))) - return await self._act(ctx, "drag", (screen, button, duration, held), frame=frame, screenshot=screenshot, + return await self._computer_act(ctx, "drag", (screen, button, duration, held), frame=frame, screenshot=screenshot, summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") @tool(annotations=_INPUT) @@ -230,12 +230,12 @@ async def computer_scroll( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Scroll the content under a point.""" - await self._ensure_ready() + await self._computer_ensure_ready() if not 1 <= amount <= self.max_scroll_amount: raise InvalidArgument(f"amount is from 1 to {self.max_scroll_amount} on this computer, got {amount}") - held = self._modifiers(modifiers) - frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) - return await self._act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, screenshot=screenshot, + held = self._computer_modifiers(modifiers) + frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + return await self._computer_act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, screenshot=screenshot, summary=f"Scrolled {direction} {amount} {where}") @tool(annotations=_INPUT) @@ -247,7 +247,7 @@ async def computer_type( ) -> Annotated[CallToolResult, InputResult]: """Type text into whatever has keyboard focus. Text that can't be typed whole is refused before anything is typed; the clipboard is left as it was.""" - await self._ensure_ready() + await self._computer_ensure_ready() if not text: raise InvalidArgument("text is empty") if len(text) > self.max_text_length: @@ -256,7 +256,7 @@ async def computer_type( if control: raise InvalidArgument(f"text has control characters {control}; press keys with computer_key") identity = self.caller_identity(ctx) - return await self._act(ctx, "type_text", (text,), frame=self._frames.latest(identity), screenshot=screenshot, + return await self._computer_act(ctx, "type_text", (text,), frame=self._computer_frames.latest(identity), screenshot=screenshot, summary=f"Typed {len(text)} characters") @tool(annotations=_INPUT) @@ -270,13 +270,13 @@ async def computer_key( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, InputResult]: """Press one key chord.""" - await self._ensure_ready() + await self._computer_ensure_ready() names = chord(keys, self.keys) if not 1 <= repeat <= self.max_key_repeat: raise InvalidArgument(f"repeat is from 1 to {self.max_key_repeat} on this computer, got {repeat}") identity = self.caller_identity(ctx) times = f" ×{repeat}" if repeat > 1 else "" - return await self._act(ctx, "press", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + return await self._computer_act(ctx, "press", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") @tool(annotations=_READ) @@ -287,14 +287,14 @@ async def computer_wait( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, WaitResult]: """Wait, then optionally return a fresh frame.""" - await self._ensure_ready() + await self._computer_ensure_ready() if not 0 <= ms <= self.max_wait_ms: raise InvalidArgument(f"ms is from 0 to {self.max_wait_ms} on this computer, got {ms}") await asyncio.sleep(ms / 1000) summary = f"Waited {ms} ms" if not screenshot: return _result(summary, {"waited_ms": ms}) - frame = await self._capture(self.caller_identity(ctx)) + frame = await self._computer_capture(self.caller_identity(ctx)) return _result(f"{summary}, then captured {frame.id}", {"waited_ms": ms, "frame": frame.data()}, frame.encode()) @tool(annotations=_INPUT) @@ -309,9 +309,9 @@ async def computer_pointer_move( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Move the pointer to a point; hover effects happen.""" - await self._ensure_ready() - frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) - return await self._act(ctx, "pointer_move", (sx, sy), frame=frame, screenshot=screenshot, + await self._computer_ensure_ready() + frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + return await self._computer_act(ctx, "pointer_move", (sx, sy), frame=frame, screenshot=screenshot, summary=f"Moved the pointer {where}") @tool(annotations=_INPUT) @@ -327,7 +327,7 @@ async def computer_pointer_down( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Press a button, optionally after moving to a point.""" - return await self._button_action(ctx, "pointer_down", "Pressed", button, x, y, frame_id, screenshot, space, + return await self._computer_button_action(ctx, "pointer_down", "Pressed", button, x, y, frame_id, screenshot, space, norm_scale) @tool(annotations=_INPUT) @@ -343,7 +343,7 @@ async def computer_pointer_up( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Release a button, optionally after moving to a point.""" - return await self._button_action(ctx, "pointer_up", "Released", button, x, y, frame_id, screenshot, space, + return await self._computer_button_action(ctx, "pointer_up", "Released", button, x, y, frame_id, screenshot, space, norm_scale) @tool(annotations=_READ) @@ -353,10 +353,10 @@ async def computer_pointer_position( frame_id: Annotated[Optional[str], Field(description="The frame to report in; defaults to your latest")] = None, ) -> Annotated[CallToolResult, PointerPositionResult]: """Where the pointer is, in a frame's pixels and in screen units, and whether it is inside that frame.""" - await self._ensure_ready() - sx, sy = await self._pointer_screen() + await self._computer_ensure_ready() + sx, sy = await self._computer_pointer() identity = self.caller_identity(ctx) - frame = self._frames.get(frame_id, identity) if frame_id is not None else self._frames.latest(identity) + frame = self._computer_frames.get(frame_id, identity) if frame_id is not None else self._computer_frames.latest(identity) data: dict = {"screen": {"x": sx, "y": sy}} if frame is None: return _result(f"The pointer is at screen ({sx}, {sy})", data) @@ -383,17 +383,17 @@ async def computer_keyboard_down( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, InputResult]: """Press keys and keep them down.""" - await self._ensure_ready() + await self._computer_ensure_ready() names = chord(keys, self.keys) if hold_ms is not None and not 0 <= hold_ms <= self.settings.max_hold_ms: raise InvalidArgument(f"hold_ms is from 0 to {self.settings.max_hold_ms}, got {hold_ms}") identity = self.caller_identity(ctx) if hold_ms is not None: - return await self._act(ctx, "key_down", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, hold=(hold_ms, names), summary=f"Held {'+'.join(names)} for {hold_ms} ms") - result = await self._act(ctx, "key_down", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + result = await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, summary=f"Pressed and held {'+'.join(names)}") - self._hold(identity, names) + self._computer_hold(identity, names) return result @tool(annotations=_INPUT) @@ -405,50 +405,50 @@ async def computer_keyboard_up( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, InputResult]: """Release held keys.""" - await self._ensure_ready() + await self._computer_ensure_ready() identity = self.caller_identity(ctx) - names = chord(keys, self.keys) if keys is not None else list(self._held.get(identity, [])) + names = chord(keys, self.keys) if keys is not None else list(self._computer_held.get(identity, [])) if not names: raise InvalidArgument("you are holding no keys") - result = await self._act(ctx, "key_up", (names,), frame=self._frames.latest(identity), screenshot=screenshot, + result = await self._computer_act(ctx, "key_up", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, summary=f"Released {'+'.join(names)}") - self._release(identity, names) + self._computer_release(identity, names) return result - async def _button_action(self, ctx, op, verb, button, x, y, frame_id, screenshot, space, norm_scale): - await self._ensure_ready() - button = self._button(button) + async def _computer_button_action(self, ctx, op, verb, button, x, y, frame_id, screenshot, space, norm_scale): + await self._computer_ensure_ready() + button = self._computer_button(button) if (x is None) != (y is None): raise InvalidArgument("give both x and y, or neither") if x is None: - return await self._act(ctx, op, (button,), frame=self._frames.latest(self.caller_identity(ctx)), + return await self._computer_act(ctx, op, (button,), frame=self._computer_frames.latest(self.caller_identity(ctx)), screenshot=screenshot, summary=f"{verb} {button}") - frame, (sx, sy), where = await self._point(ctx, x, y, frame_id, space, norm_scale) - return await self._act(ctx, op, (button,), frame=frame, screenshot=screenshot, move_to=(sx, sy), + frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + return await self._computer_act(ctx, op, (button,), frame=frame, screenshot=screenshot, move_to=(sx, sy), summary=f"{verb} {button} {where}") - async def _point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple: + async def _computer_point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple: """The frame, the screen pixel and a phrase for a point; with no point and the pointer group, the pointer.""" identity = self.caller_identity(ctx) if x is None and y is None and self.has("pointer") and frame_id is None: - sx, sy = await self._pointer_screen() - return self._frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})" + sx, sy = await self._computer_pointer() + return self._computer_frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})" if x is None or y is None: raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if self.has("pointer") else "")) - frame = self._frames.get(frame_id, identity) - self._frames.check_current(frame, await self._display_now()) + frame = self._computer_frames.get(frame_id, identity) + self._computer_frames.check_current(frame, await self._computer_display_now()) space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) fx, fy = frame_point(frame, x, y, space, scale) in_space = f" ({space} {scale})" if space == "norm" else "" return frame, frame.to_screen(fx, fy), f"at {_fmt(x, y)}{in_space} in {frame.id}" - def _button(self, button: str | None) -> str: + def _computer_button(self, button: str | None) -> str: button = button or "left" if button not in self.buttons: raise Unsupported(f"this computer has no {button} button; it has {', '.join(self.buttons)}") return button - def _modifiers(self, modifiers: list | None) -> list[str]: + def _computer_modifiers(self, modifiers: list | None) -> list[str]: if not modifiers: return [] if not isinstance(modifiers, list): diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index 1c25c62c..e5715925 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -96,7 +96,7 @@ def test_a_group_with_a_member_that_doesnt_match_the_platform_is_dropped(): assert FakeDriver(platform=LINUX).has("pointer") Driver = type("Driver", (FakeDriver,), {"move": pointer_move(os="windows")(lambda self, x, y: None)}) driver = Driver() - assert not driver.has("pointer") and "pointer_down" not in driver._operations + assert not driver.has("pointer") and "pointer_down" not in driver._computer_operations def test_unknown_predicates_are_refused(): @@ -128,7 +128,7 @@ def test_set_display_without_modes_is_refused(): def test_a_driver_method_replaces_the_linux_default(): Driver = _define({"run": run_in_session(lambda self, *a: None)}) assert declared(Driver)["run_in_session"] == [("run", {})] - assert declared(MinimalDriver)["run_in_session"] == [("_linux_run_in_session", {"os": frozenset({"linux"})})] + assert declared(MinimalDriver)["run_in_session"] == [("_computer_linux_run_in_session", {"os": frozenset({"linux"})})] def test_the_platform_is_detected(tmp_path, monkeypatch): @@ -230,7 +230,7 @@ async def test_tools_refuse_until_the_ready_checks_pass_once(): assert refused.isError and refused.structuredContent["code"] == "device_unavailable" assert refused.structuredContent["retryable"] is True assert "display: @display failed: RuntimeError: no X server" in refused.structuredContent["message"] - assert not served.isError and driver._ready + assert not served.isError and driver._computer_ready @pytest.mark.asyncio @@ -359,7 +359,7 @@ async def test_concurrent_first_calls_run_the_ready_checks_once(): driver.slow["display"] = 0.05 async with mcp_session(driver) as session: results = await asyncio.gather(*(session.call_tool("computer_wait", {"ms": 0}) for _ in range(3))) - assert all(not r.isError for r in results) and driver._ready + assert all(not r.isError for r in results) and driver._computer_ready def test_keys_and_modifiers_that_arent_lists_of_names_are_refused(): @@ -367,4 +367,16 @@ def test_keys_and_modifiers_that_arent_lists_of_names_are_refused(): with pytest.raises(InvalidArgument, match="a key is a W3C key value"): canonical(key) with pytest.raises(InvalidArgument, match="modifiers is a list"): - FakeDriver()._modifiers("Shift") + FakeDriver()._computer_modifiers("Shift") + + +@pytest.mark.parametrize("name", ["_computer_act", "_computer_frames", "computer_click", "_mounted", "_computer_new"]) +def test_a_driver_member_that_would_replace_the_base_classs_own_is_refused(name): + with pytest.raises(TypeError, match=rf"Driver redefines {name}, which ComputerDriverEnvironment uses itself"): + _define({name: lambda self: None}) + + +def test_a_drivers_own_private_helpers_and_the_hooks_are_its_own(): + Driver = _define({"_button": lambda self, d, b, down: None, "_move": lambda self: None, + "caller_identity": lambda self, context: "x", "buttons": ("left", "right")}) + assert Driver().caller_identity(None) == "x" diff --git a/packages/agentenv-protocol/tests/test_computer_frames.py b/packages/agentenv-protocol/tests/test_computer_frames.py index 58beea5b..b27e5cf3 100644 --- a/packages/agentenv-protocol/tests/test_computer_frames.py +++ b/packages/agentenv-protocol/tests/test_computer_frames.py @@ -63,7 +63,7 @@ async def test_a_region_is_a_native_crop_whose_points_map_back_exactly(): async def _captured_at(driver: FakeDriver, frame_id: str) -> str: - return driver._frames.get(frame_id, "default").data()["captured_at"] + return driver._computer_frames.get(frame_id, "default").data()["captured_at"] @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_queue.py b/packages/agentenv-protocol/tests/test_computer_queue.py index e176f0c2..96bf1920 100644 --- a/packages/agentenv-protocol/tests/test_computer_queue.py +++ b/packages/agentenv-protocol/tests/test_computer_queue.py @@ -309,7 +309,7 @@ async def test_keys_held_down_stay_down_until_released(): nothing = await session.call_tool("computer_keyboard_up", {}) assert held == ["Shift", "Control"] and driver.screen.held == [] assert partly.structuredContent["summary"] == "Released Shift" and rest.structuredContent["summary"] == "Released Control" - assert nothing.structuredContent["code"] == "invalid_argument" and driver._held == {} + assert nothing.structuredContent["code"] == "invalid_argument" and driver._computer_held == {} @pytest.mark.asyncio @@ -331,7 +331,7 @@ async def test_keys_held_past_max_hold_ms_are_released(): with anyio.fail_after(5): while driver.screen.held: await asyncio.sleep(0.01) - assert driver.screen.log[-1] == ("key_up", ("Alt",)) and driver._held == {} + assert driver.screen.log[-1] == ("key_up", ("Alt",)) and driver._computer_held == {} @pytest.mark.asyncio From 30fb5f0f695d28c7894cc6584e31e5fb86b34d06 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 12:13:00 -0700 Subject: [PATCH 08/18] fix(protocol): input bookkeeping under cancellation, failure and queueing From review: - a caller cancelled while waiting for the input queue no longer leaves its lock acquisition behind to take the queue and never give it back; - a screenshot that fails after an input was delivered reports the input as done (applied yes, no frame) rather than an error that invites a retry, and held keys are recorded as soon as they are pressed; - the automatic key release keeps waiting while other input keeps the queue busy; - a point is checked against its frame again once the call has its turn, and a display change takes a turn too, so a queued click can't land on the old geometry; - a running command is joined by its request_id however many others finish meanwhile; only finished results are evicted; - structured_output=False lists no schema for a CallToolResult tool on fastmcp. Co-Authored-By: Claude Opus 5.5 --- .../agent_env_environment.py | 3 +- .../src/agentenv_protocol/computer/driver.py | 39 +++++++--- .../agentenv_protocol/computer/extensions.py | 30 +++++--- .../src/agentenv_protocol/computer/queue.py | 22 ++++-- .../src/agentenv_protocol/computer/tools.py | 43 ++++++----- .../tests/test_computer_extensions.py | 17 +++++ .../tests/test_computer_queue.py | 74 ++++++++++++++++++- .../tests/test_tool_annotations.py | 17 +++++ 8 files changed, 191 insertions(+), 54 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index e328e2a1..b07fa6c8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -303,7 +303,8 @@ def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> extra["output_schema"] = None registered = injecting(fn) if fastmcp and _returns_call_tool_result(fn): - registered, extra["output_schema"] = _fastmcp_returning(registered) + registered, schema = _fastmcp_returning(registered) + extra["output_schema"] = None if options.structured_output is False else schema app.tool(name=descriptor.name, description=descriptor.description, **extra)(registered) def _check_tool_binding(self, app: Any) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 4f003dc1..b07c4619 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -12,7 +12,7 @@ from ..types import EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension from . import linux from .dataplane import DataPlane -from .errors import DeliveryFailed, DeviceUnavailable, Internal +from .errors import DeliveryFailed, DeviceUnavailable, InputBusy, Internal from .extensions import ( COMPUTER, COMPUTER_DISPLAY, @@ -80,7 +80,8 @@ def __init__(self, settings: ComputerSettings | None = None, platform: Platform self._computer_queue = InputQueue() self._computer_ready = False self._computer_ready_lock = asyncio.Lock() - self._computer_exec_runs: OrderedDict = OrderedDict() + self._computer_exec_running: dict = {} + self._computer_exec_finished: OrderedDict = OrderedDict() self._computer_held: dict[str, list[str]] = {} self._computer_releases: dict[str, asyncio.Task] = {} self._computer_served: dict[str, EnvironmentTool] = {} @@ -214,13 +215,18 @@ async def _computer_pointer(self) -> tuple[int, int]: except (TypeError, ValueError): raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None - async def _computer_act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, - summary: str, repeat: int = 1, hold: tuple | None = None, move_to: tuple | None = None): + async def _computer_act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, + screenshot: bool, summary: str, mapped: bool = False, repeat: int = 1, + hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None): """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result - with others' input since ``frame`` and, when asked, a fresh frame.""" + with others' input since ``frame`` and, when asked, a fresh frame. A point ``mapped`` from ``frame`` is + checked again once the call has its turn, since the display may have changed while it waited; ``then`` + runs as soon as the input is delivered.""" identity = self.caller_identity(ctx) sent = False async with self._computer_queue.turn(self.settings.queue_wait_s): + if mapped and frame is not None: + self._computer_frames.check_current(frame, await self._computer_display_now()) others = self._computer_queue.others_since(frame.inputs, identity) if frame is not None else 0 effect = None try: @@ -240,10 +246,15 @@ async def _computer_act(self, ctx: ToolContext, operation: str, args: tuple, *, finally: if sent: self._computer_queue.delivered(identity) + if then is not None: + then() data = {"applied": "yes", "delivery": "foreground", "others_since_frame": others, "effect": effect_data(effect)} if not screenshot: return _result(summary, data) - new = await self._computer_capture(identity) + try: + new = await self._computer_capture(identity) + except AgentEnvError as e: + return _result(f"{summary}; the screenshot after it failed ({e.code}: {e.message})", data) return _result(f"{summary}, then captured {new.id}", {**data, "frame": new.data()}, new.encode()) async def _computer_ensure_ready(self) -> None: @@ -287,7 +298,8 @@ def _computer_hold(self, identity: str, keys: list[str]) -> None: previous = self._computer_releases.pop(identity, None) if previous is not None: previous.cancel() - self._computer_releases[identity] = asyncio.ensure_future(self._computer_release_after(identity, self.settings.max_hold_ms)) + self._computer_releases[identity] = asyncio.ensure_future( + self._computer_release_after(identity, self.settings.max_hold_ms)) def _computer_release(self, identity: str, keys: list[str]) -> None: held = [k for k in self._computer_held.get(identity, []) if k not in keys] @@ -300,12 +312,15 @@ def _computer_release(self, identity: str, keys: list[str]) -> None: release.cancel() async def _computer_release_after(self, identity: str, ms: int) -> None: - """Release what ``identity`` still holds once ``max_hold_ms`` has passed.""" + """Release what ``identity`` still holds once ``max_hold_ms`` has passed, waiting as long as other input + keeps the queue busy.""" await asyncio.sleep(ms / 1000) - keys = list(reversed(self._computer_held.get(identity, []))) - if keys: - async with self._computer_queue.turn(self.settings.queue_wait_s): - await self._computer_call("key_up", keys) + while keys := list(reversed(self._computer_held.get(identity, []))): + try: + async with self._computer_queue.turn(self.settings.queue_wait_s): + await self._computer_call("key_up", keys) + except InputBusy: + continue self._computer_release(identity, keys) def _computer_require_init(self) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 0f2cbe23..5d1ebb8a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -20,7 +20,7 @@ COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" MAX_EXEC_TIMEOUT_S = 3600 -#: How many command results are kept for a repeated request_id. +#: How many finished command results are kept for a repeated request_id; a running command is always joined. KEEP_EXEC_RESULTS = 256 KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)} @@ -45,8 +45,9 @@ async def _computer_display_set(self, width: int, height: int, scale: Optional[f if [width, height] not in [list(m) for m in self.display_modes]: modes = ", ".join(f"{w}×{h}" for w, h in self.display_modes) raise InvalidArgument(f"{width}×{height} is not one of this computer's display modes: {modes}") - now = await self._computer_display_now() - changed = await self._computer_call("set_display", width, height, now.scale if scale is None else scale) + async with self._computer_queue.turn(self.settings.queue_wait_s): + now = await self._computer_display_now() + changed = await self._computer_call("set_display", width, height, now.scale if scale is None else scale) if not isinstance(changed, Display): changed = await self._computer_display_now() return self._computer_display_data(changed) @@ -99,19 +100,24 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st timeout = self.settings.exec_timeout_s if timeout_s is None else timeout_s if not 0 < timeout <= MAX_EXEC_TIMEOUT_S: raise InvalidArgument(f"timeout_s is above 0 and at most {MAX_EXEC_TIMEOUT_S}, got {timeout}") - runs: OrderedDict = self._computer_exec_runs - task = runs.get(request_id) + finished: OrderedDict = self._computer_exec_finished + if request_id in finished: + return finished[request_id] + running: dict = self._computer_exec_running + task = running.get(request_id) if task is None: task = asyncio.ensure_future(self._computer_run_command(argv, cwd, env, timeout)) - runs[request_id] = task + running[request_id] = task - def forget_a_failure(done: asyncio.Task) -> None: - if done.cancelled() or done.exception() is not None: - runs.pop(request_id, None) + def done(ran: asyncio.Task) -> None: + if running.get(request_id) is ran: + del running[request_id] + if not ran.cancelled() and ran.exception() is None: + finished[request_id] = ran.result() + while len(finished) > KEEP_EXEC_RESULTS: + finished.popitem(last=False) - task.add_done_callback(forget_a_failure) - while len(runs) > KEEP_EXEC_RESULTS: - runs.popitem(last=False) + task.add_done_callback(done) return await asyncio.shield(task) async def _computer_run_command(self, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py index 1f890335..311a4918 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py @@ -18,21 +18,27 @@ def __init__(self) -> None: @contextlib.asynccontextmanager async def turn(self, wait_s: float) -> AsyncIterator[None]: - """Hold the queue for one call, or refuse with input_busy after ``wait_s``. Waiting is first come, first - served; a waiter woken as its time runs out still takes its turn.""" + """Hold the queue for one call, or refuse with input_busy after ``wait_s``; waiting is first come, first + served. A wait given up, by the timeout or by the caller's cancellation, never leaves the queue held.""" waiter = asyncio.ensure_future(self._lock.acquire()) - done, _ = await asyncio.wait({waiter}, timeout=wait_s) + try: + done, _ = await asyncio.wait({waiter}, timeout=wait_s) + except BaseException: + self._abandon(waiter) + raise if not done: - waiter.cancel() - try: - await waiter - except asyncio.CancelledError: - raise InputBusy(f"waited {wait_s:g} s for earlier input to finish") from None + self._abandon(waiter) + raise InputBusy(f"waited {wait_s:g} s for earlier input to finish") try: yield finally: self._lock.release() + def _abandon(self, waiter: asyncio.Future) -> None: + """Give up a wait for the lock; if it was granted anyway, give the lock back.""" + waiter.cancel() + waiter.add_done_callback(lambda w: None if w.cancelled() or w.exception() is not None else self._lock.release()) + def delivered(self, caller: str) -> None: self._total += 1 self._by_caller[caller] += 1 diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index 8b9c6bc8..633245be 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -175,9 +175,10 @@ async def computer_click( if not 1 <= count <= self.max_click_count: raise InvalidArgument(f"count is from 1 to {self.max_click_count} on this computer, got {count}") held = self._computer_modifiers(modifiers) - frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) times = f" ×{count}" if count > 1 else "" - return await self._computer_act(ctx, "click", (sx, sy, button, count, held), frame=frame, screenshot=screenshot, + return await self._computer_act(ctx, "click", (sx, sy, button, count, held), frame=frame, mapped=mapped, + screenshot=screenshot, summary=f"Clicked {button}{times} {where}") @tool(annotations=_INPUT) @@ -211,7 +212,8 @@ async def computer_drag( if not (isinstance(point, list) and len(point) == 2): raise InvalidArgument("each path point is [x, y]") screen.append(frame.to_screen(*frame_point(frame, point[0], point[1], space, scale))) - return await self._computer_act(ctx, "drag", (screen, button, duration, held), frame=frame, screenshot=screenshot, + return await self._computer_act(ctx, "drag", (screen, button, duration, held), frame=frame, mapped=True, + screenshot=screenshot, summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") @tool(annotations=_INPUT) @@ -234,8 +236,9 @@ async def computer_scroll( if not 1 <= amount <= self.max_scroll_amount: raise InvalidArgument(f"amount is from 1 to {self.max_scroll_amount} on this computer, got {amount}") held = self._computer_modifiers(modifiers) - frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, screenshot=screenshot, + frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + return await self._computer_act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, mapped=mapped, + screenshot=screenshot, summary=f"Scrolled {direction} {amount} {where}") @tool(annotations=_INPUT) @@ -310,8 +313,8 @@ async def computer_pointer_move( ) -> Annotated[CallToolResult, InputResult]: """Move the pointer to a point; hover effects happen.""" await self._computer_ensure_ready() - frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, "pointer_move", (sx, sy), frame=frame, screenshot=screenshot, + frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + return await self._computer_act(ctx, "pointer_move", (sx, sy), frame=frame, mapped=mapped, screenshot=screenshot, summary=f"Moved the pointer {where}") @tool(annotations=_INPUT) @@ -391,10 +394,9 @@ async def computer_keyboard_down( if hold_ms is not None: return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, hold=(hold_ms, names), summary=f"Held {'+'.join(names)} for {hold_ms} ms") - result = await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, - summary=f"Pressed and held {'+'.join(names)}") - self._computer_hold(identity, names) - return result + return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), + screenshot=screenshot, summary=f"Pressed and held {'+'.join(names)}", + then=lambda: self._computer_hold(identity, names)) @tool(annotations=_INPUT) async def computer_keyboard_up( @@ -410,10 +412,9 @@ async def computer_keyboard_up( names = chord(keys, self.keys) if keys is not None else list(self._computer_held.get(identity, [])) if not names: raise InvalidArgument("you are holding no keys") - result = await self._computer_act(ctx, "key_up", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, - summary=f"Released {'+'.join(names)}") - self._computer_release(identity, names) - return result + return await self._computer_act(ctx, "key_up", (names,), frame=self._computer_frames.latest(identity), + screenshot=screenshot, summary=f"Released {'+'.join(names)}", + then=lambda: self._computer_release(identity, names)) async def _computer_button_action(self, ctx, op, verb, button, x, y, frame_id, screenshot, space, norm_scale): await self._computer_ensure_ready() @@ -423,16 +424,18 @@ async def _computer_button_action(self, ctx, op, verb, button, x, y, frame_id, s if x is None: return await self._computer_act(ctx, op, (button,), frame=self._computer_frames.latest(self.caller_identity(ctx)), screenshot=screenshot, summary=f"{verb} {button}") - frame, (sx, sy), where = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, op, (button,), frame=frame, screenshot=screenshot, move_to=(sx, sy), + frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + return await self._computer_act(ctx, op, (button,), frame=frame, mapped=mapped, screenshot=screenshot, + move_to=(sx, sy), summary=f"{verb} {button} {where}") async def _computer_point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple: - """The frame, the screen pixel and a phrase for a point; with no point and the pointer group, the pointer.""" + """The frame, the screen pixel, a phrase for a point and whether it was mapped from the frame; with no point + and the pointer group, the pointer.""" identity = self.caller_identity(ctx) if x is None and y is None and self.has("pointer") and frame_id is None: sx, sy = await self._computer_pointer() - return self._computer_frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})" + return self._computer_frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})", False if x is None or y is None: raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if self.has("pointer") else "")) frame = self._computer_frames.get(frame_id, identity) @@ -440,7 +443,7 @@ async def _computer_point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) fx, fy = frame_point(frame, x, y, space, scale) in_space = f" ({space} {scale})" if space == "norm" else "" - return frame, frame.to_screen(fx, fy), f"at {_fmt(x, y)}{in_space} in {frame.id}" + return frame, frame.to_screen(fx, fy), f"at {_fmt(x, y)}{in_space} in {frame.id}", True def _computer_button(self, button: str | None) -> str: button = button or "left" diff --git a/packages/agentenv-protocol/tests/test_computer_extensions.py b/packages/agentenv-protocol/tests/test_computer_extensions.py index 2793bdca..d3e18abf 100644 --- a/packages/agentenv-protocol/tests/test_computer_extensions.py +++ b/packages/agentenv-protocol/tests/test_computer_extensions.py @@ -177,3 +177,20 @@ async def test_only_the_latest_command_results_are_kept(monkeypatch, tmp_path): for request_id in ("a", "b", "a"): last = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id=request_id))).json() assert last["stdout"].strip() == "3", "a's first result was dropped, so a ran again" + + +@pytest.mark.asyncio +async def test_a_running_command_is_joined_however_many_others_finish_meanwhile(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr("agentenv_protocol.computer.extensions.KEEP_EXEC_RESULTS", 1) + slow = _run(argv=["sh", "-c", "sleep 0.6; echo a >> runs"], request_id="a") + async with http_client(FakeDriver()) as http: + async def others_then_retry(): + await asyncio.sleep(0.1) + for request_id in ("b", "c"): + await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["true"], request_id=request_id)) + return await http.post("/agentenv/ext/computer-exec/run", json=slow) + + first, retried = await asyncio.gather(http.post("/agentenv/ext/computer-exec/run", json=slow), + others_then_retry()) + assert first.json() == retried.json() and (tmp_path / "runs").read_text() == "a\n" diff --git a/packages/agentenv-protocol/tests/test_computer_queue.py b/packages/agentenv-protocol/tests/test_computer_queue.py index 96bf1920..548d790e 100644 --- a/packages/agentenv-protocol/tests/test_computer_queue.py +++ b/packages/agentenv-protocol/tests/test_computer_queue.py @@ -11,10 +11,11 @@ pytest.importorskip("mcp.server.fastmcp") pytest.importorskip("PIL") -from computer_fake import COLOURS, FakeDriver, FakeScreen, MinimalDriver, mcp_session # noqa: E402 +from computer_fake import COLOURS, FakeDriver, FakeScreen, MinimalDriver, http_client, mcp_session # noqa: E402 from agentenv_protocol import ROLE_META_KEY # noqa: E402 from agentenv_protocol.computer import ComputerSettings, pointer_position # noqa: E402 +from agentenv_protocol.computer.queue import InputQueue # noqa: E402 class ByRole(FakeDriver): @@ -358,3 +359,74 @@ async def test_the_minimal_driver_serves_no_group_tools(): missing = await session.call_tool("computer_pointer_move", {"x": 1, "y": 1}) assert not clicked.isError and driver.screen.log == [("click", 3, 4, "left", 1, ())] assert missing.isError and "Unknown tool" in missing.content[0].text + + +@pytest.mark.asyncio +async def test_a_caller_cancelled_while_waiting_never_leaves_the_queue_held(): + queue = InputQueue() + + async def hold(): + async with queue.turn(5): + await asyncio.sleep(0.2) + + async def wait(): + async with queue.turn(5): + pass + + holder = asyncio.ensure_future(hold()) + await asyncio.sleep(0.01) + waiter = asyncio.ensure_future(wait()) + await asyncio.sleep(0.01) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + await holder + async with queue.turn(0.5): + pass + + +@pytest.mark.asyncio +async def test_a_screenshot_that_fails_after_the_input_reports_the_input_as_done(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + driver.fail["capture"] = RuntimeError("mss died") + clicked = await session.call_tool("computer_click", {"x": 4, "y": 4, "screenshot": True}) + driver.fail["capture"] = RuntimeError("mss died") + held = await session.call_tool("computer_keyboard_down", {"keys": ["Shift"], "screenshot": True}) + assert not clicked.isError and clicked.structuredContent["applied"] == "yes" and "frame" not in clicked.structuredContent + assert "the screenshot after it failed (internal: @capture failed: RuntimeError: mss died)" in clicked.content[0].text + assert [e[0] for e in driver.screen.log] == ["click", "key_down"] + assert not held.isError and driver._computer_held == {"default": ["Shift"]} + + +@pytest.mark.asyncio +async def test_the_automatic_release_waits_out_a_busy_queue(): + driver = FakeDriver(settings=ComputerSettings(max_hold_ms=50, queue_wait_s=0.05)) + async with mcp_session(driver) as session: + await _shot(session) + await session.call_tool("computer_keyboard_down", {"keys": ["Alt"]}) + driver.slow["drag"] = 0.4 + await session.call_tool("computer_drag", {"path": [[1, 1], [9, 9]]}) + with anyio.fail_after(5): + while driver.screen.held: + await asyncio.sleep(0.01) + kinds = [e[0] for e in driver.screen.log] + assert kinds == ["key_down", "drag", "key_up"] and driver._computer_held == {} + + +@pytest.mark.asyncio +async def test_a_point_queued_while_the_display_changes_is_refused_stale(): + driver = FakeDriver() + async with mcp_session(driver) as session, http_client(driver) as http: + await _shot(session) + driver.slow["click"] = 0.3 + first = asyncio.ensure_future(session.call_tool("computer_click", {"x": 1, "y": 1})) + await asyncio.sleep(0.05) + resize = asyncio.ensure_future(http.post("/agentenv/ext/computer-display/set", json={"width": 640, "height": 400})) + await asyncio.sleep(0.05) + queued = await session.call_tool("computer_click", {"x": 1270, "y": 10}) + await first + await resize + assert queued.structuredContent["code"] == "stale_frame" and queued.structuredContent["applied"] == "no" + assert [e[1:3] for e in driver.screen.log] == [(1, 1)] diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index 23a0ab28..e2da7e5f 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -141,3 +141,20 @@ def served_tools(self, tools): with pytest.raises(ValueError, match=error): AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Bad()) + + +@pytest.mark.asyncio +async def test_structured_output_false_lists_no_schema_for_a_call_tool_result_on_fastmcp(): + @environment_card(name="opted-out") + class _OptedOut(AgentEnvEnvironment): + @tool(structured_output=False) + def shot(self) -> Annotated[CallToolResult, _Frame]: + return CallToolResult(content=[TextContent(type="text", text="ok")], structuredContent={"frame_id": "f1", "width": 1}) + + app = fastmcp.FastMCP("opted-out") + _OptedOut().mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + listed = (await client.list_tools())[0] + result = await client.call_tool("shot", {}) + assert listed.outputSchema is None and result.content[0].text == "ok" From 9aee10c80fdff968b6f3618ac8c854a96032cb0b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 12:26:23 -0700 Subject: [PATCH 09/18] perf(protocol): keep a computer's frames as PNG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found end to end: 64 raw 1280×800 frames held about 230 MB in the driver, and the bench's 1920×1080 would hold over 400 MB. A frame now keeps the PNG it is encoded to once at capture, the one every screenshot answers with anyway, and decodes it only to crop or to answer in JPEG. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/frames.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py index bf88fac4..ea99582b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -19,10 +19,12 @@ @dataclass(frozen=True) class Frame: """A capture, or a crop of one: ``origin`` is where this frame's pixels start in the root capture, and the root - capture's ``root_size`` image covers the whole ``display``.""" + capture's ``root_size`` image covers the whole ``display``. The pixels are kept as the PNG every screenshot + answers with, a tenth or less of them raw, and decoded only to crop or to answer in JPEG.""" id: str - image: PILImage.Image = field(repr=False) + png: bytes = field(repr=False) + size: tuple[int, int] display: Display origin: tuple[int, int] root_size: tuple[int, int] @@ -32,11 +34,17 @@ class Frame: @property def width(self) -> int: - return self.image.size[0] + return self.size[0] @property def height(self) -> int: - return self.image.size[1] + return self.size[1] + + @property + def image(self) -> PILImage.Image: + image = PILImage.open(io.BytesIO(self.png)) + image.load() + return image def to_screen(self, fx: float, fy: float) -> tuple[int, int]: """The screen pixel under frame pixel (fx, fy); exact at identity scale.""" @@ -64,14 +72,11 @@ def data(self) -> dict: def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str]: """The frame as base64 and its MIME type.""" + if fmt != "jpeg": + return base64.b64encode(self.png).decode("ascii"), "image/png" buffer = io.BytesIO() - if fmt == "jpeg": - self.image.convert("RGB").save(buffer, format="JPEG", quality=quality or 85) - mime = "image/jpeg" - else: - self.image.save(buffer, format="PNG") - mime = "image/png" - return base64.b64encode(buffer.getvalue()).decode("ascii"), mime + self.image.convert("RGB").save(buffer, format="JPEG", quality=quality or 85) + return base64.b64encode(buffer.getvalue()).decode("ascii"), "image/jpeg" class FrameStore: @@ -87,7 +92,8 @@ def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identit parent: Frame | None = None, origin: tuple[int, int] = (0, 0)) -> Frame: frame = Frame( id=f"f{self._next}", - image=image, + png=_png(image), + size=image.size, display=parent.display if parent is not None else display, origin=(parent.origin[0] + origin[0], parent.origin[1] + origin[1]) if parent is not None else (0, 0), root_size=parent.root_size if parent is not None else image.size, @@ -144,5 +150,11 @@ def to_pil(capture: object, display: Display) -> PILImage.Image: return image +def _png(image: PILImage.Image) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + def _size(display: Display) -> str: return f"{display.width}×{display.height} at scale {display.scale:g}" From 1c71b35b69281502b55fef29a86d413619c8ba4e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 12:30:08 -0700 Subject: [PATCH 10/18] refactor(protocol): name the computer package's tuning values Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/frames.py | 4 +++- .../src/agentenv_protocol/computer/linux.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py index ea99582b..56451a0a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -14,6 +14,8 @@ from .operations import Display, Image KEEP = 64 +#: The JPEG quality a screenshot answers with when it names none. +JPEG_QUALITY = 85 @dataclass(frozen=True) @@ -75,7 +77,7 @@ def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str if fmt != "jpeg": return base64.b64encode(self.png).decode("ascii"), "image/png" buffer = io.BytesIO() - self.image.convert("RGB").save(buffer, format="JPEG", quality=quality or 85) + self.image.convert("RGB").save(buffer, format="JPEG", quality=quality or JPEG_QUALITY) return base64.b64encode(buffer.getvalue()).decode("ascii"), "image/jpeg" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py index 303e42bc..021fbc34 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py @@ -16,6 +16,10 @@ #: or stderr open; that stream is then drained, and discarded, until the app closes it, so the app never blocks on #: a full pipe or dies writing to a closed one. DRAIN_S = 1.0 +#: How often a command is checked for having exited. +POLL_S = 0.02 +#: How much of a stream is read at a time. +READ_BYTES = 65536 _draining: set = set() @@ -56,7 +60,7 @@ async def _exited(process: asyncio.subprocess.Process, timeout_s: float) -> bool while process.returncode is None: if asyncio.get_running_loop().time() >= deadline: return False - await asyncio.sleep(0.02) + await asyncio.sleep(POLL_S) return True @@ -68,7 +72,7 @@ def __init__(self, limit: int) -> None: self.returned = False async def read(self, stream: asyncio.StreamReader) -> None: - while chunk := await stream.read(65536): + while chunk := await stream.read(READ_BYTES): if self.returned: continue room = self.limit - len(self.kept) From 659293f0c56e4df27a59eb881cd85593b93593d4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 12:37:26 -0700 Subject: [PATCH 11/18] style(protocol): realign the computer tools' calls after the rename Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/tools.py | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index 633245be..182e7c05 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -150,8 +150,8 @@ async def computer_screenshot( self._computer_frames.check_current(parent, await self._computer_display_now()) space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) box = frame_region(parent, region, space, scale) - frame = self._computer_frames.add(parent.image.crop(box), parent.display, parent.inputs, identity=identity, - parent=parent, origin=(box[0], box[1])) + frame = self._computer_frames.add(parent.image.crop(box), parent.display, parent.inputs, + identity=identity, parent=parent, origin=(box[0], box[1])) summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})" return _result(summary, frame.data(), frame.encode(format, quality)) @@ -178,8 +178,7 @@ async def computer_click( frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) times = f" ×{count}" if count > 1 else "" return await self._computer_act(ctx, "click", (sx, sy, button, count, held), frame=frame, mapped=mapped, - screenshot=screenshot, - summary=f"Clicked {button}{times} {where}") + screenshot=screenshot, summary=f"Clicked {button}{times} {where}") @tool(annotations=_INPUT) async def computer_drag( @@ -214,7 +213,7 @@ async def computer_drag( screen.append(frame.to_screen(*frame_point(frame, point[0], point[1], space, scale))) return await self._computer_act(ctx, "drag", (screen, button, duration, held), frame=frame, mapped=True, screenshot=screenshot, - summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") + summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") @tool(annotations=_INPUT) async def computer_scroll( @@ -238,8 +237,7 @@ async def computer_scroll( held = self._computer_modifiers(modifiers) frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) return await self._computer_act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, mapped=mapped, - screenshot=screenshot, - summary=f"Scrolled {direction} {amount} {where}") + screenshot=screenshot, summary=f"Scrolled {direction} {amount} {where}") @tool(annotations=_INPUT) async def computer_type( @@ -259,8 +257,8 @@ async def computer_type( if control: raise InvalidArgument(f"text has control characters {control}; press keys with computer_key") identity = self.caller_identity(ctx) - return await self._computer_act(ctx, "type_text", (text,), frame=self._computer_frames.latest(identity), screenshot=screenshot, - summary=f"Typed {len(text)} characters") + return await self._computer_act(ctx, "type_text", (text,), frame=self._computer_frames.latest(identity), + screenshot=screenshot, summary=f"Typed {len(text)} characters") @tool(annotations=_INPUT) async def computer_key( @@ -279,8 +277,8 @@ async def computer_key( raise InvalidArgument(f"repeat is from 1 to {self.max_key_repeat} on this computer, got {repeat}") identity = self.caller_identity(ctx) times = f" ×{repeat}" if repeat > 1 else "" - return await self._computer_act(ctx, "press", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, - repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") + return await self._computer_act(ctx, "press", (names,), frame=self._computer_frames.latest(identity), + screenshot=screenshot, repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") @tool(annotations=_READ) async def computer_wait( @@ -314,8 +312,8 @@ async def computer_pointer_move( """Move the pointer to a point; hover effects happen.""" await self._computer_ensure_ready() frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, "pointer_move", (sx, sy), frame=frame, mapped=mapped, screenshot=screenshot, - summary=f"Moved the pointer {where}") + return await self._computer_act(ctx, "pointer_move", (sx, sy), frame=frame, mapped=mapped, + screenshot=screenshot, summary=f"Moved the pointer {where}") @tool(annotations=_INPUT) async def computer_pointer_down( @@ -392,8 +390,9 @@ async def computer_keyboard_down( raise InvalidArgument(f"hold_ms is from 0 to {self.settings.max_hold_ms}, got {hold_ms}") identity = self.caller_identity(ctx) if hold_ms is not None: - return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, - hold=(hold_ms, names), summary=f"Held {'+'.join(names)} for {hold_ms} ms") + return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), + screenshot=screenshot, hold=(hold_ms, names), + summary=f"Held {'+'.join(names)} for {hold_ms} ms") return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), screenshot=screenshot, summary=f"Pressed and held {'+'.join(names)}", then=lambda: self._computer_hold(identity, names)) @@ -422,12 +421,12 @@ async def _computer_button_action(self, ctx, op, verb, button, x, y, frame_id, s if (x is None) != (y is None): raise InvalidArgument("give both x and y, or neither") if x is None: - return await self._computer_act(ctx, op, (button,), frame=self._computer_frames.latest(self.caller_identity(ctx)), - screenshot=screenshot, summary=f"{verb} {button}") + return await self._computer_act(ctx, op, (button,), + frame=self._computer_frames.latest(self.caller_identity(ctx)), + screenshot=screenshot, summary=f"{verb} {button}") frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) return await self._computer_act(ctx, op, (button,), frame=frame, mapped=mapped, screenshot=screenshot, - move_to=(sx, sy), - summary=f"{verb} {button} {where}") + move_to=(sx, sy), summary=f"{verb} {button} {where}") async def _computer_point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple: """The frame, the screen pixel, a phrase for a point and whether it was mapped from the frame; with no point From c92d9be66baa6d8dfc51da2edea901f6a2baab94 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 13:28:56 -0700 Subject: [PATCH 12/18] feat(protocol): let an environment choose its extensions and data-plane methods, and update a served tool served_extensions(extensions) and served_operations(operations) choose at mount which @extension handlers, extension methods and data-plane methods are served, beside served_tools; a hook naming something with no handler is refused. update_tool(name, description=...) changes a served tool's description after the mount, on the card and in tools/list, on mcp's FastMCP and on fastmcp. The card route now serves the application's current card. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 9 +- .../agent_env_environment.py | 104 ++++++++++++++++-- .../tests/test_agent_env_environment.py | 22 ++++ .../tests/test_extension_methods.py | 43 +++++++- .../tests/test_tool_annotations.py | 50 ++++++++- 5 files changed, 216 insertions(+), 12 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index fc395d54..440bdefb 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -96,7 +96,8 @@ with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and o where it can, False lists none); a tool returning `Annotated[CallToolResult, Model]` sends its result as is, image blocks included, and is listed with `Model`'s schema. An environment chooses at mount which of its `@tool` methods it serves, and with which descriptions, by overriding -`served_tools(tools)`. +`served_tools(tools)`. After the mount, `await env.update_tool(name, description=...)` changes a +served tool's description on the card and in `tools/list` alike, on both FastMCPs. `AgentEnvStarletteApplication` mounts the same handler onto a Starlette/FastAPI app instead of FastMCP; since those apps have no MCP tool registry, constructing one with `@tool` methods @@ -211,6 +212,12 @@ promise with params and no handler like `intake/v1`, is returned from `declared_ Two handlers for one method, or a handler without `op` on a URI another handler also serves, are refused at mount. +An environment chooses at mount which of its `@extension` handlers it serves by overriding +`served_extensions(extensions)`: an extension it leaves out is not served, and of one served with +`op`, only the methods its returned descriptor still lists under `params.methods`. +`served_operations(operations)` chooses the data-plane methods (`data/add` and the others it +marks) the same way. Either hook returning something with no handler is refused at mount. + ```python @extension("urn:agentenv:computer-display/v1", op="get", method="GET") async def display_get(self) -> dict: ... diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index b07fa6c8..0427d438 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -217,7 +217,8 @@ class AgentEnvApplication(ABC): def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: self.handler = handler - ext_handlers = _discover_extensions(handler) + self._app: Any = None + ext_handlers = _served_extensions(handler, _discover_extensions(handler)) environment_card = _merge_extensions(environment_card, ext_handlers + _declared_extensions(handler, ext_handlers)) tool_handlers = _served_tools(handler, _discover_tools(handler, environment_card.name)) if tool_handlers and not self.supports_tools: @@ -229,7 +230,7 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: ) if self._tool_access is not None: environment_card = _merge_extensions(environment_card, [(e,) for e in tool_access_extensions()]) - methods = _discover_methods(handler) + methods = _served_operations(handler, _discover_methods(handler)) environment_card = _merge_operations(environment_card, methods) environment_card = _merge_data_objects(environment_card, methods) self.environment_card = environment_card @@ -246,9 +247,12 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: def add_routes_to_app(self, app: Any, *, rpc_url: str = RPC_PATH, card_url: str = WELL_KNOWN_PATH) -> None: self._check_tool_binding(app) + self._app = app + if isinstance(self.handler, AgentEnvEnvironment): + self.handler._application = self self.environment_card = self._card_for(app) self._add_route(app, rpc_url, ["POST"], self._dispatch) - self._add_route(app, card_url, ["GET"], _card_handler(self.environment_card)) + self._add_route(app, card_url, ["GET"], self._serve_card) for path, method, handler in self._ext_routes: self._add_route(app, path, [method], handler) for descriptor, fn in self._tools: @@ -275,6 +279,24 @@ def _bind_tool_calls(self, app: Any) -> None: def _card_for(self, app: Any) -> EnvironmentCard: return self.environment_card + async def _serve_card(self, request: Request) -> Response: + return _json_response(self.environment_card.model_dump()) + + async def update_tool(self, name: str, *, description: str) -> None: + """Change a served tool's description after the mount: on the card and in tools/list alike.""" + if all(descriptor.name != name for descriptor, _fn in self._tools): + raise ValueError(f"no served tool is named {name!r}") + self._tools = [(d.model_copy(update={"description": description}) if d.name == name else d, fn) + for d, fn in self._tools] + caps = self.environment_card.capabilities + tools = [t.model_copy(update={"description": description}) if t.name == name else t for t in caps.tools or []] + self.environment_card = self.environment_card.model_copy( + update={"capabilities": caps.model_copy(update={"tools": tools})}) + await self._describe_registered_tool(name, description) + + async def _describe_registered_tool(self, name: str, description: str) -> None: + raise NotImplementedError(f"{type(self).__name__} does not serve MCP tools") + class AgentEnvFastMCPApplication(AgentEnvApplication): supports_tools: ClassVar[bool] = True @@ -409,6 +431,19 @@ def list_tools(*args: Any, **kwargs: Any) -> Any: functools.update_wrapper(list_tools, inner) manager.list_tools = list_tools + async def _describe_registered_tool(self, name: str, description: str) -> None: + """fastmcp replaces a tool through its public provider; mcp's FastMCP lists the tool its manager holds.""" + app = self._app + if self._binding == "middleware": + registered = await app.get_tool(name) + app.local_provider.remove_tool(name) + app.local_provider.add_tool(registered.model_copy(update={"description": description})) + return + manager = getattr(app, "_tool_manager", None) + registered = manager.get_tool(name) if manager is not None else None + if registered is not None: + registered.description = description + def _card_for(self, app: Any) -> EnvironmentCard: """Declare the MCP endpoint only when the app says where it serves it; FastMCP-shaped targets may not.""" path = getattr(getattr(app, "settings", None), "streamable_http_path", None) @@ -446,6 +481,8 @@ class AgentEnvEnvironment: #: gateway's paths and with its rules, for an env agents reach with no gateway in front. tool_access: ClassVar[bool] = False _mounted: bool = False + #: The application mounted with this environment, which ``update_tool`` changes. + _application: Any = None def _build_card(self) -> EnvironmentCard: config = dict(getattr(self, _CARD_CONFIG_ATTR, None) or {}) @@ -467,6 +504,12 @@ def mount(self, app: Any) -> Any: self._mounted = True return app + async def update_tool(self, name: str, *, description: str) -> None: + """Change a served tool's description after the mount: on the card and in tools/list alike.""" + if self._application is None: + raise RuntimeError(f"{type(self).__name__} is not mounted") + await self._application.update_tool(name, description=description) + def serve(self, transport: str = "streamable-http") -> None: if self.mcp is None: self.create_app() @@ -482,6 +525,15 @@ def declared_extensions(self) -> list[EnvironmentExtension]: """Declaration-only extensions to advertise, like ``intake/v1``: a promise with params and no handler.""" return [] + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """The ``@extension`` handlers' extensions to serve, chosen at mount: one left out is not served, and of one served + with ``op``, only the methods its returned descriptor still lists.""" + return extensions + + def served_operations(self, operations: list[str]) -> list[str]: + """The data-plane methods to serve (``data/add`` and the others the handler marks), chosen at mount.""" + return operations + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: """The ``@tool`` methods to serve, chosen at mount; a returned descriptor's description replaces the marked one.""" return tools @@ -962,6 +1014,46 @@ def _discover_tools(handler: Any, environment_name: str) -> list: return ordered +def _served_extensions(handler: Any, ext_handlers: list) -> list: + """The extension handlers the handler's ``served_extensions`` keeps, with its descriptors.""" + hook = getattr(handler, "served_extensions", None) + if hook is None: + return ext_handlers + offered = {descriptor.uri: descriptor for descriptor, *_ in ext_handlers} + chosen = {descriptor.uri: descriptor for descriptor in hook(list(offered.values()))} + unknown = sorted(set(chosen) - set(offered)) + if unknown: + raise ValueError(f"served_extensions returned extensions with no @extension handler: {unknown}") + ops = {} + for descriptor, fn, *_ in ext_handlers: + ops.setdefault(descriptor.uri, set()).add(getattr(fn, _EXT_OP_ATTR, None)) + for uri, served in chosen.items(): + if None not in ops[uri]: + added = sorted(set((served.params or {}).get("methods") or {}) - ops[uri]) + if added: + raise ValueError(f"served_extensions lists methods of {uri} with no @extension handler: {added}") + kept = [] + for descriptor, fn, path, method in ext_handlers: + served = chosen.get(descriptor.uri) + op = getattr(fn, _EXT_OP_ATTR, None) + if served is None or (op is not None and op not in (served.params or {}).get("methods", {})): + continue + kept.append((served, fn, path, method)) + return kept + + +def _served_operations(handler: Any, methods: dict) -> dict: + """The data-plane methods the handler's ``served_operations`` keeps.""" + hook = getattr(handler, "served_operations", None) + if hook is None: + return methods + chosen = set(hook(list(methods))) + unknown = sorted(chosen - set(methods)) + if unknown: + raise ValueError(f"served_operations returned operations with no handler: {unknown}") + return {name: entry for name, entry in methods.items() if name in chosen} + + def _served_tools(handler: Any, tool_handlers: list) -> list: """The tools the handler's ``served_tools`` keeps, in its order and with its descriptors; all of them without one.""" hook = getattr(handler, "served_tools", None) @@ -1262,12 +1354,6 @@ async def handler(request: Request) -> Response: return handler -def _card_handler(card: EnvironmentCard) -> Callable: - async def handler(request: Request) -> Response: - return _json_response(card.model_dump()) - return handler - - def _rpc_error(rid: Any, code: int, message: str, data: Any = None) -> Response: error: dict = {"code": code, "message": message} if data is not None: diff --git a/packages/agentenv-protocol/tests/test_agent_env_environment.py b/packages/agentenv-protocol/tests/test_agent_env_environment.py index 5c223dca..5d145b31 100644 --- a/packages/agentenv-protocol/tests/test_agent_env_environment.py +++ b/packages/agentenv-protocol/tests/test_agent_env_environment.py @@ -300,6 +300,28 @@ async def _reset(self) -> None: assert card["capabilities"]["operations"] == ["data/reset"] +@pytest.mark.asyncio +async def test_served_operations_chooses_the_data_plane_methods_served(): + class _ReadOnlyItems(_ItemsHandler): + def served_operations(self, operations: list[str]) -> list[str]: + return [op for op in operations if op != "data/add"] + + async with _client(_ReadOnlyItems) as client: + added = await client.post("/agentenv", json=_rpc("data/add", {"parts": [{"kind": "data", "data": {"items": ["a"]}}]})) + card = (await client.get("/.well-known/agent-env.json")).json() + assert added.json()["error"]["code"] == -32601 + assert card["capabilities"]["operations"] == ["data/reset", "data/get"] + + +def test_served_operations_naming_an_unmarked_method_is_refused(): + class _Bad(_ItemsHandler): + def served_operations(self, operations: list[str]) -> list[str]: + return [*operations, "data/snapshot"] + + with pytest.raises(ValueError, match=r"no handler: \['data/snapshot'\]"): + AgentEnvFastMCPApplication(environment_card=EnvironmentCard(name="x"), handler=_Bad()) + + @pytest.mark.asyncio async def test_tools_only_env_serves_card_and_tools_without_data_plane(): class _ToolsOnly: diff --git a/packages/agentenv-protocol/tests/test_extension_methods.py b/packages/agentenv-protocol/tests/test_extension_methods.py index c2940f15..73d35aa8 100644 --- a/packages/agentenv-protocol/tests/test_extension_methods.py +++ b/packages/agentenv-protocol/tests/test_extension_methods.py @@ -1,5 +1,6 @@ """Several methods per extension: handlers given ``op`` merge into one card entry with an endpoint per method, with -the environment's extension_params beside them; declaration-only extensions; the refusals at mount.""" +the environment's extension_params beside them; declaration-only extensions; served_extensions choosing what is +served; the refusals at mount.""" from __future__ import annotations from unittest.mock import patch @@ -193,3 +194,43 @@ async def a(self) -> dict: with pytest.raises(ValueError, match="both declared and served"): _card(_Both()) + + +class _ReadOnly(_Display): + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + served = [] + for e in extensions: + if e.uri == _DISPLAY: + served.append(e.model_copy(update={"params": {**e.params, "methods": {"get": e.params["methods"]["get"]}}})) + elif e.uri != "urn:agentenv:legacy/v1": + served.append(e) + return served + + +@pytest.mark.asyncio +async def test_served_extensions_chooses_the_extensions_and_methods_served(): + env = _ReadOnly() + extensions = _extensions(_card(env)) + assert set(extensions[_DISPLAY].params["methods"]) == {"get"} and extensions[_DISPLAY].params["modes"] + assert "urn:agentenv:legacy/v1" not in extensions and _POINTER in extensions + async with _http(env) as http: + got = await http.get("/agentenv/ext/computer-display/get") + unset = await http.post("/agentenv/ext/computer-display/set", json={"width": 1920, "height": 1080}) + legacy = await http.post("/agentenv/ext/legacy", json={}) + assert got.json()["width"] == 1280 and unset.status_code == 404 and legacy.status_code == 404 + + +@pytest.mark.parametrize("chosen, error", [ + (lambda exts: [*exts, EnvironmentExtension(uri="urn:agentenv:ghost/v1")], + r"extensions with no @extension handler: \['urn:agentenv:ghost/v1'\]"), + (lambda exts: [e.model_copy(update={"params": {**e.params, "methods": {**e.params["methods"], "reset": {}}}}) + if e.uri == _DISPLAY else e for e in exts], + r"methods of urn:agentenv:computer-display/v1 with no @extension handler: \['reset'\]"), +]) +def test_served_extensions_naming_an_unhandled_extension_or_method_is_refused(chosen, error): + class _Bad(_Display): + def served_extensions(self, extensions): + return chosen(extensions) + + with pytest.raises(ValueError, match=error): + _card(_Bad()) diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index e2da7e5f..b502667d 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -1,9 +1,10 @@ -"""@tool annotations, structured output and _meta on both FastMCPs, and served_tools choosing what a handler serves.""" +"""@tool annotations, structured output and _meta on both FastMCPs, served_tools choosing what a handler serves, and\nupdate_tool changing a served description.""" from __future__ import annotations from typing import Annotated import anyio +import httpx import pytest from pydantic import BaseModel @@ -158,3 +159,50 @@ def shot(self) -> Annotated[CallToolResult, _Frame]: listed = (await client.list_tools())[0] result = await client.call_tool("shot", {}) assert listed.outputSchema is None and result.content[0].text == "ok" + + +async def _card_tools(app) -> dict: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as http: + card = (await http.get("/.well-known/agent-env.json")).json() + return {t["name"]: t for t in card["capabilities"]["tools"]} + + +@pytest.mark.asyncio +async def test_update_tool_rewrites_a_description_on_the_card_and_in_tools_list_on_mcps_fastmcp(): + env = _Env() + app = env.create_app() + await env.update_tool("look", description="Look again.") + with anyio.fail_after(30): + async with create_connected_server_and_client_session(app) as session: + tools = {t.name: t for t in (await session.list_tools()).tools} + look = await session.call_tool("look", {}) + card = await _card_tools(app.streamable_http_app()) + assert tools["look"].description == card["look"]["description"] == "Look again." + assert set(card) == set(tools) == {"look", "plain", "secret", "shot"} + assert look.structuredContent == {"frame_id": "f1", "width": 1280} + + +@pytest.mark.asyncio +async def test_update_tool_rewrites_a_description_on_the_card_and_in_tools_list_on_fastmcp(): + env = _Env() + app = fastmcp.FastMCP("tools") + env.mount(app) + await env.update_tool("look", description="Look again.") + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + tools = {t.name: t for t in await client.list_tools()} + look = await client.call_tool("look", {}) + card = await _card_tools(app.http_app()) + assert tools["look"].description == card["look"]["description"] == "Look again." + assert tools["look"].annotations.readOnlyHint is True and tools["look"].meta["agentenv.io/group"] == "minimum" + assert look.structured_content == {"frame_id": "f1", "width": 1280} + + +@pytest.mark.asyncio +async def test_update_tool_needs_a_mounted_env_and_a_served_tool(): + with pytest.raises(RuntimeError, match="_Env is not mounted"): + await _Env().update_tool("look", description="x") + env = _Chooser() + env.create_app() + with pytest.raises(ValueError, match="no served tool is named 'secret'"): + await env.update_tool("secret", description="x") From 91a7ccf673b693edfdf7095b939290b29cc18442 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 13:29:26 -0700 Subject: [PATCH 13/18] refactor(protocol): build the computer driver on one runtime, result models and the card hooks ComputerDriverEnvironment keeps its state in one ComputerRuntime (self._computer: operations, frames, the input queue, readiness, held keys and command runs) instead of _computer_* attributes, and the mixins declare what they read from the driver. The card's choices move to card.py and go through served_tools, served_extensions and served_operations, so a missing operation's handlers are no longer withdrawn by setting them to None. The definition-time checks move to checks.py. Every tool result is built from its pydantic model, the same one that gives its output schema, so unset fields are left out (parent_frame_id on a full screenshot, effect when there is none). Setting the default space rewrites the point tools' descriptions through update_tool rather than mcp's tool manager, and x and y stay optional on click and scroll everywhere: a computer without computer-pointer/v1 refuses a call that leaves them out. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 11 +- .../agentenv_protocol/computer/__init__.py | 2 +- .../src/agentenv_protocol/computer/card.py | 114 +++++ .../src/agentenv_protocol/computer/checks.py | 42 ++ .../agentenv_protocol/computer/dataplane.py | 11 +- .../src/agentenv_protocol/computer/driver.py | 317 ++------------ .../agentenv_protocol/computer/extensions.py | 116 +++-- .../src/agentenv_protocol/computer/frames.py | 19 +- .../src/agentenv_protocol/computer/results.py | 94 ++++ .../src/agentenv_protocol/computer/runtime.py | 208 +++++++++ .../src/agentenv_protocol/computer/tools.py | 409 ++++++++---------- .../tests/test_computer_driver.py | 16 +- .../tests/test_computer_extensions.py | 4 +- .../tests/test_computer_frames.py | 4 +- .../tests/test_computer_queue.py | 8 +- .../tests/test_computer_spaces.py | 19 + 16 files changed, 782 insertions(+), 612 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/card.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/results.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 440bdefb..6cc25c9f 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -371,9 +371,11 @@ decorators raise `TypeError`. Predicates (`os`, `distribution`, `version`, `arch choose a method per platform, read from `/etc/os-release`, the machine and `XDG_SESSION_TYPE` or `DISPLAY`: `@capture(display="x11")` and `@capture(display="wayland")` can sit side by side. A driver's own `@run_in_session` or `@write_files` replaces the base class's Linux default, which runs -as the driver's user inside the desktop session. The base class's own members are named `_computer_*` -and `computer_*`; a driver that redefines one of them, or another private member of the base class, -is refused when it is defined, so a helper of the driver's can't silently replace one. +as the driver's user inside the desktop session. The base class keeps its state in `self._computer` +and names its own members `_computer_*` and `computer_*`; a driver that redefines one of them, or +another private member of the base class, is refused when it is defined, so a helper of the +driver's can't silently replace one. What the optional operations give is served only when the +driver has them, through the `served_tools`, `served_extensions` and `served_operations` hooks. **What the base class serves:** @@ -382,7 +384,8 @@ is refused when it is defined, so a helper of the driver's can't silently replac `computer_scroll`, `computer_type`, `computer_key` (W3C key names) and `computer_wait`, plus the group tools. Every result is structured, with a one-line summary as its text block; input results carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame - after acting. + after acting. Click and scroll leave out `x` and `y` to act at the pointer on a computer with + `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. - **Frames and spaces.** Points are pixels of a named frame (your latest by default), or a 0..`norm_scale` grid over it with `space: "norm"`. A point outside the frame or the range is refused (`out_of_bounds`), never clamped. The last 64 frames are kept, and a frame goes stale diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 7d014c4e..812c466f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -18,7 +18,7 @@ Timeout, Unsupported, ) -from .extensions import ( +from .card import ( COMPUTER, COMPUTER_DISPLAY, COMPUTER_EXEC, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py new file mode 100644 index 00000000..06a3e459 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -0,0 +1,114 @@ +"""What a driver's card says: the extensions, the tools, extensions and data-plane methods its operations support, +the extensions' params, the extensions it declares without handlers, and the descriptions that name the default +coordinate space.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..types import METHOD_ADD, EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension + +if TYPE_CHECKING: + from .driver import ComputerDriverEnvironment + +COMPUTER = "urn:agentenv:computer/v1" +COMPUTER_READY = "urn:agentenv:computer-ready/v1" +COMPUTER_DISPLAY = "urn:agentenv:computer-display/v1" +COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1" +COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1" +COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1" +COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" + +MAX_EXEC_TIMEOUT_S = 3600 +KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)} + +#: The tools a group serves. +GROUP_TOOLS = { + "pointer": ("computer_pointer_move", "computer_pointer_down", "computer_pointer_up", "computer_pointer_position", + "computer_pointer_list"), + "keyboard": ("computer_keyboard_down", "computer_keyboard_up"), +} +#: The tools whose description names the default coordinate space. +POINT_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_pointer_move", + "computer_pointer_down", "computer_pointer_up") +#: The operation or group an extension needs, and one of its methods needs; without it, it is not served. +_EXTENSION_NEEDS = {COMPUTER_KEYBOARD: "keyboard", COMPUTER_EXEC: "run_in_session"} +_METHOD_NEEDS = {(COMPUTER_DISPLAY, "set"): "set_display"} + + +def served_tools(driver: ComputerDriverEnvironment, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + points = point_tools(driver) + return [t.model_copy(update={"description": point_description(driver, t.name)}) if t.name in points else t + for t in tools if t.name not in _unserved_tools(driver)] + + +def served_extensions(driver: ComputerDriverEnvironment, + extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + served = [] + for extension in extensions: + need = _EXTENSION_NEEDS.get(extension.uri) + if need is not None and not driver.has(need): + continue + dropped = {op for (uri, op), need in _METHOD_NEEDS.items() if uri == extension.uri and not driver.has(need)} + if dropped: + methods = {op: entry for op, entry in extension.params["methods"].items() if op not in dropped} + extension = extension.model_copy(update={"params": {**extension.params, "methods": methods}}) + served.append(extension) + return served + + +def served_operations(driver: ComputerDriverEnvironment, operations: list[str]) -> list[str]: + return [op for op in operations if op != METHOD_ADD or driver.has("write_files")] + + +def extension_params(driver: ComputerDriverEnvironment, uri: str) -> dict: + if uri == COMPUTER_DISPLAY: + return {"modes": [list(m) for m in driver.display_modes]} + if uri == COMPUTER_SPACES: + return {"spaces": ["frame", "norm"]} + if uri == COMPUTER_KEYBOARD: + return {**{name: {"min": low, "max": high} for name, (low, high) in KEYBOARD_RANGES.items()}, + "layouts": list(driver.layouts)} + if uri == COMPUTER_EXEC: + return {"max_output_bytes": driver.settings.exec_output_bytes, "max_timeout_s": MAX_EXEC_TIMEOUT_S} + return {} + + +def declared_extensions(driver: ComputerDriverEnvironment) -> list[EnvironmentExtension]: + features = { + "buttons": list(driver.buttons), "max_click_count": driver.max_click_count, + "keys": driver.keys if driver.keys == "all" else list(driver.keys), "max_text_length": driver.max_text_length, + "max_wait_ms": driver.max_wait_ms, "max_drag_points": driver.max_drag_points, + "max_duration_ms": driver.max_duration_ms, "max_key_repeat": driver.max_key_repeat, + "max_scroll_amount": driver.max_scroll_amount, + } + declared = [EnvironmentExtension( + uri=COMPUTER, description="The seven computer_* tools: screenshot, click, drag, scroll, type, key, wait.", + params={**features, "delivery": "foreground", "platform": driver.platform.as_dict(), + "supports": list(driver.supports)})] + if driver.has("pointer"): + declared.append(EnvironmentExtension( + uri=COMPUTER_POINTER, description="The computer_pointer_* tools, and click and scroll at the pointer.", + params={"buttons": list(driver.buttons), "kinds": ["foreground"]})) + intake = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")]) + if driver.has("write_files"): + intake.add = [IntakeFormat(part="file", format="file-tree", load="additive")] + declared.append(intake_extension(intake, "data/add writes files under the session user's home; data/get " + "returns the env's data paths.")) + return declared + + +def point_tools(driver: ComputerDriverEnvironment) -> list[str]: + unserved = _unserved_tools(driver) + return [name for name in POINT_TOOLS if name not in unserved] + + +def point_description(driver: ComputerDriverEnvironment, name: str) -> str: + """A point tool's docstring, then the default coordinate space.""" + space, scale = driver.settings.default_space, driver.settings.default_norm_scale + note = (" Points are in frame space by default: pixels of the frame's image." if space == "frame" else + f" Points are in norm space by default: 0 to {scale} over the frame, so send x and y on that grid.") + return " ".join((getattr(type(driver), name).__doc__ or "").split()) + note + + +def _unserved_tools(driver: ComputerDriverEnvironment) -> set[str]: + return {name for group, names in GROUP_TOOLS.items() if not driver.has(group) for name in names} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py new file mode 100644 index 00000000..c65c619e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -0,0 +1,42 @@ +"""The checks a driver class passes when it is defined, beside its operations' (``operations.check_class``): its +members leave the base class's own alone, and its features are in range.""" +from __future__ import annotations + +from ..errors import AgentEnvError +from .keys import canonical +from .operations import declared + +BUTTONS = ("left", "right", "middle", "back", "forward") + + +def check_names(cls: type, base: type) -> None: + """Refuse a driver member that would replace one of ``base``'s own: its private members and the tools, which + define the contract. The hooks (``caller_identity``, ``ready_checks``, ...) and the features are the driver's.""" + reserved = {name for name in dir(base) + if (name.startswith("_") and not name.startswith("__")) or name.startswith("computer_")} + clash = sorted(name for name in vars(cls) if name in reserved or name.startswith(("_computer_", "computer_"))) + if clash: + raise TypeError(f"{cls.__qualname__} redefines {', '.join(clash)}, which {base.__name__} uses " + "itself; give the driver's own members other names") + + +def check_features(cls: type) -> None: + problems = [] + unknown = [b for b in cls.buttons if b not in BUTTONS] + if "left" not in cls.buttons or unknown: + problems.append(f"buttons include left and come from {list(BUTTONS)}, got {list(cls.buttons)}") + for name, low in (("max_click_count", 1), ("max_text_length", 1), ("max_wait_ms", 60_000), ("max_drag_points", 2), + ("max_duration_ms", 0), ("max_key_repeat", 1), ("max_scroll_amount", 1)): + value = getattr(cls, name) + if not isinstance(value, int) or value < low: + problems.append(f"{name} is an integer of at least {low}, got {value!r}") + if cls.keys != "all": + try: + if [canonical(k) for k in cls.keys] != list(cls.keys): + problems.append("keys are W3C key values in their W3C case") + except AgentEnvError as e: + problems.append(f"keys: {e.message}") + if ("set_display" in declared(cls)) != bool(cls.display_modes): + problems.append("display_modes and @set_display come together") + if problems: + raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py index ad1f7bd6..e1621dd1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py @@ -7,7 +7,7 @@ import mimetypes import tempfile from pathlib import Path -from typing import Optional +from typing import TYPE_CHECKING, Optional import httpx from pydantic import ValidationError @@ -16,6 +16,10 @@ from ..transfers import NamespaceUploader, ReadObject, WriteNamespaceGrant, download from ..types import DATA_OBJECTS_EXTENSION_URI, FilePart, FileWithBytes, FileWithUri, uploaded_file_part, uploaded_object_path +if TYPE_CHECKING: + from .runtime import ComputerRuntime + from .settings import ComputerSettings + #: The most an https URL part may fetch; a read grant carries its own limit. MAX_URL_BYTES = 1 << 30 @@ -23,6 +27,9 @@ class DataPlane: """The data plane handlers, mixed into ComputerDriverEnvironment.""" + _computer: ComputerRuntime + settings: ComputerSettings + @add_data async def _computer_data_add(self, parts: list) -> None: files = {} @@ -37,7 +44,7 @@ async def _computer_data_add(self, parts: list) -> None: raise ValueError(f"data/add on a computer takes file parts naming their path under " f"{DATA_OBJECTS_EXTENSION_URI}, got a {part.kind} part") files[path] = await _fetch(part) - await self._computer_call("write_files", files) + await self._computer.call("write_files", files) @get_data async def _computer_data_get(self, write_namespace: Optional[WriteNamespaceGrant] = None) -> list: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index b07c4619..93cdf44b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -3,45 +3,23 @@ from __future__ import annotations import asyncio -from collections import OrderedDict from typing import Any, Callable, ClassVar from ..agent_env_environment import AgentEnvEnvironment, environment_card -from ..errors import AgentEnvError from ..tool_context import ToolContext -from ..types import EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension -from . import linux +from ..types import EnvironmentExtension, EnvironmentTool +from . import card, linux +from .checks import check_features, check_names from .dataplane import DataPlane -from .errors import DeliveryFailed, DeviceUnavailable, InputBusy, Internal -from .extensions import ( - COMPUTER, - COMPUTER_DISPLAY, - COMPUTER_EXEC, - COMPUTER_KEYBOARD, - COMPUTER_POINTER, - COMPUTER_SPACES, - KEYBOARD_RANGES, - MAX_EXEC_TIMEOUT_S, - Extensions, -) -from .frames import Frame, FrameStore, to_pil -from .keys import canonical -from .operations import GROUPS, INPUT, Display, ReadyCheck, check_class, declared, default_for, resolve +from .extensions import Extensions +from .operations import ReadyCheck, check_class, default_for from .platform import Platform, detect -from .queue import InputQueue +from .runtime import ComputerRuntime from .settings import ComputerSettings, load_settings -from .tools import GROUP_TOOLS, POINT_OPTIONAL_WITH_POINTER, POINT_TOOLS, Tools, _result, effect_data +from .tools import Tools #: The caller every call comes from until agents carry an identity of their own. DEFAULT_IDENTITY = "default" -_BUTTONS = ("left", "right", "middle", "back", "forward") -#: The handlers each optional operation or group brings; without it they are withdrawn. -_HANDLERS = { - "set_display": ("_computer_display_set",), - "keyboard": ("_computer_keyboard_get", "_computer_keyboard_set"), - "run_in_session": ("_computer_exec_run",), - "write_files": ("_computer_data_add",), -} class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironment): @@ -65,36 +43,23 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen #: The platforms the driver supports, as the driver env declares them: ``[{"os": {...}, "arch": [...]}]``. supports: ClassVar[list] = [] tool_access = True + #: What the base class keeps for the driver, built by ``__init__``. + _computer: ComputerRuntime | None = None def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) - _check_names(cls) + check_names(cls, ComputerDriverEnvironment) check_class(cls) - _check_features(cls) + check_features(cls) def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None: self.settings = settings if settings is not None else load_settings() self.platform = platform if platform is not None else detect() - self._computer_operations = resolve(type(self), self.platform) - self._computer_frames = FrameStore() - self._computer_queue = InputQueue() - self._computer_ready = False - self._computer_ready_lock = asyncio.Lock() - self._computer_exec_running: dict = {} - self._computer_exec_finished: OrderedDict = OrderedDict() - self._computer_held: dict[str, list[str]] = {} - self._computer_releases: dict[str, asyncio.Task] = {} - self._computer_served: dict[str, EnvironmentTool] = {} - for capability, handlers in _HANDLERS.items(): - if not self.has(capability): - for name in handlers: - setattr(self, name, None) + self._computer = ComputerRuntime(self) def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" - if capability in GROUPS: - return all(op in self._computer_operations for op in GROUPS[capability]) - return capability in self._computer_operations + return self._computer.has(capability) def caller_identity(self, context: ToolContext) -> str: """Who a call comes from, for each caller's latest frame and others' input since a frame.""" @@ -104,227 +69,33 @@ async def ready_checks(self) -> list[ReadyCheck]: """Checks of the driver's own, run with the base class's by computer-ready/v1; never sends input.""" return [] - # The card. - - def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: - unserved = {name for group, names in GROUP_TOOLS.items() if not self.has(group) for name in names} - self._computer_served = {t.name: t.model_copy(deep=True) for t in tools if t.name not in unserved} - if not self.has("pointer"): - for name in POINT_OPTIONAL_WITH_POINTER: - schema = self._computer_served[name].inputSchema - schema["required"] = [*schema.get("required", []), "x", "y"] - self._computer_describe_tools() - return list(self._computer_served.values()) - def create_app(self) -> Any: self._computer_require_init() - app = super().create_app() - self._computer_describe_tools() - return app + return super().create_app() def mount(self, app: Any) -> Any: self._computer_require_init() - mounted = super().mount(app) - self._computer_describe_tools() - return mounted - - def _computer_describe_tools(self) -> None: - """Write the default coordinate space into the descriptions of the tools that take a point, and carry the - served descriptions and input schemas over to mcp's FastMCP's tools/list, which reads the tool functions.""" - space, scale = self.settings.default_space, self.settings.default_norm_scale - note = (" Points are in frame space by default: pixels of the frame's image." if space == "frame" else - f" Points are in norm space by default: 0 to {scale} over the frame, so send x and y on that grid.") - for name in POINT_TOOLS: - descriptor = self._computer_served.get(name) - if descriptor is not None: - descriptor.description = " ".join((getattr(type(self), name).__doc__ or "").split()) + note - manager = getattr(self.mcp, "_tool_manager", None) - for name, descriptor in self._computer_served.items(): - registered = manager.get_tool(name) if manager is not None else None - if registered is not None: - registered.description = descriptor.description - registered.parameters["required"] = sorted( - {*registered.parameters.get("required", []), *(descriptor.inputSchema or {}).get("required", [])}) - - def extension_params(self, uri: str) -> dict: - if uri == COMPUTER_DISPLAY: - return {"modes": [list(m) for m in self.display_modes]} - if uri == COMPUTER_SPACES: - return {"spaces": ["frame", "norm"]} - if uri == COMPUTER_KEYBOARD: - return {**{name: {"min": low, "max": high} for name, (low, high) in KEYBOARD_RANGES.items()}, - "layouts": list(self.layouts)} - if uri == COMPUTER_EXEC: - return {"max_output_bytes": self.settings.exec_output_bytes, "max_timeout_s": MAX_EXEC_TIMEOUT_S} - return {} - - def declared_extensions(self) -> list[EnvironmentExtension]: - features = { - "buttons": list(self.buttons), "max_click_count": self.max_click_count, - "keys": self.keys if self.keys == "all" else list(self.keys), "max_text_length": self.max_text_length, - "max_wait_ms": self.max_wait_ms, "max_drag_points": self.max_drag_points, - "max_duration_ms": self.max_duration_ms, "max_key_repeat": self.max_key_repeat, - "max_scroll_amount": self.max_scroll_amount, - } - declared = [EnvironmentExtension( - uri=COMPUTER, description="The seven computer_* tools: screenshot, click, drag, scroll, type, key, wait.", - params={**features, "delivery": "foreground", "platform": self.platform.as_dict(), - "supports": list(self.supports)})] - if self.has("pointer"): - declared.append(EnvironmentExtension( - uri=COMPUTER_POINTER, description="The computer_pointer_* tools, and click and scroll at the pointer.", - params={"buttons": list(self.buttons), "kinds": ["foreground"]})) - intake = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")]) - if self.has("write_files"): - intake.add = [IntakeFormat(part="file", format="file-tree", load="additive")] - declared.append(intake_extension(intake, "data/add writes files under the session user's home; data/get " - "returns the env's data paths.")) - return declared - - # What the tools and extensions call. - - async def _computer_call(self, operation: str, *args: Any) -> Any: - """Run one of the driver's operations: an AgentEnvError passes through, anything else is an input that may - have been delivered (delivery_failed) or an internal failure.""" - method = getattr(self, self._computer_operations[operation]) - try: - result = method(*args) - return await result if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future) else result - except AgentEnvError: - raise - except Exception as e: - error = DeliveryFailed if operation in INPUT else Internal - raise error(f"@{operation} failed: {type(e).__name__}: {e}") from e - - async def _computer_display_now(self) -> Display: - display = await self._computer_call("display") - if not isinstance(display, Display) or display.width <= 0 or display.height <= 0 or display.scale <= 0: - raise Internal(f"@display returned {display!r}, not a Display with a positive size and scale") - return display + return super().mount(app) - async def _computer_capture(self, identity: str) -> Frame: - display = await self._computer_display_now() - image = to_pil(await self._computer_call("capture"), display) - return self._computer_frames.add(image, display, self._computer_queue.snapshot(), identity=identity) + # The card: what the driver's operations support. - async def _computer_pointer(self) -> tuple[int, int]: - where = await self._computer_call("pointer_position") - try: - x, y = where - return int(x), int(y) - except (TypeError, ValueError): - raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None - - async def _computer_act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, - screenshot: bool, summary: str, mapped: bool = False, repeat: int = 1, - hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None): - """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result - with others' input since ``frame`` and, when asked, a fresh frame. A point ``mapped`` from ``frame`` is - checked again once the call has its turn, since the display may have changed while it waited; ``then`` - runs as soon as the input is delivered.""" - identity = self.caller_identity(ctx) - sent = False - async with self._computer_queue.turn(self.settings.queue_wait_s): - if mapped and frame is not None: - self._computer_frames.check_current(frame, await self._computer_display_now()) - others = self._computer_queue.others_since(frame.inputs, identity) if frame is not None else 0 - effect = None - try: - if move_to is not None: - await self._computer_call("pointer_move", *move_to) - sent = True - for _ in range(repeat): - effect = await self._computer_call(operation, *args) - sent = True - if hold is not None: - await asyncio.sleep(hold[0] / 1000) - await self._computer_call("key_up", list(reversed(hold[1]))) - except AgentEnvError as e: - if sent: - e.applied = "unknown" - raise - finally: - if sent: - self._computer_queue.delivered(identity) - if then is not None: - then() - data = {"applied": "yes", "delivery": "foreground", "others_since_frame": others, "effect": effect_data(effect)} - if not screenshot: - return _result(summary, data) - try: - new = await self._computer_capture(identity) - except AgentEnvError as e: - return _result(f"{summary}; the screenshot after it failed ({e.code}: {e.message})", data) - return _result(f"{summary}, then captured {new.id}", {**data, "frame": new.data()}, new.encode()) - - async def _computer_ensure_ready(self) -> None: - """Refuse with device_unavailable until the ready checks have passed once; run them if none has yet.""" - if self._computer_ready: - return - async with self._computer_ready_lock: - if self._computer_ready: - return - failed = [c for c in await self._computer_ready_checks() if not c.ok] - if failed: - raise DeviceUnavailable("the computer isn't ready: " + "; ".join(f"{c.name}: {c.detail}" for c in failed)) + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + return card.served_tools(self, tools) - async def _computer_ready_checks(self) -> list[ReadyCheck]: - checks = [] - try: - display = await self._computer_display_now() - checks.append(ReadyCheck("display", True, f"{display.width}×{display.height} at scale {display.scale:g}")) - except AgentEnvError as e: - display = None - checks.append(ReadyCheck("display", False, e.message)) - if display is None: - checks.append(ReadyCheck("capture", False, "the display didn't answer")) - else: - try: - image = to_pil(await self._computer_call("capture"), display) - checks.append(ReadyCheck("capture", True, f"{image.size[0]}×{image.size[1]}")) - except AgentEnvError as e: - checks.append(ReadyCheck("capture", False, e.message)) - try: - checks += list(await self.ready_checks()) - except Exception as e: - checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) - if all(c.ok for c in checks): - self._computer_ready = True - return checks + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + return card.served_extensions(self, extensions) - def _computer_hold(self, identity: str, keys: list[str]) -> None: - held = self._computer_held.setdefault(identity, []) - held += [k for k in keys if k not in held] - previous = self._computer_releases.pop(identity, None) - if previous is not None: - previous.cancel() - self._computer_releases[identity] = asyncio.ensure_future( - self._computer_release_after(identity, self.settings.max_hold_ms)) + def served_operations(self, operations: list[str]) -> list[str]: + return card.served_operations(self, operations) - def _computer_release(self, identity: str, keys: list[str]) -> None: - held = [k for k in self._computer_held.get(identity, []) if k not in keys] - if held: - self._computer_held[identity] = held - return - self._computer_held.pop(identity, None) - release = self._computer_releases.pop(identity, None) - if release is not None and release is not asyncio.current_task(): - release.cancel() + def extension_params(self, uri: str) -> dict: + return card.extension_params(self, uri) - async def _computer_release_after(self, identity: str, ms: int) -> None: - """Release what ``identity`` still holds once ``max_hold_ms`` has passed, waiting as long as other input - keeps the queue busy.""" - await asyncio.sleep(ms / 1000) - while keys := list(reversed(self._computer_held.get(identity, []))): - try: - async with self._computer_queue.turn(self.settings.queue_wait_s): - await self._computer_call("key_up", keys) - except InputBusy: - continue - self._computer_release(identity, keys) + def declared_extensions(self) -> list[EnvironmentExtension]: + return card.declared_extensions(self) def _computer_require_init(self) -> None: - if not hasattr(self, "_computer_operations"): + if self._computer is None: raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") # The Linux defaults for the OS operations. @@ -347,37 +118,3 @@ def deco(cls: type) -> type: cls.supports = list(supports) return environment_card(name=name)(cls) if name else cls return deco - - -def _check_names(cls: type) -> None: - """Refuse a driver member that would replace the base class's own: its private members and the tools, which - define the contract. The hooks (``caller_identity``, ``ready_checks``, ...) and the features are the driver's.""" - reserved = {name for name in dir(ComputerDriverEnvironment) - if (name.startswith("_") and not name.startswith("__")) or name.startswith("computer_")} - clash = sorted(name for name in vars(cls) if name in reserved or name.startswith(("_computer_", "computer_"))) - if clash: - raise TypeError(f"{cls.__qualname__} redefines {', '.join(clash)}, which ComputerDriverEnvironment uses " - "itself; give the driver's own members other names") - - -def _check_features(cls: type) -> None: - problems = [] - unknown = [b for b in cls.buttons if b not in _BUTTONS] - if "left" not in cls.buttons or unknown: - problems.append(f"buttons include left and come from {list(_BUTTONS)}, got {list(cls.buttons)}") - for name, low in (("max_click_count", 1), ("max_text_length", 1), ("max_wait_ms", 60_000), ("max_drag_points", 2), - ("max_duration_ms", 0), ("max_key_repeat", 1), ("max_scroll_amount", 1)): - value = getattr(cls, name) - if not isinstance(value, int) or value < low: - problems.append(f"{name} is an integer of at least {low}, got {value!r}") - if cls.keys != "all": - try: - if [canonical(k) for k in cls.keys] != list(cls.keys): - problems.append("keys are W3C key values in their W3C case") - except AgentEnvError as e: - problems.append(f"keys: {e.message}") - if ("set_display" in declared(cls)) != bool(cls.display_modes): - problems.append("display_modes and @set_display come together") - if problems: - raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) - diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 5d1ebb8a..37a454a1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -2,55 +2,59 @@ and commands in the session.""" from __future__ import annotations -import asyncio -from collections import OrderedDict -from typing import Optional +from typing import TYPE_CHECKING, Optional from ..agent_env_environment import extension +from . import card +from .card import ( + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_READY, + COMPUTER_SPACES, + KEYBOARD_RANGES, + MAX_EXEC_TIMEOUT_S, +) from .errors import Internal, InvalidArgument, Unsupported from .operations import Display, ExecResult from .spaces import check_space -COMPUTER = "urn:agentenv:computer/v1" -COMPUTER_READY = "urn:agentenv:computer-ready/v1" -COMPUTER_DISPLAY = "urn:agentenv:computer-display/v1" -COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1" -COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1" -COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1" -COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" - -MAX_EXEC_TIMEOUT_S = 3600 -#: How many finished command results are kept for a repeated request_id; a running command is always joined. -KEEP_EXEC_RESULTS = 256 -KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)} +if TYPE_CHECKING: + from .runtime import ComputerRuntime + from .settings import ComputerSettings class Extensions: - """The extension handlers, mixed into ComputerDriverEnvironment; one whose operation the driver lacks is - withdrawn when the driver is built.""" + """The extension handlers, mixed into ComputerDriverEnvironment; the card serves those the driver's operations + support.""" + + _computer: ComputerRuntime + settings: ComputerSettings + display_modes: tuple + layouts: tuple[str, ...] @extension(COMPUTER_READY, op="status", method="GET", description="Whether the computer is usable now, by real checks; never sends input.") async def _computer_ready_status(self) -> dict: - checks = await self._computer_ready_checks() + checks = await self._computer.ready_checks() return {"ready": all(c.ok for c in checks), "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]} @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.") async def _computer_display_get(self) -> dict: - return self._computer_display_data(await self._computer_display_now()) + return _display_data(await self._computer.display_now(), self.display_modes) @extension(COMPUTER_DISPLAY, op="set") async def _computer_display_set(self, width: int, height: int, scale: Optional[float] = None) -> dict: if [width, height] not in [list(m) for m in self.display_modes]: modes = ", ".join(f"{w}×{h}" for w, h in self.display_modes) raise InvalidArgument(f"{width}×{height} is not one of this computer's display modes: {modes}") - async with self._computer_queue.turn(self.settings.queue_wait_s): - now = await self._computer_display_now() - changed = await self._computer_call("set_display", width, height, now.scale if scale is None else scale) + async with self._computer.queue.turn(self.settings.queue_wait_s): + now = await self._computer.display_now() + changed = await self._computer.call("set_display", width, height, now.scale if scale is None else scale) if not isinstance(changed, Display): - changed = await self._computer_display_now() - return self._computer_display_data(changed) + changed = await self._computer.display_now() + return _display_data(changed, self.display_modes) @extension(COMPUTER_SPACES, op="get", method="GET", description="Get or set the default coordinate space, written into the tool descriptions.") @@ -62,7 +66,8 @@ async def _computer_spaces_set(self, default_space: str, default_norm_scale: Opt space, scale = check_space(default_space, default_norm_scale, self.settings.default_space, self.settings.default_norm_scale) self.settings = self.settings.model_copy(update={"default_space": space, "default_norm_scale": scale}) - self._computer_describe_tools() + for name in card.point_tools(self): + await self.update_tool(name, description=card.point_description(self, name)) return await self._computer_spaces_get() @extension(COMPUTER_KEYBOARD, op="get", method="GET", @@ -74,7 +79,7 @@ async def _computer_keyboard_get(self) -> dict: @extension(COMPUTER_KEYBOARD, op="set") async def _computer_keyboard_set(self, tap_hold_ms: Optional[int] = None, type_delay_ms: Optional[int] = None, - layout: Optional[str] = None, max_hold_ms: Optional[int] = None) -> dict: + layout: Optional[str] = None, max_hold_ms: Optional[int] = None) -> dict: given = {"tap_hold_ms": tap_hold_ms, "type_delay_ms": type_delay_ms, "max_hold_ms": max_hold_ms} for name, value in given.items(): low, high = KEYBOARD_RANGES[name] @@ -89,8 +94,8 @@ async def _computer_keyboard_set(self, tap_hold_ms: Optional[int] = None, type_d @extension(COMPUTER_EXEC, op="run", description="Run a command inside the desktop session as the session user; a repeated request_id " "returns the first result.") - async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None, env: Optional[dict] = None, - timeout_s: Optional[float] = None) -> dict: + async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None, + env: Optional[dict] = None, timeout_s: Optional[float] = None) -> dict: if not (isinstance(argv, list) and argv and all(isinstance(a, str) for a in argv)): raise InvalidArgument("argv is a non-empty list of strings; name a shell to use one: ['bash', '-lc', '...']") if not (isinstance(request_id, str) and request_id): @@ -100,39 +105,24 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st timeout = self.settings.exec_timeout_s if timeout_s is None else timeout_s if not 0 < timeout <= MAX_EXEC_TIMEOUT_S: raise InvalidArgument(f"timeout_s is above 0 and at most {MAX_EXEC_TIMEOUT_S}, got {timeout}") - finished: OrderedDict = self._computer_exec_finished - if request_id in finished: - return finished[request_id] - running: dict = self._computer_exec_running - task = running.get(request_id) - if task is None: - task = asyncio.ensure_future(self._computer_run_command(argv, cwd, env, timeout)) - running[request_id] = task - - def done(ran: asyncio.Task) -> None: - if running.get(request_id) is ran: - del running[request_id] - if not ran.cancelled() and ran.exception() is None: - finished[request_id] = ran.result() - while len(finished) > KEEP_EXEC_RESULTS: - finished.popitem(last=False) - - task.add_done_callback(done) - return await asyncio.shield(task) - - async def _computer_run_command(self, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: - result = await self._computer_call("run_in_session", argv, cwd, env, timeout) - if not isinstance(result, ExecResult): - raise Internal(f"@run_in_session returned {type(result).__name__}, not an ExecResult") cap = self.settings.exec_output_bytes - return { - "exit_code": result.exit_code, - "stdout": result.stdout[:cap].decode("utf-8", errors="replace"), - "stderr": result.stderr[:cap].decode("utf-8", errors="replace"), - "truncated": result.truncated or len(result.stdout) > cap or len(result.stderr) > cap, - "timed_out": result.timed_out, - } - - def _computer_display_data(self, display: Display) -> dict: - return {"width": display.width, "height": display.height, "scale": display.scale, - "modes": [list(m) for m in self.display_modes]} + return await self._computer.command(request_id, lambda: _run(self._computer, cap, argv, cwd, env, timeout)) + + +async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict], + timeout: float) -> dict: + result = await computer.call("run_in_session", argv, cwd, env, timeout) + if not isinstance(result, ExecResult): + raise Internal(f"@run_in_session returned {type(result).__name__}, not an ExecResult") + return { + "exit_code": result.exit_code, + "stdout": result.stdout[:cap].decode("utf-8", errors="replace"), + "stderr": result.stderr[:cap].decode("utf-8", errors="replace"), + "truncated": result.truncated or len(result.stdout) > cap or len(result.stderr) > cap, + "timed_out": result.timed_out, + } + + +def _display_data(display: Display, modes: tuple) -> dict: + return {"width": display.width, "height": display.height, "scale": display.scale, + "modes": [list(m) for m in modes]} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py index 56451a0a..fa004dd8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -12,6 +12,7 @@ from .errors import Internal, StaleFrame from .operations import Display, Image +from .results import DisplayData, FrameData KEEP = 64 #: The JPEG quality a screenshot answers with when it names none. @@ -62,15 +63,15 @@ def from_screen(self, sx: float, sy: float) -> tuple[int, int]: def contains(self, fx: float, fy: float) -> bool: return 0 <= fx < self.width and 0 <= fy < self.height - def data(self) -> dict: - return { - "frame_id": self.id, - "width": self.width, - "height": self.height, - "display": {"width": self.display.width, "height": self.display.height, "scale": self.display.scale}, - "parent_frame_id": self.parent_id, - "captured_at": self.captured_at.isoformat().replace("+00:00", "Z"), - } + def result(self) -> FrameData: + return FrameData( + frame_id=self.id, + width=self.width, + height=self.height, + display=DisplayData(width=self.display.width, height=self.display.height, scale=self.display.scale), + parent_frame_id=self.parent_id, + captured_at=self.captured_at.isoformat().replace("+00:00", "Z"), + ) def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str]: """The frame as base64 and its MIME type.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py new file mode 100644 index 00000000..5794b9f7 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py @@ -0,0 +1,94 @@ +"""The computer_* tools' results: each tool's output schema, and the one way a result is sent, so what a tool +answers is what its schema says.""" +from __future__ import annotations + +from typing import Literal, Optional + +from mcp.types import CallToolResult, ImageContent, TextContent +from pydantic import BaseModel + +from .operations import Effect + + +class DisplayData(BaseModel): + width: int + height: int + scale: float + + +class FrameData(BaseModel): + frame_id: str + width: int + height: int + display: DisplayData + parent_frame_id: Optional[str] = None + captured_at: str + + +class ScreenshotResult(FrameData): + summary: str + + +class EffectData(BaseModel): + focus_changed: Optional[bool] = None + pointer_moved: Optional[bool] = None + clipboard_used: Optional[bool] = None + detail: Optional[str] = None + + @classmethod + def of(cls, effect: object) -> EffectData | None: + """What an input operation returned, when it is an Effect that says something.""" + if not isinstance(effect, Effect): + return None + data = cls(**vars(effect)) + return data if data.model_dump(exclude_none=True) else None + + +class InputResult(BaseModel): + summary: str + applied: Literal["yes", "no", "unknown"] + delivery: Literal["foreground", "background"] + others_since_frame: int + effect: Optional[EffectData] = None + frame: Optional[FrameData] = None + + +class WaitResult(BaseModel): + summary: str + waited_ms: int + frame: Optional[FrameData] = None + + +class Point(BaseModel): + x: int + y: int + + +class PointerPositionResult(BaseModel): + summary: str + screen: Point + frame_id: Optional[str] = None + x: Optional[int] = None + y: Optional[int] = None + inside_frame: Optional[bool] = None + + +class PointerData(BaseModel): + id: str + kind: Literal["foreground", "background"] + label: str + default: bool + + +class PointerListResult(BaseModel): + summary: str + pointers: list[PointerData] + + +def send(result: BaseModel, image: tuple[str, str] | None = None) -> CallToolResult: + """A result as a tool answers: the summary as the text block, then the image (base64, MIME type) when there is + one, and the result as structured content with its unset fields left out.""" + content: list = [TextContent(type="text", text=result.summary)] + if image is not None: + content.append(ImageContent(type="image", data=image[0], mimeType=image[1])) + return CallToolResult(content=content, structuredContent=result.model_dump(mode="json", exclude_none=True)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py new file mode 100644 index 00000000..a2bdc7c8 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -0,0 +1,208 @@ +"""What ComputerDriverEnvironment keeps for one driver, and the steps its tools and extensions share: calling the +driver's operations, frames, the input queue, readiness, held keys and command runs. A driver reaches it as +``self._computer``; it is the base class's own, so a driver's members can't replace any of it.""" +from __future__ import annotations + +import asyncio +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Awaitable, Callable + +from mcp.types import CallToolResult + +from ..errors import AgentEnvError +from ..tool_context import ToolContext +from .errors import DeliveryFailed, DeviceUnavailable, InputBusy, Internal +from .frames import Frame, FrameStore, to_pil +from .operations import GROUPS, INPUT, Display, ReadyCheck, resolve +from .queue import InputQueue +from .results import EffectData, InputResult, send + +if TYPE_CHECKING: + from .driver import ComputerDriverEnvironment + +#: How many finished command results are kept for a repeated request_id; a running command is always joined. +KEEP_EXEC_RESULTS = 256 + + +class ComputerRuntime: + def __init__(self, driver: ComputerDriverEnvironment) -> None: + self.driver = driver + self.operations = resolve(type(driver), driver.platform) + self.frames = FrameStore() + self.queue = InputQueue() + self.ready = False + self.held: dict[str, list[str]] = {} + self._ready_lock = asyncio.Lock() + self._releases: dict[str, asyncio.Task] = {} + self._commands_running: dict[str, asyncio.Task] = {} + self._commands_finished: OrderedDict = OrderedDict() + + def has(self, capability: str) -> bool: + if capability in GROUPS: + return all(op in self.operations for op in GROUPS[capability]) + return capability in self.operations + + async def call(self, operation: str, *args: Any) -> Any: + """Run one of the driver's operations: an AgentEnvError passes through, anything else is an input that may + have been delivered (delivery_failed) or an internal failure.""" + method = getattr(self.driver, self.operations[operation]) + try: + result = method(*args) + return await result if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future) else result + except AgentEnvError: + raise + except Exception as e: + error = DeliveryFailed if operation in INPUT else Internal + raise error(f"@{operation} failed: {type(e).__name__}: {e}") from e + + async def display_now(self) -> Display: + display = await self.call("display") + if not isinstance(display, Display) or display.width <= 0 or display.height <= 0 or display.scale <= 0: + raise Internal(f"@display returned {display!r}, not a Display with a positive size and scale") + return display + + async def capture(self, identity: str) -> Frame: + display = await self.display_now() + image = to_pil(await self.call("capture"), display) + return self.frames.add(image, display, self.queue.snapshot(), identity=identity) + + async def pointer(self) -> tuple[int, int]: + where = await self.call("pointer_position") + try: + x, y = where + return int(x), int(y) + except (TypeError, ValueError): + raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None + + async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, + summary: str, mapped: bool = False, repeat: int = 1, hold: tuple | None = None, + move_to: tuple | None = None, then: Callable | None = None) -> CallToolResult: + """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result + with others' input since ``frame`` and, when asked, a fresh frame. A point ``mapped`` from ``frame`` is + checked again once the call has its turn, since the display may have changed while it waited; ``then`` + runs as soon as the input is delivered.""" + identity = self.driver.caller_identity(ctx) + sent = False + async with self.queue.turn(self.driver.settings.queue_wait_s): + if mapped and frame is not None: + self.frames.check_current(frame, await self.display_now()) + others = self.queue.others_since(frame.inputs, identity) if frame is not None else 0 + effect = None + try: + if move_to is not None: + await self.call("pointer_move", *move_to) + sent = True + for _ in range(repeat): + effect = await self.call(operation, *args) + sent = True + if hold is not None: + await asyncio.sleep(hold[0] / 1000) + await self.call("key_up", list(reversed(hold[1]))) + except AgentEnvError as e: + if sent: + e.applied = "unknown" + raise + finally: + if sent: + self.queue.delivered(identity) + if then is not None: + then() + result = InputResult(summary=summary, applied="yes", delivery="foreground", others_since_frame=others, + effect=EffectData.of(effect)) + if not screenshot: + return send(result) + try: + new = await self.capture(identity) + except AgentEnvError as e: + result.summary = f"{summary}; the screenshot after it failed ({e.code}: {e.message})" + return send(result) + result.summary, result.frame = f"{summary}, then captured {new.id}", new.result() + return send(result, new.encode()) + + async def ensure_ready(self) -> None: + """Refuse with device_unavailable until the ready checks have passed once; run them if none has yet.""" + if self.ready: + return + async with self._ready_lock: + if self.ready: + return + failed = [c for c in await self.ready_checks() if not c.ok] + if failed: + raise DeviceUnavailable("the computer isn't ready: " + "; ".join(f"{c.name}: {c.detail}" for c in failed)) + + async def ready_checks(self) -> list[ReadyCheck]: + checks = [] + try: + display = await self.display_now() + checks.append(ReadyCheck("display", True, f"{display.width}×{display.height} at scale {display.scale:g}")) + except AgentEnvError as e: + display = None + checks.append(ReadyCheck("display", False, e.message)) + if display is None: + checks.append(ReadyCheck("capture", False, "the display didn't answer")) + else: + try: + image = to_pil(await self.call("capture"), display) + checks.append(ReadyCheck("capture", True, f"{image.size[0]}×{image.size[1]}")) + except AgentEnvError as e: + checks.append(ReadyCheck("capture", False, e.message)) + try: + checks += list(await self.driver.ready_checks()) + except Exception as e: + checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) + if all(c.ok for c in checks): + self.ready = True + return checks + + def hold(self, identity: str, keys: list[str]) -> None: + held = self.held.setdefault(identity, []) + held += [k for k in keys if k not in held] + previous = self._releases.pop(identity, None) + if previous is not None: + previous.cancel() + self._releases[identity] = asyncio.ensure_future( + self._release_after(identity, self.driver.settings.max_hold_ms)) + + def release(self, identity: str, keys: list[str]) -> None: + held = [k for k in self.held.get(identity, []) if k not in keys] + if held: + self.held[identity] = held + return + self.held.pop(identity, None) + release = self._releases.pop(identity, None) + if release is not None and release is not asyncio.current_task(): + release.cancel() + + async def _release_after(self, identity: str, ms: int) -> None: + """Release what ``identity`` still holds once ``max_hold_ms`` has passed, waiting as long as other input + keeps the queue busy.""" + await asyncio.sleep(ms / 1000) + while keys := list(reversed(self.held.get(identity, []))): + try: + async with self.queue.turn(self.driver.settings.queue_wait_s): + await self.call("key_up", keys) + except InputBusy: + continue + self.release(identity, keys) + + async def command(self, request_id: str, start: Callable[[], Awaitable[dict]]) -> dict: + """The result of the command ``request_id`` names: a running one is joined, one of the last + ``KEEP_EXEC_RESULTS`` finished is answered again, and anything else is started.""" + finished, running = self._commands_finished, self._commands_running + if request_id in finished: + return finished[request_id] + task = running.get(request_id) + if task is None: + task = asyncio.ensure_future(start()) + running[request_id] = task + + def done(ran: asyncio.Task) -> None: + if running.get(request_id) is ran: + del running[request_id] + if not ran.cancelled() and ran.exception() is None: + finished[request_id] = ran.result() + while len(finished) > KEEP_EXEC_RESULTS: + finished.popitem(last=False) + + task.add_done_callback(done) + return await asyncio.shield(task) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index 182e7c05..b3f2de35 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -4,23 +4,40 @@ import asyncio import unicodedata -from typing import Annotated, Literal, Optional +from typing import TYPE_CHECKING, Annotated, Any, Callable, Literal, Optional -from mcp.types import CallToolResult, ImageContent, TextContent -from pydantic import BaseModel, Field +from mcp.types import CallToolResult +from pydantic import Field from ..agent_env_environment import tool from ..tool_context import ToolContext from .errors import InvalidArgument, Unsupported from .frames import Frame from .keys import chord -from .operations import Effect +from .results import ( + InputResult, + Point, + PointerData, + PointerListResult, + PointerPositionResult, + ScreenshotResult, + WaitResult, + send, +) from .spaces import check_space, frame_point, frame_region +if TYPE_CHECKING: + from .runtime import ComputerRuntime + from .settings import ComputerSettings + _READ = {"readOnlyHint": True, "openWorldHint": False} _INPUT = {"readOnlyHint": False, "destructiveHint": False, "idempotentHint": False, "openWorldHint": False} Number = Annotated[float, Field(description="A coordinate in the chosen space")] +AtPointer = Annotated[Optional[float], Field( + description="A coordinate in the chosen space. Required unless the computer declares computer-pointer/v1; then " + "leave out x and y to act at the pointer")] +MoveFirst = Annotated[Optional[float], Field(description="Move here first")] FrameId = Annotated[Optional[str], Field(description="The frame the point refers to; defaults to your latest")] SpaceArg = Annotated[Optional[Literal["frame", "norm"]], Field( description="frame: pixels of the frame's image; norm: 0 to norm_scale over it")] @@ -30,98 +47,23 @@ Button = Annotated[Optional[str], Field(description="Omitted, the primary press; others only where declared")] -class DisplayData(BaseModel): - width: int - height: int - scale: float - - -class FrameData(BaseModel): - frame_id: str - width: int - height: int - display: DisplayData - parent_frame_id: Optional[str] = None - captured_at: str - - -class ScreenshotResult(FrameData): - summary: str - - -class EffectData(BaseModel): - focus_changed: Optional[bool] = None - pointer_moved: Optional[bool] = None - clipboard_used: Optional[bool] = None - detail: Optional[str] = None - - -class InputResult(BaseModel): - summary: str - applied: Literal["yes", "no", "unknown"] - delivery: Literal["foreground", "background"] - others_since_frame: int - effect: Optional[EffectData] = None - frame: Optional[FrameData] = None - - -class WaitResult(BaseModel): - summary: str - waited_ms: int - frame: Optional[FrameData] = None - - -class Point(BaseModel): - x: int - y: int - - -class PointerPositionResult(BaseModel): - summary: str - screen: Point - frame_id: Optional[str] = None - x: Optional[int] = None - y: Optional[int] = None - inside_frame: Optional[bool] = None - - -class PointerData(BaseModel): - id: str - kind: Literal["foreground", "background"] - label: str - default: bool - - -class PointerListResult(BaseModel): - summary: str - pointers: list[PointerData] - - -#: The tools a group serves, and the tools whose description names the default coordinate space. -GROUP_TOOLS = { - "pointer": ("computer_pointer_move", "computer_pointer_down", "computer_pointer_up", "computer_pointer_position", - "computer_pointer_list"), - "keyboard": ("computer_keyboard_down", "computer_keyboard_up"), -} -POINT_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_pointer_move", - "computer_pointer_down", "computer_pointer_up") -POINT_OPTIONAL_WITH_POINTER = ("computer_click", "computer_scroll") - - -def _result(summary: str, data: dict, image: tuple[str, str] | None = None) -> CallToolResult: - content: list = [TextContent(type="text", text=summary)] - if image is not None: - content.append(ImageContent(type="image", data=image[0], mimeType=image[1])) - return CallToolResult(content=content, structuredContent={"summary": summary, **data}) - - -def _fmt(x: float, y: float) -> str: - return f"({x:g}, {y:g})" - - class Tools: """The tools, mixed into ComputerDriverEnvironment.""" + _computer: ComputerRuntime + settings: ComputerSettings + buttons: tuple[str, ...] + keys: Any + max_click_count: int + max_text_length: int + max_wait_ms: int + max_drag_points: int + max_duration_ms: int + max_key_repeat: int + max_scroll_amount: int + has: Callable[[str], bool] + caller_identity: Callable[[ToolContext], str] + @tool(annotations=_READ) async def computer_screenshot( self, @@ -136,31 +78,32 @@ async def computer_screenshot( ) -> Annotated[CallToolResult, ScreenshotResult]: """Capture the screen as a frame: an image plus a frame_id you can point into later, the image size, and the display resolution and scale. With region, crop an existing frame at native resolution (zoom).""" - await self._computer_ensure_ready() + computer = self._computer + await computer.ensure_ready() identity = self.caller_identity(ctx) if quality is not None and not (format == "jpeg" and 1 <= quality <= 95): raise InvalidArgument("quality is for format=jpeg, from 1 to 95") if region is None: if frame_id is not None: raise InvalidArgument("frame_id names the frame a region crops; give a region too") - frame = await self._computer_capture(identity) + frame = await computer.capture(identity) summary = f"Captured {frame.id}: {frame.width}×{frame.height} of the {_display(frame)} display" else: - parent = self._computer_frames.get(frame_id, identity) - self._computer_frames.check_current(parent, await self._computer_display_now()) - space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) - box = frame_region(parent, region, space, scale) - frame = self._computer_frames.add(parent.image.crop(box), parent.display, parent.inputs, - identity=identity, parent=parent, origin=(box[0], box[1])) + parent = computer.frames.get(frame_id, identity) + computer.frames.check_current(parent, await computer.display_now()) + box = frame_region(parent, region, *_space(self, space, norm_scale)) + frame = computer.frames.add(parent.image.crop(box), parent.display, parent.inputs, + identity=identity, parent=parent, origin=(box[0], box[1])) summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})" - return _result(summary, frame.data(), frame.encode(format, quality)) + result = ScreenshotResult(summary=summary, **frame.result().model_dump()) + return send(result, frame.encode(format, quality)) @tool(annotations=_INPUT) async def computer_click( self, ctx: ToolContext, - x: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, - y: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, + x: AtPointer = None, + y: AtPointer = None, frame_id: FrameId = None, button: Button = None, count: Annotated[int, Field(description="1 by default; double and triple clicks where declared")] = 1, @@ -170,14 +113,13 @@ async def computer_click( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Click at a point of a frame.""" - await self._computer_ensure_ready() - button = self._computer_button(button) - if not 1 <= count <= self.max_click_count: - raise InvalidArgument(f"count is from 1 to {self.max_click_count} on this computer, got {count}") - held = self._computer_modifiers(modifiers) - frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) + await self._computer.ensure_ready() + button = _button(self, button) + _check_range("count", count, 1, self.max_click_count) + held = _modifiers(self, modifiers) + frame, (sx, sy), where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) times = f" ×{count}" if count > 1 else "" - return await self._computer_act(ctx, "click", (sx, sy, button, count, held), frame=frame, mapped=mapped, + return await self._computer.act(ctx, "click", (sx, sy, button, count, held), frame=frame, mapped=mapped, screenshot=screenshot, summary=f"Clicked {button}{times} {where}") @tool(annotations=_INPUT) @@ -194,26 +136,25 @@ async def computer_drag( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Press at the first point, move through the path, and release at the last.""" - await self._computer_ensure_ready() - button = self._computer_button(button) + computer = self._computer + await computer.ensure_ready() + button = _button(self, button) if not isinstance(path, list) or not 2 <= len(path) <= self.max_drag_points: raise InvalidArgument(f"path has from 2 to {self.max_drag_points} points on this computer") duration = self.settings.drag_duration_ms if duration_ms is None else duration_ms - if not 0 <= duration <= self.max_duration_ms: - raise InvalidArgument(f"duration_ms is from 0 to {self.max_duration_ms} on this computer, got {duration}") - held = self._computer_modifiers(modifiers) - identity = self.caller_identity(ctx) - frame = self._computer_frames.get(frame_id, identity) - self._computer_frames.check_current(frame, await self._computer_display_now()) - space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) + _check_range("duration_ms", duration, 0, self.max_duration_ms) + held = _modifiers(self, modifiers) + frame = computer.frames.get(frame_id, self.caller_identity(ctx)) + computer.frames.check_current(frame, await computer.display_now()) + space, scale = _space(self, space, norm_scale) screen = [] for point in path: if not (isinstance(point, list) and len(point) == 2): raise InvalidArgument("each path point is [x, y]") screen.append(frame.to_screen(*frame_point(frame, point[0], point[1], space, scale))) - return await self._computer_act(ctx, "drag", (screen, button, duration, held), frame=frame, mapped=True, - screenshot=screenshot, - summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") + return await computer.act(ctx, "drag", (screen, button, duration, held), frame=frame, mapped=True, + screenshot=screenshot, + summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") @tool(annotations=_INPUT) async def computer_scroll( @@ -221,8 +162,8 @@ async def computer_scroll( ctx: ToolContext, direction: Annotated[Literal["up", "down", "left", "right"], Field( description="Where the view moves through the content: down shows what is further down")], - x: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, - y: Annotated[Optional[float], Field(description="A coordinate in the chosen space")] = None, + x: AtPointer = None, + y: AtPointer = None, frame_id: FrameId = None, amount: Annotated[int, Field(description="How many steps (wheel notches)")] = 3, modifiers: Modifiers = None, @@ -231,12 +172,11 @@ async def computer_scroll( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Scroll the content under a point.""" - await self._computer_ensure_ready() - if not 1 <= amount <= self.max_scroll_amount: - raise InvalidArgument(f"amount is from 1 to {self.max_scroll_amount} on this computer, got {amount}") - held = self._computer_modifiers(modifiers) - frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, mapped=mapped, + await self._computer.ensure_ready() + _check_range("amount", amount, 1, self.max_scroll_amount) + held = _modifiers(self, modifiers) + frame, (sx, sy), where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + return await self._computer.act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, mapped=mapped, screenshot=screenshot, summary=f"Scrolled {direction} {amount} {where}") @tool(annotations=_INPUT) @@ -248,7 +188,8 @@ async def computer_type( ) -> Annotated[CallToolResult, InputResult]: """Type text into whatever has keyboard focus. Text that can't be typed whole is refused before anything is typed; the clipboard is left as it was.""" - await self._computer_ensure_ready() + computer = self._computer + await computer.ensure_ready() if not text: raise InvalidArgument("text is empty") if len(text) > self.max_text_length: @@ -256,9 +197,8 @@ async def computer_type( control = sorted({f"U+{ord(c):04X}" for c in text if c not in "\n\t" and unicodedata.category(c) == "Cc"}) if control: raise InvalidArgument(f"text has control characters {control}; press keys with computer_key") - identity = self.caller_identity(ctx) - return await self._computer_act(ctx, "type_text", (text,), frame=self._computer_frames.latest(identity), - screenshot=screenshot, summary=f"Typed {len(text)} characters") + return await computer.act(ctx, "type_text", (text,), frame=computer.frames.latest(self.caller_identity(ctx)), + screenshot=screenshot, summary=f"Typed {len(text)} characters") @tool(annotations=_INPUT) async def computer_key( @@ -271,14 +211,13 @@ async def computer_key( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, InputResult]: """Press one key chord.""" - await self._computer_ensure_ready() + computer = self._computer + await computer.ensure_ready() names = chord(keys, self.keys) - if not 1 <= repeat <= self.max_key_repeat: - raise InvalidArgument(f"repeat is from 1 to {self.max_key_repeat} on this computer, got {repeat}") - identity = self.caller_identity(ctx) + _check_range("repeat", repeat, 1, self.max_key_repeat) times = f" ×{repeat}" if repeat > 1 else "" - return await self._computer_act(ctx, "press", (names,), frame=self._computer_frames.latest(identity), - screenshot=screenshot, repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") + return await computer.act(ctx, "press", (names,), frame=computer.frames.latest(self.caller_identity(ctx)), + screenshot=screenshot, repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") @tool(annotations=_READ) async def computer_wait( @@ -288,15 +227,15 @@ async def computer_wait( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, WaitResult]: """Wait, then optionally return a fresh frame.""" - await self._computer_ensure_ready() - if not 0 <= ms <= self.max_wait_ms: - raise InvalidArgument(f"ms is from 0 to {self.max_wait_ms} on this computer, got {ms}") + await self._computer.ensure_ready() + _check_range("ms", ms, 0, self.max_wait_ms) await asyncio.sleep(ms / 1000) summary = f"Waited {ms} ms" if not screenshot: - return _result(summary, {"waited_ms": ms}) - frame = await self._computer_capture(self.caller_identity(ctx)) - return _result(f"{summary}, then captured {frame.id}", {"waited_ms": ms, "frame": frame.data()}, frame.encode()) + return send(WaitResult(summary=summary, waited_ms=ms)) + frame = await self._computer.capture(self.caller_identity(ctx)) + return send(WaitResult(summary=f"{summary}, then captured {frame.id}", waited_ms=ms, frame=frame.result()), + frame.encode()) @tool(annotations=_INPUT) async def computer_pointer_move( @@ -310,9 +249,9 @@ async def computer_pointer_move( norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Move the pointer to a point; hover effects happen.""" - await self._computer_ensure_ready() - frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, "pointer_move", (sx, sy), frame=frame, mapped=mapped, + await self._computer.ensure_ready() + frame, (sx, sy), where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + return await self._computer.act(ctx, "pointer_move", (sx, sy), frame=frame, mapped=mapped, screenshot=screenshot, summary=f"Moved the pointer {where}") @tool(annotations=_INPUT) @@ -320,32 +259,32 @@ async def computer_pointer_down( self, ctx: ToolContext, button: Button = None, - x: Annotated[Optional[float], Field(description="Move here first")] = None, - y: Annotated[Optional[float], Field(description="Move here first")] = None, + x: MoveFirst = None, + y: MoveFirst = None, frame_id: FrameId = None, screenshot: Screenshot = False, space: SpaceArg = None, norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Press a button, optionally after moving to a point.""" - return await self._computer_button_action(ctx, "pointer_down", "Pressed", button, x, y, frame_id, screenshot, space, - norm_scale) + return await _button_action(self, ctx, "pointer_down", "Pressed", button, x, y, frame_id, screenshot, space, + norm_scale) @tool(annotations=_INPUT) async def computer_pointer_up( self, ctx: ToolContext, button: Button = None, - x: Annotated[Optional[float], Field(description="Move here first")] = None, - y: Annotated[Optional[float], Field(description="Move here first")] = None, + x: MoveFirst = None, + y: MoveFirst = None, frame_id: FrameId = None, screenshot: Screenshot = False, space: SpaceArg = None, norm_scale: NormScale = None, ) -> Annotated[CallToolResult, InputResult]: """Release a button, optionally after moving to a point.""" - return await self._computer_button_action(ctx, "pointer_up", "Released", button, x, y, frame_id, screenshot, space, - norm_scale) + return await _button_action(self, ctx, "pointer_up", "Released", button, x, y, frame_id, screenshot, space, + norm_scale) @tool(annotations=_READ) async def computer_pointer_position( @@ -354,24 +293,25 @@ async def computer_pointer_position( frame_id: Annotated[Optional[str], Field(description="The frame to report in; defaults to your latest")] = None, ) -> Annotated[CallToolResult, PointerPositionResult]: """Where the pointer is, in a frame's pixels and in screen units, and whether it is inside that frame.""" - await self._computer_ensure_ready() - sx, sy = await self._computer_pointer() + computer = self._computer + await computer.ensure_ready() + sx, sy = await computer.pointer() + screen = Point(x=sx, y=sy) identity = self.caller_identity(ctx) - frame = self._computer_frames.get(frame_id, identity) if frame_id is not None else self._computer_frames.latest(identity) - data: dict = {"screen": {"x": sx, "y": sy}} + frame = computer.frames.get(frame_id, identity) if frame_id is not None else computer.frames.latest(identity) if frame is None: - return _result(f"The pointer is at screen ({sx}, {sy})", data) + return send(PointerPositionResult(summary=f"The pointer is at screen ({sx}, {sy})", screen=screen)) fx, fy = frame.from_screen(sx, sy) inside = frame.contains(fx, fy) - data.update(frame_id=frame.id, x=fx, y=fy, inside_frame=inside) where = f"({fx}, {fy}) in {frame.id}" if inside else f"outside {frame.id}, at ({fx}, {fy}) in its pixels" - return _result(f"The pointer is {where}, screen ({sx}, {sy})", data) + return send(PointerPositionResult(summary=f"The pointer is {where}, screen ({sx}, {sy})", screen=screen, + frame_id=frame.id, x=fx, y=fy, inside_frame=inside)) @tool(annotations=_READ) async def computer_pointer_list(self) -> Annotated[CallToolResult, PointerListResult]: """The pointers you may use, and which one is your default.""" - pointers = [{"id": "default", "kind": "foreground", "label": "The computer's pointer", "default": True}] - return _result("1 pointer: default (foreground)", {"pointers": pointers}) + pointer = PointerData(id="default", kind="foreground", label="The computer's pointer", default=True) + return send(PointerListResult(summary="1 pointer: default (foreground)", pointers=[pointer])) @tool(annotations=_INPUT) async def computer_keyboard_down( @@ -384,18 +324,19 @@ async def computer_keyboard_down( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, InputResult]: """Press keys and keep them down.""" - await self._computer_ensure_ready() + computer = self._computer + await computer.ensure_ready() names = chord(keys, self.keys) if hold_ms is not None and not 0 <= hold_ms <= self.settings.max_hold_ms: raise InvalidArgument(f"hold_ms is from 0 to {self.settings.max_hold_ms}, got {hold_ms}") identity = self.caller_identity(ctx) + frame = computer.frames.latest(identity) if hold_ms is not None: - return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), - screenshot=screenshot, hold=(hold_ms, names), - summary=f"Held {'+'.join(names)} for {hold_ms} ms") - return await self._computer_act(ctx, "key_down", (names,), frame=self._computer_frames.latest(identity), - screenshot=screenshot, summary=f"Pressed and held {'+'.join(names)}", - then=lambda: self._computer_hold(identity, names)) + return await computer.act(ctx, "key_down", (names,), frame=frame, screenshot=screenshot, + hold=(hold_ms, names), summary=f"Held {'+'.join(names)} for {hold_ms} ms") + return await computer.act(ctx, "key_down", (names,), frame=frame, screenshot=screenshot, + summary=f"Pressed and held {'+'.join(names)}", + then=lambda: computer.hold(identity, names)) @tool(annotations=_INPUT) async def computer_keyboard_up( @@ -406,66 +347,76 @@ async def computer_keyboard_up( screenshot: Screenshot = False, ) -> Annotated[CallToolResult, InputResult]: """Release held keys.""" - await self._computer_ensure_ready() + computer = self._computer + await computer.ensure_ready() identity = self.caller_identity(ctx) - names = chord(keys, self.keys) if keys is not None else list(self._computer_held.get(identity, [])) + names = chord(keys, self.keys) if keys is not None else list(computer.held.get(identity, [])) if not names: raise InvalidArgument("you are holding no keys") - return await self._computer_act(ctx, "key_up", (names,), frame=self._computer_frames.latest(identity), - screenshot=screenshot, summary=f"Released {'+'.join(names)}", - then=lambda: self._computer_release(identity, names)) - - async def _computer_button_action(self, ctx, op, verb, button, x, y, frame_id, screenshot, space, norm_scale): - await self._computer_ensure_ready() - button = self._computer_button(button) - if (x is None) != (y is None): - raise InvalidArgument("give both x and y, or neither") - if x is None: - return await self._computer_act(ctx, op, (button,), - frame=self._computer_frames.latest(self.caller_identity(ctx)), - screenshot=screenshot, summary=f"{verb} {button}") - frame, (sx, sy), where, mapped = await self._computer_point(ctx, x, y, frame_id, space, norm_scale) - return await self._computer_act(ctx, op, (button,), frame=frame, mapped=mapped, screenshot=screenshot, - move_to=(sx, sy), summary=f"{verb} {button} {where}") - - async def _computer_point(self, ctx, x, y, frame_id, space, norm_scale) -> tuple: - """The frame, the screen pixel, a phrase for a point and whether it was mapped from the frame; with no point - and the pointer group, the pointer.""" - identity = self.caller_identity(ctx) - if x is None and y is None and self.has("pointer") and frame_id is None: - sx, sy = await self._computer_pointer() - return self._computer_frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})", False - if x is None or y is None: - raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if self.has("pointer") else "")) - frame = self._computer_frames.get(frame_id, identity) - self._computer_frames.check_current(frame, await self._computer_display_now()) - space, scale = check_space(space, norm_scale, self.settings.default_space, self.settings.default_norm_scale) - fx, fy = frame_point(frame, x, y, space, scale) - in_space = f" ({space} {scale})" if space == "norm" else "" - return frame, frame.to_screen(fx, fy), f"at {_fmt(x, y)}{in_space} in {frame.id}", True - - def _computer_button(self, button: str | None) -> str: - button = button or "left" - if button not in self.buttons: - raise Unsupported(f"this computer has no {button} button; it has {', '.join(self.buttons)}") - return button - - def _computer_modifiers(self, modifiers: list | None) -> list[str]: - if not modifiers: - return [] - if not isinstance(modifiers, list): - raise InvalidArgument("modifiers is a list of W3C key names") - return chord(modifiers, self.keys) + return await computer.act(ctx, "key_up", (names,), frame=computer.frames.latest(identity), + screenshot=screenshot, summary=f"Released {'+'.join(names)}", + then=lambda: computer.release(identity, names)) + + +async def _button_action(driver: Tools, ctx: ToolContext, op: str, verb: str, button: str | None, x: float | None, + y: float | None, frame_id: str | None, screenshot: bool, space: str | None, + norm_scale: int | None) -> CallToolResult: + computer = driver._computer + await computer.ensure_ready() + button = _button(driver, button) + if (x is None) != (y is None): + raise InvalidArgument("give both x and y, or neither") + if x is None: + return await computer.act(ctx, op, (button,), frame=computer.frames.latest(driver.caller_identity(ctx)), + screenshot=screenshot, summary=f"{verb} {button}") + frame, (sx, sy), where, mapped = await _point(driver, ctx, x, y, frame_id, space, norm_scale) + return await computer.act(ctx, op, (button,), frame=frame, mapped=mapped, screenshot=screenshot, + move_to=(sx, sy), summary=f"{verb} {button} {where}") + + +async def _point(driver: Tools, ctx: ToolContext, x: float | None, y: float | None, frame_id: str | None, + space: str | None, norm_scale: int | None) -> tuple[Frame | None, tuple[int, int], str, bool]: + """The frame, the screen pixel, a phrase for a point and whether it was mapped from the frame; with no point + and the pointer group, the pointer.""" + computer = driver._computer + identity = driver.caller_identity(ctx) + if x is None and y is None and driver.has("pointer") and frame_id is None: + sx, sy = await computer.pointer() + return computer.frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})", False + if x is None or y is None: + raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if driver.has("pointer") else "")) + frame = computer.frames.get(frame_id, identity) + computer.frames.check_current(frame, await computer.display_now()) + space, scale = _space(driver, space, norm_scale) + fx, fy = frame_point(frame, x, y, space, scale) + in_space = f" ({space} {scale})" if space == "norm" else "" + return frame, frame.to_screen(fx, fy), f"at ({x:g}, {y:g}){in_space} in {frame.id}", True + + +def _space(driver: Tools, space: str | None, norm_scale: int | None) -> tuple[str, int]: + return check_space(space, norm_scale, driver.settings.default_space, driver.settings.default_norm_scale) + + +def _button(driver: Tools, button: str | None) -> str: + button = button or "left" + if button not in driver.buttons: + raise Unsupported(f"this computer has no {button} button; it has {', '.join(driver.buttons)}") + return button + + +def _modifiers(driver: Tools, modifiers: list | None) -> list[str]: + if not modifiers: + return [] + if not isinstance(modifiers, list): + raise InvalidArgument("modifiers is a list of W3C key names") + return chord(modifiers, driver.keys) + + +def _check_range(name: str, value: int, low: int, high: int) -> None: + if not low <= value <= high: + raise InvalidArgument(f"{name} is from {low} to {high} on this computer, got {value}") def _display(frame: Frame) -> str: d = frame.display return f"{d.width}×{d.height}" + (f" at scale {d.scale:g}" if d.scale != 1 else "") - - -def effect_data(effect: object) -> dict | None: - if not isinstance(effect, Effect): - return None - data = {k: v for k, v in vars(effect).items() if v is not None} - return data or None - diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index e5715925..e029f62b 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -42,6 +42,7 @@ from agentenv_protocol.computer.operations import declared, resolve # noqa: E402 from agentenv_protocol.computer.platform import detect, normalize # noqa: E402 from agentenv_protocol.computer.settings import load_settings # noqa: E402 +from agentenv_protocol.computer.tools import _modifiers # noqa: E402 MACOS = Platform(os="macos", arch="arm64") @@ -96,7 +97,7 @@ def test_a_group_with_a_member_that_doesnt_match_the_platform_is_dropped(): assert FakeDriver(platform=LINUX).has("pointer") Driver = type("Driver", (FakeDriver,), {"move": pointer_move(os="windows")(lambda self, x, y: None)}) driver = Driver() - assert not driver.has("pointer") and "pointer_down" not in driver._computer_operations + assert not driver.has("pointer") and "pointer_down" not in driver._computer.operations def test_unknown_predicates_are_refused(): @@ -230,7 +231,7 @@ async def test_tools_refuse_until_the_ready_checks_pass_once(): assert refused.isError and refused.structuredContent["code"] == "device_unavailable" assert refused.structuredContent["retryable"] is True assert "display: @display failed: RuntimeError: no X server" in refused.structuredContent["message"] - assert not served.isError and driver._computer_ready + assert not served.isError and driver._computer.ready @pytest.mark.asyncio @@ -349,8 +350,11 @@ async def test_a_driver_mounted_on_an_app_serves_the_same_tools(): app = driver.mount(FastMCP("mounted")) async with create_connected_server_and_client_session(app) as session: tools = {t.name: t for t in (await session.list_tools()).tools} - assert tools["computer_click"].inputSchema["required"] == ["x", "y"] + refused = await session.call_tool("computer_click", {}) + assert "required" not in tools["computer_click"].inputSchema and "computer_pointer_move" not in tools assert "frame space by default" in tools["computer_scroll"].description + assert refused.structuredContent["code"] == "invalid_argument" + assert refused.structuredContent["message"] == "give x and y" @pytest.mark.asyncio @@ -359,7 +363,7 @@ async def test_concurrent_first_calls_run_the_ready_checks_once(): driver.slow["display"] = 0.05 async with mcp_session(driver) as session: results = await asyncio.gather(*(session.call_tool("computer_wait", {"ms": 0}) for _ in range(3))) - assert all(not r.isError for r in results) and driver._computer_ready + assert all(not r.isError for r in results) and driver._computer.ready def test_keys_and_modifiers_that_arent_lists_of_names_are_refused(): @@ -367,10 +371,10 @@ def test_keys_and_modifiers_that_arent_lists_of_names_are_refused(): with pytest.raises(InvalidArgument, match="a key is a W3C key value"): canonical(key) with pytest.raises(InvalidArgument, match="modifiers is a list"): - FakeDriver()._computer_modifiers("Shift") + _modifiers(FakeDriver(), "Shift") -@pytest.mark.parametrize("name", ["_computer_act", "_computer_frames", "computer_click", "_mounted", "_computer_new"]) +@pytest.mark.parametrize("name", ["_computer", "_computer_ready_status", "_computer_new", "computer_click", "_application", "_mounted"]) def test_a_driver_member_that_would_replace_the_base_classs_own_is_refused(name): with pytest.raises(TypeError, match=rf"Driver redefines {name}, which ComputerDriverEnvironment uses itself"): _define({name: lambda self: None}) diff --git a/packages/agentenv-protocol/tests/test_computer_extensions.py b/packages/agentenv-protocol/tests/test_computer_extensions.py index d3e18abf..a76b766f 100644 --- a/packages/agentenv-protocol/tests/test_computer_extensions.py +++ b/packages/agentenv-protocol/tests/test_computer_extensions.py @@ -171,7 +171,7 @@ async def test_a_background_apps_output_is_drained_after_the_call_returns(monkey @pytest.mark.asyncio async def test_only_the_latest_command_results_are_kept(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) - monkeypatch.setattr("agentenv_protocol.computer.extensions.KEEP_EXEC_RESULTS", 1) + monkeypatch.setattr("agentenv_protocol.computer.runtime.KEEP_EXEC_RESULTS", 1) count = ["sh", "-c", "echo x >> runs; wc -l < runs"] async with http_client(FakeDriver()) as http: for request_id in ("a", "b", "a"): @@ -182,7 +182,7 @@ async def test_only_the_latest_command_results_are_kept(monkeypatch, tmp_path): @pytest.mark.asyncio async def test_a_running_command_is_joined_however_many_others_finish_meanwhile(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) - monkeypatch.setattr("agentenv_protocol.computer.extensions.KEEP_EXEC_RESULTS", 1) + monkeypatch.setattr("agentenv_protocol.computer.runtime.KEEP_EXEC_RESULTS", 1) slow = _run(argv=["sh", "-c", "sleep 0.6; echo a >> runs"], request_id="a") async with http_client(FakeDriver()) as http: async def others_then_retry(): diff --git a/packages/agentenv-protocol/tests/test_computer_frames.py b/packages/agentenv-protocol/tests/test_computer_frames.py index b27e5cf3..6c5eb9b7 100644 --- a/packages/agentenv-protocol/tests/test_computer_frames.py +++ b/packages/agentenv-protocol/tests/test_computer_frames.py @@ -28,7 +28,7 @@ async def test_a_screenshot_is_a_native_frame_with_its_geometry(): second = await session.call_tool("computer_screenshot", {"format": "jpeg", "quality": 40}) data = first.structuredContent assert data["frame_id"] == "f1" and (data["width"], data["height"]) == (1280, 800) - assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and data["parent_frame_id"] is None + assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and "parent_frame_id" not in data assert data["captured_at"].endswith("Z") and data["summary"] == "Captured f1: 1280×800 of the 1280×800 display" assert first.content[0].text == data["summary"] and first.content[1].mimeType == "image/png" assert _image(first).size == (1280, 800) @@ -63,7 +63,7 @@ async def test_a_region_is_a_native_crop_whose_points_map_back_exactly(): async def _captured_at(driver: FakeDriver, frame_id: str) -> str: - return driver._computer_frames.get(frame_id, "default").data()["captured_at"] + return driver._computer.frames.get(frame_id, "default").result().captured_at @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_queue.py b/packages/agentenv-protocol/tests/test_computer_queue.py index 548d790e..d8eb3838 100644 --- a/packages/agentenv-protocol/tests/test_computer_queue.py +++ b/packages/agentenv-protocol/tests/test_computer_queue.py @@ -310,7 +310,7 @@ async def test_keys_held_down_stay_down_until_released(): nothing = await session.call_tool("computer_keyboard_up", {}) assert held == ["Shift", "Control"] and driver.screen.held == [] assert partly.structuredContent["summary"] == "Released Shift" and rest.structuredContent["summary"] == "Released Control" - assert nothing.structuredContent["code"] == "invalid_argument" and driver._computer_held == {} + assert nothing.structuredContent["code"] == "invalid_argument" and driver._computer.held == {} @pytest.mark.asyncio @@ -332,7 +332,7 @@ async def test_keys_held_past_max_hold_ms_are_released(): with anyio.fail_after(5): while driver.screen.held: await asyncio.sleep(0.01) - assert driver.screen.log[-1] == ("key_up", ("Alt",)) and driver._computer_held == {} + assert driver.screen.log[-1] == ("key_up", ("Alt",)) and driver._computer.held == {} @pytest.mark.asyncio @@ -397,7 +397,7 @@ async def test_a_screenshot_that_fails_after_the_input_reports_the_input_as_done assert not clicked.isError and clicked.structuredContent["applied"] == "yes" and "frame" not in clicked.structuredContent assert "the screenshot after it failed (internal: @capture failed: RuntimeError: mss died)" in clicked.content[0].text assert [e[0] for e in driver.screen.log] == ["click", "key_down"] - assert not held.isError and driver._computer_held == {"default": ["Shift"]} + assert not held.isError and driver._computer.held == {"default": ["Shift"]} @pytest.mark.asyncio @@ -412,7 +412,7 @@ async def test_the_automatic_release_waits_out_a_busy_queue(): while driver.screen.held: await asyncio.sleep(0.01) kinds = [e[0] for e in driver.screen.log] - assert kinds == ["key_down", "drag", "key_up"] and driver._computer_held == {} + assert kinds == ["key_down", "drag", "key_up"] and driver._computer.held == {} @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_spaces.py b/packages/agentenv-protocol/tests/test_computer_spaces.py index ebafee3f..d32314e6 100644 --- a/packages/agentenv-protocol/tests/test_computer_spaces.py +++ b/packages/agentenv-protocol/tests/test_computer_spaces.py @@ -2,10 +2,12 @@ clamped outside the frame or the range, with the default space set per deploy.""" from __future__ import annotations +import httpx import pytest pytest.importorskip("mcp.server.fastmcp") pytest.importorskip("PIL") +fastmcp = pytest.importorskip("fastmcp") from computer_fake import FakeDriver, FakeScreen, http_client, mcp_session # noqa: E402 from PIL import Image as PILImage # noqa: E402 @@ -83,3 +85,20 @@ async def test_setting_the_default_space_rewrites_the_descriptions(): card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click") assert card_click["description"] == after["computer_click"] assert refused.status_code == 400 and refused.json()["error"]["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_setting_the_default_space_rewrites_the_descriptions_on_fastmcp(): + app = fastmcp.FastMCP("desktop") + FakeDriver().mount(app) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app.http_app()), base_url="http://t") as http: + await http.post("/agentenv/ext/computer-spaces/set", json={"default_space": "norm", "default_norm_scale": 1000}) + card = (await http.get("/.well-known/agent-env.json")).json() + async with fastmcp.Client(app) as client: + listed = {t.name: t for t in await client.list_tools()} + await client.call_tool("computer_screenshot", {}) + clicked = await client.call_tool("computer_click", {"x": 500, "y": 500}) + card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click") + assert "norm space by default: 0 to 1000" in listed["computer_click"].description + assert card_click["description"] == listed["computer_click"].description + assert clicked.structured_content["summary"].startswith("Clicked left at (500, 500) (norm 1000) in f1") From d7686da4280ca935fc0bfe66c9c939151b5418d9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 14:07:19 -0700 Subject: [PATCH 14/18] fix(protocol): rewrite only the point tools a driver serves when the default space changes computer-spaces/v1 set chose the tools to update from the driver's operations, so a driver whose served_tools left out a point tool answered 500, after the new space was already set. It now reads the tools from mounted_card(), the card the mounted application serves, which the SDK adds beside update_tool. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 5 +++-- .../agentenv_protocol/agent_env_environment.py | 9 ++++++++- .../src/agentenv_protocol/computer/card.py | 15 +++------------ .../src/agentenv_protocol/computer/extensions.py | 9 ++++++--- .../tests/test_computer_spaces.py | 15 +++++++++++++++ .../tests/test_tool_annotations.py | 3 +++ 6 files changed, 38 insertions(+), 18 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 6cc25c9f..67c65714 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -96,8 +96,9 @@ with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and o where it can, False lists none); a tool returning `Annotated[CallToolResult, Model]` sends its result as is, image blocks included, and is listed with `Model`'s schema. An environment chooses at mount which of its `@tool` methods it serves, and with which descriptions, by overriding -`served_tools(tools)`. After the mount, `await env.update_tool(name, description=...)` changes a -served tool's description on the card and in `tools/list` alike, on both FastMCPs. +`served_tools(tools)`. After the mount, `env.mounted_card()` is the card the application serves, and +`await env.update_tool(name, description=...)` changes a served tool's description on it and in +`tools/list` alike, on both FastMCPs. `AgentEnvStarletteApplication` mounts the same handler onto a Starlette/FastAPI app instead of FastMCP; since those apps have no MCP tool registry, constructing one with `@tool` methods diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index 0427d438..02b1c889 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -504,11 +504,18 @@ def mount(self, app: Any) -> Any: self._mounted = True return app + def mounted_card(self) -> EnvironmentCard: + """The card the mounted application serves: what ``served_*`` kept, as ``update_tool`` left it.""" + return self._mounted_application().environment_card + async def update_tool(self, name: str, *, description: str) -> None: """Change a served tool's description after the mount: on the card and in tools/list alike.""" + await self._mounted_application().update_tool(name, description=description) + + def _mounted_application(self) -> Any: if self._application is None: raise RuntimeError(f"{type(self).__name__} is not mounted") - await self._application.update_tool(name, description=description) + return self._application def serve(self, transport: str = "streamable-http") -> None: if self.mcp is None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py index 06a3e459..671af965 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -36,9 +36,9 @@ def served_tools(driver: ComputerDriverEnvironment, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: - points = point_tools(driver) - return [t.model_copy(update={"description": point_description(driver, t.name)}) if t.name in points else t - for t in tools if t.name not in _unserved_tools(driver)] + unserved = {name for group, names in GROUP_TOOLS.items() if not driver.has(group) for name in names} + return [t.model_copy(update={"description": point_description(driver, t.name)}) if t.name in POINT_TOOLS else t + for t in tools if t.name not in unserved] def served_extensions(driver: ComputerDriverEnvironment, @@ -97,18 +97,9 @@ def declared_extensions(driver: ComputerDriverEnvironment) -> list[EnvironmentEx return declared -def point_tools(driver: ComputerDriverEnvironment) -> list[str]: - unserved = _unserved_tools(driver) - return [name for name in POINT_TOOLS if name not in unserved] - - def point_description(driver: ComputerDriverEnvironment, name: str) -> str: """A point tool's docstring, then the default coordinate space.""" space, scale = driver.settings.default_space, driver.settings.default_norm_scale note = (" Points are in frame space by default: pixels of the frame's image." if space == "frame" else f" Points are in norm space by default: 0 to {scale} over the frame, so send x and y on that grid.") return " ".join((getattr(type(driver), name).__doc__ or "").split()) + note - - -def _unserved_tools(driver: ComputerDriverEnvironment) -> set[str]: - return {name for group, names in GROUP_TOOLS.items() if not driver.has(group) for name in names} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 37a454a1..4ce81239 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -2,7 +2,7 @@ and commands in the session.""" from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Callable, Optional from ..agent_env_environment import extension from . import card @@ -20,6 +20,7 @@ from .spaces import check_space if TYPE_CHECKING: + from ..types import EnvironmentCard from .runtime import ComputerRuntime from .settings import ComputerSettings @@ -32,6 +33,7 @@ class Extensions: settings: ComputerSettings display_modes: tuple layouts: tuple[str, ...] + mounted_card: Callable[[], EnvironmentCard] @extension(COMPUTER_READY, op="status", method="GET", description="Whether the computer is usable now, by real checks; never sends input.") @@ -66,8 +68,9 @@ async def _computer_spaces_set(self, default_space: str, default_norm_scale: Opt space, scale = check_space(default_space, default_norm_scale, self.settings.default_space, self.settings.default_norm_scale) self.settings = self.settings.model_copy(update={"default_space": space, "default_norm_scale": scale}) - for name in card.point_tools(self): - await self.update_tool(name, description=card.point_description(self, name)) + for tool in self.mounted_card().capabilities.tools or []: + if tool.name in card.POINT_TOOLS: + await self.update_tool(tool.name, description=card.point_description(self, tool.name)) return await self._computer_spaces_get() @extension(COMPUTER_KEYBOARD, op="get", method="GET", diff --git a/packages/agentenv-protocol/tests/test_computer_spaces.py b/packages/agentenv-protocol/tests/test_computer_spaces.py index d32314e6..9352dbf1 100644 --- a/packages/agentenv-protocol/tests/test_computer_spaces.py +++ b/packages/agentenv-protocol/tests/test_computer_spaces.py @@ -12,6 +12,7 @@ from computer_fake import FakeDriver, FakeScreen, http_client, mcp_session # noqa: E402 from PIL import Image as PILImage # noqa: E402 +from agentenv_protocol import EnvironmentTool # noqa: E402 from agentenv_protocol.computer import ComputerSettings, InvalidArgument, OutOfBounds # noqa: E402 from agentenv_protocol.computer.frames import FrameStore # noqa: E402 from agentenv_protocol.computer.operations import Display # noqa: E402 @@ -102,3 +103,17 @@ async def test_setting_the_default_space_rewrites_the_descriptions_on_fastmcp(): assert "norm space by default: 0 to 1000" in listed["computer_click"].description assert card_click["description"] == listed["computer_click"].description assert clicked.structured_content["summary"].startswith("Clicked left at (500, 500) (norm 1000) in f1") + + +@pytest.mark.asyncio +async def test_setting_the_default_space_rewrites_only_the_point_tools_the_driver_serves(): + class NoScroll(FakeDriver): + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + return [t for t in super().served_tools(tools) if t.name != "computer_scroll"] + + driver = NoScroll() + async with http_client(driver) as http, mcp_session(driver) as session: + applied = await http.post("/agentenv/ext/computer-spaces/set", json={"default_space": "norm"}) + listed = {t.name: t.description for t in (await session.list_tools()).tools} + assert applied.status_code == 200 and "computer_scroll" not in listed + assert "norm space by default" in listed["computer_click"] and "norm space by default" in listed["computer_drag"] diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index b502667d..feeb1c58 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -179,6 +179,7 @@ async def test_update_tool_rewrites_a_description_on_the_card_and_in_tools_list_ card = await _card_tools(app.streamable_http_app()) assert tools["look"].description == card["look"]["description"] == "Look again." assert set(card) == set(tools) == {"look", "plain", "secret", "shot"} + assert {t.name: t.description for t in env.mounted_card().capabilities.tools}["look"] == "Look again." assert look.structuredContent == {"frame_id": "f1", "width": 1280} @@ -202,6 +203,8 @@ async def test_update_tool_rewrites_a_description_on_the_card_and_in_tools_list_ async def test_update_tool_needs_a_mounted_env_and_a_served_tool(): with pytest.raises(RuntimeError, match="_Env is not mounted"): await _Env().update_tool("look", description="x") + with pytest.raises(RuntimeError, match="_Env is not mounted"): + _Env().mounted_card() env = _Chooser() env.create_app() with pytest.raises(ValueError, match="no served tool is named 'secret'"): From 99cfd866ede0a7f4f6dad1a88829461e846e347e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 14:41:01 -0700 Subject: [PATCH 15/18] docs(protocol): add an X11 computer driver example with an image to run it examples/computer_x11 is the driver the end-to-end runs used: mss for capture and XTEST through python-xlib for input, the pointer and keyboard groups, an XTEST ready check, and a reconnect when the X connection drops. Its X calls run off the event loop. The image runs it on Xvfb beside a Tk practice app that records what reaches it, built from this checkout's SDK. test_computer_x11_example runs the driver against fake mss and Xlib modules, so it keeps up with the SDK without an X server. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 4 +- .../examples/computer_x11/Dockerfile | 19 ++ .../examples/computer_x11/README.md | 60 ++++ .../examples/computer_x11/driver.py | 276 ++++++++++++++++++ .../examples/computer_x11/fixture.py | 173 +++++++++++ .../examples/computer_x11/start.sh | 9 + .../tests/test_computer_x11_example.py | 164 +++++++++++ 7 files changed, 704 insertions(+), 1 deletion(-) create mode 100644 packages/agentenv-protocol/examples/computer_x11/Dockerfile create mode 100644 packages/agentenv-protocol/examples/computer_x11/README.md create mode 100644 packages/agentenv-protocol/examples/computer_x11/driver.py create mode 100644 packages/agentenv-protocol/examples/computer_x11/fixture.py create mode 100644 packages/agentenv-protocol/examples/computer_x11/start.sh create mode 100644 packages/agentenv-protocol/tests/test_computer_x11_example.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 67c65714..82f360b4 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -318,7 +318,9 @@ A computer driver is what runs inside a computer an agent uses through screensho one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, which needs the `computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow). The driver marks the methods that capture the screen and send input; the base class serves the -computer contract from them, so the tools, results and errors can't drift from it. +computer contract from them, so the tools, results and errors can't drift from it. A complete +driver for an X11 desktop, with an image to run it in, is in +[`examples/computer_x11/`](https://github.com/scaleapi/agentenv-framework/tree/main/packages/agentenv-protocol/examples/computer_x11). ```python from agentenv_protocol.computer import ( diff --git a/packages/agentenv-protocol/examples/computer_x11/Dockerfile b/packages/agentenv-protocol/examples/computer_x11/Dockerfile new file mode 100644 index 00000000..02b4b7e3 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/Dockerfile @@ -0,0 +1,19 @@ +# Built from packages/agentenv-protocol, so the driver runs on this checkout's SDK: +# docker build -f examples/computer_x11/Dockerfile -t x11-desktop . +FROM public.ecr.aws/docker/library/ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-tk xvfb x11-utils fonts-dejavu-core procps \ + libxcb1 libxcb-render0 libxcb-shm0 libxcb-randr0 libxcb-xfixes0 libxcb-xinerama0 \ + && rm -rf /var/lib/apt/lists/* +COPY pyproject.toml README.md LICENSE NOTICE THIRD_PARTY_NOTICES.md /opt/protocol/ +COPY src /opt/protocol/src +RUN python3 -m venv /opt/driver \ + && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer]" "mss>=9,<11" "python-xlib>=0.33,<0.34" +RUN useradd --create-home --uid 10001 user +COPY examples/computer_x11/start.sh examples/computer_x11/driver.py examples/computer_x11/fixture.py /opt/desktop/ +USER user +WORKDIR /home/user +ENV DISPLAY=:99 XDG_SESSION_TYPE=x11 +EXPOSE 18765 +CMD ["/bin/sh", "/opt/desktop/start.sh"] diff --git a/packages/agentenv-protocol/examples/computer_x11/README.md b/packages/agentenv-protocol/examples/computer_x11/README.md new file mode 100644 index 00000000..c69a20f0 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/README.md @@ -0,0 +1,60 @@ +# An X11 computer driver + +[`driver.py`](driver.py) is a complete computer driver for an X11 desktop, written on +`ComputerDriverEnvironment` from `agentenv_protocol.computer`: `mss` captures the screen and XTEST, +through `python-xlib`, sends the input. It implements the seven required operations and the pointer +and keyboard groups, declares five buttons, triple clicks and 50-point drags, and adds a ready check +of its own for XTEST. The base class serves everything else from those: the `computer_*` tools, +frames and coordinate spaces, the input queue, the framework extensions and the card, and its Linux +defaults give the driver `computer-exec/v1` and `data/add`. + +The image runs the driver on Xvfb beside [`fixture.py`](fixture.py), the Fixture Desk: a Tk window +with a name field, a checkbox, a 100-item list, a grid of cells, a drawing pad and a Save button. It +records whatever reaches it in `~/fixture/state.json`, so what an agent did can be read back from +the app rather than from the driver's answers, and shows the text of `~/inbox/message.txt`, which +`data/add` can write. [`start.sh`](start.sh) restarts Xvfb and the app if either dies. + +## Run it + +```bash +cd packages/agentenv-protocol +docker build -f examples/computer_x11/Dockerfile -t x11-desktop . +docker run --rm -p 18765:18765 x11-desktop +``` + +The card is at `http://localhost:18765/.well-known/agent-env.json`, readiness at +`/agentenv/ext/computer-ready/status`, and the tools at the MCP endpoint `/mcp`: + +```python +import asyncio + +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + + +async def main() -> None: + async with streamablehttp_client("http://localhost:18765/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + shot = await session.call_tool("computer_screenshot", {}) + print(shot.structuredContent["summary"]) + await session.call_tool("computer_click", {"x": 300, "y": 52}) + typed = await session.call_tool("computer_type", {"text": "Ada Lovelace", "screenshot": True}) + print(typed.structuredContent["summary"]) + + +asyncio.run(main()) +``` + +`AGENTENV_COMPUTER_SETTINGS` takes the driver's settings as one JSON value, for example +`-e AGENTENV_COMPUTER_SETTINGS='{"default_space": "norm", "data_paths": ["fixture"]}'`. + +## In agent-env + +As a bundle env (`envs//Dockerfile`), the image is built from the env's own folder, so copy +the three files there and install `agentenv-framework-protocol[computer]` in place of this checkout. +Name the env with `environment_name` in its `env.toml`. A task then deploys it with `deploy_env` +and gives it to an agent with `deploy_agent`, as for any MCP env. + +`tests/test_computer_x11_example.py` runs the driver against fake `mss` and `Xlib` modules, so it +keeps up with the SDK without an X server. diff --git a/packages/agentenv-protocol/examples/computer_x11/driver.py b/packages/agentenv-protocol/examples/computer_x11/driver.py new file mode 100644 index 00000000..1a42a908 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/driver.py @@ -0,0 +1,276 @@ +"""An X11 computer driver: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off the +event loop, one at a time, and a dropped X connection is reopened on the next call, so the driver outlives a +restarted X server.""" +from __future__ import annotations + +import asyncio +import math +import threading +import time +from typing import Any, Callable + +import mss +from Xlib import X, XK +from Xlib import display as xdisplay +from Xlib.error import ConnectionClosedError, DisplayConnectionError +from Xlib.ext import xtest + +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + DeviceUnavailable, + Display, + Image, + ReadyCheck, + Unsupported, + capture, + click, + computer_driver, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + scroll, + type_text, +) + +BUTTONS = {"left": 1, "middle": 2, "right": 3, "back": 8, "forward": 9} +SCROLL = {"up": 4, "down": 5, "left": 6, "right": 7} +#: W3C key values and their X keysym names; any other single character is typed as its own keysym. +KEYSYMS = { + "Enter": "Return", "Tab": "Tab", "Escape": "Escape", "Backspace": "BackSpace", "Delete": "Delete", + "ArrowUp": "Up", "ArrowDown": "Down", "ArrowLeft": "Left", "ArrowRight": "Right", "Home": "Home", "End": "End", + "PageUp": "Prior", "PageDown": "Next", "Insert": "Insert", "Control": "Control_L", "Shift": "Shift_L", + "Alt": "Alt_L", "Meta": "Super_L", "Super": "Super_L", "CapsLock": "Caps_Lock", "ContextMenu": "Menu", + "PrintScreen": "Print", "Pause": "Pause", " ": "space", **{f"F{n}": f"F{n}" for n in range(1, 25)}, +} +TYPED = {"\n": "Return", "\t": "Tab"} +DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) + + +@computer_driver(name="x11-desktop", + supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +class X11Driver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle", "back", "forward") + max_click_count = 3 + max_drag_points = 50 + + def __init__(self) -> None: + super().__init__() + self._x = None + self._x_lock = threading.Lock() + + @display + async def screen(self) -> Display: + def read(d): + s = d.screen() + return Display(s.width_in_pixels, s.height_in_pixels, 1.0) + return await self._on_x(read) + + @capture + async def grab(self) -> Image: + return await asyncio.to_thread(_grab) + + async def ready_checks(self) -> list[ReadyCheck]: + try: + has = await self._on_x(lambda d: d.has_extension("XTEST")) + except DeviceUnavailable as e: + return [ReadyCheck("xtest", False, e.message)] + return [ReadyCheck("xtest", bool(has), "XTEST available" if has else "XTEST missing")] + + @click + async def click_at(self, x, y, button, count, modifiers) -> None: + def act(d): + held = _press(d, modifiers) + _move(d, x, y) + for i in range(count): + _button(d, button, True) + _button(d, button, False) + if i + 1 < count: + d.sync() + time.sleep(0.04) + _release(d, reversed(held)) + await self._on_x(act) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers) -> None: + steps = _steps(path) + pause = duration_ms / 1000 / len(steps) + + def start(d): + held = _press(d, modifiers) + _move(d, *path[0]) + _button(d, button, True) + return held + + def finish(d): + _button(d, button, False) + _release(d, reversed(held)) + + held = await self._on_x(start) + for x, y in steps: + await self._on_x(lambda d, x=x, y=y: _move(d, x, y)) + await asyncio.sleep(pause) + await self._on_x(finish) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers) -> None: + def act(d): + held = _press(d, modifiers) + _move(d, x, y) + for _ in range(amount): + xtest.fake_input(d, X.ButtonPress, SCROLL[direction]) + xtest.fake_input(d, X.ButtonRelease, SCROLL[direction]) + _release(d, reversed(held)) + await self._on_x(act) + + @pointer_move + async def move(self, x, y) -> None: + await self._on_x(lambda d: _move(d, x, y)) + + @pointer_down + async def down(self, button) -> None: + await self._on_x(lambda d: _button(d, button, True)) + + @pointer_up + async def up(self, button) -> None: + await self._on_x(lambda d: _button(d, button, False)) + + @pointer_position + async def where(self) -> tuple[int, int]: + def read(d): + p = d.screen().root.query_pointer() + return p.root_x, p.root_y + return await self._on_x(read) + + @type_text + async def type_in(self, text) -> None: + for ch in text: + sym = XK.string_to_keysym(TYPED[ch]) if ch in TYPED else _keysym(ch) + hold = self.settings.tap_hold_ms / 1000 + await self._on_x(lambda d, sym=sym: _tap(d, sym, hold)) + await asyncio.sleep(self.settings.type_delay_ms / 1000) + + @press + async def press_keys(self, keys) -> None: + hold = self.settings.tap_hold_ms / 1000 + + def act(d): + codes = _press(d, keys) + d.sync() + time.sleep(hold) + _release(d, reversed(codes)) + await self._on_x(act) + + @key_down + async def keys_down(self, keys) -> None: + await self._on_x(lambda d: _press(d, keys)) + + @key_up + async def keys_up(self, keys) -> None: + await self._on_x(lambda d: _release(d, _keycodes(d, keys))) + + async def _on_x(self, fn: Callable[[Any], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn) + + def _x_call(self, fn: Callable[[Any], Any]) -> Any: + """``fn(display)`` on a live connection, reconnecting once if it dropped.""" + with self._x_lock: + for attempt in (1, 2): + try: + if self._x is None: + self._x = xdisplay.Display() + result = fn(self._x) + self._x.sync() + return result + except DROPPED as e: + self._x = None + if attempt == 2: + raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + + +def _grab() -> Image: + try: + with mss.MSS() as m: + shot = m.grab(m.monitors[0]) + return Image(shot.width, shot.height, bytes(shot.bgra), "BGRA") + except Exception as e: + raise DeviceUnavailable(f"capture failed: {type(e).__name__}: {e}") from None + + +def _steps(path: list) -> list[tuple[float, float]]: + """The points a drag moves through after its start, about every 20 pixels.""" + steps = [] + for (x0, y0), (x1, y1) in zip(path, path[1:]): + n = max(2, int(math.hypot(x1 - x0, y1 - y0) / 20)) + steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] + return steps + + +def _move(d, x, y) -> None: + xtest.fake_input(d, X.MotionNotify, x=int(x), y=int(y)) + + +def _button(d, button: str, down: bool) -> None: + xtest.fake_input(d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) + + +def _press(d, keys: list[str]) -> list[int]: + codes = _keycodes(d, keys) + for code in codes: + xtest.fake_input(d, X.KeyPress, code) + return codes + + +def _release(d, codes) -> None: + for code in codes: + xtest.fake_input(d, X.KeyRelease, code) + + +def _tap(d, keysym: int, hold_s: float) -> None: + code, shifted = _keycode(d, keysym) + shift = d.keysym_to_keycode(XK.string_to_keysym("Shift_L")) + if shifted: + xtest.fake_input(d, X.KeyPress, shift) + xtest.fake_input(d, X.KeyPress, code) + d.sync() + time.sleep(hold_s) + xtest.fake_input(d, X.KeyRelease, code) + if shifted: + xtest.fake_input(d, X.KeyRelease, shift) + + +def _keycodes(d, keys: list[str]) -> list[int]: + return [_keycode(d, _keysym(k))[0] for k in keys] + + +def _keysym(key: str) -> int: + if key in KEYSYMS: + return XK.string_to_keysym(KEYSYMS[key]) + if len(key) == 1: + code = ord(key) + return code if code < 0x100 else 0x01000000 | code + raise Unsupported(f"this driver has no X keysym for {key!r}") + + +def _keycode(d, keysym: int) -> tuple[int, bool]: + """The keycode for ``keysym`` and whether it needs Shift; with no keycode for it, a spare one is mapped to it.""" + code = d.keysym_to_keycode(keysym) + if code: + return code, d.keycode_to_keysym(code, 0) != keysym and d.keycode_to_keysym(code, 1) == keysym + first = d.display.info.min_keycode + mapping = d.get_keyboard_mapping(first, d.display.info.max_keycode - first + 1) + spare = next((first + i for i, syms in enumerate(mapping) if not any(syms)), None) + if spare is None: + raise Unsupported(f"no spare keycode to map keysym {keysym:#x} to") + d.change_keyboard_mapping(spare, [(keysym,) * len(mapping[0])]) + d.sync() + return spare, False + + +if __name__ == "__main__": + X11Driver().serve() diff --git a/packages/agentenv-protocol/examples/computer_x11/fixture.py b/packages/agentenv-protocol/examples/computer_x11/fixture.py new file mode 100644 index 00000000..78cd2699 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/fixture.py @@ -0,0 +1,173 @@ +"""The Fixture Desk: a Tk window whose every effect is recorded in ~/fixture/state.json, and whose widgets' screen +boxes are written to ~/fixture/layout.json, so a test can aim at them and read back what reached them.""" +import json +import os +import tkinter as tk +from pathlib import Path + +HOME = Path.home() / "fixture" +HOME.mkdir(exist_ok=True) +COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff") + +root = tk.Tk() +root.title("Fixture Desk") +root.geometry("1280x800+0+0") +root.configure(bg="#f0f0f0") +state = {"name": "", "saved_count": 0, "subscribed": False, "selected": None, "cells": {}, "drags": [], + "keys": [], "scroll_top": 0.0, "right_clicks": 0, "double_clicks": 0, "loaded": None} + + +def save_state(): + tmp = HOME / ".state.json.part" + tmp.write_text(json.dumps(state)) + os.replace(tmp, HOME / "state.json") + + +tk.Label(root, text="Name:", font=("DejaVu Sans", 16), bg="#f0f0f0").place(x=40, y=40) +name = tk.Entry(root, font=("DejaVu Sans", 16), width=30) +name.place(x=140, y=40) + + +def on_key(event): + state["name"] = name.get() + state["keys"] = (state["keys"] + [event.keysym])[-50:] + save_state() + + +name.bind("", on_key) + + +def on_save(): + state["name"] = name.get() + state["saved_count"] += 1 + save_state() + + +save = tk.Button(root, text="Save", font=("DejaVu Sans", 16), command=on_save, width=8) +save.place(x=560, y=34) + +subscribed = tk.BooleanVar(value=False) + + +def on_subscribe(): + state["subscribed"] = subscribed.get() + save_state() + + +subscribe = tk.Checkbutton(root, text="Subscribe", font=("DejaVu Sans", 16), variable=subscribed, + command=on_subscribe, bg="#f0f0f0") +subscribe.place(x=40, y=100) + +frame = tk.Frame(root) +frame.place(x=40, y=160, width=300, height=400) +listbox = tk.Listbox(frame, font=("DejaVu Sans", 14), exportselection=False) +listbox.pack(side="left", fill="both", expand=True) +bar = tk.Scrollbar(frame, command=listbox.yview) +bar.pack(side="right", fill="y") + + +def on_scroll(first, last): + bar.set(first, last) + state["scroll_top"] = float(first) + save_state() + + +listbox.configure(yscrollcommand=on_scroll) +for i in range(1, 101): + listbox.insert("end", f"Item {i}") + + +def on_select(event): + chosen = listbox.curselection() + state["selected"] = listbox.get(chosen[0]) if chosen else None + save_state() + + +listbox.bind("<>", on_select) + +grid = tk.Canvas(root, width=400, height=400, bg="#ffffff", highlightthickness=0) +grid.place(x=400, y=160) +CELL = 50 +for row in range(8): + for col in range(8): + grid.create_rectangle(col * CELL, row * CELL, (col + 1) * CELL, (row + 1) * CELL, fill=COLOURS[0], + outline="#999999", tags=(f"cell-{row}-{col}",)) + + +def on_cell(event, button): + row, col = event.y // CELL, event.x // CELL + key = f"{row},{col}" + count = (state["cells"].get(key, 0) + 1) % len(COLOURS) + state["cells"][key] = count + grid.itemconfigure(f"cell-{row}-{col}", fill=COLOURS[count]) + if button == 3: + state["right_clicks"] += 1 + save_state() + + +def on_double(event): + state["double_clicks"] += 1 + save_state() + + +grid.bind("", lambda e: on_cell(e, 1)) +grid.bind("", lambda e: on_cell(e, 3)) +grid.bind("", on_double) + +pad = tk.Canvas(root, width=400, height=400, bg="#fffbe6", highlightthickness=1) +pad.place(x=840, y=160) +stroke = [] + + +def on_press(event): + stroke.clear() + stroke.append((event.x, event.y)) + + +def on_motion(event): + if stroke: + pad.create_line(stroke[-1][0], stroke[-1][1], event.x, event.y, width=3) + stroke.append((event.x, event.y)) + + +def on_release(event): + if stroke: + stroke.append((event.x, event.y)) + state["drags"] = (state["drags"] + [{"start": stroke[0], "end": stroke[-1], "points": len(stroke)}])[-20:] + save_state() + stroke.clear() + + +pad.bind("", on_press) +pad.bind("", on_motion) +pad.bind("", on_release) + +loaded = tk.Label(root, text="", font=("DejaVu Sans", 14), bg="#f0f0f0") +loaded.place(x=40, y=600) + + +def poll_loaded(): + path = Path.home() / "inbox" / "message.txt" + if path.exists(): + text = path.read_text() + if text != state["loaded"]: + state["loaded"] = text + loaded.configure(text=f"Loaded: {text}") + save_state() + root.after(200, poll_loaded) + + +def write_layout(): + root.update_idletasks() + boxes = {} + for key, widget in (("name", name), ("save", save), ("subscribe", subscribe), ("list", listbox), + ("grid", grid), ("pad", pad)): + boxes[key] = [widget.winfo_rootx(), widget.winfo_rooty(), widget.winfo_width(), widget.winfo_height()] + boxes["list_row_height"] = listbox.bbox(0)[3] if listbox.bbox(0) else 0 + (HOME / "layout.json").write_text(json.dumps(boxes)) + save_state() + + +root.after(300, write_layout) +root.after(500, poll_loaded) +root.mainloop() diff --git a/packages/agentenv-protocol/examples/computer_x11/start.sh b/packages/agentenv-protocol/examples/computer_x11/start.sh new file mode 100644 index 00000000..31dcb19d --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/start.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# The desktop: Xvfb and the fixture each restart if they die, then the driver serves on MCP_PORT. +(while true; do Xvfb :99 -screen 0 1280x800x24 -nolisten tcp -ac >/tmp/xvfb.log 2>&1; sleep 0.5; done) & +for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done +(while true; do + for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done + python3 /opt/desktop/fixture.py >/tmp/fixture.log 2>&1; sleep 0.5 + done) & +exec /opt/driver/bin/python /opt/desktop/driver.py diff --git a/packages/agentenv-protocol/tests/test_computer_x11_example.py b/packages/agentenv-protocol/tests/test_computer_x11_example.py new file mode 100644 index 00000000..b9759c06 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_x11_example.py @@ -0,0 +1,164 @@ +"""The X11 example driver against fake mss and Xlib modules: it passes the definition checks, serves the whole +computer contract, and turns tool calls into the XTEST events they mean.""" +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import LINUX, http_client, mcp_session # noqa: E402 + +from agentenv_protocol.computer import COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_POINTER # noqa: E402 + +DRIVER = Path(__file__).resolve().parents[1] / "examples" / "computer_x11" / "driver.py" +WIDTH, HEIGHT = 640, 400 + + +class ConnectionClosedError(Exception): + pass + + +class FakeX: + """An X display: its screen and pointer, the XTEST events sent to it, and ``drops`` syncs that fail.""" + + def __init__(self) -> None: + self.events: list = [] + self.pointer = (0, 0) + self.opened = 0 + self.drops = 0 + self.remapped = None + self.display = types.SimpleNamespace(info=types.SimpleNamespace(min_keycode=8, max_keycode=255)) + + def screen(self): + query = lambda: types.SimpleNamespace(root_x=self.pointer[0], root_y=self.pointer[1]) # noqa: E731 + return types.SimpleNamespace(width_in_pixels=WIDTH, height_in_pixels=HEIGHT, + root=types.SimpleNamespace(query_pointer=query)) + + def sync(self) -> None: + if self.drops: + self.drops -= 1 + raise ConnectionClosedError("display") + + def has_extension(self, name: str) -> bool: + return name == "XTEST" + + def keysym_to_keycode(self, keysym: int) -> int: + return keysym + 8 if keysym < 0x10000 else 0 + + def keycode_to_keysym(self, code: int, index: int) -> int: + return code - 8 if index == 0 else 0 + + def get_keyboard_mapping(self, first: int, count: int) -> list: + return [(0, 0)] * count + + def change_keyboard_mapping(self, first: int, keysyms: list) -> None: + self.remapped = (first, keysyms) + + +class FakeMSS: + monitors = [{"left": 0, "top": 0, "width": WIDTH, "height": HEIGHT}] + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + return None + + def grab(self, monitor: dict): + return types.SimpleNamespace(width=WIDTH, height=HEIGHT, bgra=bytes(WIDTH * HEIGHT * 4)) + + +@pytest.fixture +def x11(monkeypatch): + """The example's driver on a fake X display, and the keysyms the fake XK gave out by name.""" + fake, keysyms = FakeX(), {} + + def open_display(): + fake.opened += 1 + return fake + + def fake_input(d, kind, detail=0, x=0, y=0): + d.events.append((kind, (x, y)) if kind == "move" else (kind, detail)) + + def module(name: str, **attrs) -> types.ModuleType: + m = types.ModuleType(name) + m.__dict__.update(attrs) + monkeypatch.setitem(sys.modules, name, m) + return m + + X = module("Xlib.X", MotionNotify="move", ButtonPress="button-down", ButtonRelease="button-up", + KeyPress="key-down", KeyRelease="key-up") + XK = module("Xlib.XK", string_to_keysym=lambda name: keysyms.setdefault(name, 0xff00 + len(keysyms))) + display = module("Xlib.display", Display=open_display) + module("Xlib.error", ConnectionClosedError=ConnectionClosedError, DisplayConnectionError=ConnectionClosedError) + xtest = module("Xlib.ext.xtest", fake_input=fake_input) + module("Xlib.ext", xtest=xtest) + module("Xlib", X=X, XK=XK, display=display) + module("mss", MSS=FakeMSS) + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) + monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + spec = importlib.util.spec_from_file_location("computer_x11_driver", DRIVER) + example = importlib.util.module_from_spec(spec) + spec.loader.exec_module(example) + return example.X11Driver(), fake, keysyms + + +@pytest.mark.asyncio +async def test_the_example_serves_the_whole_computer_contract(x11): + driver, _x, _keysyms = x11 + async with http_client(driver) as http, mcp_session(driver) as session: + card = (await http.get("/.well-known/agent-env.json")).json() + tools = {t.name for t in (await session.list_tools()).tools} + ready = (await http.get("/agentenv/ext/computer-ready/status")).json() + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + assert card["name"] == "x11-desktop" and len(tools) == 14 and "computer_keyboard_down" in tools + assert {COMPUTER_POINTER, COMPUTER_KEYBOARD, COMPUTER_EXEC} <= set(extensions) + assert extensions["urn:agentenv:computer/v1"]["params"]["max_click_count"] == 3 + assert ready["ready"] and [c["name"] for c in ready["checks"]] == ["display", "capture", "xtest"] + + +@pytest.mark.asyncio +async def test_tool_calls_become_the_xtest_events_they_mean(x11): + driver, x, keysyms = x11 + + async def events(session, tool: str, args: dict) -> list: + x.events.clear() + result = await session.call_tool(tool, args) + assert not result.isError, result.content[0].text + return list(x.events) + + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + clicked = await events(session, "computer_click", {"x": 10, "y": 20, "button": "right", "modifiers": ["Shift"]}) + typed = await events(session, "computer_type", {"text": "a東\n"}) + dragged = await events(session, "computer_drag", {"path": [[0, 0], [40, 0]], "duration_ms": 0}) + pressed = await events(session, "computer_key", {"keys": ["Control", "s"]}) + shift, control, enter = (keysyms[name] + 8 for name in ("Shift_L", "Control_L", "Return")) + assert clicked == [("key-down", shift), ("move", (10, 20)), ("button-down", 3), ("button-up", 3), ("key-up", shift)] + assert typed == [("key-down", 0x69), ("key-up", 0x69), ("key-down", 8), ("key-up", 8), + ("key-down", enter), ("key-up", enter)] + assert x.remapped == (8, [(0x01006771, 0x01006771)]) + assert dragged == [("move", (0, 0)), ("button-down", 1), ("move", (20, 0)), ("move", (40, 0)), ("button-up", 1)] + assert pressed == [("key-down", control), ("key-down", 0x7b), ("key-up", 0x7b), ("key-up", control)] + + +@pytest.mark.asyncio +async def test_a_dropped_x_connection_is_reopened_and_a_dead_one_is_device_unavailable(x11): + driver, x, _keysyms = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + x.drops = 1 + moved = await session.call_tool("computer_pointer_position", {}) + reopened = x.opened + x.drops = 2 + refused = await session.call_tool("computer_pointer_position", {}) + assert not moved.isError and reopened == 2 + assert refused.structuredContent["code"] == "device_unavailable" + assert "the X server is unavailable: ConnectionClosedError" in refused.structuredContent["message"] From 03dec85234a9ea0969c2cdde484fdc04a09c8a17 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 15:01:58 -0700 Subject: [PATCH 16/18] fix(protocol): make the X11 example driver safe to retry and true to the keys it is given - Input is no longer resent when the X connection drops during it, which could click or type twice; the connection is checked with a round trip before input is sent, so a dead X server still answers device_unavailable, and a drop mid-input answers delivery_failed (applied unknown). - A drag that fails or is cancelled lets go of its button and modifiers. - A character on a key's second level, such as A or !, is pressed with Shift, in computer_key and computer_keyboard_down as in typing. - Only keys an action pressed are released, so a key held by computer_keyboard_down stays down through typing, clicks, scrolls and chords; computer_keyboard_up releases what each key's press pressed. - A character the keymap lacks borrows a spare keycode, kept for that character and reused, or the least recently used one once all are taken, instead of using up a keycode per character; text that can't be typed at all is refused before any of it is sent. - The click gap and drag spacing are named constants. Co-Authored-By: Claude Opus 5.5 --- .../examples/computer_x11/driver.py | 304 +++++++++++------- .../tests/test_computer_x11_example.py | 169 ++++++++-- 2 files changed, 328 insertions(+), 145 deletions(-) diff --git a/packages/agentenv-protocol/examples/computer_x11/driver.py b/packages/agentenv-protocol/examples/computer_x11/driver.py index 1a42a908..565b2951 100644 --- a/packages/agentenv-protocol/examples/computer_x11/driver.py +++ b/packages/agentenv-protocol/examples/computer_x11/driver.py @@ -1,12 +1,14 @@ """An X11 computer driver: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off the -event loop, one at a time, and a dropped X connection is reopened on the next call, so the driver outlives a -restarted X server.""" +event loop, one at a time, on a connection checked before each call and reopened when it drops, so the driver outlives +a restarted X server.""" from __future__ import annotations import asyncio +import contextlib import math import threading import time +from collections import OrderedDict from typing import Any, Callable import mss @@ -17,6 +19,7 @@ from agentenv_protocol.computer import ( ComputerDriverEnvironment, + DeliveryFailed, DeviceUnavailable, Display, Image, @@ -49,6 +52,10 @@ "PrintScreen": "Print", "Pause": "Pause", " ": "space", **{f"F{n}": f"F{n}" for n in range(1, 25)}, } TYPED = {"\n": "Return", "\t": "Tab"} +#: The pause between the presses of a double or triple click, in seconds. +CLICK_GAP_S = 0.04 +#: How far apart a drag's pointer moves are, in pixels. +DRAG_STEP_PX = 20 DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) @@ -61,15 +68,17 @@ class X11Driver(ComputerDriverEnvironment): def __init__(self) -> None: super().__init__() - self._x = None + self._x: _Connection | None = None self._x_lock = threading.Lock() + #: The keycodes each key held by @key_down pressed, which @key_up releases. + self._held: dict[str, list[int]] = {} @display async def screen(self) -> Display: - def read(d): - s = d.screen() + def read(c): + s = c.d.screen() return Display(s.width_in_pixels, s.height_in_pixels, 1.0) - return await self._on_x(read) + return await self._read(read) @capture async def grab(self) -> Image: @@ -77,121 +86,240 @@ async def grab(self) -> Image: async def ready_checks(self) -> list[ReadyCheck]: try: - has = await self._on_x(lambda d: d.has_extension("XTEST")) + has = await self._read(lambda c: c.d.has_extension("XTEST")) except DeviceUnavailable as e: return [ReadyCheck("xtest", False, e.message)] return [ReadyCheck("xtest", bool(has), "XTEST available" if has else "XTEST missing")] @click async def click_at(self, x, y, button, count, modifiers) -> None: - def act(d): - held = _press(d, modifiers) - _move(d, x, y) + syms = [_keysym(k) for k in modifiers] + + def act(c): + held = c.hold(syms) + c.move(x, y) for i in range(count): - _button(d, button, True) - _button(d, button, False) + c.button(button, True) + c.button(button, False) if i + 1 < count: - d.sync() - time.sleep(0.04) - _release(d, reversed(held)) - await self._on_x(act) + c.d.sync() + time.sleep(CLICK_GAP_S) + c.release(reversed(held)) + await self._send(act) @drag async def drag_along(self, path, button, duration_ms, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] steps = _steps(path) pause = duration_ms / 1000 / len(steps) + held: list[int] = [] - def start(d): - held = _press(d, modifiers) - _move(d, *path[0]) - _button(d, button, True) - return held + def start(c): + held.extend(c.hold(syms)) + c.move(*path[0]) + c.button(button, True) - def finish(d): - _button(d, button, False) - _release(d, reversed(held)) + def finish(c): + c.button(button, False) + c.release(reversed(held)) - held = await self._on_x(start) - for x, y in steps: - await self._on_x(lambda d, x=x, y=y: _move(d, x, y)) - await asyncio.sleep(pause) - await self._on_x(finish) + try: + await self._send(start) + for px, py in steps: + await self._send(lambda c, px=px, py=py: c.move(px, py)) + await asyncio.sleep(pause) + except BaseException: + with contextlib.suppress(Exception): + await self._send(finish) + raise + await self._send(finish) @scroll async def scroll_at(self, x, y, direction, amount, modifiers) -> None: - def act(d): - held = _press(d, modifiers) - _move(d, x, y) + syms = [_keysym(k) for k in modifiers] + + def act(c): + held = c.hold(syms) + c.move(x, y) for _ in range(amount): - xtest.fake_input(d, X.ButtonPress, SCROLL[direction]) - xtest.fake_input(d, X.ButtonRelease, SCROLL[direction]) - _release(d, reversed(held)) - await self._on_x(act) + xtest.fake_input(c.d, X.ButtonPress, SCROLL[direction]) + xtest.fake_input(c.d, X.ButtonRelease, SCROLL[direction]) + c.release(reversed(held)) + await self._send(act) @pointer_move async def move(self, x, y) -> None: - await self._on_x(lambda d: _move(d, x, y)) + await self._send(lambda c: c.move(x, y)) @pointer_down async def down(self, button) -> None: - await self._on_x(lambda d: _button(d, button, True)) + await self._send(lambda c: c.button(button, True)) @pointer_up async def up(self, button) -> None: - await self._on_x(lambda d: _button(d, button, False)) + await self._send(lambda c: c.button(button, False)) @pointer_position async def where(self) -> tuple[int, int]: - def read(d): - p = d.screen().root.query_pointer() + def read(c): + p = c.d.screen().root.query_pointer() return p.root_x, p.root_y - return await self._on_x(read) + return await self._read(read) @type_text async def type_in(self, text) -> None: - for ch in text: - sym = XK.string_to_keysym(TYPED[ch]) if ch in TYPED else _keysym(ch) + syms = [XK.string_to_keysym(TYPED[ch]) if ch in TYPED else _keysym(ch) for ch in text] + await self._read(lambda c: c.check(syms)) + for sym in syms: hold = self.settings.tap_hold_ms / 1000 - await self._on_x(lambda d, sym=sym: _tap(d, sym, hold)) + await self._send(lambda c, sym=sym: c.tap(sym, hold)) await asyncio.sleep(self.settings.type_delay_ms / 1000) @press async def press_keys(self, keys) -> None: + syms = [_keysym(k) for k in keys] hold = self.settings.tap_hold_ms / 1000 - def act(d): - codes = _press(d, keys) - d.sync() + def act(c): + pressed = c.hold(syms) + c.d.sync() time.sleep(hold) - _release(d, reversed(codes)) - await self._on_x(act) + c.release(reversed(pressed)) + await self._send(act) @key_down async def keys_down(self, keys) -> None: - await self._on_x(lambda d: _press(d, keys)) + def act(c): + for key in keys: + self._held.setdefault(key, []).extend(c.hold([_keysym(key)])) + await self._send(act) @key_up async def keys_up(self, keys) -> None: - await self._on_x(lambda d: _release(d, _keycodes(d, keys))) + def act(c): + for key in keys: + c.release(reversed(self._held.pop(key, None) or c.codes(_keysym(key))[-1:])) + await self._send(act) - async def _on_x(self, fn: Callable[[Any], Any]) -> Any: - return await asyncio.to_thread(self._x_call, fn) + async def _read(self, fn: Callable[[_Connection], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn, False) - def _x_call(self, fn: Callable[[Any], Any]) -> Any: - """``fn(display)`` on a live connection, reconnecting once if it dropped.""" + async def _send(self, fn: Callable[[_Connection], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn, True) + + def _x_call(self, fn: Callable[[_Connection], Any], sends_input: bool) -> Any: + """``fn`` on a live connection. When the connection drops, a read is retried once on a new one; input is not + sent again, since some of it may have arrived.""" with self._x_lock: for attempt in (1, 2): + c = self._connected() try: - if self._x is None: - self._x = xdisplay.Display() - result = fn(self._x) - self._x.sync() + result = fn(c) + c.d.sync() return result except DROPPED as e: self._x = None + if sends_input: + raise DeliveryFailed(f"the X connection dropped while sending input: {type(e).__name__}") from None if attempt == 2: raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + def _connected(self) -> _Connection: + """The connection, opened or reopened as needed and checked with a round trip before anything is sent.""" + for attempt in (1, 2): + try: + if self._x is None: + self._x = _Connection() + self._held.clear() + self._x.d.sync() + return self._x + except DROPPED as e: + self._x = None + if attempt == 2: + raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + + +class _Connection: + """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare + keycode mapped to it, kept for that character and reused, or the least recently used one once all are taken.""" + + def __init__(self) -> None: + self.d = xdisplay.Display() + first = self.d.display.info.min_keycode + mapping = self.d.get_keyboard_mapping(first, self.d.display.info.max_keycode - first + 1) + self.width = len(mapping[0]) + self.spare = [first + i for i, syms in enumerate(mapping) if not any(syms)] + self.mapped: OrderedDict[int, int] = OrderedDict() + self.shift = self.d.keysym_to_keycode(XK.string_to_keysym("Shift_L")) + + def move(self, x, y) -> None: + xtest.fake_input(self.d, X.MotionNotify, x=int(x), y=int(y)) + + def button(self, button: str, down: bool) -> None: + xtest.fake_input(self.d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) + + def hold(self, keysyms: list[int]) -> list[int]: + """Press what the keysyms need that isn't down already, Shift first where a character needs it; the keycodes + this pressed, in order, so the caller releases only those.""" + down = self._down() + pressed = [] + for keysym in keysyms: + for code in self.codes(keysym): + if code not in down: + xtest.fake_input(self.d, X.KeyPress, code) + down.add(code) + pressed.append(code) + return pressed + + def release(self, codes) -> None: + for code in codes: + xtest.fake_input(self.d, X.KeyRelease, code) + + def tap(self, keysym: int, hold_s: float) -> None: + """Type one character: its key pressed and released, with Shift around it when it needs Shift and nothing + holds Shift already.""" + *modifiers, code = self.codes(keysym) + down = self._down() + pressed = [m for m in modifiers if m not in down] + for m in pressed: + xtest.fake_input(self.d, X.KeyPress, m) + xtest.fake_input(self.d, X.KeyPress, code) + self.d.sync() + time.sleep(hold_s) + xtest.fake_input(self.d, X.KeyRelease, code) + self.release(reversed(pressed)) + + def codes(self, keysym: int) -> list[int]: + """The keycodes that make ``keysym``: Shift and the key when it is the key's second level, else one key.""" + codes = self._on_keymap(keysym) + if codes is not None: + return codes + self.check([keysym]) + code = self.spare.pop() if self.spare else self.mapped.popitem(last=False)[1] + self.d.change_keyboard_mapping(code, [(keysym,) * self.width]) + self.d.sync() + self.mapped[keysym] = code + return [code] + + def check(self, keysyms: list[int]) -> None: + """Refuse, before anything is sent, keysyms that can't be typed: one off the keymap with no spare keycode.""" + if not (self.spare or self.mapped) and any(self._on_keymap(k) is None for k in keysyms): + raise Unsupported("this X server has no spare keycode for the characters its keymap lacks") + + def _on_keymap(self, keysym: int) -> list[int] | None: + if keysym in self.mapped: + self.mapped.move_to_end(keysym) + return [self.mapped[keysym]] + code = self.d.keysym_to_keycode(keysym) + if code and self.d.keycode_to_keysym(code, 0) == keysym: + return [code] + if code and self.d.keycode_to_keysym(code, 1) == keysym: + return [self.shift, code] + return None + + def _down(self) -> set[int]: + return {8 * i + bit for i, byte in enumerate(self.d.query_keymap()) for bit in range(8) if byte >> bit & 1} + def _grab() -> Image: try: @@ -203,51 +331,14 @@ def _grab() -> Image: def _steps(path: list) -> list[tuple[float, float]]: - """The points a drag moves through after its start, about every 20 pixels.""" + """The points a drag moves through after its start, about every DRAG_STEP_PX pixels.""" steps = [] for (x0, y0), (x1, y1) in zip(path, path[1:]): - n = max(2, int(math.hypot(x1 - x0, y1 - y0) / 20)) + n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX)) steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] return steps -def _move(d, x, y) -> None: - xtest.fake_input(d, X.MotionNotify, x=int(x), y=int(y)) - - -def _button(d, button: str, down: bool) -> None: - xtest.fake_input(d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) - - -def _press(d, keys: list[str]) -> list[int]: - codes = _keycodes(d, keys) - for code in codes: - xtest.fake_input(d, X.KeyPress, code) - return codes - - -def _release(d, codes) -> None: - for code in codes: - xtest.fake_input(d, X.KeyRelease, code) - - -def _tap(d, keysym: int, hold_s: float) -> None: - code, shifted = _keycode(d, keysym) - shift = d.keysym_to_keycode(XK.string_to_keysym("Shift_L")) - if shifted: - xtest.fake_input(d, X.KeyPress, shift) - xtest.fake_input(d, X.KeyPress, code) - d.sync() - time.sleep(hold_s) - xtest.fake_input(d, X.KeyRelease, code) - if shifted: - xtest.fake_input(d, X.KeyRelease, shift) - - -def _keycodes(d, keys: list[str]) -> list[int]: - return [_keycode(d, _keysym(k))[0] for k in keys] - - def _keysym(key: str) -> int: if key in KEYSYMS: return XK.string_to_keysym(KEYSYMS[key]) @@ -257,20 +348,5 @@ def _keysym(key: str) -> int: raise Unsupported(f"this driver has no X keysym for {key!r}") -def _keycode(d, keysym: int) -> tuple[int, bool]: - """The keycode for ``keysym`` and whether it needs Shift; with no keycode for it, a spare one is mapped to it.""" - code = d.keysym_to_keycode(keysym) - if code: - return code, d.keycode_to_keysym(code, 0) != keysym and d.keycode_to_keysym(code, 1) == keysym - first = d.display.info.min_keycode - mapping = d.get_keyboard_mapping(first, d.display.info.max_keycode - first + 1) - spare = next((first + i for i, syms in enumerate(mapping) if not any(syms)), None) - if spare is None: - raise Unsupported(f"no spare keycode to map keysym {keysym:#x} to") - d.change_keyboard_mapping(spare, [(keysym,) * len(mapping[0])]) - d.sync() - return spare, False - - if __name__ == "__main__": X11Driver().serve() diff --git a/packages/agentenv-protocol/tests/test_computer_x11_example.py b/packages/agentenv-protocol/tests/test_computer_x11_example.py index b9759c06..bc20cc4c 100644 --- a/packages/agentenv-protocol/tests/test_computer_x11_example.py +++ b/packages/agentenv-protocol/tests/test_computer_x11_example.py @@ -1,7 +1,9 @@ """The X11 example driver against fake mss and Xlib modules: it passes the definition checks, serves the whole -computer contract, and turns tool calls into the XTEST events they mean.""" +computer contract, and turns tool calls into the XTEST events they mean, without resending input, leaving keys or +buttons down, or letting go of keys it didn't press.""" from __future__ import annotations +import asyncio import importlib.util import sys import types @@ -18,6 +20,9 @@ DRIVER = Path(__file__).resolve().parents[1] / "examples" / "computer_x11" / "driver.py" WIDTH, HEIGHT = 640, 400 +TOKYO = [0x01000000 | ord(ch) for ch in "東京"] +#: The fake keymap puts the named keys, such as Shift_L, from this keycode up; printable ASCII is its keysym + 8. +NAMED_KEYS = 200 class ConnectionClosedError(Exception): @@ -25,14 +30,19 @@ class ConnectionClosedError(Exception): class FakeX: - """An X display: its screen and pointer, the XTEST events sent to it, and ``drops`` syncs that fail.""" + """An X display: its screen and pointer, a US-like keymap whose capital letters are the second level of their + keys, the keys down, the XTEST events sent, ``drops`` syncs and the ``fail_at``-th event that fail, and the + ``spare`` keycodes the keymap leaves empty.""" def __init__(self) -> None: self.events: list = [] + self.down: set[int] = set() + self.remapped: list = [] self.pointer = (0, 0) self.opened = 0 self.drops = 0 - self.remapped = None + self.fail_at = None + self.spare = {8} self.display = types.SimpleNamespace(info=types.SimpleNamespace(min_keycode=8, max_keycode=255)) def screen(self): @@ -49,16 +59,26 @@ def has_extension(self, name: str) -> bool: return name == "XTEST" def keysym_to_keycode(self, keysym: int) -> int: - return keysym + 8 if keysym < 0x10000 else 0 + if ord("A") <= keysym <= ord("Z"): + keysym += 0x20 + if 0xff00 <= keysym < 0xff38: + return NAMED_KEYS + keysym - 0xff00 + return keysym + 8 if 0x20 <= keysym < 0x7f else 0 def keycode_to_keysym(self, code: int, index: int) -> int: - return code - 8 if index == 0 else 0 + sym = 0xff00 + code - NAMED_KEYS if code >= NAMED_KEYS else code - 8 + if index == 0: + return sym + return sym - 0x20 if index == 1 and ord("a") <= sym <= ord("z") else 0 def get_keyboard_mapping(self, first: int, count: int) -> list: - return [(0, 0)] * count + return [(0, 0) if first + i in self.spare else (1, 0) for i in range(count)] def change_keyboard_mapping(self, first: int, keysyms: list) -> None: - self.remapped = (first, keysyms) + self.remapped.append((first, keysyms[0][0])) + + def query_keymap(self) -> list[int]: + return [sum(1 << bit for bit in range(8) if 8 * i + bit in self.down) for i in range(32)] class FakeMSS: @@ -76,7 +96,7 @@ def grab(self, monitor: dict): @pytest.fixture def x11(monkeypatch): - """The example's driver on a fake X display, and the keysyms the fake XK gave out by name.""" + """The example's driver on a fake X display, and the keycodes the fake keymap gives named keys.""" fake, keysyms = FakeX(), {} def open_display(): @@ -84,7 +104,16 @@ def open_display(): return fake def fake_input(d, kind, detail=0, x=0, y=0): + if d.fail_at is not None: + d.fail_at -= 1 + if d.fail_at == 0: + d.fail_at = None + raise ConnectionClosedError("display") d.events.append((kind, (x, y)) if kind == "move" else (kind, detail)) + if kind == "key-down": + d.down.add(detail) + elif kind == "key-up": + d.down.discard(detail) def module(name: str, **attrs) -> types.ModuleType: m = types.ModuleType(name) @@ -107,12 +136,19 @@ def module(name: str, **attrs) -> types.ModuleType: spec = importlib.util.spec_from_file_location("computer_x11_driver", DRIVER) example = importlib.util.module_from_spec(spec) spec.loader.exec_module(example) - return example.X11Driver(), fake, keysyms + return example.X11Driver(), fake, lambda name: fake.keysym_to_keycode(XK.string_to_keysym(name)) + + +async def _events(fake: FakeX, session, tool: str, args: dict) -> list: + fake.events.clear() + result = await session.call_tool(tool, args) + assert not result.isError, result.content[0].text + return list(fake.events) @pytest.mark.asyncio async def test_the_example_serves_the_whole_computer_contract(x11): - driver, _x, _keysyms = x11 + driver, _fake, _code = x11 async with http_client(driver) as http, mcp_session(driver) as session: card = (await http.get("/.well-known/agent-env.json")).json() tools = {t.name for t in (await session.list_tools()).tools} @@ -126,38 +162,109 @@ async def test_the_example_serves_the_whole_computer_contract(x11): @pytest.mark.asyncio async def test_tool_calls_become_the_xtest_events_they_mean(x11): - driver, x, keysyms = x11 - - async def events(session, tool: str, args: dict) -> list: - x.events.clear() - result = await session.call_tool(tool, args) - assert not result.isError, result.content[0].text - return list(x.events) - + driver, fake, code = x11 async with mcp_session(driver) as session: await session.call_tool("computer_screenshot", {}) - clicked = await events(session, "computer_click", {"x": 10, "y": 20, "button": "right", "modifiers": ["Shift"]}) - typed = await events(session, "computer_type", {"text": "a東\n"}) - dragged = await events(session, "computer_drag", {"path": [[0, 0], [40, 0]], "duration_ms": 0}) - pressed = await events(session, "computer_key", {"keys": ["Control", "s"]}) - shift, control, enter = (keysyms[name] + 8 for name in ("Shift_L", "Control_L", "Return")) + clicked = await _events(fake, session, "computer_click", + {"x": 10, "y": 20, "button": "right", "modifiers": ["Shift"]}) + typed = await _events(fake, session, "computer_type", {"text": "aA\n"}) + dragged = await _events(fake, session, "computer_drag", {"path": [[0, 0], [40, 0]], "duration_ms": 0}) + chord = await _events(fake, session, "computer_key", {"keys": ["Control", "s"]}) + capital = await _events(fake, session, "computer_key", {"keys": ["A"]}) + shift, a = code("Shift_L"), ord("a") + 8 assert clicked == [("key-down", shift), ("move", (10, 20)), ("button-down", 3), ("button-up", 3), ("key-up", shift)] - assert typed == [("key-down", 0x69), ("key-up", 0x69), ("key-down", 8), ("key-up", 8), - ("key-down", enter), ("key-up", enter)] - assert x.remapped == (8, [(0x01006771, 0x01006771)]) + assert typed == [("key-down", a), ("key-up", a), ("key-down", shift), ("key-down", a), ("key-up", a), + ("key-up", shift), ("key-down", code("Return")), ("key-up", code("Return"))] assert dragged == [("move", (0, 0)), ("button-down", 1), ("move", (20, 0)), ("move", (40, 0)), ("button-up", 1)] - assert pressed == [("key-down", control), ("key-down", 0x7b), ("key-up", 0x7b), ("key-up", control)] + assert chord == [("key-down", code("Control_L")), ("key-down", ord("s") + 8), ("key-up", ord("s") + 8), + ("key-up", code("Control_L"))] + assert capital == [("key-down", shift), ("key-down", a), ("key-up", a), ("key-up", shift)] + assert not fake.down + + +@pytest.mark.asyncio +async def test_keys_held_by_keyboard_down_stay_down_through_other_input(x11): + driver, fake, code = x11 + shift, a = code("Shift_L"), ord("a") + 8 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + held = await _events(fake, session, "computer_keyboard_down", {"keys": ["Shift"]}) + typed = await _events(fake, session, "computer_type", {"text": "A"}) + clicked = await _events(fake, session, "computer_click", {"x": 1, "y": 1, "modifiers": ["Shift"]}) + chord = await _events(fake, session, "computer_key", {"keys": ["Shift", "a"]}) + released = await _events(fake, session, "computer_keyboard_up", {}) + assert held == [("key-down", shift)] + assert typed == [("key-down", a), ("key-up", a)] and chord == [("key-down", a), ("key-up", a)] + assert clicked == [("move", (1, 1)), ("button-down", 1), ("button-up", 1)] + assert released == [("key-up", shift)] and not fake.down + + +@pytest.mark.asyncio +async def test_characters_off_the_keymap_share_the_spare_keycodes(x11): + driver, fake, _code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + repeated = await _events(fake, session, "computer_type", {"text": "東東"}) + mapped_once = list(fake.remapped) + recycled = await _events(fake, session, "computer_type", {"text": "京東"}) + assert repeated == [("key-down", 8), ("key-up", 8)] * 2 and mapped_once == [(8, TOKYO[0])] + assert recycled == [("key-down", 8), ("key-up", 8)] * 2 + assert fake.remapped == [(8, TOKYO[0]), (8, TOKYO[1]), (8, TOKYO[0])] + + +@pytest.mark.asyncio +async def test_text_that_cant_be_typed_whole_is_refused_before_any_of_it(x11): + driver, fake, _code = x11 + fake.spare = set() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.events.clear() + refused = await session.call_tool("computer_type", {"text": "a東"}) + assert refused.structuredContent["code"] == "unsupported" and refused.structuredContent["applied"] == "no" + assert fake.events == [] + + +@pytest.mark.asyncio +async def test_input_is_not_sent_again_when_the_connection_drops_during_it(x11): + driver, fake, _code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.events.clear() + fake.fail_at = 2 + dropped = await session.call_tool("computer_click", {"x": 5, "y": 5}) + assert dropped.structuredContent["code"] == "delivery_failed" + assert dropped.structuredContent["applied"] == "unknown" + assert fake.events == [("move", (5, 5))] and fake.opened == 1 + + +@pytest.mark.asyncio +async def test_a_drag_that_fails_or_is_cancelled_lets_go_of_its_button_and_modifiers(x11): + driver, fake, code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.events.clear() + fake.fail_at = 4 + failed = await session.call_tool("computer_drag", {"path": [[0, 0], [40, 0]], "modifiers": ["Shift"]}) + assert failed.structuredContent["code"] == "delivery_failed" + assert fake.events[-2:] == [("button-up", 1), ("key-up", code("Shift_L"))] and not fake.down + fake.events.clear() + dragging = asyncio.ensure_future(driver.drag_along([[0, 0], [400, 0]], "left", 10_000, ["Control"])) + await asyncio.sleep(0.2) + dragging.cancel() + with pytest.raises(asyncio.CancelledError): + await dragging + assert fake.events[-2:] == [("button-up", 1), ("key-up", code("Control_L"))] and not fake.down @pytest.mark.asyncio async def test_a_dropped_x_connection_is_reopened_and_a_dead_one_is_device_unavailable(x11): - driver, x, _keysyms = x11 + driver, fake, _code = x11 async with mcp_session(driver) as session: await session.call_tool("computer_screenshot", {}) - x.drops = 1 + fake.drops = 1 moved = await session.call_tool("computer_pointer_position", {}) - reopened = x.opened - x.drops = 2 + reopened = fake.opened + fake.drops = 2 refused = await session.call_tool("computer_pointer_position", {}) assert not moved.isError and reopened == 2 assert refused.structuredContent["code"] == "device_unavailable" From ba1a5f5802b827f5f5947ae266f7557a9b7293d3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 15:29:28 -0700 Subject: [PATCH 17/18] fix(protocol): read what a computer call's turn can change once it has the turn, and release timed holds - A click or scroll at the pointer reads the pointer once its input turn comes, not before it waits, so one queued behind a drag acts where the drag left the pointer. - A screenshot takes the input count before it captures, so input that lands while it runs counts as after the frame in others_since_frame. - A timed computer_keyboard_down that is cancelled during its hold still releases its keys. - The X11 example never maps a keycode a key holds down to another character, and finds every keycode a chord needs before pressing any, so a chord it can't hold whole is refused with nothing sent. Co-Authored-By: Claude Opus 5.5 --- .../examples/computer_x11/driver.py | 48 ++++++++++++------- .../src/agentenv_protocol/computer/runtime.py | 34 +++++++++---- .../src/agentenv_protocol/computer/tools.py | 35 +++++++------- .../tests/test_computer_queue.py | 45 +++++++++++++++++ .../tests/test_computer_x11_example.py | 17 +++++++ 5 files changed, 138 insertions(+), 41 deletions(-) diff --git a/packages/agentenv-protocol/examples/computer_x11/driver.py b/packages/agentenv-protocol/examples/computer_x11/driver.py index 565b2951..f09ecf88 100644 --- a/packages/agentenv-protocol/examples/computer_x11/driver.py +++ b/packages/agentenv-protocol/examples/computer_x11/driver.py @@ -241,7 +241,8 @@ def _connected(self) -> _Connection: class _Connection: """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare - keycode mapped to it, kept for that character and reused, or the least recently used one once all are taken.""" + keycode mapped to it, kept for that character and reused; once all are taken, the least recently used one that + no key holds down is mapped again.""" def __init__(self) -> None: self.d = xdisplay.Display() @@ -260,15 +261,18 @@ def button(self, button: str, down: bool) -> None: def hold(self, keysyms: list[int]) -> list[int]: """Press what the keysyms need that isn't down already, Shift first where a character needs it; the keycodes - this pressed, in order, so the caller releases only those.""" + this pressed, in order, so the caller releases only those. Every keycode is found before the first press, so + keysyms that can't all be held at once are refused with nothing sent.""" down = self._down() - pressed = [] + planned: list[int] = [] for keysym in keysyms: - for code in self.codes(keysym): - if code not in down: - xtest.fake_input(self.d, X.KeyPress, code) - down.add(code) - pressed.append(code) + planned += self.codes(keysym, busy=down | set(planned)) + pressed = [] + for code in planned: + if code not in down: + xtest.fake_input(self.d, X.KeyPress, code) + down.add(code) + pressed.append(code) return pressed def release(self, codes) -> None: @@ -278,8 +282,8 @@ def release(self, codes) -> None: def tap(self, keysym: int, hold_s: float) -> None: """Type one character: its key pressed and released, with Shift around it when it needs Shift and nothing holds Shift already.""" - *modifiers, code = self.codes(keysym) down = self._down() + *modifiers, code = self.codes(keysym, busy=down) pressed = [m for m in modifiers if m not in down] for m in pressed: xtest.fake_input(self.d, X.KeyPress, m) @@ -289,22 +293,34 @@ def tap(self, keysym: int, hold_s: float) -> None: xtest.fake_input(self.d, X.KeyRelease, code) self.release(reversed(pressed)) - def codes(self, keysym: int) -> list[int]: - """The keycodes that make ``keysym``: Shift and the key when it is the key's second level, else one key.""" + def codes(self, keysym: int, busy: set[int] = frozenset()) -> list[int]: + """The keycodes that make ``keysym``: Shift and the key when it is the key's second level, else one key; a + keysym off the keymap is mapped to a spare keycode not in ``busy``.""" codes = self._on_keymap(keysym) if codes is not None: return codes - self.check([keysym]) - code = self.spare.pop() if self.spare else self.mapped.popitem(last=False)[1] + code = self._free(busy) self.d.change_keyboard_mapping(code, [(keysym,) * self.width]) self.d.sync() self.mapped[keysym] = code return [code] def check(self, keysyms: list[int]) -> None: - """Refuse, before anything is sent, keysyms that can't be typed: one off the keymap with no spare keycode.""" - if not (self.spare or self.mapped) and any(self._on_keymap(k) is None for k in keysyms): - raise Unsupported("this X server has no spare keycode for the characters its keymap lacks") + """Refuse, before anything is sent, text with a character off the keymap when no spare keycode is free.""" + if any(self._on_keymap(k) is None for k in keysyms): + self._free(self._down(), take=False) + + def _free(self, busy: set[int], take: bool = True) -> int: + """A spare keycode not in ``busy``, the least recently used mapped one when none is unmapped.""" + if self.spare: + return self.spare.pop() if take else self.spare[-1] + for keysym, code in self.mapped.items(): + if code not in busy: + if take: + del self.mapped[keysym] + return code + raise Unsupported("no spare keycode is free for a character off the X keymap: the server has none, or keys " + "hold them all down") def _on_keymap(self, keysym: int) -> list[int] | None: if keysym in self.mapped: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index a2bdc7c8..36223531 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import contextlib from collections import OrderedDict from typing import TYPE_CHECKING, Any, Awaitable, Callable @@ -62,9 +63,12 @@ async def display_now(self) -> Display: return display async def capture(self, identity: str) -> Frame: + """A new frame of the screen. The input count is taken before the capture, so input that lands while it runs + counts as after the frame, which the frame may not show.""" + inputs = self.queue.snapshot() display = await self.display_now() image = to_pil(await self.call("capture"), display) - return self.frames.add(image, display, self.queue.snapshot(), identity=identity) + return self.frames.add(image, display, inputs, identity=identity) async def pointer(self) -> tuple[int, int]: where = await self.call("pointer_position") @@ -75,17 +79,21 @@ async def pointer(self) -> tuple[int, int]: raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, - summary: str, mapped: bool = False, repeat: int = 1, hold: tuple | None = None, - move_to: tuple | None = None, then: Callable | None = None) -> CallToolResult: + summary: str, mapped: bool = False, at_pointer: bool = False, repeat: int = 1, + hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None) -> CallToolResult: """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result - with others' input since ``frame`` and, when asked, a fresh frame. A point ``mapped`` from ``frame`` is - checked again once the call has its turn, since the display may have changed while it waited; ``then`` - runs as soon as the input is delivered.""" + with others' input since ``frame`` and, when asked, a fresh frame. Whatever the call's turn can change is read + once it has the turn: a point ``mapped`` from ``frame`` is checked again, since the display may have changed + while it waited, and ``at_pointer`` reads the pointer and puts it before ``args``. ``then`` runs as soon as + the input is delivered.""" identity = self.driver.caller_identity(ctx) sent = False async with self.queue.turn(self.driver.settings.queue_wait_s): if mapped and frame is not None: self.frames.check_current(frame, await self.display_now()) + if at_pointer: + sx, sy = await self.pointer() + args, summary = (sx, sy, *args), f"{summary} at the pointer, screen ({sx}, {sy})" others = self.queue.others_since(frame.inputs, identity) if frame is not None else 0 effect = None try: @@ -96,8 +104,7 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra effect = await self.call(operation, *args) sent = True if hold is not None: - await asyncio.sleep(hold[0] / 1000) - await self.call("key_up", list(reversed(hold[1]))) + await self._hold_for(*hold) except AgentEnvError as e: if sent: e.applied = "unknown" @@ -119,6 +126,17 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra result.summary, result.frame = f"{summary}, then captured {new.id}", new.result() return send(result, new.encode()) + async def _hold_for(self, ms: int, keys: list[str]) -> None: + """Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile.""" + release = ("key_up", list(reversed(keys))) + try: + await asyncio.sleep(ms / 1000) + except asyncio.CancelledError: + with contextlib.suppress(Exception): + await asyncio.shield(self.call(*release)) + raise + await self.call(*release) + async def ensure_ready(self) -> None: """Refuse with device_unavailable until the ready checks have passed once; run them if none has yet.""" if self.ready: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index b3f2de35..9b99c91b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -117,10 +117,11 @@ async def computer_click( button = _button(self, button) _check_range("count", count, 1, self.max_click_count) held = _modifiers(self, modifiers) - frame, (sx, sy), where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + frame, screen, where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) times = f" ×{count}" if count > 1 else "" - return await self._computer.act(ctx, "click", (sx, sy, button, count, held), frame=frame, mapped=mapped, - screenshot=screenshot, summary=f"Clicked {button}{times} {where}") + return await self._computer.act(ctx, "click", (*screen, button, count, held), frame=frame, mapped=mapped, + at_pointer=not screen, screenshot=screenshot, + summary=f"Clicked {button}{times}{where}") @tool(annotations=_INPUT) async def computer_drag( @@ -175,9 +176,10 @@ async def computer_scroll( await self._computer.ensure_ready() _check_range("amount", amount, 1, self.max_scroll_amount) held = _modifiers(self, modifiers) - frame, (sx, sy), where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) - return await self._computer.act(ctx, "scroll", (sx, sy, direction, amount, held), frame=frame, mapped=mapped, - screenshot=screenshot, summary=f"Scrolled {direction} {amount} {where}") + frame, screen, where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + return await self._computer.act(ctx, "scroll", (*screen, direction, amount, held), frame=frame, mapped=mapped, + at_pointer=not screen, screenshot=screenshot, + summary=f"Scrolled {direction} {amount}{where}") @tool(annotations=_INPUT) async def computer_type( @@ -250,9 +252,9 @@ async def computer_pointer_move( ) -> Annotated[CallToolResult, InputResult]: """Move the pointer to a point; hover effects happen.""" await self._computer.ensure_ready() - frame, (sx, sy), where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) - return await self._computer.act(ctx, "pointer_move", (sx, sy), frame=frame, mapped=mapped, - screenshot=screenshot, summary=f"Moved the pointer {where}") + frame, screen, where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + return await self._computer.act(ctx, "pointer_move", screen, frame=frame, mapped=mapped, + screenshot=screenshot, summary=f"Moved the pointer{where}") @tool(annotations=_INPUT) async def computer_pointer_down( @@ -369,20 +371,19 @@ async def _button_action(driver: Tools, ctx: ToolContext, op: str, verb: str, bu if x is None: return await computer.act(ctx, op, (button,), frame=computer.frames.latest(driver.caller_identity(ctx)), screenshot=screenshot, summary=f"{verb} {button}") - frame, (sx, sy), where, mapped = await _point(driver, ctx, x, y, frame_id, space, norm_scale) + frame, screen, where, mapped = await _point(driver, ctx, x, y, frame_id, space, norm_scale) return await computer.act(ctx, op, (button,), frame=frame, mapped=mapped, screenshot=screenshot, - move_to=(sx, sy), summary=f"{verb} {button} {where}") + move_to=screen, summary=f"{verb} {button}{where}") async def _point(driver: Tools, ctx: ToolContext, x: float | None, y: float | None, frame_id: str | None, - space: str | None, norm_scale: int | None) -> tuple[Frame | None, tuple[int, int], str, bool]: - """The frame, the screen pixel, a phrase for a point and whether it was mapped from the frame; with no point - and the pointer group, the pointer.""" + space: str | None, norm_scale: int | None) -> tuple[Frame | None, tuple[int, ...], str, bool]: + """The frame, the screen pixel, a phrase for the point and whether it was mapped from the frame. With no point + and the pointer group, no pixel and no phrase: the call acts at the pointer, read once it has its input turn.""" computer = driver._computer identity = driver.caller_identity(ctx) if x is None and y is None and driver.has("pointer") and frame_id is None: - sx, sy = await computer.pointer() - return computer.frames.latest(identity), (sx, sy), f"at the pointer, screen ({sx}, {sy})", False + return computer.frames.latest(identity), (), "", False if x is None or y is None: raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if driver.has("pointer") else "")) frame = computer.frames.get(frame_id, identity) @@ -390,7 +391,7 @@ async def _point(driver: Tools, ctx: ToolContext, x: float | None, y: float | No space, scale = _space(driver, space, norm_scale) fx, fy = frame_point(frame, x, y, space, scale) in_space = f" ({space} {scale})" if space == "norm" else "" - return frame, frame.to_screen(fx, fy), f"at ({x:g}, {y:g}){in_space} in {frame.id}", True + return frame, frame.to_screen(fx, fy), f" at ({x:g}, {y:g}){in_space} in {frame.id}", True def _space(driver: Tools, space: str | None, norm_scale: int | None) -> tuple[str, int]: diff --git a/packages/agentenv-protocol/tests/test_computer_queue.py b/packages/agentenv-protocol/tests/test_computer_queue.py index d8eb3838..93ca1f07 100644 --- a/packages/agentenv-protocol/tests/test_computer_queue.py +++ b/packages/agentenv-protocol/tests/test_computer_queue.py @@ -16,6 +16,7 @@ from agentenv_protocol import ROLE_META_KEY # noqa: E402 from agentenv_protocol.computer import ComputerSettings, pointer_position # noqa: E402 from agentenv_protocol.computer.queue import InputQueue # noqa: E402 +from agentenv_protocol.tool_context import EMPTY # noqa: E402 class ByRole(FakeDriver): @@ -430,3 +431,47 @@ async def test_a_point_queued_while_the_display_changes_is_refused_stale(): await resize assert queued.structuredContent["code"] == "stale_frame" and queued.structuredContent["applied"] == "no" assert [e[1:3] for e in driver.screen.log] == [(1, 1)] + + +@pytest.mark.asyncio +async def test_a_click_at_the_pointer_reads_the_pointer_when_its_turn_comes(): + driver = FakeDriver() + driver.slow["drag"] = 0.3 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_pointer_move", {"x": 5, "y": 5}) + dragging = asyncio.ensure_future(session.call_tool("computer_drag", {"path": [[10, 10], [100, 100]]})) + await asyncio.sleep(0.1) + clicked = await session.call_tool("computer_click", {}) + await dragging + assert clicked.structuredContent["summary"] == "Clicked left at the pointer, screen (100, 100)" + assert driver.screen.log[-1][:3] == ("click", 100, 100) + + +@pytest.mark.asyncio +async def test_input_that_lands_while_a_screenshot_is_taken_counts_as_after_it(): + driver = ByRole() + async with mcp_session(driver) as session: + await _shot(session, **{ROLE_META_KEY: "bob"}) + driver.slow["capture"] = 0.3 + shot = asyncio.ensure_future(_shot(session, **{ROLE_META_KEY: "alice"})) + await asyncio.sleep(0.1) + await session.call_tool("computer_click", {"x": 1, "y": 1}, meta={ROLE_META_KEY: "bob"}) + frame = await shot + del driver.slow["capture"] + clicked = await session.call_tool("computer_click", {"x": 2, "y": 2, "frame_id": frame["frame_id"]}, + meta={ROLE_META_KEY: "alice"}) + assert clicked.structuredContent["others_since_frame"] == 1 + + +@pytest.mark.asyncio +async def test_a_timed_hold_that_is_cancelled_still_lets_go_of_its_keys(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + holding = asyncio.ensure_future(driver.computer_keyboard_down(EMPTY, keys=["Shift"], hold_ms=5000)) + await asyncio.sleep(0.1) + holding.cancel() + with pytest.raises(asyncio.CancelledError): + await holding + assert driver.screen.log[-2:] == [("key_down", ("Shift",)), ("key_up", ("Shift",))] diff --git a/packages/agentenv-protocol/tests/test_computer_x11_example.py b/packages/agentenv-protocol/tests/test_computer_x11_example.py index bc20cc4c..d7ad391f 100644 --- a/packages/agentenv-protocol/tests/test_computer_x11_example.py +++ b/packages/agentenv-protocol/tests/test_computer_x11_example.py @@ -212,6 +212,23 @@ async def test_characters_off_the_keymap_share_the_spare_keycodes(x11): assert fake.remapped == [(8, TOKYO[0]), (8, TOKYO[1]), (8, TOKYO[0])] +@pytest.mark.asyncio +async def test_a_held_characters_keycode_is_not_taken_by_another(x11): + driver, fake, _code = x11 + fake.spare = {8, 9} + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + held = await _events(fake, session, "computer_keyboard_down", {"keys": ["東"]}) + fake.events.clear() + chord = await session.call_tool("computer_key", {"keys": ["京", "日"]}) + refused = list(fake.events) + typed = await _events(fake, session, "computer_type", {"text": "京"}) + released = await _events(fake, session, "computer_keyboard_up", {}) + assert held == [("key-down", 9)] and typed == [("key-down", 8), ("key-up", 8)] + assert chord.structuredContent["code"] == "unsupported" and chord.structuredContent["applied"] == "no" + assert refused == [] and released == [("key-up", 9)] and not fake.down + + @pytest.mark.asyncio async def test_text_that_cant_be_typed_whole_is_refused_before_any_of_it(x11): driver, fake, _code = x11 From 1e0d2bf7e872cf7a3b37ecf026249532704cc40e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 15:44:48 -0700 Subject: [PATCH 18/18] fix(protocol): find every keycode a computer_keyboard_down needs before pressing any in the X11 example keys_down held its keys one at a time, so keys it couldn't hold all at once were refused after the first were already down. It now plans them together through hold_each, which still returns the keycodes each key pressed for computer_keyboard_up. Co-Authored-By: Claude Opus 5.5 --- .../examples/computer_x11/driver.py | 26 ++++++++++++------- .../tests/test_computer_x11_example.py | 4 ++- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/examples/computer_x11/driver.py b/packages/agentenv-protocol/examples/computer_x11/driver.py index f09ecf88..aa19b533 100644 --- a/packages/agentenv-protocol/examples/computer_x11/driver.py +++ b/packages/agentenv-protocol/examples/computer_x11/driver.py @@ -189,9 +189,11 @@ def act(c): @key_down async def keys_down(self, keys) -> None: + syms = [_keysym(k) for k in keys] + def act(c): - for key in keys: - self._held.setdefault(key, []).extend(c.hold([_keysym(key)])) + for key, pressed in zip(keys, c.hold_each(syms)): + self._held.setdefault(key, []).extend(pressed) await self._send(act) @key_up @@ -260,19 +262,23 @@ def button(self, button: str, down: bool) -> None: xtest.fake_input(self.d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) def hold(self, keysyms: list[int]) -> list[int]: - """Press what the keysyms need that isn't down already, Shift first where a character needs it; the keycodes - this pressed, in order, so the caller releases only those. Every keycode is found before the first press, so - keysyms that can't all be held at once are refused with nothing sent.""" + """``hold_each``'s keycodes, in the order they were pressed.""" + return [code for pressed in self.hold_each(keysyms) for code in pressed] + + def hold_each(self, keysyms: list[int]) -> list[list[int]]: + """Press what the keysyms need that isn't down already, Shift first where a character needs it; for each + keysym, the keycodes this pressed, so the caller releases only those. Every keycode is found before the first + press, so keysyms that can't all be held at once are refused with nothing sent.""" down = self._down() - planned: list[int] = [] + planned: list[list[int]] = [] for keysym in keysyms: - planned += self.codes(keysym, busy=down | set(planned)) + planned.append(self.codes(keysym, busy=down | {code for codes in planned for code in codes})) pressed = [] - for code in planned: - if code not in down: + for codes in planned: + pressed.append([code for code in codes if code not in down]) + for code in pressed[-1]: xtest.fake_input(self.d, X.KeyPress, code) down.add(code) - pressed.append(code) return pressed def release(self, codes) -> None: diff --git a/packages/agentenv-protocol/tests/test_computer_x11_example.py b/packages/agentenv-protocol/tests/test_computer_x11_example.py index d7ad391f..2700e276 100644 --- a/packages/agentenv-protocol/tests/test_computer_x11_example.py +++ b/packages/agentenv-protocol/tests/test_computer_x11_example.py @@ -221,11 +221,13 @@ async def test_a_held_characters_keycode_is_not_taken_by_another(x11): held = await _events(fake, session, "computer_keyboard_down", {"keys": ["東"]}) fake.events.clear() chord = await session.call_tool("computer_key", {"keys": ["京", "日"]}) + held_too = await session.call_tool("computer_keyboard_down", {"keys": ["京", "日"]}) refused = list(fake.events) typed = await _events(fake, session, "computer_type", {"text": "京"}) released = await _events(fake, session, "computer_keyboard_up", {}) assert held == [("key-down", 9)] and typed == [("key-down", 8), ("key-up", 8)] - assert chord.structuredContent["code"] == "unsupported" and chord.structuredContent["applied"] == "no" + for call in (chord, held_too): + assert call.structuredContent["code"] == "unsupported" and call.structuredContent["applied"] == "no" assert refused == [] and released == [("key-up", 9)] and not fake.down