diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 8f9cf600..82f360b4 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -90,10 +90,34 @@ step names it. Steps that name it show and hide it as they do any tool. Steps fo that role's earlier named steps, and a full reset (`role: "*", tools: "*"`) clears all of them, so grant a hidden tool after such steps. Trigger tool actions call it as usual. +`@tool(..., annotations={"readOnlyHint": True}, structured_output=True, meta={...})` lists the tool +with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and on fastmcp alike. +`structured_output` follows mcp's FastMCP (None derives an output schema from the return annotation +where it can, False lists none); a tool returning `Annotated[CallToolResult, Model]` sends its +result as is, image blocks included, and is listed with `Model`'s schema. An environment chooses +at mount which of its `@tool` methods it serves, and with which descriptions, by overriding +`served_tools(tools)`. After the mount, `env.mounted_card()` is the card the application serves, and +`await env.update_tool(name, description=...)` changes a served tool's description on it and in +`tools/list` alike, on both FastMCPs. + `AgentEnvStarletteApplication` mounts the same handler onto a Starlette/FastAPI app instead of FastMCP; since those apps have no MCP tool registry, constructing one with `@tool` methods raises. +### Errors + +Raise `AgentEnvError(code, message, *, retryable=False, applied="no", hint=None, status=400)` to +answer a caller with a code rather than a crash. From an `@extension` method it answers with +`status` and the body `{"ok": false, "error": {code, message, retryable, applied, hint?}}`; from a +`@tool` it answers `isError` with `": "` as the text and the same fields as the +structured content. `applied` says whether the call's effect happened (`yes`, `no`, or `unknown` +when it may have been delivered before something failed), and callers retry on their own only when +`retryable` is true and `applied` is `no`. Any other exception in an extension still answers +`extension_failed` with status 500, now with `applied: "unknown"`. On the client side, +`invoke_extension` raises `ExtensionCallError`, an `httpx.HTTPStatusError` carrying `code`, +`detail`, `retryable`, `applied` and `hint`; a body without those fields, such as a proxy's error +page, leaves `code` None and `applied` `unknown`. + ### Who is calling A tool that needs the caller declares a parameter annotated `ToolContext` (any name) and receives @@ -180,6 +204,40 @@ extension's, and `client.invoke_extension(base_url, card, uri, params, method=.. its HTTP verb. Without `method`, `invoke_extension` calls the first method listed; an extension with several methods, such as a gateway's `urn:agentenv:clock/v1`, should always be called by name. +An extension with several methods is served by giving each handler an `op`. Handlers sharing a URI +merge into one card entry, each method under `params.methods.` with its own verb and endpoint, +by default `/agentenv/ext//`, where `` is the URN's name without +`urn:agentenv:` and the version. Extension-level params, such as features and limits, come from the +environment's `extension_params(uri)`, merged beside `methods`. A declaration-only extension, a +promise with params and no handler like `intake/v1`, is returned from `declared_extensions()`. +Two handlers for one method, or a handler without `op` on a URI another handler also serves, are +refused at mount. + +An environment chooses at mount which of its `@extension` handlers it serves by overriding +`served_extensions(extensions)`: an extension it leaves out is not served, and of one served with +`op`, only the methods its returned descriptor still lists under `params.methods`. +`served_operations(operations)` chooses the data-plane methods (`data/add` and the others it +marks) the same way. Either hook returning something with no handler is refused at mount. + +```python +@extension("urn:agentenv:computer-display/v1", op="get", method="GET") +async def display_get(self) -> dict: ... + +@extension("urn:agentenv:computer-display/v1", op="set") +async def display_set(self, width: int, height: int) -> dict: ... +``` + +### Tool access without a gateway + +An environment agents reach directly, with no gateway in front, can enforce per-role tool access +itself: set `tool_access = True` on the class. It then filters `tools/list` and refuses `tools/call` +(`permission_denied`) by the caller's role, read from `AgentEnv-Role` or a gateway-stamped `_meta` +role, and serves `urn:agentenv:disable-tool/v1` and `urn:agentenv:enable-tool/v1` at the gateway's +paths (`POST /tools/disable`, `POST /tools/enable`) with its bodies (`{role, tools: [...] | "*"}` to +`{role, disabled, allowed}`) and its rules, hidden-by-default tools included. So agent-env's +`modify_env_tool_access` step works on such an env unchanged. A call with no role is `default`, as +at the gateway. + Where the binding lives depends on the app. On mcp's own FastMCP it wraps the tool manager's dispatch; on the standalone `fastmcp` package it joins the app's middleware chain as the outermost layer, so a handler's own middleware runs under it too. An app with neither seam mounts unbound, with a warning, unless a tool declares a `ToolContext` parameter or the handler defines `on_tool_call`: then the mount is refused rather than serving an empty caller. ### Uploading an export through a grant @@ -254,6 +312,116 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session: - **Tools:** `list_tools()` returns the tools the role may use as `EnvironmentTool`, the type a card's `capabilities.tools` holds. Each `inputSchema` is the JSON Schema a model API takes as the tool's parameters. +## Computer drivers + +A computer driver is what runs inside a computer an agent uses through screenshots and input. Write +one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, which needs the +`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow). +The driver marks the methods that capture the screen and send input; the base class serves the +computer contract from them, so the tools, results and errors can't drift from it. A complete +driver for an X11 desktop, with an image to run it in, is in +[`examples/computer_x11/`](https://github.com/scaleapi/agentenv-framework/tree/main/packages/agentenv-protocol/examples/computer_x11). + +```python +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, Display, computer_driver, + capture, click, display, drag, press, scroll, type_text, +) + + +@computer_driver(name="my-driver", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}]) +class MyDriver(ComputerDriverEnvironment): + buttons = ("left", "right") # features, on the card + max_click_count = 2 + + @display + async def size(self) -> Display: ... # width, height, scale + @capture(display="x11") + async def grab(self): ... # the whole display, native resolution + @click + async def click_at(self, x, y, button, count, modifiers): ... + @drag + async def drag_along(self, path, button, duration_ms, modifiers): ... + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): ... + @type_text + async def type_in(self, text): ... + @press + async def press_keys(self, keys): ... + + +if __name__ == "__main__": + MyDriver().serve() +``` + +**Operations.** Operations get points in screen pixels, after the base class has mapped, checked +and queued the call, and may be sync or async. `@capture` returns a Pillow image or an +`Image(width, height, data, mode)` (PNG or JPEG bytes, or raw `RGB`, `RGBA`, `BGRA` or `BGRX` +pixels), at the display's size times its scale. Input operations may return an `Effect`. + +| Decorator | Required | Gives | +|---|---|---| +| `@display`, `@capture`, `@click`, `@drag`, `@scroll`, `@type_text`, `@press` | yes | the seven `computer_*` tools and `computer/v1` | +| `@set_display` | no, with `display_modes` | `computer-display/v1` `set` (`get` is always served) | +| `@pointer_move`, `@pointer_down`, `@pointer_up`, `@pointer_position` | all or none | `computer-pointer/v1` and its five tools; click and scroll at the pointer | +| `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | +| `@run_in_session` | no; a Linux default | `computer-exec/v1` | +| `@write_files` | no; a Linux default | `data/add` | + +The class is checked when it is defined: a missing required operation, half a group, two methods +for one operation that some platform could both match, or features that don't match the +decorators raise `TypeError`. Predicates (`os`, `distribution`, `version`, `arch`, `display`) +choose a method per platform, read from `/etc/os-release`, the machine and `XDG_SESSION_TYPE` or +`DISPLAY`: `@capture(display="x11")` and `@capture(display="wayland")` can sit side by side. A +driver's own `@run_in_session` or `@write_files` replaces the base class's Linux default, which runs +as the driver's user inside the desktop session. The base class keeps its state in `self._computer` +and names its own members `_computer_*` and `computer_*`; a driver that redefines one of them, or +another private member of the base class, is refused when it is defined, so a helper of the +driver's can't silently replace one. What the optional operations give is served only when the +driver has them, through the `served_tools`, `served_extensions` and `served_operations` hooks. + +**What the base class serves:** + +- **Tools.** `computer_screenshot` (a frame: `frame_id`, image size, display resolution and scale; + `region` crops a frame at native resolution), `computer_click`, `computer_drag`, + `computer_scroll`, `computer_type`, `computer_key` (W3C key names) and `computer_wait`, plus the + group tools. Every result is structured, with a one-line summary as its text block; input results + carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame + after acting. Click and scroll leave out `x` and `y` to act at the pointer on a computer with + `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. +- **Frames and spaces.** Points are pixels of a named frame (your latest by default), or a + 0..`norm_scale` grid over it with `space: "norm"`. A point outside the frame or the range is + refused (`out_of_bounds`), never clamped. The last 64 frames are kept, and a frame goes stale + (`stale_frame`) when the display geometry it was captured at changes. +- **The input queue.** Input calls run one at a time in arrival order; one that waits past + `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. +- **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a + capture, plus the driver's own `ready_checks()`; tools refuse `device_unavailable` until they + pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool + descriptions), `computer-keyboard/v1` and `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, + request_id}`: output capped per stream, a repeated `request_id` answered with the first result). +- **The data plane.** `data/add` takes file parts that name their path, relative to the session + user's home, under the `data-objects/v1` marker (`{"path": ...}`), as inline bytes, an https URL, + or a `transfers.ReadObject` read grant under the marker's `read`, and writes them atomically. + `data/get` returns the files under the settings' `data_paths`, inline or uploaded under a + `write_namespace` grant. +- **The card** (the name from `ENVIRONMENT_NAME`, then `@computer_driver`'s), and per-role tool + access (`tool_access` is on). + +**Errors.** A driver raises the errors table's exceptions, all `AgentEnvError`s: +`InvalidArgument`, `OutOfBounds`, `StaleFrame`, `InputBusy`, `PermissionDenied`, `Unsupported`, +`BackgroundUnavailable`, `TargetUnavailable`, `DeviceUnavailable` and `LimitExceeded` mean nothing +was sent (`applied: "no"`); `Timeout`, `DeliveryFailed` and `Internal` mean it may have been. Any +other exception from an input operation is reported `delivery_failed` with `applied: "unknown"`. + +**Settings** reach the driver as one JSON value in `AGENTENV_COMPUTER_SETTINGS` +(`ComputerSettings`): `data_paths`, `default_space`, `default_norm_scale`, `queue_wait_s`, +`tap_hold_ms`, `type_delay_ms`, `max_hold_ms`, `layout`, `drag_duration_ms`, `exec_output_bytes` +and `exec_timeout_s`. A driver reads them as `self.settings`, for example its tap hold in `@press`. + +Every caller is `default` until agents carry an identity of their own; `caller_identity(context)` +names the caller whose latest frame and others' input a call uses. + ## A2A agent framework The distribution exposes two unrelated decorators named `extension`: diff --git a/packages/agentenv-protocol/examples/computer_x11/Dockerfile b/packages/agentenv-protocol/examples/computer_x11/Dockerfile new file mode 100644 index 00000000..02b4b7e3 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/Dockerfile @@ -0,0 +1,19 @@ +# Built from packages/agentenv-protocol, so the driver runs on this checkout's SDK: +# docker build -f examples/computer_x11/Dockerfile -t x11-desktop . +FROM public.ecr.aws/docker/library/ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-tk xvfb x11-utils fonts-dejavu-core procps \ + libxcb1 libxcb-render0 libxcb-shm0 libxcb-randr0 libxcb-xfixes0 libxcb-xinerama0 \ + && rm -rf /var/lib/apt/lists/* +COPY pyproject.toml README.md LICENSE NOTICE THIRD_PARTY_NOTICES.md /opt/protocol/ +COPY src /opt/protocol/src +RUN python3 -m venv /opt/driver \ + && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer]" "mss>=9,<11" "python-xlib>=0.33,<0.34" +RUN useradd --create-home --uid 10001 user +COPY examples/computer_x11/start.sh examples/computer_x11/driver.py examples/computer_x11/fixture.py /opt/desktop/ +USER user +WORKDIR /home/user +ENV DISPLAY=:99 XDG_SESSION_TYPE=x11 +EXPOSE 18765 +CMD ["/bin/sh", "/opt/desktop/start.sh"] diff --git a/packages/agentenv-protocol/examples/computer_x11/README.md b/packages/agentenv-protocol/examples/computer_x11/README.md new file mode 100644 index 00000000..c69a20f0 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/README.md @@ -0,0 +1,60 @@ +# An X11 computer driver + +[`driver.py`](driver.py) is a complete computer driver for an X11 desktop, written on +`ComputerDriverEnvironment` from `agentenv_protocol.computer`: `mss` captures the screen and XTEST, +through `python-xlib`, sends the input. It implements the seven required operations and the pointer +and keyboard groups, declares five buttons, triple clicks and 50-point drags, and adds a ready check +of its own for XTEST. The base class serves everything else from those: the `computer_*` tools, +frames and coordinate spaces, the input queue, the framework extensions and the card, and its Linux +defaults give the driver `computer-exec/v1` and `data/add`. + +The image runs the driver on Xvfb beside [`fixture.py`](fixture.py), the Fixture Desk: a Tk window +with a name field, a checkbox, a 100-item list, a grid of cells, a drawing pad and a Save button. It +records whatever reaches it in `~/fixture/state.json`, so what an agent did can be read back from +the app rather than from the driver's answers, and shows the text of `~/inbox/message.txt`, which +`data/add` can write. [`start.sh`](start.sh) restarts Xvfb and the app if either dies. + +## Run it + +```bash +cd packages/agentenv-protocol +docker build -f examples/computer_x11/Dockerfile -t x11-desktop . +docker run --rm -p 18765:18765 x11-desktop +``` + +The card is at `http://localhost:18765/.well-known/agent-env.json`, readiness at +`/agentenv/ext/computer-ready/status`, and the tools at the MCP endpoint `/mcp`: + +```python +import asyncio + +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + + +async def main() -> None: + async with streamablehttp_client("http://localhost:18765/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + shot = await session.call_tool("computer_screenshot", {}) + print(shot.structuredContent["summary"]) + await session.call_tool("computer_click", {"x": 300, "y": 52}) + typed = await session.call_tool("computer_type", {"text": "Ada Lovelace", "screenshot": True}) + print(typed.structuredContent["summary"]) + + +asyncio.run(main()) +``` + +`AGENTENV_COMPUTER_SETTINGS` takes the driver's settings as one JSON value, for example +`-e AGENTENV_COMPUTER_SETTINGS='{"default_space": "norm", "data_paths": ["fixture"]}'`. + +## In agent-env + +As a bundle env (`envs//Dockerfile`), the image is built from the env's own folder, so copy +the three files there and install `agentenv-framework-protocol[computer]` in place of this checkout. +Name the env with `environment_name` in its `env.toml`. A task then deploys it with `deploy_env` +and gives it to an agent with `deploy_agent`, as for any MCP env. + +`tests/test_computer_x11_example.py` runs the driver against fake `mss` and `Xlib` modules, so it +keeps up with the SDK without an X server. diff --git a/packages/agentenv-protocol/examples/computer_x11/driver.py b/packages/agentenv-protocol/examples/computer_x11/driver.py new file mode 100644 index 00000000..aa19b533 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/driver.py @@ -0,0 +1,374 @@ +"""An X11 computer driver: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off the +event loop, one at a time, on a connection checked before each call and reopened when it drops, so the driver outlives +a restarted X server.""" +from __future__ import annotations + +import asyncio +import contextlib +import math +import threading +import time +from collections import OrderedDict +from typing import Any, Callable + +import mss +from Xlib import X, XK +from Xlib import display as xdisplay +from Xlib.error import ConnectionClosedError, DisplayConnectionError +from Xlib.ext import xtest + +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + DeliveryFailed, + DeviceUnavailable, + Display, + Image, + ReadyCheck, + Unsupported, + capture, + click, + computer_driver, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + scroll, + type_text, +) + +BUTTONS = {"left": 1, "middle": 2, "right": 3, "back": 8, "forward": 9} +SCROLL = {"up": 4, "down": 5, "left": 6, "right": 7} +#: W3C key values and their X keysym names; any other single character is typed as its own keysym. +KEYSYMS = { + "Enter": "Return", "Tab": "Tab", "Escape": "Escape", "Backspace": "BackSpace", "Delete": "Delete", + "ArrowUp": "Up", "ArrowDown": "Down", "ArrowLeft": "Left", "ArrowRight": "Right", "Home": "Home", "End": "End", + "PageUp": "Prior", "PageDown": "Next", "Insert": "Insert", "Control": "Control_L", "Shift": "Shift_L", + "Alt": "Alt_L", "Meta": "Super_L", "Super": "Super_L", "CapsLock": "Caps_Lock", "ContextMenu": "Menu", + "PrintScreen": "Print", "Pause": "Pause", " ": "space", **{f"F{n}": f"F{n}" for n in range(1, 25)}, +} +TYPED = {"\n": "Return", "\t": "Tab"} +#: The pause between the presses of a double or triple click, in seconds. +CLICK_GAP_S = 0.04 +#: How far apart a drag's pointer moves are, in pixels. +DRAG_STEP_PX = 20 +DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) + + +@computer_driver(name="x11-desktop", + supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +class X11Driver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle", "back", "forward") + max_click_count = 3 + max_drag_points = 50 + + def __init__(self) -> None: + super().__init__() + self._x: _Connection | None = None + self._x_lock = threading.Lock() + #: The keycodes each key held by @key_down pressed, which @key_up releases. + self._held: dict[str, list[int]] = {} + + @display + async def screen(self) -> Display: + def read(c): + s = c.d.screen() + return Display(s.width_in_pixels, s.height_in_pixels, 1.0) + return await self._read(read) + + @capture + async def grab(self) -> Image: + return await asyncio.to_thread(_grab) + + async def ready_checks(self) -> list[ReadyCheck]: + try: + has = await self._read(lambda c: c.d.has_extension("XTEST")) + except DeviceUnavailable as e: + return [ReadyCheck("xtest", False, e.message)] + return [ReadyCheck("xtest", bool(has), "XTEST available" if has else "XTEST missing")] + + @click + async def click_at(self, x, y, button, count, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] + + def act(c): + held = c.hold(syms) + c.move(x, y) + for i in range(count): + c.button(button, True) + c.button(button, False) + if i + 1 < count: + c.d.sync() + time.sleep(CLICK_GAP_S) + c.release(reversed(held)) + await self._send(act) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] + steps = _steps(path) + pause = duration_ms / 1000 / len(steps) + held: list[int] = [] + + def start(c): + held.extend(c.hold(syms)) + c.move(*path[0]) + c.button(button, True) + + def finish(c): + c.button(button, False) + c.release(reversed(held)) + + try: + await self._send(start) + for px, py in steps: + await self._send(lambda c, px=px, py=py: c.move(px, py)) + await asyncio.sleep(pause) + except BaseException: + with contextlib.suppress(Exception): + await self._send(finish) + raise + await self._send(finish) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] + + def act(c): + held = c.hold(syms) + c.move(x, y) + for _ in range(amount): + xtest.fake_input(c.d, X.ButtonPress, SCROLL[direction]) + xtest.fake_input(c.d, X.ButtonRelease, SCROLL[direction]) + c.release(reversed(held)) + await self._send(act) + + @pointer_move + async def move(self, x, y) -> None: + await self._send(lambda c: c.move(x, y)) + + @pointer_down + async def down(self, button) -> None: + await self._send(lambda c: c.button(button, True)) + + @pointer_up + async def up(self, button) -> None: + await self._send(lambda c: c.button(button, False)) + + @pointer_position + async def where(self) -> tuple[int, int]: + def read(c): + p = c.d.screen().root.query_pointer() + return p.root_x, p.root_y + return await self._read(read) + + @type_text + async def type_in(self, text) -> None: + syms = [XK.string_to_keysym(TYPED[ch]) if ch in TYPED else _keysym(ch) for ch in text] + await self._read(lambda c: c.check(syms)) + for sym in syms: + hold = self.settings.tap_hold_ms / 1000 + await self._send(lambda c, sym=sym: c.tap(sym, hold)) + await asyncio.sleep(self.settings.type_delay_ms / 1000) + + @press + async def press_keys(self, keys) -> None: + syms = [_keysym(k) for k in keys] + hold = self.settings.tap_hold_ms / 1000 + + def act(c): + pressed = c.hold(syms) + c.d.sync() + time.sleep(hold) + c.release(reversed(pressed)) + await self._send(act) + + @key_down + async def keys_down(self, keys) -> None: + syms = [_keysym(k) for k in keys] + + def act(c): + for key, pressed in zip(keys, c.hold_each(syms)): + self._held.setdefault(key, []).extend(pressed) + await self._send(act) + + @key_up + async def keys_up(self, keys) -> None: + def act(c): + for key in keys: + c.release(reversed(self._held.pop(key, None) or c.codes(_keysym(key))[-1:])) + await self._send(act) + + async def _read(self, fn: Callable[[_Connection], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn, False) + + async def _send(self, fn: Callable[[_Connection], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn, True) + + def _x_call(self, fn: Callable[[_Connection], Any], sends_input: bool) -> Any: + """``fn`` on a live connection. When the connection drops, a read is retried once on a new one; input is not + sent again, since some of it may have arrived.""" + with self._x_lock: + for attempt in (1, 2): + c = self._connected() + try: + result = fn(c) + c.d.sync() + return result + except DROPPED as e: + self._x = None + if sends_input: + raise DeliveryFailed(f"the X connection dropped while sending input: {type(e).__name__}") from None + if attempt == 2: + raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + + def _connected(self) -> _Connection: + """The connection, opened or reopened as needed and checked with a round trip before anything is sent.""" + for attempt in (1, 2): + try: + if self._x is None: + self._x = _Connection() + self._held.clear() + self._x.d.sync() + return self._x + except DROPPED as e: + self._x = None + if attempt == 2: + raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + + +class _Connection: + """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare + keycode mapped to it, kept for that character and reused; once all are taken, the least recently used one that + no key holds down is mapped again.""" + + def __init__(self) -> None: + self.d = xdisplay.Display() + first = self.d.display.info.min_keycode + mapping = self.d.get_keyboard_mapping(first, self.d.display.info.max_keycode - first + 1) + self.width = len(mapping[0]) + self.spare = [first + i for i, syms in enumerate(mapping) if not any(syms)] + self.mapped: OrderedDict[int, int] = OrderedDict() + self.shift = self.d.keysym_to_keycode(XK.string_to_keysym("Shift_L")) + + def move(self, x, y) -> None: + xtest.fake_input(self.d, X.MotionNotify, x=int(x), y=int(y)) + + def button(self, button: str, down: bool) -> None: + xtest.fake_input(self.d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) + + def hold(self, keysyms: list[int]) -> list[int]: + """``hold_each``'s keycodes, in the order they were pressed.""" + return [code for pressed in self.hold_each(keysyms) for code in pressed] + + def hold_each(self, keysyms: list[int]) -> list[list[int]]: + """Press what the keysyms need that isn't down already, Shift first where a character needs it; for each + keysym, the keycodes this pressed, so the caller releases only those. Every keycode is found before the first + press, so keysyms that can't all be held at once are refused with nothing sent.""" + down = self._down() + planned: list[list[int]] = [] + for keysym in keysyms: + planned.append(self.codes(keysym, busy=down | {code for codes in planned for code in codes})) + pressed = [] + for codes in planned: + pressed.append([code for code in codes if code not in down]) + for code in pressed[-1]: + xtest.fake_input(self.d, X.KeyPress, code) + down.add(code) + return pressed + + def release(self, codes) -> None: + for code in codes: + xtest.fake_input(self.d, X.KeyRelease, code) + + def tap(self, keysym: int, hold_s: float) -> None: + """Type one character: its key pressed and released, with Shift around it when it needs Shift and nothing + holds Shift already.""" + down = self._down() + *modifiers, code = self.codes(keysym, busy=down) + pressed = [m for m in modifiers if m not in down] + for m in pressed: + xtest.fake_input(self.d, X.KeyPress, m) + xtest.fake_input(self.d, X.KeyPress, code) + self.d.sync() + time.sleep(hold_s) + xtest.fake_input(self.d, X.KeyRelease, code) + self.release(reversed(pressed)) + + def codes(self, keysym: int, busy: set[int] = frozenset()) -> list[int]: + """The keycodes that make ``keysym``: Shift and the key when it is the key's second level, else one key; a + keysym off the keymap is mapped to a spare keycode not in ``busy``.""" + codes = self._on_keymap(keysym) + if codes is not None: + return codes + code = self._free(busy) + self.d.change_keyboard_mapping(code, [(keysym,) * self.width]) + self.d.sync() + self.mapped[keysym] = code + return [code] + + def check(self, keysyms: list[int]) -> None: + """Refuse, before anything is sent, text with a character off the keymap when no spare keycode is free.""" + if any(self._on_keymap(k) is None for k in keysyms): + self._free(self._down(), take=False) + + def _free(self, busy: set[int], take: bool = True) -> int: + """A spare keycode not in ``busy``, the least recently used mapped one when none is unmapped.""" + if self.spare: + return self.spare.pop() if take else self.spare[-1] + for keysym, code in self.mapped.items(): + if code not in busy: + if take: + del self.mapped[keysym] + return code + raise Unsupported("no spare keycode is free for a character off the X keymap: the server has none, or keys " + "hold them all down") + + def _on_keymap(self, keysym: int) -> list[int] | None: + if keysym in self.mapped: + self.mapped.move_to_end(keysym) + return [self.mapped[keysym]] + code = self.d.keysym_to_keycode(keysym) + if code and self.d.keycode_to_keysym(code, 0) == keysym: + return [code] + if code and self.d.keycode_to_keysym(code, 1) == keysym: + return [self.shift, code] + return None + + def _down(self) -> set[int]: + return {8 * i + bit for i, byte in enumerate(self.d.query_keymap()) for bit in range(8) if byte >> bit & 1} + + +def _grab() -> Image: + try: + with mss.MSS() as m: + shot = m.grab(m.monitors[0]) + return Image(shot.width, shot.height, bytes(shot.bgra), "BGRA") + except Exception as e: + raise DeviceUnavailable(f"capture failed: {type(e).__name__}: {e}") from None + + +def _steps(path: list) -> list[tuple[float, float]]: + """The points a drag moves through after its start, about every DRAG_STEP_PX pixels.""" + steps = [] + for (x0, y0), (x1, y1) in zip(path, path[1:]): + n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX)) + steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] + return steps + + +def _keysym(key: str) -> int: + if key in KEYSYMS: + return XK.string_to_keysym(KEYSYMS[key]) + if len(key) == 1: + code = ord(key) + return code if code < 0x100 else 0x01000000 | code + raise Unsupported(f"this driver has no X keysym for {key!r}") + + +if __name__ == "__main__": + X11Driver().serve() diff --git a/packages/agentenv-protocol/examples/computer_x11/fixture.py b/packages/agentenv-protocol/examples/computer_x11/fixture.py new file mode 100644 index 00000000..78cd2699 --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/fixture.py @@ -0,0 +1,173 @@ +"""The Fixture Desk: a Tk window whose every effect is recorded in ~/fixture/state.json, and whose widgets' screen +boxes are written to ~/fixture/layout.json, so a test can aim at them and read back what reached them.""" +import json +import os +import tkinter as tk +from pathlib import Path + +HOME = Path.home() / "fixture" +HOME.mkdir(exist_ok=True) +COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff") + +root = tk.Tk() +root.title("Fixture Desk") +root.geometry("1280x800+0+0") +root.configure(bg="#f0f0f0") +state = {"name": "", "saved_count": 0, "subscribed": False, "selected": None, "cells": {}, "drags": [], + "keys": [], "scroll_top": 0.0, "right_clicks": 0, "double_clicks": 0, "loaded": None} + + +def save_state(): + tmp = HOME / ".state.json.part" + tmp.write_text(json.dumps(state)) + os.replace(tmp, HOME / "state.json") + + +tk.Label(root, text="Name:", font=("DejaVu Sans", 16), bg="#f0f0f0").place(x=40, y=40) +name = tk.Entry(root, font=("DejaVu Sans", 16), width=30) +name.place(x=140, y=40) + + +def on_key(event): + state["name"] = name.get() + state["keys"] = (state["keys"] + [event.keysym])[-50:] + save_state() + + +name.bind("", on_key) + + +def on_save(): + state["name"] = name.get() + state["saved_count"] += 1 + save_state() + + +save = tk.Button(root, text="Save", font=("DejaVu Sans", 16), command=on_save, width=8) +save.place(x=560, y=34) + +subscribed = tk.BooleanVar(value=False) + + +def on_subscribe(): + state["subscribed"] = subscribed.get() + save_state() + + +subscribe = tk.Checkbutton(root, text="Subscribe", font=("DejaVu Sans", 16), variable=subscribed, + command=on_subscribe, bg="#f0f0f0") +subscribe.place(x=40, y=100) + +frame = tk.Frame(root) +frame.place(x=40, y=160, width=300, height=400) +listbox = tk.Listbox(frame, font=("DejaVu Sans", 14), exportselection=False) +listbox.pack(side="left", fill="both", expand=True) +bar = tk.Scrollbar(frame, command=listbox.yview) +bar.pack(side="right", fill="y") + + +def on_scroll(first, last): + bar.set(first, last) + state["scroll_top"] = float(first) + save_state() + + +listbox.configure(yscrollcommand=on_scroll) +for i in range(1, 101): + listbox.insert("end", f"Item {i}") + + +def on_select(event): + chosen = listbox.curselection() + state["selected"] = listbox.get(chosen[0]) if chosen else None + save_state() + + +listbox.bind("<>", on_select) + +grid = tk.Canvas(root, width=400, height=400, bg="#ffffff", highlightthickness=0) +grid.place(x=400, y=160) +CELL = 50 +for row in range(8): + for col in range(8): + grid.create_rectangle(col * CELL, row * CELL, (col + 1) * CELL, (row + 1) * CELL, fill=COLOURS[0], + outline="#999999", tags=(f"cell-{row}-{col}",)) + + +def on_cell(event, button): + row, col = event.y // CELL, event.x // CELL + key = f"{row},{col}" + count = (state["cells"].get(key, 0) + 1) % len(COLOURS) + state["cells"][key] = count + grid.itemconfigure(f"cell-{row}-{col}", fill=COLOURS[count]) + if button == 3: + state["right_clicks"] += 1 + save_state() + + +def on_double(event): + state["double_clicks"] += 1 + save_state() + + +grid.bind("", lambda e: on_cell(e, 1)) +grid.bind("", lambda e: on_cell(e, 3)) +grid.bind("", on_double) + +pad = tk.Canvas(root, width=400, height=400, bg="#fffbe6", highlightthickness=1) +pad.place(x=840, y=160) +stroke = [] + + +def on_press(event): + stroke.clear() + stroke.append((event.x, event.y)) + + +def on_motion(event): + if stroke: + pad.create_line(stroke[-1][0], stroke[-1][1], event.x, event.y, width=3) + stroke.append((event.x, event.y)) + + +def on_release(event): + if stroke: + stroke.append((event.x, event.y)) + state["drags"] = (state["drags"] + [{"start": stroke[0], "end": stroke[-1], "points": len(stroke)}])[-20:] + save_state() + stroke.clear() + + +pad.bind("", on_press) +pad.bind("", on_motion) +pad.bind("", on_release) + +loaded = tk.Label(root, text="", font=("DejaVu Sans", 14), bg="#f0f0f0") +loaded.place(x=40, y=600) + + +def poll_loaded(): + path = Path.home() / "inbox" / "message.txt" + if path.exists(): + text = path.read_text() + if text != state["loaded"]: + state["loaded"] = text + loaded.configure(text=f"Loaded: {text}") + save_state() + root.after(200, poll_loaded) + + +def write_layout(): + root.update_idletasks() + boxes = {} + for key, widget in (("name", name), ("save", save), ("subscribe", subscribe), ("list", listbox), + ("grid", grid), ("pad", pad)): + boxes[key] = [widget.winfo_rootx(), widget.winfo_rooty(), widget.winfo_width(), widget.winfo_height()] + boxes["list_row_height"] = listbox.bbox(0)[3] if listbox.bbox(0) else 0 + (HOME / "layout.json").write_text(json.dumps(boxes)) + save_state() + + +root.after(300, write_layout) +root.after(500, poll_loaded) +root.mainloop() diff --git a/packages/agentenv-protocol/examples/computer_x11/start.sh b/packages/agentenv-protocol/examples/computer_x11/start.sh new file mode 100644 index 00000000..31dcb19d --- /dev/null +++ b/packages/agentenv-protocol/examples/computer_x11/start.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# The desktop: Xvfb and the fixture each restart if they die, then the driver serves on MCP_PORT. +(while true; do Xvfb :99 -screen 0 1280x800x24 -nolisten tcp -ac >/tmp/xvfb.log 2>&1; sleep 0.5; done) & +for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done +(while true; do + for i in $(seq 100); do xdpyinfo -display :99 >/dev/null 2>&1 && break; sleep 0.1; done + python3 /opt/desktop/fixture.py >/tmp/fixture.log 2>&1; sleep 0.5 + done) & +exec /opt/driver/bin/python /opt/desktop/driver.py diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 979f1422..277d3a5f 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -35,6 +35,12 @@ agent = [ "regex>=2023.12.25", "uvicorn>=0.30", ] +# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP and crops and encodes frames. +computer = [ + "mcp>=1.25,<2", + "Pillow>=10.4", + "uvicorn>=0.30", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -43,6 +49,7 @@ dev = [ "pytest-asyncio>=0.23.0", "mcp>=1.25,<2", "fastmcp>=3.4,<4", # the standalone-fastmcp mount path + "Pillow>=10.4", # the computer extra's frame crops and encodings ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py index c68ef7f2..fd209caf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py @@ -1,6 +1,8 @@ """Open data-plane protocol and server SDK for agent environments.""" from . import client +from .client import ExtensionCallError +from .errors import Applied, AgentEnvError from .preflight import intake_fit_check from .tool_client import ToolResult, ToolSession, ToolSessionError from .agent_env_environment import ( @@ -144,6 +146,9 @@ "AddDataResponse", "GetDataResponse", "error_body", + "AgentEnvError", + "Applied", + "ExtensionCallError", "PROTOCOL_VERSION", "RPC_PATH", "WELL_KNOWN_PATH", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index c1e9b9c6..02b1c889 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -15,12 +15,15 @@ from abc import ABC, abstractmethod from typing import Annotated, Any, Callable, ClassVar, Literal, NamedTuple, Union, get_args, get_origin, get_type_hints -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from pydantic.fields import FieldInfo from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route +from .errors import AgentEnvError +from .tool_access import DISABLE as TOOL_DISABLE, ENABLE as TOOL_ENABLE, PATHS as TOOL_ACCESS_PATHS, ToolAccess, caller_role +from .tool_access import extensions as tool_access_extensions from .tool_context import EMPTY, ToolContext, bound, context_from_http, context_from_mcp from .transfers import WriteNamespaceGrant from .types import DATA_OBJECTS_EXTENSION_URI, MCP_TRANSPORT, METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH, AddDataRequest, AddDataResponse, EnvironmentCapabilities, EnvironmentCard, EnvironmentExtension, EnvironmentInterface, EnvironmentTool, GetDataResponse, ResetDataResponse, error_body @@ -30,8 +33,10 @@ _OP_ATTR = "_aee_op" _EXT_ATTR = "_aee_ext" _EXT_ROUTE_ATTR = "_aee_ext_route" +_EXT_OP_ATTR = "_aee_ext_op" _TOOL_ATTR = "_aee_tool" _HIDDEN_BY_DEFAULT_ATTR = "_aee_hidden_by_default" +_TOOL_OPTIONS_ATTR = "_aee_tool_options" _CARD_CONFIG_ATTR = "_aee_card_config" #: Set on the tool dispatch installed by ``_bind_tool_calls``; a guard downstream checks it at boot. _TOOL_CONTEXT_BOUND_ATTR = "__agentenv_tool_context__" @@ -68,7 +73,7 @@ def get_data(fn: Callable) -> Callable: return fn -def extension(uri: str, *, description: str | None = None, params: dict | None = None, +def extension(uri: str, *, op: str | None = None, description: str | None = None, params: dict | None = None, required: bool | None = None, method: str = "POST", path: str | None = None) -> Callable: """Mark a handler method as the invocation handler for a card extension. @@ -86,7 +91,15 @@ def extension(uri: str, *, description: str | None = None, params: dict | None = where the request JSON-Schema is derived from the signature. Pass an explicit ``params`` to override the advertisement. A handler that wants the raw params dict can declare ``**params``. + + With ``op``, the handler is one method of an extension that has several: handlers sharing the URI + merge into one card entry, each method under ``methods.`` with its own verb and ``endpoint``, + by default ``/ext//`` where ```` is the URN's name without + ``urn:agentenv:`` and the version. Extension-level params come from the environment's + ``extension_params(uri)``, merged beside ``methods``, so ``params`` is refused with ``op``. """ + if op is not None and params is not None: + raise TypeError("an extension method's advertisement comes from extension_params(uri), not params=") # Normalize the HTTP verb once, here, so the advertised card method, the # route registration, and the handler's request-binding (which checks for # the exact string "GET") all agree. Without this, ``@extension(method="get")`` @@ -95,20 +108,35 @@ def extension(uri: str, *, description: str | None = None, params: dict | None = http_method = method.upper() def deco(fn: Callable) -> Callable: - op = fn.__name__ or "invoke" - route_path = path or f"{RPC_PATH}/ext/{op}" - advertised = params if params is not None else { - "endpoint": route_path, - "methods": {op: {"method": http_method, "request": _schema_from_signature(fn)}}, - } + if op is None: + name = fn.__name__ or "invoke" + route_path = path or f"{RPC_PATH}/ext/{name}" + advertised = params if params is not None else { + "endpoint": route_path, + "methods": {name: {"method": http_method, "request": _schema_from_signature(fn)}}, + } + else: + route_path = path or f"{RPC_PATH}/ext/{_extension_slug(uri)}/{op}" + advertised = {"methods": {op: {"method": http_method, "endpoint": route_path, "request": _schema_from_signature(fn)}}} setattr(fn, _EXT_ATTR, EnvironmentExtension(uri=uri, description=description, params=advertised, required=required)) setattr(fn, _EXT_ROUTE_ATTR, (route_path, http_method)) + setattr(fn, _EXT_OP_ATTR, op) return fn return deco -def tool(name: str | None = None, *, description: str | None = None, hidden_by_default: bool = False) -> Callable: +def _extension_slug(uri: str) -> str: + """``urn:agentenv:computer-display/v1`` -> ``computer-display``; another namespace keeps its name as a path.""" + name = uri.removeprefix("urn:").removeprefix("agentenv:") + head, _, version = name.rpartition("/") + if head and version[:1] == "v" and version[1:].isdigit(): + name = head + return name.replace(":", "/") + + +def tool(name: str | None = None, *, description: str | None = None, hidden_by_default: bool = False, + annotations: dict | None = None, structured_output: bool | None = None, meta: dict | None = None) -> Callable: """Mark a handler method as an MCP tool. At mount the bound method is registered as a real MCP tool on the server app and advertised @@ -131,6 +159,12 @@ def tool(name: str | None = None, *, description: str | None = None, hidden_by_d ``{HIDDEN_BY_DEFAULT_META_KEY: true}``, and a gateway hides it from every role until a tool-access step names it. Steps for every tool (``tools="*"``) never show it, though they still clear earlier named steps as they do for any tool. + + ``annotations`` (MCP ``ToolAnnotations`` fields, such as ``{"readOnlyHint": True}``) and ``meta`` are listed with + the tool. ``structured_output`` follows mcp's FastMCP: None derives an output schema from the return annotation + where it can, True requires one, False lists none; on a fastmcp app it is fastmcp's own inference, off with + False. A tool returning ``Annotated[CallToolResult, Model]`` sends its result as is on either, listed with + ``Model``'s schema. """ if callable(name): raise TypeError("use @tool(...) with parentheses, not bare @tool") @@ -143,11 +177,18 @@ def deco(fn: Callable) -> Callable: )) if hidden_by_default: setattr(fn, _HIDDEN_BY_DEFAULT_ATTR, True) + setattr(fn, _TOOL_OPTIONS_ATTR, _ToolOptions(annotations, structured_output, meta)) return fn return deco +class _ToolOptions(NamedTuple): + annotations: dict | None + structured_output: bool | None + meta: dict | None + + def environment_card(_cls: type | None = None, **config: Any) -> Callable: """Configure the class's EnvironmentCard; kwargs are EnvironmentCard fields. @@ -176,18 +217,28 @@ class AgentEnvApplication(ABC): def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: self.handler = handler - ext_handlers = _discover_extensions(handler) - environment_card = _merge_extensions(environment_card, ext_handlers) - tool_handlers = _discover_tools(handler, environment_card.name) + self._app: Any = None + ext_handlers = _served_extensions(handler, _discover_extensions(handler)) + environment_card = _merge_extensions(environment_card, ext_handlers + _declared_extensions(handler, ext_handlers)) + tool_handlers = _served_tools(handler, _discover_tools(handler, environment_card.name)) if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") environment_card = _merge_tools(environment_card, tool_handlers) - methods = _discover_methods(handler) + self._tool_access = ( + ToolAccess(descriptor.name for descriptor, fn in tool_handlers if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False)) + if self.supports_tools and getattr(handler, "tool_access", False) else None + ) + if self._tool_access is not None: + environment_card = _merge_extensions(environment_card, [(e,) for e in tool_access_extensions()]) + methods = _served_operations(handler, _discover_methods(handler)) environment_card = _merge_operations(environment_card, methods) environment_card = _merge_data_objects(environment_card, methods) self.environment_card = environment_card self._dispatch = _jsonrpc_handler(methods) self._ext_routes = [(path, method, _extension_handler(fn, method)) for _descriptor, fn, path, method in ext_handlers] + if self._tool_access is not None: + self._ext_routes += [(TOOL_ACCESS_PATHS[action], "POST", _tool_access_handler(self._tool_access, action)) + for action in (TOOL_DISABLE, TOOL_ENABLE)] # Register with the advertised descriptor (card-declared wins in _merge_tools) so # tools/list cannot drift from the card; falls back to the discovered descriptor. caps_tools = (environment_card.capabilities.tools if environment_card.capabilities else None) or [] @@ -196,9 +247,12 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: def add_routes_to_app(self, app: Any, *, rpc_url: str = RPC_PATH, card_url: str = WELL_KNOWN_PATH) -> None: self._check_tool_binding(app) + self._app = app + if isinstance(self.handler, AgentEnvEnvironment): + self.handler._application = self self.environment_card = self._card_for(app) self._add_route(app, rpc_url, ["POST"], self._dispatch) - self._add_route(app, card_url, ["GET"], _card_handler(self.environment_card)) + self._add_route(app, card_url, ["GET"], self._serve_card) for path, method, handler in self._ext_routes: self._add_route(app, path, [method], handler) for descriptor, fn in self._tools: @@ -225,6 +279,24 @@ def _bind_tool_calls(self, app: Any) -> None: def _card_for(self, app: Any) -> EnvironmentCard: return self.environment_card + async def _serve_card(self, request: Request) -> Response: + return _json_response(self.environment_card.model_dump()) + + async def update_tool(self, name: str, *, description: str) -> None: + """Change a served tool's description after the mount: on the card and in tools/list alike.""" + if all(descriptor.name != name for descriptor, _fn in self._tools): + raise ValueError(f"no served tool is named {name!r}") + self._tools = [(d.model_copy(update={"description": description}) if d.name == name else d, fn) + for d, fn in self._tools] + caps = self.environment_card.capabilities + tools = [t.model_copy(update={"description": description}) if t.name == name else t for t in caps.tools or []] + self.environment_card = self.environment_card.model_copy( + update={"capabilities": caps.model_copy(update={"tools": tools})}) + await self._describe_registered_tool(name, description) + + async def _describe_registered_tool(self, name: str, description: str) -> None: + raise NotImplementedError(f"{type(self).__name__} does not serve MCP tools") + class AgentEnvFastMCPApplication(AgentEnvApplication): supports_tools: ClassVar[bool] = True @@ -238,9 +310,24 @@ def _add_route(self, app: Any, path: str, methods: list, handler: Callable) -> N app.custom_route(path, methods=methods)(handler) def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> None: - # passed only when set, so a FastMCP-shaped target without the parameter still mounts unflagged tools - extra = {"meta": {HIDDEN_BY_DEFAULT_META_KEY: True}} if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False) else {} - app.tool(name=descriptor.name, description=descriptor.description, **extra)(injecting(fn)) + # passed only when set, so a FastMCP-shaped target without the parameters still mounts plain tools + options = getattr(fn, _TOOL_OPTIONS_ATTR, None) or _ToolOptions(None, None, None) + fastmcp = self._binding == "middleware" + extra: dict = {} + meta = {**(options.meta or {}), **({HIDDEN_BY_DEFAULT_META_KEY: True} if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False) else {})} + if meta: + extra["meta"] = meta + if options.annotations is not None: + extra["annotations"] = options.annotations if fastmcp else _mcp_tool_annotations(options.annotations) + if options.structured_output is not None and not fastmcp: + extra["structured_output"] = options.structured_output + elif options.structured_output is False: + extra["output_schema"] = None + registered = injecting(fn) + if fastmcp and _returns_call_tool_result(fn): + registered, schema = _fastmcp_returning(registered) + extra["output_schema"] = None if options.structured_output is False else schema + app.tool(name=descriptor.name, description=descriptor.description, **extra)(registered) def _check_tool_binding(self, app: Any) -> None: """Decide, before any route or tool is added, how every tools/call on ``app`` gets its ToolContext, and @@ -258,7 +345,7 @@ def _check_tool_binding(self, app: Any) -> None: if isinstance(middleware, list) and callable(getattr(app, "add_middleware", None)): if any(getattr(entry, _TOOL_CONTEXT_BOUND_ATTR, False) for entry in middleware): raise RuntimeError("this fastmcp app already carries an AgentEnv tool-context middleware; mount one handler per app") - self._middleware = _tool_context_middleware(self._hook) + self._middleware = _tool_context_middleware(self._hook, self._tool_access) self._binding = "middleware" return manager = getattr(app, "_tool_manager", None) @@ -268,10 +355,12 @@ def _check_tool_binding(self, app: Any) -> None: self._binding = "dispatch" return readers = [descriptor.name for descriptor, fn in self._tools if _tool_context_slot(fn)] - if readers or self._hook is not None: + readers += ["on_tool_call"] if self._hook is not None else [] + readers += ["tool_access"] if self._tool_access is not None else [] + if readers: raise RuntimeError( "this app has no tool dispatch to bind a ToolContext around (mcp>=1.25,<2 keeps one at " - f"FastMCP._tool_manager; fastmcp exposes a middleware list), and {', '.join(readers) or 'on_tool_call'} " + f"FastMCP._tool_manager; fastmcp exposes a middleware list), and {', '.join(readers)} " "would read an empty caller" ) self._binding = None @@ -302,22 +391,58 @@ def _bind_tool_dispatch(self, app: Any) -> None: manager = app._tool_manager inner = manager.call_tool hook = self._hook + access = self._tool_access async def call_tool(name: str, arguments: dict, *args: Any, **kwargs: Any) -> Any: if manager.get_tool(name) is None: return await inner(name, arguments, *args, **kwargs) context = kwargs.get("context", args[0] if args else None) with bound(context_from_mcp(context, name, arguments)) as tool_context: - if hook is not None: - answer = await _maybe_await(hook(tool_context)) - if answer is not None: - _check_call_tool_result(answer) - return answer - return await inner(name, arguments, *args, **kwargs) + try: + if access is not None: + access.check_call(tool_context) + if hook is not None: + answer = await _maybe_await(hook(tool_context)) + if answer is not None: + _check_call_tool_result(answer) + return answer + return await inner(name, arguments, *args, **kwargs) + except Exception as e: + error = _agentenv_error(e) + if error is None: + raise + return _error_result(error) functools.update_wrapper(call_tool, inner) setattr(call_tool, _TOOL_CONTEXT_BOUND_ATTR, True) manager.call_tool = call_tool + if access is not None: + self._filter_tool_list(app, manager, access) + + @staticmethod + def _filter_tool_list(app: Any, manager: Any, access: ToolAccess) -> None: + """tools/list under the live request's caller, read through ``FastMCP.get_context``, as tools/call reads it.""" + inner = manager.list_tools + + def list_tools(*args: Any, **kwargs: Any) -> Any: + role = caller_role(context_from_mcp(app.get_context(), "", {})) + return [t for t in inner(*args, **kwargs) if not access.is_disabled(role, t.name)] + + functools.update_wrapper(list_tools, inner) + manager.list_tools = list_tools + + async def _describe_registered_tool(self, name: str, description: str) -> None: + """fastmcp replaces a tool through its public provider; mcp's FastMCP lists the tool its manager holds.""" + app = self._app + if self._binding == "middleware": + registered = await app.get_tool(name) + app.local_provider.remove_tool(name) + app.local_provider.add_tool(registered.model_copy(update={"description": description})) + return + manager = getattr(app, "_tool_manager", None) + registered = manager.get_tool(name) if manager is not None else None + if registered is not None: + registered.description = description def _card_for(self, app: Any) -> EnvironmentCard: """Declare the MCP endpoint only when the app says where it serves it; FastMCP-shaped targets may not.""" @@ -352,7 +477,12 @@ class AgentEnvEnvironment: """ mcp: Any = None + #: Filter tools/list and refuse tools/call per caller role, and serve disable-tool/v1 and enable-tool/v1 at the + #: gateway's paths and with its rules, for an env agents reach with no gateway in front. + tool_access: ClassVar[bool] = False _mounted: bool = False + #: The application mounted with this environment, which ``update_tool`` changes. + _application: Any = None def _build_card(self) -> EnvironmentCard: config = dict(getattr(self, _CARD_CONFIG_ATTR, None) or {}) @@ -374,6 +504,19 @@ def mount(self, app: Any) -> Any: self._mounted = True return app + def mounted_card(self) -> EnvironmentCard: + """The card the mounted application serves: what ``served_*`` kept, as ``update_tool`` left it.""" + return self._mounted_application().environment_card + + async def update_tool(self, name: str, *, description: str) -> None: + """Change a served tool's description after the mount: on the card and in tools/list alike.""" + await self._mounted_application().update_tool(name, description=description) + + def _mounted_application(self) -> Any: + if self._application is None: + raise RuntimeError(f"{type(self).__name__} is not mounted") + return self._application + def serve(self, transport: str = "streamable-http") -> None: if self.mcp is None: self.create_app() @@ -381,6 +524,27 @@ def serve(self, transport: str = "streamable-http") -> None: self.mount(self.mcp) self.mcp.run(transport=transport) + def extension_params(self, uri: str) -> dict: + """Extension-level params (features, limits) advertised beside the methods of an extension served with ``op``.""" + return {} + + def declared_extensions(self) -> list[EnvironmentExtension]: + """Declaration-only extensions to advertise, like ``intake/v1``: a promise with params and no handler.""" + return [] + + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """The ``@extension`` handlers' extensions to serve, chosen at mount: one left out is not served, and of one served + with ``op``, only the methods its returned descriptor still lists.""" + return extensions + + def served_operations(self, operations: list[str]) -> list[str]: + """The data-plane methods to serve (``data/add`` and the others the handler marks), chosen at mount.""" + return operations + + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + """The ``@tool`` methods to serve, chosen at mount; a returned descriptor's description replaces the marked one.""" + return tools + def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -605,27 +769,43 @@ def _check_call_tool_result(value: Any) -> None: raise TypeError(f"on_tool_call must return None or mcp.types.CallToolResult, got {type(value).__name__}") -def _tool_context_middleware(hook: Callable | None) -> Any: +def _tool_context_middleware(hook: Callable | None, access: ToolAccess | None = None) -> Any: """A fastmcp ``Middleware`` that runs every tools/call under its ToolContext, the handler's ``on_tool_call`` - first. A ``Middleware`` subclass rather than a bare ``(context, call_next)`` callable because fastmcp runs the - typed ``on_call_tool`` exactly once per call, where a bare callable runs once per dispatch phase. fastmcp is - not a dependency of this package: it is imported here, where an app built with it has just been handed over.""" + first, and with ``access`` filters tools/list and refuses tools/call per caller role. A ``Middleware`` subclass + rather than a bare ``(context, call_next)`` callable because fastmcp runs the typed hooks exactly once per call, + where a bare callable runs once per dispatch phase. fastmcp is not a dependency of this package: it is imported + here, where an app built with it has just been handed over.""" try: from fastmcp.server.middleware import Middleware except ImportError as e: raise RuntimeError("this app carries a fastmcp middleware list but fastmcp cannot be imported") from e class ToolContextMiddleware(Middleware): + async def on_list_tools(self, context: Any, call_next: Callable) -> Any: + tools = await call_next(context) + if access is None: + return tools + role = caller_role(context_from_mcp(context.fastmcp_context, "", {})) + return [t for t in tools if not access.is_disabled(role, t.name)] + async def on_call_tool(self, context: Any, call_next: Callable) -> Any: message = context.message arguments = dict(getattr(message, "arguments", None) or {}) with bound(context_from_mcp(context.fastmcp_context, message.name, arguments)) as tool_context: - if hook is not None: - answer = await _maybe_await(hook(tool_context)) - if answer is not None: - _check_call_tool_result(answer) - return _fastmcp_tool_result(answer) - return await call_next(context) + try: + if access is not None: + access.check_call(tool_context) + if hook is not None: + answer = await _maybe_await(hook(tool_context)) + if answer is not None: + _check_call_tool_result(answer) + return _fastmcp_tool_result(answer) + return await call_next(context) + except Exception as e: + error = _agentenv_error(e) + if error is None: + raise + return _fastmcp_tool_result(_error_result(error)) middleware = ToolContextMiddleware() setattr(middleware, _TOOL_CONTEXT_BOUND_ATTR, True) @@ -646,6 +826,57 @@ def _fastmcp_tool_result(answer: Any) -> Any: return ToolResult(content=list(answer.content), structured_content=structured, is_error=bool(is_error)) +def _mcp_tool_annotations(annotations: Any) -> Any: + from mcp.types import ToolAnnotations + return ToolAnnotations.model_validate(annotations) + + +def _returns_call_tool_result(fn: Callable) -> bool: + returns = _resolved_hints(fn).get("return") + if get_origin(returns) is Annotated: + returns = get_args(returns)[0] + result_type = getattr(sys.modules.get("mcp.types"), "CallToolResult", None) + return result_type is not None and isinstance(returns, type) and issubclass(returns, result_type) + + +def _fastmcp_returning(fn: Callable) -> tuple: + """A tool returning ``CallToolResult`` as fastmcp sends it as is: a twin returning fastmcp's ``ToolResult``, and the + output schema of ``Annotated[CallToolResult, Model]``'s model (None for a bare ``CallToolResult``).""" + from fastmcp.tools import ToolResult + signature, annotations = _reduced_signature(fn, "") + returns = annotations.get("return") + schema = TypeAdapter(get_args(returns)[1]).json_schema() if get_origin(returns) is Annotated else None + + if inspect.iscoroutinefunction(fn): + async def wrapper(*args: Any, **kwargs: Any) -> Any: + return _fastmcp_tool_result(await fn(*args, **kwargs)) + else: + def wrapper(*args: Any, **kwargs: Any) -> Any: + return _fastmcp_tool_result(fn(*args, **kwargs)) + functools.update_wrapper(wrapper, fn) + wrapper.__signature__ = signature.replace(return_annotation=ToolResult) + wrapper.__annotations__ = {**annotations, "return": ToolResult} + return wrapper, schema + + +def _agentenv_error(exc: BaseException) -> AgentEnvError | None: + """The AgentEnvError a tool raised, found under the ToolError both FastMCPs wrap it in.""" + while exc is not None: + if isinstance(exc, AgentEnvError): + return exc + exc = exc.__cause__ + return None + + +def _error_result(error: AgentEnvError) -> Any: + from mcp.types import CallToolResult, TextContent + return CallToolResult( + isError=True, + content=[TextContent(type="text", text=f"{error.code}: {error.message}")], + structuredContent=error.fields(), + ) + + def _schema_from_signature(fn: Callable) -> dict: """Derive a JSON-Schema object for a handler's call params from its signature (Annotated[Field] descriptions included). Injected parameters (``ToolContext``, FastMCP ``Context``) are not wire parameters and are left out.""" @@ -697,20 +928,59 @@ def _discover_methods(handler: Any) -> dict: def _discover_extensions(handler: Any) -> list: - """Collect (descriptor, bound_method, route_path, http_method) for every @extension method.""" - seen_uris: set = set() + """Collect (descriptor, bound_method, route_path, http_method) for every @extension method. The methods of an + extension served with ``op`` share one descriptor: their ``methods`` merged beside ``extension_params(uri)``.""" + ops_by_uri: dict = {} seen_paths: set = set() ordered: list = [] for attr, descriptor in _marked_members(handler, _EXT_ATTR): route_path, http_method = getattr(attr, _EXT_ROUTE_ATTR) - if descriptor.uri in seen_uris: + op = getattr(attr, _EXT_OP_ATTR, None) + ops = ops_by_uri.setdefault(descriptor.uri, set()) + if None in ops or (op is None and ops): raise ValueError(f"Multiple handlers for extension {descriptor.uri}") + if op in ops: + raise ValueError(f"Multiple handlers for extension {descriptor.uri} method {op}") if route_path in seen_paths: raise ValueError(f"Multiple handlers for extension path {route_path}") - seen_uris.add(descriptor.uri) + ops.add(op) seen_paths.add(route_path) ordered.append((descriptor, attr, route_path, http_method)) - return ordered + merged = { + uri: _merged_extension(handler, [d for d, *_ in ordered if d.uri == uri]) + for uri, ops in ops_by_uri.items() if None not in ops + } + return [(merged.get(d.uri, d), attr, path, method) for d, attr, path, method in ordered] + + +def _merged_extension(handler: Any, descriptors: list) -> EnvironmentExtension: + uri = descriptors[0].uri + methods = {name: entry for d in descriptors for name, entry in d.params["methods"].items()} + hook = getattr(handler, "extension_params", None) + params = {**(hook(uri) if hook is not None else {}), "methods": methods} + return EnvironmentExtension( + uri=uri, + description=_one_value(uri, "description", [d.description for d in descriptors]), + params=params, + required=_one_value(uri, "required", [d.required for d in descriptors]), + ) + + +def _one_value(uri: str, field: str, values: list) -> Any: + given = {v for v in values if v is not None} + if len(given) > 1: + raise ValueError(f"the methods of extension {uri} declare different {field} values: {sorted(map(str, given))}") + return given.pop() if given else None + + +def _declared_extensions(handler: Any, ext_handlers: list) -> list: + hook = getattr(handler, "declared_extensions", None) + declared = list(hook()) if hook is not None else [] + served = {descriptor.uri for descriptor, *_ in ext_handlers} + clash = sorted(d.uri for d in declared if d.uri in served) + if clash: + raise ValueError(f"extensions both declared and served by a handler: {clash}") + return [(d,) for d in declared] def _merge_extensions(card: EnvironmentCard, ext_handlers: list) -> EnvironmentCard: @@ -751,6 +1021,62 @@ def _discover_tools(handler: Any, environment_name: str) -> list: return ordered +def _served_extensions(handler: Any, ext_handlers: list) -> list: + """The extension handlers the handler's ``served_extensions`` keeps, with its descriptors.""" + hook = getattr(handler, "served_extensions", None) + if hook is None: + return ext_handlers + offered = {descriptor.uri: descriptor for descriptor, *_ in ext_handlers} + chosen = {descriptor.uri: descriptor for descriptor in hook(list(offered.values()))} + unknown = sorted(set(chosen) - set(offered)) + if unknown: + raise ValueError(f"served_extensions returned extensions with no @extension handler: {unknown}") + ops = {} + for descriptor, fn, *_ in ext_handlers: + ops.setdefault(descriptor.uri, set()).add(getattr(fn, _EXT_OP_ATTR, None)) + for uri, served in chosen.items(): + if None not in ops[uri]: + added = sorted(set((served.params or {}).get("methods") or {}) - ops[uri]) + if added: + raise ValueError(f"served_extensions lists methods of {uri} with no @extension handler: {added}") + kept = [] + for descriptor, fn, path, method in ext_handlers: + served = chosen.get(descriptor.uri) + op = getattr(fn, _EXT_OP_ATTR, None) + if served is None or (op is not None and op not in (served.params or {}).get("methods", {})): + continue + kept.append((served, fn, path, method)) + return kept + + +def _served_operations(handler: Any, methods: dict) -> dict: + """The data-plane methods the handler's ``served_operations`` keeps.""" + hook = getattr(handler, "served_operations", None) + if hook is None: + return methods + chosen = set(hook(list(methods))) + unknown = sorted(chosen - set(methods)) + if unknown: + raise ValueError(f"served_operations returned operations with no handler: {unknown}") + return {name: entry for name, entry in methods.items() if name in chosen} + + +def _served_tools(handler: Any, tool_handlers: list) -> list: + """The tools the handler's ``served_tools`` keeps, in its order and with its descriptors; all of them without one.""" + hook = getattr(handler, "served_tools", None) + if hook is None: + return tool_handlers + chosen = list(hook([descriptor for descriptor, _fn in tool_handlers])) + by_name = {descriptor.name: fn for descriptor, fn in tool_handlers} + names = [descriptor.name for descriptor in chosen] + unknown = sorted(set(names) - set(by_name)) + if unknown: + raise ValueError(f"served_tools returned tools with no @tool method: {unknown}") + if len(set(names)) != len(names): + raise ValueError(f"served_tools returned a tool twice: {sorted({n for n in names if names.count(n) > 1})}") + return [(descriptor, by_name[descriptor.name]) for descriptor in chosen] + + def _merge_tools(card: EnvironmentCard, tool_handlers: list) -> EnvironmentCard: """Advertise decorator-declared tools under card.capabilities; card-declared names win.""" if not tool_handlers: @@ -961,22 +1287,42 @@ async def handler(request: Request) -> Response: raise ValueError("request body must be a JSON object") params = _coerce_params(fn, params) except Exception as e: - return _json_response(error_body("invalid_params", str(e)), 400) + return _error_response(AgentEnvError("invalid_params", str(e))) try: signature.bind(**params) except TypeError as e: - return _json_response(error_body("invalid_params", str(e)), 400) + return _error_response(AgentEnvError("invalid_params", str(e))) with bound(context_from_http(request, op, params)) as tool_context: call_kwargs = {**params, slot: tool_context} if slot else params try: result = await _maybe_await(fn(**call_kwargs)) + except AgentEnvError as e: + return _error_response(e) except Exception as e: logger.exception("extension invocation failed") - return _json_response(error_body("extension_failed", str(e)), 500) + return _error_response(AgentEnvError("extension_failed", str(e), applied="unknown", status=500)) return _json_response(result if result is not None else {}) return handler +def _error_response(error: AgentEnvError) -> Response: + return _json_response(error_body(**error.fields()), error.status) + + +def _tool_access_handler(access: ToolAccess, action: str) -> Callable: + async def handler(request: Request) -> Response: + raw = await request.body() + try: + body = json.loads(raw) if raw else None + except ValueError: + return _error_response(AgentEnvError("invalid_params", "request body must be JSON")) + try: + return _json_response(access.change(action, body)) + except AgentEnvError as e: + return _error_response(e) + return handler + + class _Op(NamedTuple): name: str rpc_method: str @@ -1015,12 +1361,6 @@ async def handler(request: Request) -> Response: return handler -def _card_handler(card: EnvironmentCard) -> Callable: - async def handler(request: Request) -> Response: - return _json_response(card.model_dump()) - return handler - - def _rpc_error(rid: Any, code: int, message: str, data: Any = None) -> Response: error: dict = {"code": code, "message": message} if data is not None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/client.py b/packages/agentenv-protocol/src/agentenv_protocol/client.py index 9d3be78a..e934b772 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/client.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/client.py @@ -12,6 +12,29 @@ logger = logging.getLogger(__name__) +class ExtensionCallError(httpx.HTTPStatusError): + """A non-2xx answer from an extension method, with the error fields the server sent. A body without them + (a proxy's own error page, an older server) leaves ``code`` None and ``applied`` ``"unknown"``.""" + + def __init__(self, response: httpx.Response) -> None: + error = _error_fields(response) + self.code: str | None = error.get("code") + self.detail: str = error.get("message") or response.reason_phrase + self.retryable: bool = bool(error.get("retryable", False)) + self.applied: str = error.get("applied", "unknown") + self.hint: str | None = error.get("hint") + super().__init__(f"{response.status_code} {self.code or 'error'}: {self.detail}", request=response.request, response=response) + + +def _error_fields(response: httpx.Response) -> dict: + try: + body = response.json() + except ValueError: + return {} + error = body.get("error") if isinstance(body, dict) else None + return error if isinstance(error, dict) else {} + + async def _rpc(base_url: str, method: str, params: dict, timeout: int, verify: bool) -> dict: request = {"jsonrpc": "2.0", "id": 1, "method": method, "params": params} async with httpx.AsyncClient(verify=verify) as client: @@ -168,7 +191,7 @@ async def invoke_extension(base_url: str, card: dict, uri: str, params: dict | N Reads the endpoint and HTTP verb of the extension method named `method` from the card, so the card alone is enough to invoke; without `method`, the first method listed. Returns the parsed - JSON result; raises on a non-2xx response. + JSON result; a non-2xx response raises `ExtensionCallError`. """ ext = find_extension(card, uri) if ext is None: @@ -187,5 +210,6 @@ async def invoke_extension(base_url: str, card: dict, uri: str, params: dict | N response = await client.get(url, params=params or {}, timeout=timeout) else: response = await client.request(op["method"], url, json=params or {}, timeout=timeout) - response.raise_for_status() + if response.status_code >= 400: + raise ExtensionCallError(response) return response.json() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py new file mode 100644 index 00000000..812c466f --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -0,0 +1,67 @@ +"""The computer driver SDK: subclass ComputerDriverEnvironment, mark the methods that capture and send input, and +serve() it. Needs the ``computer`` extra: ``pip install 'agentenv-framework-protocol[computer]'``.""" + +from .driver import DEFAULT_IDENTITY, ComputerDriverEnvironment, computer_driver +from .errors import ( + BackgroundUnavailable, + ComputerError, + DeliveryFailed, + DeviceUnavailable, + InputBusy, + Internal, + InvalidArgument, + LimitExceeded, + OutOfBounds, + PermissionDenied, + StaleFrame, + TargetUnavailable, + Timeout, + Unsupported, +) +from .card import ( + COMPUTER, + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_POINTER, + COMPUTER_READY, + COMPUTER_SPACES, +) +from .operations import ( + Display, + Effect, + ExecResult, + Image, + ReadyCheck, + capture, + click, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + run_in_session, + scroll, + set_display, + type_text, + write_files, +) +from .platform import Platform +from .settings import SETTINGS_ENV, ComputerSettings + +__all__ = [ + "ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", + "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", + "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", + "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", + "run_in_session", "write_files", + "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", + "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed", + "Internal", + "COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD", + "COMPUTER_EXEC", +] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py new file mode 100644 index 00000000..671af965 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -0,0 +1,105 @@ +"""What a driver's card says: the extensions, the tools, extensions and data-plane methods its operations support, +the extensions' params, the extensions it declares without handlers, and the descriptions that name the default +coordinate space.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..types import METHOD_ADD, EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension + +if TYPE_CHECKING: + from .driver import ComputerDriverEnvironment + +COMPUTER = "urn:agentenv:computer/v1" +COMPUTER_READY = "urn:agentenv:computer-ready/v1" +COMPUTER_DISPLAY = "urn:agentenv:computer-display/v1" +COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1" +COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1" +COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1" +COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" + +MAX_EXEC_TIMEOUT_S = 3600 +KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)} + +#: The tools a group serves. +GROUP_TOOLS = { + "pointer": ("computer_pointer_move", "computer_pointer_down", "computer_pointer_up", "computer_pointer_position", + "computer_pointer_list"), + "keyboard": ("computer_keyboard_down", "computer_keyboard_up"), +} +#: The tools whose description names the default coordinate space. +POINT_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_pointer_move", + "computer_pointer_down", "computer_pointer_up") +#: The operation or group an extension needs, and one of its methods needs; without it, it is not served. +_EXTENSION_NEEDS = {COMPUTER_KEYBOARD: "keyboard", COMPUTER_EXEC: "run_in_session"} +_METHOD_NEEDS = {(COMPUTER_DISPLAY, "set"): "set_display"} + + +def served_tools(driver: ComputerDriverEnvironment, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + unserved = {name for group, names in GROUP_TOOLS.items() if not driver.has(group) for name in names} + return [t.model_copy(update={"description": point_description(driver, t.name)}) if t.name in POINT_TOOLS else t + for t in tools if t.name not in unserved] + + +def served_extensions(driver: ComputerDriverEnvironment, + extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + served = [] + for extension in extensions: + need = _EXTENSION_NEEDS.get(extension.uri) + if need is not None and not driver.has(need): + continue + dropped = {op for (uri, op), need in _METHOD_NEEDS.items() if uri == extension.uri and not driver.has(need)} + if dropped: + methods = {op: entry for op, entry in extension.params["methods"].items() if op not in dropped} + extension = extension.model_copy(update={"params": {**extension.params, "methods": methods}}) + served.append(extension) + return served + + +def served_operations(driver: ComputerDriverEnvironment, operations: list[str]) -> list[str]: + return [op for op in operations if op != METHOD_ADD or driver.has("write_files")] + + +def extension_params(driver: ComputerDriverEnvironment, uri: str) -> dict: + if uri == COMPUTER_DISPLAY: + return {"modes": [list(m) for m in driver.display_modes]} + if uri == COMPUTER_SPACES: + return {"spaces": ["frame", "norm"]} + if uri == COMPUTER_KEYBOARD: + return {**{name: {"min": low, "max": high} for name, (low, high) in KEYBOARD_RANGES.items()}, + "layouts": list(driver.layouts)} + if uri == COMPUTER_EXEC: + return {"max_output_bytes": driver.settings.exec_output_bytes, "max_timeout_s": MAX_EXEC_TIMEOUT_S} + return {} + + +def declared_extensions(driver: ComputerDriverEnvironment) -> list[EnvironmentExtension]: + features = { + "buttons": list(driver.buttons), "max_click_count": driver.max_click_count, + "keys": driver.keys if driver.keys == "all" else list(driver.keys), "max_text_length": driver.max_text_length, + "max_wait_ms": driver.max_wait_ms, "max_drag_points": driver.max_drag_points, + "max_duration_ms": driver.max_duration_ms, "max_key_repeat": driver.max_key_repeat, + "max_scroll_amount": driver.max_scroll_amount, + } + declared = [EnvironmentExtension( + uri=COMPUTER, description="The seven computer_* tools: screenshot, click, drag, scroll, type, key, wait.", + params={**features, "delivery": "foreground", "platform": driver.platform.as_dict(), + "supports": list(driver.supports)})] + if driver.has("pointer"): + declared.append(EnvironmentExtension( + uri=COMPUTER_POINTER, description="The computer_pointer_* tools, and click and scroll at the pointer.", + params={"buttons": list(driver.buttons), "kinds": ["foreground"]})) + intake = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")]) + if driver.has("write_files"): + intake.add = [IntakeFormat(part="file", format="file-tree", load="additive")] + declared.append(intake_extension(intake, "data/add writes files under the session user's home; data/get " + "returns the env's data paths.")) + return declared + + +def point_description(driver: ComputerDriverEnvironment, name: str) -> str: + """A point tool's docstring, then the default coordinate space.""" + space, scale = driver.settings.default_space, driver.settings.default_norm_scale + note = (" Points are in frame space by default: pixels of the frame's image." if space == "frame" else + f" Points are in norm space by default: 0 to {scale} over the frame, so send x and y on that grid.") + return " ".join((getattr(type(driver), name).__doc__ or "").split()) + note diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py new file mode 100644 index 00000000..c65c619e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -0,0 +1,42 @@ +"""The checks a driver class passes when it is defined, beside its operations' (``operations.check_class``): its +members leave the base class's own alone, and its features are in range.""" +from __future__ import annotations + +from ..errors import AgentEnvError +from .keys import canonical +from .operations import declared + +BUTTONS = ("left", "right", "middle", "back", "forward") + + +def check_names(cls: type, base: type) -> None: + """Refuse a driver member that would replace one of ``base``'s own: its private members and the tools, which + define the contract. The hooks (``caller_identity``, ``ready_checks``, ...) and the features are the driver's.""" + reserved = {name for name in dir(base) + if (name.startswith("_") and not name.startswith("__")) or name.startswith("computer_")} + clash = sorted(name for name in vars(cls) if name in reserved or name.startswith(("_computer_", "computer_"))) + if clash: + raise TypeError(f"{cls.__qualname__} redefines {', '.join(clash)}, which {base.__name__} uses " + "itself; give the driver's own members other names") + + +def check_features(cls: type) -> None: + problems = [] + unknown = [b for b in cls.buttons if b not in BUTTONS] + if "left" not in cls.buttons or unknown: + problems.append(f"buttons include left and come from {list(BUTTONS)}, got {list(cls.buttons)}") + for name, low in (("max_click_count", 1), ("max_text_length", 1), ("max_wait_ms", 60_000), ("max_drag_points", 2), + ("max_duration_ms", 0), ("max_key_repeat", 1), ("max_scroll_amount", 1)): + value = getattr(cls, name) + if not isinstance(value, int) or value < low: + problems.append(f"{name} is an integer of at least {low}, got {value!r}") + if cls.keys != "all": + try: + if [canonical(k) for k in cls.keys] != list(cls.keys): + problems.append("keys are W3C key values in their W3C case") + except AgentEnvError as e: + problems.append(f"keys: {e.message}") + if ("set_display" in declared(cls)) != bool(cls.display_modes): + problems.append("display_modes and @set_display come together") + if problems: + raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py new file mode 100644 index 00000000..e1621dd1 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py @@ -0,0 +1,116 @@ +"""The data plane on a computer: ``data/add`` writes files under the session user's home, ``data/get`` returns the +env's ``data_paths``. Each file part names its path, relative to the home, under the ``data-objects/v1`` marker.""" +from __future__ import annotations + +import base64 +import binascii +import mimetypes +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +import httpx +from pydantic import ValidationError + +from ..agent_env_environment import add_data, get_data +from ..transfers import NamespaceUploader, ReadObject, WriteNamespaceGrant, download +from ..types import DATA_OBJECTS_EXTENSION_URI, FilePart, FileWithBytes, FileWithUri, uploaded_file_part, uploaded_object_path + +if TYPE_CHECKING: + from .runtime import ComputerRuntime + from .settings import ComputerSettings + +#: The most an https URL part may fetch; a read grant carries its own limit. +MAX_URL_BYTES = 1 << 30 + + +class DataPlane: + """The data plane handlers, mixed into ComputerDriverEnvironment.""" + + _computer: ComputerRuntime + settings: ComputerSettings + + @add_data + async def _computer_data_add(self, parts: list) -> None: + files = {} + for part in parts: + try: + path = uploaded_object_path(part) if isinstance(part, FilePart) else None + except ValueError: + marker = part.metadata[DATA_OBJECTS_EXTENSION_URI] + raise ValueError(f"the part's path {marker.get('path')!r} is not a normalized path relative to " + "the home") from None + if path is None: + raise ValueError(f"data/add on a computer takes file parts naming their path under " + f"{DATA_OBJECTS_EXTENSION_URI}, got a {part.kind} part") + files[path] = await _fetch(part) + await self._computer.call("write_files", files) + + @get_data + async def _computer_data_get(self, write_namespace: Optional[WriteNamespaceGrant] = None) -> list: + files = _data_files(Path.home(), self.settings.data_paths) + if write_namespace is not None: + uploader = NamespaceUploader(write_namespace) + parts = [] + for relative, source in files: + await uploader.upload(relative, source) + parts.append(uploaded_file_part(relative, name=source.name, mime_type=_mime(source))) + return parts + return [ + FilePart(file=FileWithBytes(bytes=base64.b64encode(source.read_bytes()).decode("ascii"), + name=source.name, mimeType=_mime(source)), + metadata={DATA_OBJECTS_EXTENSION_URI: {"path": relative}}) + for relative, source in files + ] + + +async def _fetch(part: FilePart) -> bytes: + """A part's bytes: inline, from a read grant under the marker, or from an https URL.""" + marker = part.metadata[DATA_OBJECTS_EXTENSION_URI] + if "read" in marker: + try: + grant = ReadObject.model_validate(marker["read"]) + except ValidationError: + raise ValueError("the part's read grant is not a valid ReadObject") from None + with tempfile.TemporaryDirectory() as scratch: + destination = Path(scratch) / "object" + await download(grant, destination) + return destination.read_bytes() + if isinstance(part.file, FileWithBytes): + try: + return base64.b64decode(part.file.bytes, validate=True) + except (binascii.Error, ValueError): + raise ValueError("the part's bytes are not base64") from None + return await _get_url(part.file) + + +async def _get_url(file: FileWithUri) -> bytes: + if not file.uri.startswith("https://"): + raise ValueError("a file part's uri is an https URL") + data = bytearray() + async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: + async with client.stream("GET", file.uri) as response: + response.raise_for_status() + async for chunk in response.aiter_bytes(): + data += chunk + if len(data) > MAX_URL_BYTES: + raise ValueError(f"the file at the part's uri is over {MAX_URL_BYTES} bytes") + return bytes(data) + + +def _data_files(home: Path, data_paths: list[str]) -> list[tuple[str, Path]]: + """Every file under the data paths, by its path relative to the home; a path not there yet is skipped.""" + root = home.resolve() + found: dict[str, Path] = {} + for relative in data_paths: + target = home / relative + if not target.exists() or not target.resolve().is_relative_to(root): + continue + for path in sorted(target.rglob("*")) if target.is_dir() else [target]: + if path.is_file() and path.resolve().is_relative_to(root): + found[path.relative_to(home).as_posix()] = path + return list(found.items()) + + +def _mime(path: Path) -> str: + return mimetypes.guess_type(path.name)[0] or "application/octet-stream" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py new file mode 100644 index 00000000..93cdf44b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -0,0 +1,120 @@ +"""ComputerDriverEnvironment: a driver writes screen operations, and the base class serves the computer contract +from them: the tools agents call, the framework extensions agentenv calls, the data plane and the card.""" +from __future__ import annotations + +import asyncio +from typing import Any, Callable, ClassVar + +from ..agent_env_environment import AgentEnvEnvironment, environment_card +from ..tool_context import ToolContext +from ..types import EnvironmentExtension, EnvironmentTool +from . import card, linux +from .checks import check_features, check_names +from .dataplane import DataPlane +from .extensions import Extensions +from .operations import ReadyCheck, check_class, default_for +from .platform import Platform, detect +from .runtime import ComputerRuntime +from .settings import ComputerSettings, load_settings +from .tools import Tools + +#: The caller every call comes from until agents carry an identity of their own. +DEFAULT_IDENTITY = "default" + + +class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironment): + """Subclass it and mark the methods that capture and send input (see ``operations``); the class is checked when + it is defined. Features are class attributes, on the card. A subclass's ``__init__`` calls ``super().__init__()``. + """ + + buttons: ClassVar[tuple[str, ...]] = ("left",) + max_click_count: ClassVar[int] = 1 + #: ``"all"`` for every W3C key, or the key values this computer can press. + keys: ClassVar[Any] = "all" + max_text_length: ClassVar[int] = 10_000 + max_wait_ms: ClassVar[int] = 60_000 + max_drag_points: ClassVar[int] = 2 + max_duration_ms: ClassVar[int] = 10_000 + max_key_repeat: ClassVar[int] = 100 + max_scroll_amount: ClassVar[int] = 100 + #: The (width, height) modes computer-display/v1 set takes; with ``@set_display`` only. + display_modes: ClassVar[tuple] = () + layouts: ClassVar[tuple[str, ...]] = ("us",) + #: The platforms the driver supports, as the driver env declares them: ``[{"os": {...}, "arch": [...]}]``. + supports: ClassVar[list] = [] + tool_access = True + #: What the base class keeps for the driver, built by ``__init__``. + _computer: ComputerRuntime | None = None + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + check_names(cls, ComputerDriverEnvironment) + check_class(cls) + check_features(cls) + + def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None: + self.settings = settings if settings is not None else load_settings() + self.platform = platform if platform is not None else detect() + self._computer = ComputerRuntime(self) + + def has(self, capability: str) -> bool: + """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" + return self._computer.has(capability) + + def caller_identity(self, context: ToolContext) -> str: + """Who a call comes from, for each caller's latest frame and others' input since a frame.""" + return DEFAULT_IDENTITY + + async def ready_checks(self) -> list[ReadyCheck]: + """Checks of the driver's own, run with the base class's by computer-ready/v1; never sends input.""" + return [] + + def create_app(self) -> Any: + self._computer_require_init() + return super().create_app() + + def mount(self, app: Any) -> Any: + self._computer_require_init() + return super().mount(app) + + # The card: what the driver's operations support. + + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + return card.served_tools(self, tools) + + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + return card.served_extensions(self, extensions) + + def served_operations(self, operations: list[str]) -> list[str]: + return card.served_operations(self, operations) + + def extension_params(self, uri: str) -> dict: + return card.extension_params(self, uri) + + def declared_extensions(self) -> list[EnvironmentExtension]: + return card.declared_extensions(self) + + def _computer_require_init(self) -> None: + if self._computer is None: + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + + # The Linux defaults for the OS operations. + + @default_for("run_in_session", os="linux") + async def _computer_linux_run_in_session(self, argv: list, cwd: str | None, env: dict | None, timeout_s: float): + return await linux.run_in_session(argv, cwd, env, timeout_s, output_bytes=self.settings.exec_output_bytes) + + @default_for("write_files", os="linux") + async def _computer_linux_write_files(self, files: dict) -> None: + await asyncio.to_thread(linux.write_files, files) + + +def computer_driver(name: str | None = None, *, supports: list | None = None) -> Callable: + """Name a driver, its card's name unless ENVIRONMENT_NAME gives one, and declare the platforms it supports.""" + def deco(cls: type) -> type: + if not (isinstance(cls, type) and issubclass(cls, ComputerDriverEnvironment)): + raise TypeError("@computer_driver decorates a ComputerDriverEnvironment subclass") + if supports is not None: + cls.supports = list(supports) + return environment_card(name=name)(cls) if name else cls + return deco diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py new file mode 100644 index 00000000..854b992e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py @@ -0,0 +1,71 @@ +"""The computer contract's error codes as exceptions a driver raises: ``raise OutOfBounds("...")``.""" +from __future__ import annotations + +from typing import ClassVar + +from ..errors import AgentEnvError, Applied + + +class ComputerError(AgentEnvError): + """One row of the errors table; a subclass fixes its code, status, whether it may be retried and whether the + action may have happened.""" + + CODE: ClassVar[str] = "internal" + STATUS: ClassVar[int] = 500 + RETRYABLE: ClassVar[bool] = False + APPLIED: ClassVar[Applied] = "unknown" + + def __init__(self, message: str, *, hint: str | None = None) -> None: + super().__init__(self.CODE, message, retryable=self.RETRYABLE, applied=self.APPLIED, hint=hint, status=self.STATUS) + + +class InvalidArgument(ComputerError): + CODE, STATUS, APPLIED = "invalid_argument", 400, "no" + + +class OutOfBounds(ComputerError): + CODE, STATUS, APPLIED = "out_of_bounds", 400, "no" + + +class StaleFrame(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "stale_frame", 409, "no", True + + +class InputBusy(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "input_busy", 409, "no", True + + +class PermissionDenied(ComputerError): + CODE, STATUS, APPLIED = "permission_denied", 403, "no" + + +class Unsupported(ComputerError): + CODE, STATUS, APPLIED = "unsupported", 501, "no" + + +class BackgroundUnavailable(ComputerError): + CODE, STATUS, APPLIED = "background_unavailable", 501, "no" + + +class TargetUnavailable(ComputerError): + CODE, STATUS, APPLIED = "target_unavailable", 404, "no" + + +class DeviceUnavailable(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "device_unavailable", 503, "no", True + + +class LimitExceeded(ComputerError): + CODE, STATUS, APPLIED, RETRYABLE = "limit_exceeded", 429, "no", True + + +class Timeout(ComputerError): + CODE, STATUS = "timeout", 504 + + +class DeliveryFailed(ComputerError): + CODE, STATUS = "delivery_failed", 502 + + +class Internal(ComputerError): + pass diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py new file mode 100644 index 00000000..4ce81239 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -0,0 +1,131 @@ +"""The framework extensions agentenv calls, never agents: readiness, display, coordinate spaces, keyboard settings +and commands in the session.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable, Optional + +from ..agent_env_environment import extension +from . import card +from .card import ( + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_READY, + COMPUTER_SPACES, + KEYBOARD_RANGES, + MAX_EXEC_TIMEOUT_S, +) +from .errors import Internal, InvalidArgument, Unsupported +from .operations import Display, ExecResult +from .spaces import check_space + +if TYPE_CHECKING: + from ..types import EnvironmentCard + from .runtime import ComputerRuntime + from .settings import ComputerSettings + + +class Extensions: + """The extension handlers, mixed into ComputerDriverEnvironment; the card serves those the driver's operations + support.""" + + _computer: ComputerRuntime + settings: ComputerSettings + display_modes: tuple + layouts: tuple[str, ...] + mounted_card: Callable[[], EnvironmentCard] + + @extension(COMPUTER_READY, op="status", method="GET", + description="Whether the computer is usable now, by real checks; never sends input.") + async def _computer_ready_status(self) -> dict: + checks = await self._computer.ready_checks() + return {"ready": all(c.ok for c in checks), + "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]} + + @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.") + async def _computer_display_get(self) -> dict: + return _display_data(await self._computer.display_now(), self.display_modes) + + @extension(COMPUTER_DISPLAY, op="set") + async def _computer_display_set(self, width: int, height: int, scale: Optional[float] = None) -> dict: + if [width, height] not in [list(m) for m in self.display_modes]: + modes = ", ".join(f"{w}×{h}" for w, h in self.display_modes) + raise InvalidArgument(f"{width}×{height} is not one of this computer's display modes: {modes}") + async with self._computer.queue.turn(self.settings.queue_wait_s): + now = await self._computer.display_now() + changed = await self._computer.call("set_display", width, height, now.scale if scale is None else scale) + if not isinstance(changed, Display): + changed = await self._computer.display_now() + return _display_data(changed, self.display_modes) + + @extension(COMPUTER_SPACES, op="get", method="GET", + description="Get or set the default coordinate space, written into the tool descriptions.") + async def _computer_spaces_get(self) -> dict: + return {"default_space": self.settings.default_space, "default_norm_scale": self.settings.default_norm_scale} + + @extension(COMPUTER_SPACES, op="set") + async def _computer_spaces_set(self, default_space: str, default_norm_scale: Optional[int] = None) -> dict: + space, scale = check_space(default_space, default_norm_scale, self.settings.default_space, + self.settings.default_norm_scale) + self.settings = self.settings.model_copy(update={"default_space": space, "default_norm_scale": scale}) + for tool in self.mounted_card().capabilities.tools or []: + if tool.name in card.POINT_TOOLS: + await self.update_tool(tool.name, description=card.point_description(self, tool.name)) + return await self._computer_spaces_get() + + @extension(COMPUTER_KEYBOARD, op="get", method="GET", + description="Get or set the tap hold, typing delay, layout and maximum hold.") + async def _computer_keyboard_get(self) -> dict: + s = self.settings + return {"tap_hold_ms": s.tap_hold_ms, "type_delay_ms": s.type_delay_ms, "layout": s.layout, + "max_hold_ms": s.max_hold_ms} + + @extension(COMPUTER_KEYBOARD, op="set") + async def _computer_keyboard_set(self, tap_hold_ms: Optional[int] = None, type_delay_ms: Optional[int] = None, + layout: Optional[str] = None, max_hold_ms: Optional[int] = None) -> dict: + given = {"tap_hold_ms": tap_hold_ms, "type_delay_ms": type_delay_ms, "max_hold_ms": max_hold_ms} + for name, value in given.items(): + low, high = KEYBOARD_RANGES[name] + if value is not None and not low <= value <= high: + raise InvalidArgument(f"{name} is from {low} to {high}, got {value}") + if layout is not None and layout not in self.layouts: + raise Unsupported(f"this computer types in the layouts {list(self.layouts)}, not {layout!r}") + update = {k: v for k, v in {**given, "layout": layout}.items() if v is not None} + self.settings = self.settings.model_copy(update=update) + return await self._computer_keyboard_get() + + @extension(COMPUTER_EXEC, op="run", + description="Run a command inside the desktop session as the session user; a repeated request_id " + "returns the first result.") + async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[str] = None, + env: Optional[dict] = None, timeout_s: Optional[float] = None) -> dict: + if not (isinstance(argv, list) and argv and all(isinstance(a, str) for a in argv)): + raise InvalidArgument("argv is a non-empty list of strings; name a shell to use one: ['bash', '-lc', '...']") + if not (isinstance(request_id, str) and request_id): + raise InvalidArgument("request_id is a non-empty string") + if env is not None and not all(isinstance(k, str) and isinstance(v, str) for k, v in env.items()): + raise InvalidArgument("env maps names to string values") + timeout = self.settings.exec_timeout_s if timeout_s is None else timeout_s + if not 0 < timeout <= MAX_EXEC_TIMEOUT_S: + raise InvalidArgument(f"timeout_s is above 0 and at most {MAX_EXEC_TIMEOUT_S}, got {timeout}") + cap = self.settings.exec_output_bytes + return await self._computer.command(request_id, lambda: _run(self._computer, cap, argv, cwd, env, timeout)) + + +async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict], + timeout: float) -> dict: + result = await computer.call("run_in_session", argv, cwd, env, timeout) + if not isinstance(result, ExecResult): + raise Internal(f"@run_in_session returned {type(result).__name__}, not an ExecResult") + return { + "exit_code": result.exit_code, + "stdout": result.stdout[:cap].decode("utf-8", errors="replace"), + "stderr": result.stderr[:cap].decode("utf-8", errors="replace"), + "truncated": result.truncated or len(result.stdout) > cap or len(result.stderr) > cap, + "timed_out": result.timed_out, + } + + +def _display_data(display: Display, modes: tuple) -> dict: + return {"width": display.width, "height": display.height, "scale": display.scale, + "modes": [list(m) for m in modes]} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py new file mode 100644 index 00000000..fa004dd8 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -0,0 +1,163 @@ +"""Frames: every screenshot, kept by id so any caller can point into it later, and its mapping to screen pixels.""" +from __future__ import annotations + +import base64 +import io +import math +from collections import OrderedDict +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from PIL import Image as PILImage + +from .errors import Internal, StaleFrame +from .operations import Display, Image +from .results import DisplayData, FrameData + +KEEP = 64 +#: The JPEG quality a screenshot answers with when it names none. +JPEG_QUALITY = 85 + + +@dataclass(frozen=True) +class Frame: + """A capture, or a crop of one: ``origin`` is where this frame's pixels start in the root capture, and the root + capture's ``root_size`` image covers the whole ``display``. The pixels are kept as the PNG every screenshot + answers with, a tenth or less of them raw, and decoded only to crop or to answer in JPEG.""" + + id: str + png: bytes = field(repr=False) + size: tuple[int, int] + display: Display + origin: tuple[int, int] + root_size: tuple[int, int] + parent_id: str | None + captured_at: datetime + inputs: tuple = field(repr=False) + + @property + def width(self) -> int: + return self.size[0] + + @property + def height(self) -> int: + return self.size[1] + + @property + def image(self) -> PILImage.Image: + image = PILImage.open(io.BytesIO(self.png)) + image.load() + return image + + def to_screen(self, fx: float, fy: float) -> tuple[int, int]: + """The screen pixel under frame pixel (fx, fy); exact at identity scale.""" + rx, ry = self.origin[0] + fx, self.origin[1] + fy + return (int(math.floor(rx * self.display.width / self.root_size[0])), + int(math.floor(ry * self.display.height / self.root_size[1]))) + + def from_screen(self, sx: float, sy: float) -> tuple[int, int]: + rx = sx * self.root_size[0] / self.display.width + ry = sy * self.root_size[1] / self.display.height + return int(math.floor(rx - self.origin[0])), int(math.floor(ry - self.origin[1])) + + def contains(self, fx: float, fy: float) -> bool: + return 0 <= fx < self.width and 0 <= fy < self.height + + def result(self) -> FrameData: + return FrameData( + frame_id=self.id, + width=self.width, + height=self.height, + display=DisplayData(width=self.display.width, height=self.display.height, scale=self.display.scale), + parent_frame_id=self.parent_id, + captured_at=self.captured_at.isoformat().replace("+00:00", "Z"), + ) + + def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str]: + """The frame as base64 and its MIME type.""" + if fmt != "jpeg": + return base64.b64encode(self.png).decode("ascii"), "image/png" + buffer = io.BytesIO() + self.image.convert("RGB").save(buffer, format="JPEG", quality=quality or JPEG_QUALITY) + return base64.b64encode(buffer.getvalue()).decode("ascii"), "image/jpeg" + + +class FrameStore: + """The last ``KEEP`` frames by id, ``f``, and each caller's latest; a frame no longer kept is unknown.""" + + def __init__(self, keep: int = KEEP) -> None: + self._keep = keep + self._frames: OrderedDict[str, Frame] = OrderedDict() + self._next = 1 + self._latest: dict[str, str] = {} + + def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identity: str, + parent: Frame | None = None, origin: tuple[int, int] = (0, 0)) -> Frame: + frame = Frame( + id=f"f{self._next}", + png=_png(image), + size=image.size, + display=parent.display if parent is not None else display, + origin=(parent.origin[0] + origin[0], parent.origin[1] + origin[1]) if parent is not None else (0, 0), + root_size=parent.root_size if parent is not None else image.size, + parent_id=parent.id if parent is not None else None, + captured_at=parent.captured_at if parent is not None else datetime.now(timezone.utc), + inputs=parent.inputs if parent is not None else inputs, + ) + self._next += 1 + self._frames[frame.id] = frame + while len(self._frames) > self._keep: + self._frames.popitem(last=False) + self._latest[identity] = frame.id + return frame + + def get(self, frame_id: str | None, identity: str) -> Frame: + """The named frame, or the caller's latest when none is named.""" + if frame_id is None: + frame_id = self._latest.get(identity) + if frame_id is None: + raise StaleFrame("there is no frame yet: take a screenshot first") + frame = self._frames.get(frame_id) + if frame is None: + raise StaleFrame(f"frame {frame_id} is unknown or no longer kept: take a new screenshot") + return frame + + def latest(self, identity: str) -> Frame | None: + return self._frames.get(self._latest.get(identity, "")) + + def check_current(self, frame: Frame, now: Display) -> None: + if (frame.display.width, frame.display.height, frame.display.scale) != (now.width, now.height, now.scale): + raise StaleFrame( + f"frame {frame.id} was captured at {_size(frame.display)} and the display is now {_size(now)}: " + "take a new screenshot") + + +def to_pil(capture: object, display: Display) -> PILImage.Image: + """A capture as a Pillow image, checked to be the display at native resolution.""" + if isinstance(capture, PILImage.Image): + image = capture + elif isinstance(capture, Image): + if capture.mode in ("png", "jpeg"): + image = PILImage.open(io.BytesIO(capture.data)) + image.load() + elif capture.mode in ("BGRA", "BGRX"): + image = PILImage.frombytes("RGB", (capture.width, capture.height), capture.data, "raw", "BGRX") + else: + image = PILImage.frombytes(capture.mode, (capture.width, capture.height), capture.data) + else: + raise Internal(f"@capture returned {type(capture).__name__}, not an Image") + expected = (round(display.width * display.scale), round(display.height * display.scale)) + if image.size != expected: + raise Internal(f"@capture returned {image.size[0]}×{image.size[1]}, but the display at scale " + f"{display.scale:g} is {expected[0]}×{expected[1]}") + return image + + +def _png(image: PILImage.Image) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +def _size(display: Display) -> str: + return f"{display.width}×{display.height} at scale {display.scale:g}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py new file mode 100644 index 00000000..b8bab2aa --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py @@ -0,0 +1,66 @@ +"""W3C UI Events key values: a single character, or a named key, case-insensitive and passed on in its W3C case. +Model spellings (ctrl, Return, ctrl+s) are an adapter's to map; a refused name gets a hint, never a remapping.""" +from __future__ import annotations + +import difflib + +from .errors import InvalidArgument, Unsupported + +NAMED = ( + "Alt", "AltGraph", "CapsLock", "Control", "Fn", "FnLock", "Hyper", "Meta", "NumLock", "ScrollLock", "Shift", + "Super", "Symbol", "SymbolLock", + "Enter", "Tab", + "ArrowDown", "ArrowLeft", "ArrowRight", "ArrowUp", "End", "Home", "PageDown", "PageUp", + "Backspace", "Clear", "Copy", "CrSel", "Cut", "Delete", "EraseEof", "ExSel", "Insert", "Paste", "Redo", "Undo", + "Accept", "Again", "Attn", "Cancel", "ContextMenu", "Escape", "Execute", "Find", "Help", "Pause", "Play", + "Props", "Select", "ZoomIn", "ZoomOut", + "BrightnessDown", "BrightnessUp", "Eject", "LogOff", "Power", "PowerOff", "PrintScreen", "Hibernate", "Standby", + "WakeUp", + *(f"F{n}" for n in range(1, 25)), + "AudioVolumeDown", "AudioVolumeMute", "AudioVolumeUp", "MediaPlayPause", "MediaStop", "MediaTrackNext", + "MediaTrackPrevious", + "BrowserBack", "BrowserFavorites", "BrowserForward", "BrowserHome", "BrowserRefresh", "BrowserSearch", + "BrowserStop", +) +_BY_LOWER = {name.lower(): name for name in NAMED} +_SPELLINGS = { + "ctrl": "Control", "control_l": "Control", "cmd": "Meta", "command": "Meta", "win": "Meta", "windows": "Meta", + "super_l": "Meta", "option": "Alt", "opt": "Alt", "return": "Enter", "esc": "Escape", "del": "Delete", + "space": "' '", "spacebar": "' '", "up": "ArrowUp", "down": "ArrowDown", + "left": "ArrowLeft", "right": "ArrowRight", "pgup": "PageUp", "pgdn": "PageDown", "ins": "Insert", + "menu": "ContextMenu", "print": "PrintScreen", +} + + +def canonical(key: object) -> str: + """``key`` as its W3C value, or InvalidArgument with a hint naming the closest valid one.""" + if not isinstance(key, str) or not key: + raise InvalidArgument(f"a key is a W3C key value, got {key!r}") + if len(key) == 1: + return key + name = _BY_LOWER.get(key.lower()) + if name is not None: + return name + raise InvalidArgument(f"{key!r} is not a W3C key value", hint=_hint(key)) + + +def chord(keys: object, supported: object) -> list[str]: + """A chord's keys in W3C case, refused whole when one isn't a key or isn't one ``supported`` names.""" + if not isinstance(keys, list) or not keys: + raise InvalidArgument("keys is a non-empty list of W3C key values, such as ['Control', 's']") + names = [canonical(k) for k in keys] + if supported != "all": + lacking = [n for n in names if n not in supported] + if lacking: + raise Unsupported(f"this computer doesn't support the key(s) {lacking}") + return names + + +def _hint(key: str) -> str: + lower = key.lower() + if "+" in key or " " in key.strip(): + return "one key per entry: keys is a list, such as ['Control', 's']" + if lower in _SPELLINGS: + return f"did you mean {_SPELLINGS[lower]}?" + close = difflib.get_close_matches(lower, list(_BY_LOWER), n=1, cutoff=0.6) + return f"did you mean {_BY_LOWER[close[0]]}?" if close else "named keys are W3C UI Events key values, such as Enter" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py new file mode 100644 index 00000000..021fbc34 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py @@ -0,0 +1,100 @@ +"""The base class's Linux defaults for the OS operations. The driver runs as the session user inside the desktop +session, so both act directly: a command inherits the session's environment, a file lands as that user.""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +import tempfile +from pathlib import Path + +from .errors import InvalidArgument +from .operations import ExecResult + +#: How long output is still kept after the command exits. An app it started in the background may hold its stdout +#: or stderr open; that stream is then drained, and discarded, until the app closes it, so the app never blocks on +#: a full pipe or dies writing to a closed one. +DRAIN_S = 1.0 +#: How often a command is checked for having exited. +POLL_S = 0.02 +#: How much of a stream is read at a time. +READ_BYTES = 65536 +_draining: set = set() + + +async def run_in_session(argv: list[str], cwd: str | None, env: dict[str, str] | None, timeout_s: float, *, + output_bytes: int) -> ExecResult: + """Run ``argv`` with the driver's own environment plus ``env``, in ``cwd`` (relative to the home), keeping at + most ``output_bytes`` of each stream; past ``timeout_s`` its process group is killed.""" + home = Path.home() + workdir = home / cwd if cwd else home + try: + process = await asyncio.create_subprocess_exec( + *argv, cwd=workdir, env={**os.environ, **(env or {})}, stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, start_new_session=True) + except (FileNotFoundError, PermissionError, NotADirectoryError) as e: + raise InvalidArgument(f"cannot run {argv[0]!r}: {e.strerror or e}") from None + outputs = [_Capped(output_bytes), _Capped(output_bytes)] + readers = [asyncio.ensure_future(out.read(stream)) for out, stream in zip(outputs, (process.stdout, process.stderr))] + timed_out = not await _exited(process, timeout_s) + if timed_out: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + await _exited(process, DRAIN_S) + _, pending = await asyncio.wait(readers, timeout=DRAIN_S) + for out in outputs: + out.returned = True + for reader in pending: + _draining.add(reader) + reader.add_done_callback(_draining.discard) + stdout, stderr = outputs + return ExecResult(exit_code=None if timed_out else process.returncode, stdout=bytes(stdout.kept), + stderr=bytes(stderr.kept), timed_out=timed_out, truncated=stdout.cut or stderr.cut) + + +async def _exited(process: asyncio.subprocess.Process, timeout_s: float) -> bool: + """Whether the process exits within ``timeout_s``. Read from ``returncode``, which is set when the process exits: + ``Process.wait()`` also waits for the pipes to close, which a background child can hold open.""" + deadline = asyncio.get_running_loop().time() + timeout_s + while process.returncode is None: + if asyncio.get_running_loop().time() >= deadline: + return False + await asyncio.sleep(POLL_S) + return True + + +class _Capped: + def __init__(self, limit: int) -> None: + self.limit = limit + self.kept = bytearray() + self.cut = False + self.returned = False + + async def read(self, stream: asyncio.StreamReader) -> None: + while chunk := await stream.read(READ_BYTES): + if self.returned: + continue + room = self.limit - len(self.kept) + if len(chunk) > room: + self.cut = True + self.kept += chunk[:max(room, 0)] + + +def write_files(files: dict[str, bytes]) -> None: + """Write each file under the home atomically: a temporary file beside it, then a rename.""" + home = Path.home().resolve() + for relative, data in files.items(): + destination = home / relative + if not destination.resolve().is_relative_to(home): + raise InvalidArgument(f"{relative!r} is outside the home") + destination.parent.mkdir(parents=True, exist_ok=True) + handle, temporary = tempfile.mkstemp(dir=destination.parent, prefix=f".{destination.name}.", suffix=".part") + try: + with os.fdopen(handle, "wb") as out: + out.write(data) + os.replace(temporary, destination) + except BaseException: + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) + raise diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py new file mode 100644 index 00000000..0f7419da --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -0,0 +1,183 @@ +"""The operations a driver implements, marked with decorators; the base class builds the contract from them. + +Each operation is called with screen pixels, positionally, in the order below, and may be sync or async: + + display() -> Display set_display(width, height, scale) -> Display + capture() -> Image | PIL.Image.Image the whole display, at native resolution + click(x, y, button, count, modifiers) -> Effect | None + drag(path, button, duration_ms, modifiers) -> Effect | None path: [(x, y), ...] + scroll(x, y, direction, amount, modifiers) -> Effect | None + type_text(text) -> Effect | None press(keys) -> Effect | None, one chord + pointer_move(x, y), pointer_down(button), pointer_up(button), pointer_position() -> (x, y) + key_down(keys), key_up(keys) + run_in_session(argv, cwd, env, timeout_s) -> ExecResult write_files({path: bytes}) +""" +from __future__ import annotations + +import inspect +from dataclasses import dataclass +from typing import Any, Callable + +from .platform import Platform, matches, normalize, overlap + +REQUIRED = ("display", "capture", "click", "drag", "scroll", "type_text", "press") +GROUPS = { + "pointer": ("pointer_move", "pointer_down", "pointer_up", "pointer_position"), + "keyboard": ("key_down", "key_up"), +} +OPTIONAL = ("set_display", "run_in_session", "write_files") +INPUT = frozenset({"click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", + "key_down", "key_up"}) + +_OP_ATTR = "_aec_operation" + + +@dataclass(frozen=True) +class Display: + """The display: its resolution in screen pixels, and how many image pixels a capture has per screen pixel.""" + width: int + height: int + scale: float = 1.0 + + +@dataclass(frozen=True) +class Image: + """A capture: encoded ``png`` or ``jpeg`` bytes, or raw pixels in a Pillow raw mode (``RGB``, ``RGBA``, ``BGRA``, + ``BGRX``).""" + width: int + height: int + data: bytes + mode: str = "png" + + +@dataclass(frozen=True) +class Effect: + """What a driver can confirm an input did; unknown fields stay None.""" + focus_changed: bool | None = None + pointer_moved: bool | None = None + clipboard_used: bool | None = None + detail: str | None = None + + +@dataclass(frozen=True) +class ExecResult: + exit_code: int | None + stdout: bytes = b"" + stderr: bytes = b"" + timed_out: bool = False + truncated: bool = False + + +@dataclass(frozen=True) +class ReadyCheck: + name: str + ok: bool + detail: str = "" + + +@dataclass(frozen=True) +class _Mark: + operation: str + predicates: dict + default: bool = False + + +def _marker(operation: str) -> Callable: + def decorate(fn: Callable | None = None, /, **predicates: Any) -> Callable: + mark = _Mark(operation, normalize(predicates)) + + def deco(f: Callable) -> Callable: + setattr(f, _OP_ATTR, mark) + return f + + return deco(fn) if fn is not None else deco + + decorate.__name__ = decorate.__qualname__ = operation + decorate.__doc__ = (f"Mark the method that performs ``{operation}``. Use bare, or with predicates " + f"(os, distribution, version, arch, display) to choose it per platform.") + return decorate + + +display = _marker("display") +set_display = _marker("set_display") +capture = _marker("capture") +click = _marker("click") +drag = _marker("drag") +scroll = _marker("scroll") +type_text = _marker("type_text") +press = _marker("press") +pointer_move = _marker("pointer_move") +pointer_down = _marker("pointer_down") +pointer_up = _marker("pointer_up") +pointer_position = _marker("pointer_position") +key_down = _marker("key_down") +key_up = _marker("key_up") +run_in_session = _marker("run_in_session") +write_files = _marker("write_files") + + +def default_for(operation: str, **predicates: Any) -> Callable: + """Mark one of the base class's per-OS defaults, which a driver's own method for the operation replaces.""" + def deco(f: Callable) -> Callable: + setattr(f, _OP_ATTR, _Mark(operation, normalize(predicates), default=True)) + return f + return deco + + +def declared(cls: type) -> dict[str, list[tuple[str, dict]]]: + """Every operation the class marks, as (method name, predicates), with the base class's defaults dropped for + an operation the driver implements itself.""" + found: dict[str, list[tuple[str, _Mark]]] = {} + for name in dir(cls): + if name.startswith("__"): + continue + mark = getattr(inspect.getattr_static(cls, name, None), _OP_ATTR, None) + if isinstance(mark, _Mark): + found.setdefault(mark.operation, []).append((name, mark)) + out = {} + for operation, marks in found.items(): + own = [(name, mark) for name, mark in marks if not mark.default] + out[operation] = [(name, mark.predicates) for name, mark in (own or marks)] + return out + + +def check_class(cls: type) -> None: + """The class-definition checks: every required operation, whole groups, one method per operation per + platform. Raises TypeError naming what is wrong.""" + ops = declared(cls) + problems = [] + missing = [op for op in REQUIRED if op not in ops] + if missing: + problems.append(f"missing required operation(s): {', '.join('@' + op for op in missing)}") + for group, members in GROUPS.items(): + have = [op for op in members if op in ops] + if have and len(have) != len(members): + lacking = [op for op in members if op not in ops] + problems.append(f"the {group} group needs all of {', '.join('@' + op for op in members)}; " + f"missing {', '.join('@' + op for op in lacking)}") + for op, methods in ops.items(): + for i, (a, pa) in enumerate(methods): + for b, pb in methods[i + 1:]: + if overlap(pa, pb): + problems.append(f"@{op} is marked on both {a} and {b}, and some platform matches both") + if problems: + raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) + + +def resolve(cls: type, running: Platform) -> dict[str, str]: + """The method each operation uses on ``running``, by name. A required operation with no method for it raises + RuntimeError; a group is kept only when all of it matches.""" + chosen = {} + for op, methods in declared(cls).items(): + hits = [name for name, predicates in methods if matches(predicates, running)] + if hits: + chosen[op] = hits[0] + missing = [op for op in REQUIRED if op not in chosen] + if missing: + raise RuntimeError(f"{cls.__qualname__} has no method for {', '.join('@' + op for op in missing)} " + f"on this platform ({running.as_dict()})") + for members in GROUPS.values(): + if not all(op in chosen for op in members): + for op in members: + chosen.pop(op, None) + return chosen diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py new file mode 100644 index 00000000..949d17e0 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py @@ -0,0 +1,76 @@ +"""The platform a driver runs on, and the predicates (``os``, ``distribution``, ``version``, ``arch``, ``display``) +that choose one method per operation for it.""" +from __future__ import annotations + +import os +import platform as _platform +import sys +from dataclasses import dataclass +from pathlib import Path + +PREDICATE_KEYS = ("os", "distribution", "version", "arch", "display") + +_ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"} +_OS = {"linux": "linux", "darwin": "macos", "win32": "windows"} + + +@dataclass(frozen=True) +class Platform: + os: str | None = None + distribution: str | None = None + version: str | None = None + arch: str | None = None + display: str | None = None + + def as_dict(self) -> dict: + return {k: getattr(self, k) for k in PREDICATE_KEYS if getattr(self, k) is not None} + + +def detect(environ: dict | None = None, os_release: Path = Path("/etc/os-release")) -> Platform: + """The running platform: the OS, ``/etc/os-release``'s ID and VERSION_ID on Linux, the machine, and the + display from ``XDG_SESSION_TYPE``, else ``DISPLAY`` (x11) or ``WAYLAND_DISPLAY`` (wayland).""" + environ = os.environ if environ is None else environ + name = _OS.get(sys.platform, sys.platform) + release = _os_release(os_release) if name == "linux" else {} + display = None + if name == "linux": + session = (environ.get("XDG_SESSION_TYPE") or "").strip().lower() + if session in ("x11", "wayland"): + display = session + elif environ.get("DISPLAY"): + display = "x11" + elif environ.get("WAYLAND_DISPLAY"): + display = "wayland" + machine = _platform.machine().lower() + return Platform(os=name, distribution=release.get("ID"), version=release.get("VERSION_ID"), + arch=_ARCH.get(machine, machine or None), display=display) + + +def _os_release(path: Path) -> dict: + try: + text = path.read_text() + except OSError: + return {} + fields = {} + for line in text.splitlines(): + key, sep, value = line.partition("=") + if sep and key.strip(): + fields[key.strip()] = value.strip().strip("\"'") + return fields + + +def normalize(predicates: dict) -> dict[str, frozenset]: + """Predicates as sets of allowed values; an unknown key is refused.""" + unknown = sorted(set(predicates) - set(PREDICATE_KEYS)) + if unknown: + raise TypeError(f"unknown predicate(s) {unknown}; predicates are {list(PREDICATE_KEYS)}") + return {k: frozenset([v] if isinstance(v, str) else v) for k, v in predicates.items()} + + +def matches(predicates: dict[str, frozenset], running: Platform) -> bool: + return all(getattr(running, key) in allowed for key, allowed in predicates.items()) + + +def overlap(a: dict[str, frozenset], b: dict[str, frozenset]) -> bool: + """Whether some platform could satisfy both: no key both constrain to disjoint values.""" + return all(a[k] & b[k] for k in set(a) & set(b)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py new file mode 100644 index 00000000..311a4918 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/queue.py @@ -0,0 +1,51 @@ +"""The input queue: input calls run one at a time in arrival order, and delivered inputs are counted per caller so +a result can say how many others landed since the frame it named.""" +from __future__ import annotations + +import asyncio +import contextlib +from collections import Counter +from typing import AsyncIterator + +from .errors import InputBusy + + +class InputQueue: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._total = 0 + self._by_caller: Counter = Counter() + + @contextlib.asynccontextmanager + async def turn(self, wait_s: float) -> AsyncIterator[None]: + """Hold the queue for one call, or refuse with input_busy after ``wait_s``; waiting is first come, first + served. A wait given up, by the timeout or by the caller's cancellation, never leaves the queue held.""" + waiter = asyncio.ensure_future(self._lock.acquire()) + try: + done, _ = await asyncio.wait({waiter}, timeout=wait_s) + except BaseException: + self._abandon(waiter) + raise + if not done: + self._abandon(waiter) + raise InputBusy(f"waited {wait_s:g} s for earlier input to finish") + try: + yield + finally: + self._lock.release() + + def _abandon(self, waiter: asyncio.Future) -> None: + """Give up a wait for the lock; if it was granted anyway, give the lock back.""" + waiter.cancel() + waiter.add_done_callback(lambda w: None if w.cancelled() or w.exception() is not None else self._lock.release()) + + def delivered(self, caller: str) -> None: + self._total += 1 + self._by_caller[caller] += 1 + + def snapshot(self) -> tuple: + return self._total, dict(self._by_caller) + + def others_since(self, snapshot: tuple, caller: str) -> int: + total, by_caller = snapshot + return (self._total - total) - (self._by_caller[caller] - by_caller.get(caller, 0)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py new file mode 100644 index 00000000..5794b9f7 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py @@ -0,0 +1,94 @@ +"""The computer_* tools' results: each tool's output schema, and the one way a result is sent, so what a tool +answers is what its schema says.""" +from __future__ import annotations + +from typing import Literal, Optional + +from mcp.types import CallToolResult, ImageContent, TextContent +from pydantic import BaseModel + +from .operations import Effect + + +class DisplayData(BaseModel): + width: int + height: int + scale: float + + +class FrameData(BaseModel): + frame_id: str + width: int + height: int + display: DisplayData + parent_frame_id: Optional[str] = None + captured_at: str + + +class ScreenshotResult(FrameData): + summary: str + + +class EffectData(BaseModel): + focus_changed: Optional[bool] = None + pointer_moved: Optional[bool] = None + clipboard_used: Optional[bool] = None + detail: Optional[str] = None + + @classmethod + def of(cls, effect: object) -> EffectData | None: + """What an input operation returned, when it is an Effect that says something.""" + if not isinstance(effect, Effect): + return None + data = cls(**vars(effect)) + return data if data.model_dump(exclude_none=True) else None + + +class InputResult(BaseModel): + summary: str + applied: Literal["yes", "no", "unknown"] + delivery: Literal["foreground", "background"] + others_since_frame: int + effect: Optional[EffectData] = None + frame: Optional[FrameData] = None + + +class WaitResult(BaseModel): + summary: str + waited_ms: int + frame: Optional[FrameData] = None + + +class Point(BaseModel): + x: int + y: int + + +class PointerPositionResult(BaseModel): + summary: str + screen: Point + frame_id: Optional[str] = None + x: Optional[int] = None + y: Optional[int] = None + inside_frame: Optional[bool] = None + + +class PointerData(BaseModel): + id: str + kind: Literal["foreground", "background"] + label: str + default: bool + + +class PointerListResult(BaseModel): + summary: str + pointers: list[PointerData] + + +def send(result: BaseModel, image: tuple[str, str] | None = None) -> CallToolResult: + """A result as a tool answers: the summary as the text block, then the image (base64, MIME type) when there is + one, and the result as structured content with its unset fields left out.""" + content: list = [TextContent(type="text", text=result.summary)] + if image is not None: + content.append(ImageContent(type="image", data=image[0], mimeType=image[1])) + return CallToolResult(content=content, structuredContent=result.model_dump(mode="json", exclude_none=True)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py new file mode 100644 index 00000000..36223531 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -0,0 +1,226 @@ +"""What ComputerDriverEnvironment keeps for one driver, and the steps its tools and extensions share: calling the +driver's operations, frames, the input queue, readiness, held keys and command runs. A driver reaches it as +``self._computer``; it is the base class's own, so a driver's members can't replace any of it.""" +from __future__ import annotations + +import asyncio +import contextlib +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Awaitable, Callable + +from mcp.types import CallToolResult + +from ..errors import AgentEnvError +from ..tool_context import ToolContext +from .errors import DeliveryFailed, DeviceUnavailable, InputBusy, Internal +from .frames import Frame, FrameStore, to_pil +from .operations import GROUPS, INPUT, Display, ReadyCheck, resolve +from .queue import InputQueue +from .results import EffectData, InputResult, send + +if TYPE_CHECKING: + from .driver import ComputerDriverEnvironment + +#: How many finished command results are kept for a repeated request_id; a running command is always joined. +KEEP_EXEC_RESULTS = 256 + + +class ComputerRuntime: + def __init__(self, driver: ComputerDriverEnvironment) -> None: + self.driver = driver + self.operations = resolve(type(driver), driver.platform) + self.frames = FrameStore() + self.queue = InputQueue() + self.ready = False + self.held: dict[str, list[str]] = {} + self._ready_lock = asyncio.Lock() + self._releases: dict[str, asyncio.Task] = {} + self._commands_running: dict[str, asyncio.Task] = {} + self._commands_finished: OrderedDict = OrderedDict() + + def has(self, capability: str) -> bool: + if capability in GROUPS: + return all(op in self.operations for op in GROUPS[capability]) + return capability in self.operations + + async def call(self, operation: str, *args: Any) -> Any: + """Run one of the driver's operations: an AgentEnvError passes through, anything else is an input that may + have been delivered (delivery_failed) or an internal failure.""" + method = getattr(self.driver, self.operations[operation]) + try: + result = method(*args) + return await result if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future) else result + except AgentEnvError: + raise + except Exception as e: + error = DeliveryFailed if operation in INPUT else Internal + raise error(f"@{operation} failed: {type(e).__name__}: {e}") from e + + async def display_now(self) -> Display: + display = await self.call("display") + if not isinstance(display, Display) or display.width <= 0 or display.height <= 0 or display.scale <= 0: + raise Internal(f"@display returned {display!r}, not a Display with a positive size and scale") + return display + + async def capture(self, identity: str) -> Frame: + """A new frame of the screen. The input count is taken before the capture, so input that lands while it runs + counts as after the frame, which the frame may not show.""" + inputs = self.queue.snapshot() + display = await self.display_now() + image = to_pil(await self.call("capture"), display) + return self.frames.add(image, display, inputs, identity=identity) + + async def pointer(self) -> tuple[int, int]: + where = await self.call("pointer_position") + try: + x, y = where + return int(x), int(y) + except (TypeError, ValueError): + raise Internal(f"@pointer_position returned {where!r}, not (x, y)") from None + + async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, + summary: str, mapped: bool = False, at_pointer: bool = False, repeat: int = 1, + hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None) -> CallToolResult: + """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result + with others' input since ``frame`` and, when asked, a fresh frame. Whatever the call's turn can change is read + once it has the turn: a point ``mapped`` from ``frame`` is checked again, since the display may have changed + while it waited, and ``at_pointer`` reads the pointer and puts it before ``args``. ``then`` runs as soon as + the input is delivered.""" + identity = self.driver.caller_identity(ctx) + sent = False + async with self.queue.turn(self.driver.settings.queue_wait_s): + if mapped and frame is not None: + self.frames.check_current(frame, await self.display_now()) + if at_pointer: + sx, sy = await self.pointer() + args, summary = (sx, sy, *args), f"{summary} at the pointer, screen ({sx}, {sy})" + others = self.queue.others_since(frame.inputs, identity) if frame is not None else 0 + effect = None + try: + if move_to is not None: + await self.call("pointer_move", *move_to) + sent = True + for _ in range(repeat): + effect = await self.call(operation, *args) + sent = True + if hold is not None: + await self._hold_for(*hold) + except AgentEnvError as e: + if sent: + e.applied = "unknown" + raise + finally: + if sent: + self.queue.delivered(identity) + if then is not None: + then() + result = InputResult(summary=summary, applied="yes", delivery="foreground", others_since_frame=others, + effect=EffectData.of(effect)) + if not screenshot: + return send(result) + try: + new = await self.capture(identity) + except AgentEnvError as e: + result.summary = f"{summary}; the screenshot after it failed ({e.code}: {e.message})" + return send(result) + result.summary, result.frame = f"{summary}, then captured {new.id}", new.result() + return send(result, new.encode()) + + async def _hold_for(self, ms: int, keys: list[str]) -> None: + """Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile.""" + release = ("key_up", list(reversed(keys))) + try: + await asyncio.sleep(ms / 1000) + except asyncio.CancelledError: + with contextlib.suppress(Exception): + await asyncio.shield(self.call(*release)) + raise + await self.call(*release) + + async def ensure_ready(self) -> None: + """Refuse with device_unavailable until the ready checks have passed once; run them if none has yet.""" + if self.ready: + return + async with self._ready_lock: + if self.ready: + return + failed = [c for c in await self.ready_checks() if not c.ok] + if failed: + raise DeviceUnavailable("the computer isn't ready: " + "; ".join(f"{c.name}: {c.detail}" for c in failed)) + + async def ready_checks(self) -> list[ReadyCheck]: + checks = [] + try: + display = await self.display_now() + checks.append(ReadyCheck("display", True, f"{display.width}×{display.height} at scale {display.scale:g}")) + except AgentEnvError as e: + display = None + checks.append(ReadyCheck("display", False, e.message)) + if display is None: + checks.append(ReadyCheck("capture", False, "the display didn't answer")) + else: + try: + image = to_pil(await self.call("capture"), display) + checks.append(ReadyCheck("capture", True, f"{image.size[0]}×{image.size[1]}")) + except AgentEnvError as e: + checks.append(ReadyCheck("capture", False, e.message)) + try: + checks += list(await self.driver.ready_checks()) + except Exception as e: + checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) + if all(c.ok for c in checks): + self.ready = True + return checks + + def hold(self, identity: str, keys: list[str]) -> None: + held = self.held.setdefault(identity, []) + held += [k for k in keys if k not in held] + previous = self._releases.pop(identity, None) + if previous is not None: + previous.cancel() + self._releases[identity] = asyncio.ensure_future( + self._release_after(identity, self.driver.settings.max_hold_ms)) + + def release(self, identity: str, keys: list[str]) -> None: + held = [k for k in self.held.get(identity, []) if k not in keys] + if held: + self.held[identity] = held + return + self.held.pop(identity, None) + release = self._releases.pop(identity, None) + if release is not None and release is not asyncio.current_task(): + release.cancel() + + async def _release_after(self, identity: str, ms: int) -> None: + """Release what ``identity`` still holds once ``max_hold_ms`` has passed, waiting as long as other input + keeps the queue busy.""" + await asyncio.sleep(ms / 1000) + while keys := list(reversed(self.held.get(identity, []))): + try: + async with self.queue.turn(self.driver.settings.queue_wait_s): + await self.call("key_up", keys) + except InputBusy: + continue + self.release(identity, keys) + + async def command(self, request_id: str, start: Callable[[], Awaitable[dict]]) -> dict: + """The result of the command ``request_id`` names: a running one is joined, one of the last + ``KEEP_EXEC_RESULTS`` finished is answered again, and anything else is started.""" + finished, running = self._commands_finished, self._commands_running + if request_id in finished: + return finished[request_id] + task = running.get(request_id) + if task is None: + task = asyncio.ensure_future(start()) + running[request_id] = task + + def done(ran: asyncio.Task) -> None: + if running.get(request_id) is ran: + del running[request_id] + if not ran.cancelled() and ran.exception() is None: + finished[request_id] = ran.result() + while len(finished) > KEEP_EXEC_RESULTS: + finished.popitem(last=False) + + task.add_done_callback(done) + return await asyncio.shield(task) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py new file mode 100644 index 00000000..320ed333 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py @@ -0,0 +1,43 @@ +"""The settings a computer driver reads at start, from one JSON value in ``AGENTENV_COMPUTER_SETTINGS``.""" +from __future__ import annotations + +import json +import os +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from ..transfers import RelativePath + +SETTINGS_ENV = "AGENTENV_COMPUTER_SETTINGS" + +Space = Literal["frame", "norm"] + + +class ComputerSettings(BaseModel): + """``data_paths`` are relative to the session user's home and are what ``data/get`` returns; the rest are the + defaults the card reports and the framework extensions change.""" + + model_config = ConfigDict(extra="forbid", validate_assignment=True) + + data_paths: list[RelativePath] = Field(default_factory=list) + default_space: Space = "frame" + default_norm_scale: int = Field(default=1000, ge=1, le=1_000_000) + queue_wait_s: float = Field(default=30.0, gt=0, le=3600) + tap_hold_ms: int = Field(default=50, ge=0, le=1000) + type_delay_ms: int = Field(default=10, ge=0, le=1000) + max_hold_ms: int = Field(default=10_000, ge=0, le=600_000) + layout: str = "us" + drag_duration_ms: int = Field(default=500, ge=0) + exec_output_bytes: int = Field(default=1 << 20, gt=0) + exec_timeout_s: float = Field(default=60.0, gt=0, le=3600) + + +def load_settings(environ: dict | None = None) -> ComputerSettings: + raw = (os.environ if environ is None else environ).get(SETTINGS_ENV) + if not raw: + return ComputerSettings() + try: + return ComputerSettings.model_validate(json.loads(raw)) + except (ValueError, ValidationError) as e: + raise ValueError(f"{SETTINGS_ENV} is not valid computer settings: {e}") from None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py new file mode 100644 index 00000000..d3465112 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/spaces.py @@ -0,0 +1,49 @@ +"""Coordinate spaces: points in a frame's pixels (``frame``) or on a 0..norm_scale grid over it (``norm``), mapped +to the frame's pixels and refused, never clamped, when outside it.""" +from __future__ import annotations + +import math + +from .errors import InvalidArgument, OutOfBounds +from .frames import Frame +from .settings import Space + +SPACES = ("frame", "norm") + + +def frame_point(frame: Frame, x: float, y: float, space: Space, norm_scale: int) -> tuple[float, float]: + if space == "norm": + if not (0 <= x <= norm_scale and 0 <= y <= norm_scale): + raise OutOfBounds(f"({x:g}, {y:g}) is outside the norm range 0..{norm_scale} of frame {frame.id}") + fx, fy = min(x * frame.width / norm_scale, frame.width - 1), min(y * frame.height / norm_scale, frame.height - 1) + return fx, fy + if not frame.contains(x, y): + raise OutOfBounds(f"({x:g}, {y:g}) is outside frame {frame.id} ({frame.width}×{frame.height})") + return x, y + + +def frame_region(frame: Frame, region: list, space: Space, norm_scale: int) -> tuple[int, int, int, int]: + """``[x0, y0, x1, y1]`` as a pixel box within the frame, end exclusive.""" + if len(region) != 4: + raise InvalidArgument(f"region is [x0, y0, x1, y1], got {len(region)} numbers") + x0, y0, x1, y1 = region + if space == "norm": + if not all(0 <= v <= norm_scale for v in region): + raise OutOfBounds(f"region {list(region)} is outside the norm range 0..{norm_scale} of frame {frame.id}") + x0, x1 = x0 * frame.width / norm_scale, x1 * frame.width / norm_scale + y0, y1 = y0 * frame.height / norm_scale, y1 * frame.height / norm_scale + box = (int(math.floor(x0)), int(math.floor(y0)), int(math.ceil(x1)), int(math.ceil(y1))) + if not (0 <= box[0] < box[2] <= frame.width and 0 <= box[1] < box[3] <= frame.height): + raise OutOfBounds(f"region {list(region)} is not a non-empty box within frame {frame.id} " + f"({frame.width}×{frame.height})") + return box + + +def check_space(space: str | None, norm_scale: int | None, default_space: Space, default_scale: int) -> tuple: + space = space or default_space + if space not in SPACES: + raise InvalidArgument(f"space is 'frame' or 'norm', got {space!r}") + scale = default_scale if norm_scale is None else norm_scale + if not (isinstance(scale, int) and 1 <= scale <= 1_000_000): + raise InvalidArgument(f"norm_scale is an integer from 1 to 1000000, got {norm_scale!r}") + return space, scale diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py new file mode 100644 index 00000000..9b99c91b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -0,0 +1,423 @@ +"""The seven computer_* tools and the pointer and keyboard groups' tools: fixed names, parameters and results, built +from the driver's operations. Every result is structured, with a one-line summary repeated as its text block.""" +from __future__ import annotations + +import asyncio +import unicodedata +from typing import TYPE_CHECKING, Annotated, Any, Callable, Literal, Optional + +from mcp.types import CallToolResult +from pydantic import Field + +from ..agent_env_environment import tool +from ..tool_context import ToolContext +from .errors import InvalidArgument, Unsupported +from .frames import Frame +from .keys import chord +from .results import ( + InputResult, + Point, + PointerData, + PointerListResult, + PointerPositionResult, + ScreenshotResult, + WaitResult, + send, +) +from .spaces import check_space, frame_point, frame_region + +if TYPE_CHECKING: + from .runtime import ComputerRuntime + from .settings import ComputerSettings + +_READ = {"readOnlyHint": True, "openWorldHint": False} +_INPUT = {"readOnlyHint": False, "destructiveHint": False, "idempotentHint": False, "openWorldHint": False} + +Number = Annotated[float, Field(description="A coordinate in the chosen space")] +AtPointer = Annotated[Optional[float], Field( + description="A coordinate in the chosen space. Required unless the computer declares computer-pointer/v1; then " + "leave out x and y to act at the pointer")] +MoveFirst = Annotated[Optional[float], Field(description="Move here first")] +FrameId = Annotated[Optional[str], Field(description="The frame the point refers to; defaults to your latest")] +SpaceArg = Annotated[Optional[Literal["frame", "norm"]], Field( + description="frame: pixels of the frame's image; norm: 0 to norm_scale over it")] +NormScale = Annotated[Optional[int], Field(description="The norm grid's size, for space=norm")] +Modifiers = Annotated[Optional[list[str]], Field(description="W3C key names held for this action, such as ['Shift']")] +Screenshot = Annotated[bool, Field(description="Return a fresh frame after acting")] +Button = Annotated[Optional[str], Field(description="Omitted, the primary press; others only where declared")] + + +class Tools: + """The tools, mixed into ComputerDriverEnvironment.""" + + _computer: ComputerRuntime + settings: ComputerSettings + buttons: tuple[str, ...] + keys: Any + max_click_count: int + max_text_length: int + max_wait_ms: int + max_drag_points: int + max_duration_ms: int + max_key_repeat: int + max_scroll_amount: int + has: Callable[[str], bool] + caller_identity: Callable[[ToolContext], str] + + @tool(annotations=_READ) + async def computer_screenshot( + self, + ctx: ToolContext, + region: Annotated[Optional[list[float]], Field( + description="[x0, y0, x1, y1] within a frame: return that part at native resolution as a new frame")] = None, + frame_id: Annotated[Optional[str], Field(description="The frame region refers to; defaults to your latest")] = None, + format: Annotated[Literal["png", "jpeg"], Field(description="The image encoding")] = "png", + quality: Annotated[Optional[int], Field(description="JPEG quality, 1 to 95")] = None, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, ScreenshotResult]: + """Capture the screen as a frame: an image plus a frame_id you can point into later, the image size, and the + display resolution and scale. With region, crop an existing frame at native resolution (zoom).""" + computer = self._computer + await computer.ensure_ready() + identity = self.caller_identity(ctx) + if quality is not None and not (format == "jpeg" and 1 <= quality <= 95): + raise InvalidArgument("quality is for format=jpeg, from 1 to 95") + if region is None: + if frame_id is not None: + raise InvalidArgument("frame_id names the frame a region crops; give a region too") + frame = await computer.capture(identity) + summary = f"Captured {frame.id}: {frame.width}×{frame.height} of the {_display(frame)} display" + else: + parent = computer.frames.get(frame_id, identity) + computer.frames.check_current(parent, await computer.display_now()) + box = frame_region(parent, region, *_space(self, space, norm_scale)) + frame = computer.frames.add(parent.image.crop(box), parent.display, parent.inputs, + identity=identity, parent=parent, origin=(box[0], box[1])) + summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})" + result = ScreenshotResult(summary=summary, **frame.result().model_dump()) + return send(result, frame.encode(format, quality)) + + @tool(annotations=_INPUT) + async def computer_click( + self, + ctx: ToolContext, + x: AtPointer = None, + y: AtPointer = None, + frame_id: FrameId = None, + button: Button = None, + count: Annotated[int, Field(description="1 by default; double and triple clicks where declared")] = 1, + modifiers: Modifiers = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Click at a point of a frame.""" + await self._computer.ensure_ready() + button = _button(self, button) + _check_range("count", count, 1, self.max_click_count) + held = _modifiers(self, modifiers) + frame, screen, where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + times = f" ×{count}" if count > 1 else "" + return await self._computer.act(ctx, "click", (*screen, button, count, held), frame=frame, mapped=mapped, + at_pointer=not screen, screenshot=screenshot, + summary=f"Clicked {button}{times}{where}") + + @tool(annotations=_INPUT) + async def computer_drag( + self, + ctx: ToolContext, + path: Annotated[list[list[float]], Field(description="Points [[x, y], ...]: at least a start and an end")], + frame_id: FrameId = None, + button: Button = None, + duration_ms: Annotated[Optional[int], Field(description="How long the move takes")] = None, + modifiers: Modifiers = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Press at the first point, move through the path, and release at the last.""" + computer = self._computer + await computer.ensure_ready() + button = _button(self, button) + if not isinstance(path, list) or not 2 <= len(path) <= self.max_drag_points: + raise InvalidArgument(f"path has from 2 to {self.max_drag_points} points on this computer") + duration = self.settings.drag_duration_ms if duration_ms is None else duration_ms + _check_range("duration_ms", duration, 0, self.max_duration_ms) + held = _modifiers(self, modifiers) + frame = computer.frames.get(frame_id, self.caller_identity(ctx)) + computer.frames.check_current(frame, await computer.display_now()) + space, scale = _space(self, space, norm_scale) + screen = [] + for point in path: + if not (isinstance(point, list) and len(point) == 2): + raise InvalidArgument("each path point is [x, y]") + screen.append(frame.to_screen(*frame_point(frame, point[0], point[1], space, scale))) + return await computer.act(ctx, "drag", (screen, button, duration, held), frame=frame, mapped=True, + screenshot=screenshot, + summary=f"Dragged {button} through {len(path)} points of {frame.id} in {duration} ms") + + @tool(annotations=_INPUT) + async def computer_scroll( + self, + ctx: ToolContext, + direction: Annotated[Literal["up", "down", "left", "right"], Field( + description="Where the view moves through the content: down shows what is further down")], + x: AtPointer = None, + y: AtPointer = None, + frame_id: FrameId = None, + amount: Annotated[int, Field(description="How many steps (wheel notches)")] = 3, + modifiers: Modifiers = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Scroll the content under a point.""" + await self._computer.ensure_ready() + _check_range("amount", amount, 1, self.max_scroll_amount) + held = _modifiers(self, modifiers) + frame, screen, where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + return await self._computer.act(ctx, "scroll", (*screen, direction, amount, held), frame=frame, mapped=mapped, + at_pointer=not screen, screenshot=screenshot, + summary=f"Scrolled {direction} {amount}{where}") + + @tool(annotations=_INPUT) + async def computer_type( + self, + ctx: ToolContext, + text: Annotated[str, Field(description="Any Unicode; \\n types Enter and \\t types Tab")], + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Type text into whatever has keyboard focus. Text that can't be typed whole is refused before anything is + typed; the clipboard is left as it was.""" + computer = self._computer + await computer.ensure_ready() + if not text: + raise InvalidArgument("text is empty") + if len(text) > self.max_text_length: + raise InvalidArgument(f"text is at most {self.max_text_length} characters on this computer, got {len(text)}") + control = sorted({f"U+{ord(c):04X}" for c in text if c not in "\n\t" and unicodedata.category(c) == "Cc"}) + if control: + raise InvalidArgument(f"text has control characters {control}; press keys with computer_key") + return await computer.act(ctx, "type_text", (text,), frame=computer.frames.latest(self.caller_identity(ctx)), + screenshot=screenshot, summary=f"Typed {len(text)} characters") + + @tool(annotations=_INPUT) + async def computer_key( + self, + ctx: ToolContext, + keys: Annotated[list[str], Field( + description="One chord, W3C key names pressed in order and released in reverse: ['Control', 's'] or " + "['Enter']")], + repeat: Annotated[int, Field(description="How many times to press the chord")] = 1, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Press one key chord.""" + computer = self._computer + await computer.ensure_ready() + names = chord(keys, self.keys) + _check_range("repeat", repeat, 1, self.max_key_repeat) + times = f" ×{repeat}" if repeat > 1 else "" + return await computer.act(ctx, "press", (names,), frame=computer.frames.latest(self.caller_identity(ctx)), + screenshot=screenshot, repeat=repeat, summary=f"Pressed {'+'.join(names)}{times}") + + @tool(annotations=_READ) + async def computer_wait( + self, + ctx: ToolContext, + ms: Annotated[int, Field(description="How long to wait, in milliseconds")] = 1000, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, WaitResult]: + """Wait, then optionally return a fresh frame.""" + await self._computer.ensure_ready() + _check_range("ms", ms, 0, self.max_wait_ms) + await asyncio.sleep(ms / 1000) + summary = f"Waited {ms} ms" + if not screenshot: + return send(WaitResult(summary=summary, waited_ms=ms)) + frame = await self._computer.capture(self.caller_identity(ctx)) + return send(WaitResult(summary=f"{summary}, then captured {frame.id}", waited_ms=ms, frame=frame.result()), + frame.encode()) + + @tool(annotations=_INPUT) + async def computer_pointer_move( + self, + ctx: ToolContext, + x: Number, + y: Number, + frame_id: FrameId = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Move the pointer to a point; hover effects happen.""" + await self._computer.ensure_ready() + frame, screen, where, mapped = await _point(self, ctx, x, y, frame_id, space, norm_scale) + return await self._computer.act(ctx, "pointer_move", screen, frame=frame, mapped=mapped, + screenshot=screenshot, summary=f"Moved the pointer{where}") + + @tool(annotations=_INPUT) + async def computer_pointer_down( + self, + ctx: ToolContext, + button: Button = None, + x: MoveFirst = None, + y: MoveFirst = None, + frame_id: FrameId = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Press a button, optionally after moving to a point.""" + return await _button_action(self, ctx, "pointer_down", "Pressed", button, x, y, frame_id, screenshot, space, + norm_scale) + + @tool(annotations=_INPUT) + async def computer_pointer_up( + self, + ctx: ToolContext, + button: Button = None, + x: MoveFirst = None, + y: MoveFirst = None, + frame_id: FrameId = None, + screenshot: Screenshot = False, + space: SpaceArg = None, + norm_scale: NormScale = None, + ) -> Annotated[CallToolResult, InputResult]: + """Release a button, optionally after moving to a point.""" + return await _button_action(self, ctx, "pointer_up", "Released", button, x, y, frame_id, screenshot, space, + norm_scale) + + @tool(annotations=_READ) + async def computer_pointer_position( + self, + ctx: ToolContext, + frame_id: Annotated[Optional[str], Field(description="The frame to report in; defaults to your latest")] = None, + ) -> Annotated[CallToolResult, PointerPositionResult]: + """Where the pointer is, in a frame's pixels and in screen units, and whether it is inside that frame.""" + computer = self._computer + await computer.ensure_ready() + sx, sy = await computer.pointer() + screen = Point(x=sx, y=sy) + identity = self.caller_identity(ctx) + frame = computer.frames.get(frame_id, identity) if frame_id is not None else computer.frames.latest(identity) + if frame is None: + return send(PointerPositionResult(summary=f"The pointer is at screen ({sx}, {sy})", screen=screen)) + fx, fy = frame.from_screen(sx, sy) + inside = frame.contains(fx, fy) + where = f"({fx}, {fy}) in {frame.id}" if inside else f"outside {frame.id}, at ({fx}, {fy}) in its pixels" + return send(PointerPositionResult(summary=f"The pointer is {where}, screen ({sx}, {sy})", screen=screen, + frame_id=frame.id, x=fx, y=fy, inside_frame=inside)) + + @tool(annotations=_READ) + async def computer_pointer_list(self) -> Annotated[CallToolResult, PointerListResult]: + """The pointers you may use, and which one is your default.""" + pointer = PointerData(id="default", kind="foreground", label="The computer's pointer", default=True) + return send(PointerListResult(summary="1 pointer: default (foreground)", pointers=[pointer])) + + @tool(annotations=_INPUT) + async def computer_keyboard_down( + self, + ctx: ToolContext, + keys: Annotated[list[str], Field(description="W3C key names, pressed in the order given")], + hold_ms: Annotated[Optional[int], Field( + description="Hold this long, then release in reverse order; without it they stay down until " + "computer_keyboard_up")] = None, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Press keys and keep them down.""" + computer = self._computer + await computer.ensure_ready() + names = chord(keys, self.keys) + if hold_ms is not None and not 0 <= hold_ms <= self.settings.max_hold_ms: + raise InvalidArgument(f"hold_ms is from 0 to {self.settings.max_hold_ms}, got {hold_ms}") + identity = self.caller_identity(ctx) + frame = computer.frames.latest(identity) + if hold_ms is not None: + return await computer.act(ctx, "key_down", (names,), frame=frame, screenshot=screenshot, + hold=(hold_ms, names), summary=f"Held {'+'.join(names)} for {hold_ms} ms") + return await computer.act(ctx, "key_down", (names,), frame=frame, screenshot=screenshot, + summary=f"Pressed and held {'+'.join(names)}", + then=lambda: computer.hold(identity, names)) + + @tool(annotations=_INPUT) + async def computer_keyboard_up( + self, + ctx: ToolContext, + keys: Annotated[Optional[list[str]], Field( + description="The keys to release, in order; without it, every key you are holding")] = None, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Release held keys.""" + computer = self._computer + await computer.ensure_ready() + identity = self.caller_identity(ctx) + names = chord(keys, self.keys) if keys is not None else list(computer.held.get(identity, [])) + if not names: + raise InvalidArgument("you are holding no keys") + return await computer.act(ctx, "key_up", (names,), frame=computer.frames.latest(identity), + screenshot=screenshot, summary=f"Released {'+'.join(names)}", + then=lambda: computer.release(identity, names)) + + +async def _button_action(driver: Tools, ctx: ToolContext, op: str, verb: str, button: str | None, x: float | None, + y: float | None, frame_id: str | None, screenshot: bool, space: str | None, + norm_scale: int | None) -> CallToolResult: + computer = driver._computer + await computer.ensure_ready() + button = _button(driver, button) + if (x is None) != (y is None): + raise InvalidArgument("give both x and y, or neither") + if x is None: + return await computer.act(ctx, op, (button,), frame=computer.frames.latest(driver.caller_identity(ctx)), + screenshot=screenshot, summary=f"{verb} {button}") + frame, screen, where, mapped = await _point(driver, ctx, x, y, frame_id, space, norm_scale) + return await computer.act(ctx, op, (button,), frame=frame, mapped=mapped, screenshot=screenshot, + move_to=screen, summary=f"{verb} {button}{where}") + + +async def _point(driver: Tools, ctx: ToolContext, x: float | None, y: float | None, frame_id: str | None, + space: str | None, norm_scale: int | None) -> tuple[Frame | None, tuple[int, ...], str, bool]: + """The frame, the screen pixel, a phrase for the point and whether it was mapped from the frame. With no point + and the pointer group, no pixel and no phrase: the call acts at the pointer, read once it has its input turn.""" + computer = driver._computer + identity = driver.caller_identity(ctx) + if x is None and y is None and driver.has("pointer") and frame_id is None: + return computer.frames.latest(identity), (), "", False + if x is None or y is None: + raise InvalidArgument("give x and y" + (", or neither to act at the pointer" if driver.has("pointer") else "")) + frame = computer.frames.get(frame_id, identity) + computer.frames.check_current(frame, await computer.display_now()) + space, scale = _space(driver, space, norm_scale) + fx, fy = frame_point(frame, x, y, space, scale) + in_space = f" ({space} {scale})" if space == "norm" else "" + return frame, frame.to_screen(fx, fy), f" at ({x:g}, {y:g}){in_space} in {frame.id}", True + + +def _space(driver: Tools, space: str | None, norm_scale: int | None) -> tuple[str, int]: + return check_space(space, norm_scale, driver.settings.default_space, driver.settings.default_norm_scale) + + +def _button(driver: Tools, button: str | None) -> str: + button = button or "left" + if button not in driver.buttons: + raise Unsupported(f"this computer has no {button} button; it has {', '.join(driver.buttons)}") + return button + + +def _modifiers(driver: Tools, modifiers: list | None) -> list[str]: + if not modifiers: + return [] + if not isinstance(modifiers, list): + raise InvalidArgument("modifiers is a list of W3C key names") + return chord(modifiers, driver.keys) + + +def _check_range(name: str, value: int, low: int, high: int) -> None: + if not low <= value <= high: + raise InvalidArgument(f"{name} is from {low} to {high} on this computer, got {value}") + + +def _display(frame: Frame) -> str: + d = frame.display + return f"{d.width}×{d.height}" + (f" at scale {d.scale:g}" if d.scale != 1 else "") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/errors.py new file mode 100644 index 00000000..9ba47e4d --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/errors.py @@ -0,0 +1,29 @@ +"""Errors a tool or extension raises to answer its caller with a code, not a crash.""" +from __future__ import annotations + +from typing import Literal + +#: Whether the call's effect happened: ``unknown`` when it may have been delivered before something failed. +Applied = Literal["yes", "no", "unknown"] + + +class AgentEnvError(Exception): + """Raised in an extension, answers with ``status`` and these fields as the error body; raised in a tool, + answers ``isError`` with ``": "`` as text and the fields as structured content. Callers retry + on their own only when ``retryable`` and ``applied == "no"``.""" + + def __init__(self, code: str, message: str, *, retryable: bool = False, applied: Applied = "no", + hint: str | None = None, status: int = 400) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retryable = retryable + self.applied = applied + self.hint = hint + self.status = status + + def fields(self) -> dict: + fields = {"code": self.code, "message": self.message, "retryable": self.retryable, "applied": self.applied} + if self.hint is not None: + fields["hint"] = self.hint + return fields diff --git a/packages/agentenv-protocol/src/agentenv_protocol/tool_access.py b/packages/agentenv-protocol/src/agentenv_protocol/tool_access.py new file mode 100644 index 00000000..5391465d --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/tool_access.py @@ -0,0 +1,98 @@ +"""Per-role tool visibility for an environment agents reach with no gateway in front, under the gateway's rules and +at its paths, so ``modify_env_tool_access`` and an env's ``[tools]`` behave the same with or without one.""" +from __future__ import annotations + +from typing import Iterable + +from .errors import AgentEnvError +from .tool_context import DEFAULT_ROLE, ToolContext +from .types import EnvironmentExtension + +WILDCARD = "*" +DISABLE_TOOL_EXTENSION_URI = "urn:agentenv:disable-tool/v1" +ENABLE_TOOL_EXTENSION_URI = "urn:agentenv:enable-tool/v1" +DISABLE = "disable" +ENABLE = "enable" +PATHS = {DISABLE: "/tools/disable", ENABLE: "/tools/enable"} + +_REQUEST = { + "type": "object", + "properties": { + "role": {"type": "string"}, + "tools": {"anyOf": [{"type": "array", "items": {"type": "string"}}, {"const": WILDCARD}]}, + }, + "required": ["role", "tools"], +} + + +class ToolAccess: + """Rules per role, each a tool name or ``*`` mapped to disabled or not. A call's role is looked up first, then + ``*``; the first rule naming the tool decides, then a ``*`` rule, except that a tool hidden by default skips + ``*`` rules and stays hidden until a rule names it.""" + + def __init__(self, hidden_by_default: Iterable[str] = ()) -> None: + self.rules: dict[str, dict[str, bool]] = {} + self.hidden_by_default = set(hidden_by_default) + + def is_disabled(self, role: str, name: str) -> bool: + hidden = name in self.hidden_by_default + for r in (role, WILDCARD): + rules = self.rules.get(r, {}) + if name in rules: + return rules[name] + if WILDCARD in rules and not hidden: + return rules[WILDCARD] + return hidden + + def apply(self, role: str, tool: str, disabled: bool) -> None: + if role == WILDCARD and tool == WILDCARD: + self.rules.clear() + self.rules[WILDCARD] = {WILDCARD: disabled} + elif role == WILDCARD: + for rules in self.rules.values(): + rules[tool] = disabled + self.rules.setdefault(WILDCARD, {})[tool] = disabled + elif tool == WILDCARD: + self.rules[role] = {WILDCARD: disabled} + else: + self.rules.setdefault(role, {})[tool] = disabled + + def change(self, action: str, body: object) -> dict: + """A ``disable-tool/v1`` or ``enable-tool/v1`` request applied: ``{role, tools: [...] | "*"}`` to + ``{role, disabled, allowed}``, the role's own rules after the change.""" + role = body.get("role") if isinstance(body, dict) else None + if not isinstance(role, str) or not role: + raise AgentEnvError("invalid_params", "missing or invalid 'role' field") + tools = body.get("tools") + if tools == WILDCARD: + names = [WILDCARD] + elif isinstance(tools, list) and all(isinstance(t, str) for t in tools): + names = tools + else: + raise AgentEnvError("invalid_params", "'tools' must be a list of strings or the string '*'") + for name in names: + self.apply(role, name, action == DISABLE) + rules = self.rules.get(role, {}) + return {"role": role, "disabled": sorted(k for k, v in rules.items() if v), + "allowed": sorted(k for k, v in rules.items() if not v)} + + def check_call(self, context: ToolContext) -> None: + role = caller_role(context) + if self.is_disabled(role, context.tool): + raise AgentEnvError("permission_denied", f"Tool '{context.tool}' is disabled for role '{role}'", status=403) + + +def caller_role(context: ToolContext) -> str: + """The role a rule is keyed by: the caller's, or the gateway's unforwarded ``default``.""" + return context.caller.role or DEFAULT_ROLE + + +def extensions() -> list[EnvironmentExtension]: + return [ + EnvironmentExtension(uri=uri, description=description, required=False, params={ + "endpoint": PATHS[action], "methods": {action: {"method": "POST", "request": _REQUEST}}}) + for uri, action, description in ( + (DISABLE_TOOL_EXTENSION_URI, DISABLE, "Disable tools for a role."), + (ENABLE_TOOL_EXTENSION_URI, ENABLE, "Enable tools for a role."), + ) + ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/types.py b/packages/agentenv-protocol/src/agentenv_protocol/types.py index 1f95e52c..56a860ea 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/types.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/types.py @@ -1,15 +1,15 @@ """Wire types for the v1 data-plane protocol.""" from __future__ import annotations -from typing import Annotated, Literal, Optional, Union +from typing import Annotated, Any, Literal, Optional, Union from pydantic import BaseModel, Field, TypeAdapter, ValidationError from .transfers import RelativePath -def error_body(code: str, message: str) -> dict: - return {"ok": False, "error": {"code": code, "message": message}} +def error_body(code: str, message: str, **fields: Any) -> dict: + return {"ok": False, "error": {"code": code, "message": message, **fields}} PROTOCOL_VERSION = "1.0" diff --git a/packages/agentenv-protocol/tests/computer_fake.py b/packages/agentenv-protocol/tests/computer_fake.py new file mode 100644 index 00000000..6133104c --- /dev/null +++ b/packages/agentenv-protocol/tests/computer_fake.py @@ -0,0 +1,218 @@ +"""An in-memory screen and a driver on it that implements every operation, so the tests exercise the whole base +class with no display. A click paints its button's colour at its point; everything that reaches the screen is +logged; ``fail`` makes the next call of an operation raise.""" +from __future__ import annotations + +import asyncio +import contextlib + +import httpx +from mcp.shared.memory import create_connected_server_and_client_session +from PIL import Image as PILImage + +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + ComputerSettings, + Display, + Effect, + Platform, + capture, + click, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + scroll, + set_display, + type_text, +) + +LINUX = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") +BACKGROUND = (255, 255, 255) +COLOURS = {"left": (255, 0, 0), "right": (0, 0, 255), "middle": (0, 255, 0)} + + +class FakeScreen: + def __init__(self, width: int = 1280, height: int = 800, scale: float = 1.0) -> None: + self.log: list = [] + self.pointer = (0, 0) + self.held: list = [] + self.scrolled = [0, 0] + self.resize(width, height, scale) + + def resize(self, width: int, height: int, scale: float) -> None: + self.display = Display(width, height, scale) + self.image = PILImage.new("RGB", (round(width * scale), round(height * scale)), BACKGROUND) + + def paint(self, x: int, y: int, colour: tuple) -> None: + """Colour the image pixels under screen pixel (x, y).""" + s = self.display.scale + for px in range(int(x * s), max(int((x + 1) * s), int(x * s) + 1)): + for py in range(int(y * s), max(int((y + 1) * s), int(y * s) + 1)): + self.image.putpixel((px, py), colour) + + def colour_at(self, x: int, y: int) -> tuple: + s = self.display.scale + return self.image.getpixel((int(x * s), int(y * s))) + + +class FakeDriver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle") + max_click_count = 3 + max_drag_points = 10 + display_modes = ((1280, 800), (1920, 1080), (640, 400)) + + def __init__(self, screen: FakeScreen | None = None, settings: ComputerSettings | None = None, + platform: Platform = LINUX) -> None: + super().__init__(settings=settings or ComputerSettings(), platform=platform) + self.screen = screen or FakeScreen() + self.fail: dict = {} + self.slow: dict = {} + + async def _step(self, op: str) -> None: + if op in self.slow: + await asyncio.sleep(self.slow[op]) + error = self.fail.pop(op, None) + if error is not None: + raise error + + @display + async def screen_size(self) -> Display: + await self._step("display") + return self.screen.display + + @set_display + async def resize(self, width: int, height: int, scale: float) -> Display: + await self._step("set_display") + self.screen.resize(width, height, scale) + return self.screen.display + + @capture + async def grab(self): + await self._step("capture") + return self.screen.image.copy() + + @click + async def click_at(self, x, y, button, count, modifiers): + await self._step("click") + self.screen.log.append(("click", x, y, button, count, tuple(modifiers))) + self.screen.pointer = (x, y) + self.screen.paint(x, y, COLOURS[button]) + return Effect(pointer_moved=True) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers): + await self._step("drag") + self.screen.log.append(("drag", tuple(map(tuple, path)), button, duration_ms, tuple(modifiers))) + self.screen.pointer = tuple(path[-1]) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): + await self._step("scroll") + self.screen.log.append(("scroll", x, y, direction, amount, tuple(modifiers))) + dx, dy = {"up": (0, -1), "down": (0, 1), "left": (-1, 0), "right": (1, 0)}[direction] + self.screen.scrolled = [self.screen.scrolled[0] + dx * amount, self.screen.scrolled[1] + dy * amount] + + @type_text + async def type_in(self, text): + await self._step("type_text") + self.screen.log.append(("type", text)) + + @press + async def press_keys(self, keys): + await self._step("press") + self.screen.log.append(("press", tuple(keys), self.settings.tap_hold_ms)) + + @pointer_move + async def move(self, x, y): + await self._step("pointer_move") + self.screen.log.append(("move", x, y)) + self.screen.pointer = (x, y) + + @pointer_down + async def down(self, button): + await self._step("pointer_down") + self.screen.log.append(("down", button)) + + @pointer_up + async def up(self, button): + await self._step("pointer_up") + self.screen.log.append(("up", button)) + + @pointer_position + async def where(self): + await self._step("pointer_position") + return self.screen.pointer + + @key_down + async def keys_down(self, keys): + await self._step("key_down") + self.screen.log.append(("key_down", tuple(keys))) + self.screen.held += [k for k in keys if k not in self.screen.held] + + @key_up + async def keys_up(self, keys): + await self._step("key_up") + self.screen.log.append(("key_up", tuple(keys))) + self.screen.held = [k for k in self.screen.held if k not in keys] + + +class MinimalDriver(ComputerDriverEnvironment): + """Only the required operations, synchronous ones: no pointer or keyboard group, no display modes.""" + + def __init__(self, screen: FakeScreen | None = None, platform: Platform = LINUX) -> None: + super().__init__(settings=ComputerSettings(), platform=platform) + self.screen = screen or FakeScreen(640, 400) + + @display + def screen_size(self) -> Display: + return self.screen.display + + @capture + def grab(self): + return self.screen.image.copy() + + @click + def click_at(self, x, y, button, count, modifiers): + self.screen.log.append(("click", x, y, button, count, tuple(modifiers))) + self.screen.paint(x, y, COLOURS[button]) + + @drag + def drag_along(self, path, button, duration_ms, modifiers): + self.screen.log.append(("drag", tuple(map(tuple, path)))) + + @scroll + def scroll_at(self, x, y, direction, amount, modifiers): + self.screen.log.append(("scroll", x, y, direction, amount)) + + @type_text + def type_in(self, text): + self.screen.log.append(("type", text)) + + @press + def press_keys(self, keys): + self.screen.log.append(("press", tuple(keys))) + + +@contextlib.asynccontextmanager +async def mcp_session(driver: ComputerDriverEnvironment): + """An MCP client session on the driver's app, in memory.""" + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +@contextlib.asynccontextmanager +async def http_client(driver: ComputerDriverEnvironment): + """An HTTP client on the driver's app, for its card, extensions and data plane.""" + app = (driver.mcp or driver.create_app()).streamable_http_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + yield client + + +def rpc(method: str, params: dict | None = None) -> dict: + return {"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}} diff --git a/packages/agentenv-protocol/tests/test_agent_env_environment.py b/packages/agentenv-protocol/tests/test_agent_env_environment.py index 5c223dca..5d145b31 100644 --- a/packages/agentenv-protocol/tests/test_agent_env_environment.py +++ b/packages/agentenv-protocol/tests/test_agent_env_environment.py @@ -300,6 +300,28 @@ async def _reset(self) -> None: assert card["capabilities"]["operations"] == ["data/reset"] +@pytest.mark.asyncio +async def test_served_operations_chooses_the_data_plane_methods_served(): + class _ReadOnlyItems(_ItemsHandler): + def served_operations(self, operations: list[str]) -> list[str]: + return [op for op in operations if op != "data/add"] + + async with _client(_ReadOnlyItems) as client: + added = await client.post("/agentenv", json=_rpc("data/add", {"parts": [{"kind": "data", "data": {"items": ["a"]}}]})) + card = (await client.get("/.well-known/agent-env.json")).json() + assert added.json()["error"]["code"] == -32601 + assert card["capabilities"]["operations"] == ["data/reset", "data/get"] + + +def test_served_operations_naming_an_unmarked_method_is_refused(): + class _Bad(_ItemsHandler): + def served_operations(self, operations: list[str]) -> list[str]: + return [*operations, "data/snapshot"] + + with pytest.raises(ValueError, match=r"no handler: \['data/snapshot'\]"): + AgentEnvFastMCPApplication(environment_card=EnvironmentCard(name="x"), handler=_Bad()) + + @pytest.mark.asyncio async def test_tools_only_env_serves_card_and_tools_without_data_plane(): class _ToolsOnly: diff --git a/packages/agentenv-protocol/tests/test_client.py b/packages/agentenv-protocol/tests/test_client.py index 844119a6..13088bf0 100644 --- a/packages/agentenv-protocol/tests/test_client.py +++ b/packages/agentenv-protocol/tests/test_client.py @@ -8,7 +8,7 @@ import httpx import pytest -from agentenv_protocol import DataPart, client as protocol_v1 +from agentenv_protocol import DataPart, ExtensionCallError, client as protocol_v1 _BASE = "http://gw/svc/mcp-items" @@ -223,6 +223,31 @@ async def test_invoke_extension_get_uses_query_params(): assert client.get.call_args.args[0] == f"{_BASE}/agentenv/ext/ping" +def _error_response(status: int, **kwargs) -> httpx.Response: + return httpx.Response(status, request=httpx.Request("POST", f"{_BASE}/agentenv/ext/set_errors"), **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response, fields", [ + (_error_response(429, json={"ok": False, "error": { + "code": "limit_exceeded", "message": "3 leases held", "retryable": True, "applied": "no", "hint": "release one"}}), + ("limit_exceeded", "3 leases held", True, "no", "release one")), + # A server from before the structured fields: its code, and nothing claimed about the effect. + (_error_response(500, json={"ok": False, "error": {"code": "extension_failed", "message": "kaboom"}}), + ("extension_failed", "kaboom", False, "unknown", None)), + # A proxy's own error page. + (_error_response(502, text="Bad Gateway"), (None, "Bad Gateway", False, "unknown", None)), + (_error_response(404, json=["not", "an", "object"]), (None, "Not Found", False, "unknown", None)), +]) +async def test_invoke_extension_raises_the_error_fields_it_was_sent(response, fields): + with _patch(_client(response)), pytest.raises(ExtensionCallError) as raised: + await protocol_v1.invoke_extension(_BASE, _CARD_SET_ERRORS, "urn:agentenv:set-errors/v1", {}) + error = raised.value + assert (error.code, error.detail, error.retryable, error.applied, error.hint) == fields + assert isinstance(error, httpx.HTTPStatusError) and error.response is response + assert str(error).startswith(f"{response.status_code} {fields[0] or 'error'}: ") + + @pytest.mark.asyncio async def test_invoke_extension_unknown_uri_raises(): with pytest.raises(ValueError, match="not advertised"): diff --git a/packages/agentenv-protocol/tests/test_computer_dataplane.py b/packages/agentenv-protocol/tests/test_computer_dataplane.py new file mode 100644 index 00000000..212d9912 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_dataplane.py @@ -0,0 +1,153 @@ +"""The data plane on a computer: data/add writes file parts atomically under the session user's home, from inline +bytes, an https URL or a read grant, with paths checked; data/get returns the data paths, inline or uploaded.""" +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, http_client, rpc # noqa: E402 + +from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI # noqa: E402 +from agentenv_protocol.computer import ComputerSettings # noqa: E402 + + +@pytest.fixture +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +def _part(path: str, data: bytes = b"hello", **marker) -> dict: + return {"kind": "file", "file": {"bytes": base64.b64encode(data).decode()}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": path, **marker}}} + + +async def _add(driver: FakeDriver, *parts: dict) -> dict: + async with http_client(driver) as http: + return (await http.post("/agentenv", json=rpc("data/add", {"parts": list(parts)}))).json() + + +@pytest.mark.asyncio +async def test_file_parts_land_under_the_home_atomically(home): + (home / "notes").mkdir() + (home / "notes" / "a.txt").write_text("old") + answer = await _add(FakeDriver(), _part("notes/a.txt", b"new"), _part("deep/er/b.bin", b"\x00\x01")) + assert answer["result"] == {} + assert (home / "notes" / "a.txt").read_text() == "new" and (home / "deep" / "er" / "b.bin").read_bytes() == b"\x00\x01" + assert sorted(p.name for p in (home / "notes").iterdir()) == ["a.txt"], "no temporary file is left behind" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("part, error", [ + (_part("../outside"), "is not a normalized path relative to the home"), + (_part("/etc/passwd"), "is not a normalized path relative to the home"), + ({"kind": "text", "text": "hi"}, "takes file parts naming their path"), + ({"kind": "file", "file": {"bytes": "aGk="}}, "takes file parts naming their path"), + ({"kind": "file", "file": {"bytes": "not base64!"}, "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "a"}}}, + "the part's bytes are not base64"), + ({"kind": "file", "file": {"uri": "http://plain/x"}, "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "a"}}}, + "a file part's uri is an https URL"), + (_part("a", read={"not": "a grant"}), "the part's read grant is not a valid ReadObject"), +]) +async def test_a_bad_part_is_refused_and_nothing_is_written(home, part, error): + answer = await _add(FakeDriver(), part) + assert answer["error"]["data"]["code"] == "add_failed" and error in answer["error"]["message"] + assert list(home.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_a_symlink_out_of_the_home_is_refused(home, tmp_path_factory): + outside = tmp_path_factory.mktemp("outside") + (home / "link").symlink_to(outside) + answer = await _add(FakeDriver(), _part("link/escaped.txt")) + assert "'link/escaped.txt' is outside the home" in answer["error"]["message"] + assert list(outside.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_an_https_url_part_is_fetched_by_the_driver(home): + transport = httpx.MockTransport(lambda request: httpx.Response(200, content=b"from the web")) + real = httpx.AsyncClient + with patch("agentenv_protocol.computer.dataplane.httpx.AsyncClient", + lambda **kwargs: real(**{"transport": transport, **kwargs})): + answer = await _add(FakeDriver(), {"kind": "file", "file": {"uri": "https://example.test/f"}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "web.txt"}}}) + assert answer["result"] == {} and (home / "web.txt").read_bytes() == b"from the web" + + +@pytest.mark.asyncio +async def test_an_https_url_over_the_limit_is_refused(home): + transport = httpx.MockTransport(lambda request: httpx.Response(200, content=b"x" * 64)) + real = httpx.AsyncClient + with patch("agentenv_protocol.computer.dataplane.httpx.AsyncClient", + lambda **kwargs: real(**{"transport": transport, **kwargs})), \ + patch("agentenv_protocol.computer.dataplane.MAX_URL_BYTES", 10): + answer = await _add(FakeDriver(), {"kind": "file", "file": {"uri": "https://example.test/f"}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "big"}}}) + assert "is over 10 bytes" in answer["error"]["message"] and not (home / "big").exists() + + +@pytest.mark.asyncio +async def test_a_read_grant_part_is_downloaded_through_the_transfers_helper(home): + grant = {"media_type": "text/plain", "max_bytes": 100, + "read": {"kind": "http-get", "url": "https://store.test/o", + "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat()}} + + async def fake_download(source, destination): + assert source.read.url == "https://store.test/o" + destination.write_bytes(b"granted") + + with patch("agentenv_protocol.computer.dataplane.download", fake_download): + answer = await _add(FakeDriver(), _part("granted.txt", b"ignored", read=grant)) + assert answer["result"] == {} and (home / "granted.txt").read_bytes() == b"granted" + + +@pytest.mark.asyncio +async def test_data_get_returns_the_data_paths_inline(home, tmp_path_factory): + (home / "app" / "db").mkdir(parents=True) + (home / "app" / "db" / "state.json").write_text('{"n": 1}') + (home / "app" / "db" / "blob.bin").write_bytes(b"\x01") + (home / "notes.md").write_text("# notes") + outside = tmp_path_factory.mktemp("outside") + (outside / "secret").write_text("no") + (home / "app" / "db" / "escape").symlink_to(outside / "secret") + driver = FakeDriver(settings=ComputerSettings(data_paths=["app", "notes.md", "missing", "app/db"])) + async with http_client(driver) as http: + parts = (await http.post("/agentenv", json=rpc("data/get"))).json()["result"]["parts"] + by_path = {p["metadata"][DATA_OBJECTS_EXTENSION_URI]["path"]: p["file"] for p in parts} + assert sorted(by_path) == ["app/db/blob.bin", "app/db/state.json", "notes.md"] + assert base64.b64decode(by_path["app/db/state.json"]["bytes"]) == b'{"n": 1}' + assert by_path["app/db/state.json"]["mimeType"] == "application/json" and by_path["notes.md"]["name"] == "notes.md" + assert by_path["app/db/blob.bin"]["mimeType"] == "application/octet-stream" + + +@pytest.mark.asyncio +async def test_data_get_uploads_under_a_write_namespace_grant(home): + (home / "notes.md").write_text("# notes") + grant = {"root_path": "runs/1", "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + "max_objects": 5, "max_object_bytes": 100, "max_total_bytes": 500, + "write": {"kind": "http-post-policy", "url": "https://store.test/", "fields": {}, + "path_field": "key", "file_field": "file"}} + upload = AsyncMock() + with patch("agentenv_protocol.computer.dataplane.NamespaceUploader") as uploader: + uploader.return_value.upload = upload + driver = FakeDriver(settings=ComputerSettings(data_paths=["notes.md"])) + async with http_client(driver) as http: + parts = (await http.post("/agentenv", json=rpc("data/get", {"write_namespace": grant}))).json()["result"]["parts"] + assert upload.await_args.args[0] == "notes.md" and upload.await_args.args[1] == home / "notes.md" + assert parts == [{"kind": "file", "file": {"uri": "notes.md", "mimeType": "text/markdown", "name": "notes.md"}, + "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "notes.md"}}}] + + +@pytest.mark.asyncio +async def test_a_failed_write_leaves_no_temporary_file(home): + with patch("agentenv_protocol.computer.linux.os.replace", side_effect=OSError("disk full")): + answer = await _add(FakeDriver(), _part("notes/a.txt")) + assert "disk full" in answer["error"]["message"] and list((home / "notes").iterdir()) == [] diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py new file mode 100644 index 00000000..e029f62b --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -0,0 +1,386 @@ +"""ComputerDriverEnvironment as a class: the checks at class definition, operations chosen per platform, settings, +the generated card, readiness and how driver failures are reported.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +import asyncio # noqa: E402 +import base64 # noqa: E402 +import io # noqa: E402 + +from computer_fake import LINUX, FakeDriver, FakeScreen, MinimalDriver, http_client, mcp_session # noqa: E402 +from mcp.server.fastmcp import FastMCP # noqa: E402 +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol.computer import ( # noqa: E402 + SETTINGS_ENV, + ComputerDriverEnvironment, + ComputerSettings, + Display, + Image, + InvalidArgument, + OutOfBounds, + Platform, + ReadyCheck, + capture, + click, + computer_driver, + display, + drag, + pointer_move, + press, + run_in_session, + scroll, + set_display, + type_text, +) +from agentenv_protocol.computer.keys import canonical # noqa: E402 +from agentenv_protocol.computer.operations import declared, resolve # noqa: E402 +from agentenv_protocol.computer.platform import detect, normalize # noqa: E402 +from agentenv_protocol.computer.settings import load_settings # noqa: E402 +from agentenv_protocol.computer.tools import _modifiers # noqa: E402 + +MACOS = Platform(os="macos", arch="arm64") + + +def _required(namespace: dict) -> dict: + """The required operations as no-op methods, with ``namespace`` added on top.""" + ops = { + "screen": display(lambda self: Display(10, 10)), + "grab": capture(lambda self: PILImage.new("RGB", (10, 10))), + "click_at": click(lambda self, *a: None), + "drag_along": drag(lambda self, *a: None), + "scroll_at": scroll(lambda self, *a: None), + "type_in": type_text(lambda self, *a: None), + "press_keys": press(lambda self, *a: None), + } + return {**ops, **namespace} + + +def _define(namespace: dict, **features) -> type: + return type("Driver", (ComputerDriverEnvironment,), {**_required(namespace), **features}) + + +def test_a_driver_missing_required_operations_is_refused_when_defined(): + with pytest.raises(TypeError, match=r"missing required operation\(s\): @drag, @press"): + type("Driver", (ComputerDriverEnvironment,), + {k: v for k, v in _required({}).items() if k not in ("drag_along", "press_keys")}) + + +def test_half_a_group_is_refused(): + with pytest.raises(TypeError, match="the pointer group needs all of .*missing @pointer_down, @pointer_up, " + "@pointer_position"): + _define({"move": pointer_move(lambda self, x, y: None)}) + + +def test_two_methods_one_platform_could_both_match_are_refused(): + with pytest.raises(TypeError, match="@capture is marked on both"): + _define({"grab_x11": capture(display="x11")(lambda self: None)}) + + +def test_predicates_choose_one_method_per_platform(): + Driver = _define({ + "grab": capture(display="x11")(lambda self: "x11"), + "grab_wayland": capture(display="wayland")(lambda self: "wayland"), + }) + assert resolve(Driver, LINUX)["capture"] == "grab" + assert resolve(Driver, Platform(os="linux", display="wayland"))["capture"] == "grab_wayland" + with pytest.raises(RuntimeError, match=r"no method for @capture on this platform \(\{'os': 'macos'"): + Driver(platform=MACOS) + + +def test_a_group_with_a_member_that_doesnt_match_the_platform_is_dropped(): + assert FakeDriver(platform=LINUX).has("pointer") + Driver = type("Driver", (FakeDriver,), {"move": pointer_move(os="windows")(lambda self, x, y: None)}) + driver = Driver() + assert not driver.has("pointer") and "pointer_down" not in driver._computer.operations + + +def test_unknown_predicates_are_refused(): + with pytest.raises(TypeError, match=r"unknown predicate\(s\) \['kernel'\]"): + capture(kernel="6.8") + assert normalize({"arch": ["amd64", "arm64"], "os": "linux"}) == {"arch": frozenset({"amd64", "arm64"}), + "os": frozenset({"linux"})} + + +@pytest.mark.parametrize("features, error", [ + ({"buttons": ("right",)}, "buttons include left"), + ({"buttons": ("left", "thumb")}, "buttons include left"), + ({"max_click_count": 0}, "max_click_count is an integer of at least 1"), + ({"max_wait_ms": 1000}, "max_wait_ms is an integer of at least 60000"), + ({"keys": ("enter",)}, "keys are W3C key values in their W3C case"), + ({"keys": ("Return",)}, "keys: 'Return' is not a W3C key value"), + ({"display_modes": ((10, 10),)}, "display_modes and @set_display come together"), +]) +def test_features_are_checked_when_the_class_is_defined(features, error): + with pytest.raises(TypeError, match=error): + _define({}, **features) + + +def test_set_display_without_modes_is_refused(): + with pytest.raises(TypeError, match="display_modes and @set_display come together"): + _define({"resize": set_display(lambda self, w, h, s: None)}) + + +def test_a_driver_method_replaces_the_linux_default(): + Driver = _define({"run": run_in_session(lambda self, *a: None)}) + assert declared(Driver)["run_in_session"] == [("run", {})] + assert declared(MinimalDriver)["run_in_session"] == [("_computer_linux_run_in_session", {"os": frozenset({"linux"})})] + + +def test_the_platform_is_detected(tmp_path, monkeypatch): + release = tmp_path / "os-release" + release.write_text('NAME="Ubuntu"\nID=ubuntu\nVERSION_ID="24.04"\n# comment\nbare\n') + monkeypatch.setattr("agentenv_protocol.computer.platform.sys.platform", "linux") + monkeypatch.setattr("agentenv_protocol.computer.platform._platform.machine", lambda: "aarch64") + assert detect({"DISPLAY": ":0"}, release) == Platform("linux", "ubuntu", "24.04", "arm64", "x11") + assert detect({"XDG_SESSION_TYPE": "Wayland", "DISPLAY": ":0"}, release).display == "wayland" + assert detect({"WAYLAND_DISPLAY": "wayland-0"}, release).display == "wayland" + assert detect({}, tmp_path / "missing").as_dict() == {"os": "linux", "arch": "arm64"} + monkeypatch.setattr("agentenv_protocol.computer.platform.sys.platform", "darwin") + assert detect({"DISPLAY": ":0"}, release) == Platform("macos", None, None, "arm64", None) + + +def test_settings_come_from_one_json_value(): + assert load_settings({}) == ComputerSettings() + settings = load_settings({SETTINGS_ENV: '{"data_paths": ["notes"], "default_space": "norm", "tap_hold_ms": 20}'}) + assert settings.data_paths == ["notes"] and settings.default_space == "norm" and settings.tap_hold_ms == 20 + for raw in ('{"data_paths": ["../out"]}', '{"tap_hold_ms": -1}', '{"unknown": 1}', "not json"): + with pytest.raises(ValueError, match=SETTINGS_ENV): + load_settings({SETTINGS_ENV: raw}) + + +def test_a_driver_reads_its_settings_and_platform_from_the_environment(monkeypatch): + monkeypatch.setenv(SETTINGS_ENV, '{"queue_wait_s": 2}') + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) + driver = _define({})() + assert driver.settings.queue_wait_s == 2 and driver.platform == LINUX + + +@pytest.mark.asyncio +async def test_the_card_declares_the_contract_the_driver_implements(): + async with http_client(FakeDriver()) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + computer = extensions["urn:agentenv:computer/v1"]["params"] + assert computer["buttons"] == ["left", "right", "middle"] and computer["max_click_count"] == 3 + assert computer["keys"] == "all" and computer["platform"] == LINUX.as_dict() and computer["delivery"] == "foreground" + assert extensions["urn:agentenv:computer-pointer/v1"]["params"]["buttons"] == ["left", "right", "middle"] + assert set(extensions["urn:agentenv:computer-keyboard/v1"]["params"]["methods"]) == {"get", "set"} + assert set(extensions["urn:agentenv:computer-display/v1"]["params"]["methods"]) == {"get", "set"} + assert extensions["urn:agentenv:computer-exec/v1"]["params"]["max_timeout_s"] == 3600 + for uri in ("urn:agentenv:computer-ready/v1", "urn:agentenv:computer-spaces/v1", "urn:agentenv:intake/v1", + "urn:agentenv:data-objects/v1", "urn:agentenv:disable-tool/v1", "urn:agentenv:enable-tool/v1"): + assert uri in extensions + assert card["capabilities"]["operations"] == ["data/add", "data/get"] + assert {t["name"] for t in card["capabilities"]["tools"]} >= {"computer_click", "computer_pointer_list"} + assert card["additionalInterfaces"] == [{"url": "/mcp", "transport": "mcp"}] + + +@pytest.mark.asyncio +async def test_a_minimal_driver_declares_only_what_it_has(): + async with http_client(MinimalDriver(platform=MACOS)) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + display_set = await http.post("/agentenv/ext/computer-display/set", json={"width": 1, "height": 1}) + exec_run = await http.post("/agentenv/ext/computer-exec/run", json={"argv": ["true"], "request_id": "r"}) + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + assert "urn:agentenv:computer-pointer/v1" not in extensions and "urn:agentenv:computer-keyboard/v1" not in extensions + assert "urn:agentenv:computer-exec/v1" not in extensions + assert set(extensions["urn:agentenv:computer-display/v1"]["params"]["methods"]) == {"get"} + assert extensions["urn:agentenv:intake/v1"]["params"] == {"get": [{"part": "file", "format": "file-tree"}]} + assert card["capabilities"]["operations"] == ["data/get"] + assert {t["name"] for t in card["capabilities"]["tools"]} == { + "computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", "computer_key", + "computer_wait"} + assert display_set.status_code == 404 and exec_run.status_code == 404 + + +def test_computer_driver_names_the_driver_and_declares_its_platforms(): + @computer_driver(name="reference", supports=[{"os": {"type": "linux"}, "arch": ["amd64"]}]) + class Reference(MinimalDriver): + pass + + driver = Reference() + card = driver._build_card() + assert card.name == "reference" and Reference.supports[0]["arch"] == ["amd64"] + assert computer_driver()(Reference) is Reference + with pytest.raises(TypeError, match="decorates a ComputerDriverEnvironment subclass"): + computer_driver(name="x")(object) + + +def test_a_driver_that_skips_super_init_is_told_so(): + class Forgetful(MinimalDriver): + def __init__(self) -> None: + pass + + with pytest.raises(TypeError, match=r"Forgetful.__init__ must call super\(\).__init__\(\)"): + Forgetful().create_app() + + +@pytest.mark.asyncio +async def test_tools_refuse_until_the_ready_checks_pass_once(): + driver = FakeDriver() + driver.fail["display"] = RuntimeError("no X server") + async with mcp_session(driver) as session: + refused = await session.call_tool("computer_screenshot", {}) + served = await session.call_tool("computer_screenshot", {}) + assert refused.isError and refused.structuredContent["code"] == "device_unavailable" + assert refused.structuredContent["retryable"] is True + assert "display: @display failed: RuntimeError: no X server" in refused.structuredContent["message"] + assert not served.isError and driver._computer.ready + + +@pytest.mark.asyncio +async def test_the_drivers_own_ready_checks_count(): + class Checked(FakeDriver): + healthy = False + + async def ready_checks(self): + return [ReadyCheck("session", self.healthy, "logged in" if self.healthy else "no session")] + + class Raising(FakeDriver): + async def ready_checks(self): + raise RuntimeError("probe crashed") + + driver = Checked() + async with http_client(driver) as http: + before = (await http.get("/agentenv/ext/computer-ready/status")).json() + driver.healthy = True + after = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert before["ready"] is False and before["checks"][-1] == {"name": "session", "ok": False, "detail": "no session"} + assert after["ready"] is True + async with http_client(Raising()) as http: + raised = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert raised["checks"][-1] == {"name": "driver", "ok": False, "detail": "RuntimeError: probe crashed"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("captured, error", [ + (lambda screen: "pixels", "@capture returned str, not an Image"), + (lambda screen: PILImage.new("RGB", (5, 5)), "@capture returned 5×5, but the display at scale 1 is 1280×800"), +]) +async def test_a_bad_capture_is_an_internal_error(captured, error): + class Bad(FakeDriver): + @capture + async def grab(self): + return captured(self.screen) + + async with http_client(Bad()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert status["ready"] is False and error in status["checks"][1]["detail"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result", [None, Display(0, 10), Display(10, 10, 0)]) +async def test_a_bad_display_is_an_internal_error(result): + class Bad(FakeDriver): + @display + async def screen_size(self): + return result + + async with http_client(Bad()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert status["checks"][0]["ok"] is False and "@display returned" in status["checks"][0]["detail"] + assert status["checks"][1] == {"name": "capture", "ok": False, "detail": "the display didn't answer"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["png", "jpeg", "RGB", "BGRA"]) +async def test_a_capture_may_be_encoded_or_raw_pixels(mode): + screen = FakeScreen(4, 2) + screen.image.putpixel((1, 0), (10, 20, 30)) + + class Raw(FakeDriver): + @capture + async def grab(self): + if mode in ("png", "jpeg"): + buffer = io.BytesIO() + self.screen.image.save(buffer, format=mode.upper()) + return Image(4, 2, buffer.getvalue(), mode) + if mode == "BGRA": + r, g, b = self.screen.image.split() + return Image(4, 2, PILImage.merge("RGBA", (b, g, r, r)).tobytes(), "BGRA") + return Image(4, 2, self.screen.image.tobytes(), "RGB") + + async with mcp_session(Raw(screen)) as session: + shot = await session.call_tool("computer_screenshot", {}) + assert not shot.isError and shot.structuredContent["width"] == 4 + decoded = PILImage.open(io.BytesIO(base64.b64decode(shot.content[1].data))).convert("RGB") + assert decoded.size == (4, 2) + if mode != "jpeg": + assert decoded.getpixel((1, 0)) == (10, 20, 30) + + +@pytest.mark.asyncio +async def test_a_failing_input_operation_may_have_been_delivered_and_others_were_not(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + driver.fail["click"] = RuntimeError("XTEST gone") + crashed = await session.call_tool("computer_click", {"x": 1, "y": 1}) + driver.fail["click"] = OutOfBounds("the window moved") + refused = await session.call_tool("computer_click", {"x": 1, "y": 1}) + driver.fail["capture"] = RuntimeError("mss failed") + broken = await session.call_tool("computer_screenshot", {}) + assert crashed.structuredContent == {"code": "delivery_failed", "message": "@click failed: RuntimeError: XTEST gone", + "retryable": False, "applied": "unknown"} + assert refused.structuredContent["code"] == "out_of_bounds" and refused.structuredContent["applied"] == "no" + assert broken.structuredContent["code"] == "internal" and broken.structuredContent["applied"] == "unknown" + + +@pytest.mark.asyncio +async def test_a_failure_after_part_of_a_call_was_delivered_is_applied_unknown(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + driver.fail["pointer_down"] = OutOfBounds("the window moved") + moved_then_refused = await session.call_tool("computer_pointer_down", {"x": 3, "y": 4}) + assert moved_then_refused.structuredContent["code"] == "out_of_bounds" + assert moved_then_refused.structuredContent["applied"] == "unknown" + assert driver.screen.log[-1] == ("move", 3, 4) + + +@pytest.mark.asyncio +async def test_a_driver_mounted_on_an_app_serves_the_same_tools(): + driver = MinimalDriver() + app = driver.mount(FastMCP("mounted")) + async with create_connected_server_and_client_session(app) as session: + tools = {t.name: t for t in (await session.list_tools()).tools} + refused = await session.call_tool("computer_click", {}) + assert "required" not in tools["computer_click"].inputSchema and "computer_pointer_move" not in tools + assert "frame space by default" in tools["computer_scroll"].description + assert refused.structuredContent["code"] == "invalid_argument" + assert refused.structuredContent["message"] == "give x and y" + + +@pytest.mark.asyncio +async def test_concurrent_first_calls_run_the_ready_checks_once(): + driver = FakeDriver() + driver.slow["display"] = 0.05 + async with mcp_session(driver) as session: + results = await asyncio.gather(*(session.call_tool("computer_wait", {"ms": 0}) for _ in range(3))) + assert all(not r.isError for r in results) and driver._computer.ready + + +def test_keys_and_modifiers_that_arent_lists_of_names_are_refused(): + for key in (3, ""): + with pytest.raises(InvalidArgument, match="a key is a W3C key value"): + canonical(key) + with pytest.raises(InvalidArgument, match="modifiers is a list"): + _modifiers(FakeDriver(), "Shift") + + +@pytest.mark.parametrize("name", ["_computer", "_computer_ready_status", "_computer_new", "computer_click", "_application", "_mounted"]) +def test_a_driver_member_that_would_replace_the_base_classs_own_is_refused(name): + with pytest.raises(TypeError, match=rf"Driver redefines {name}, which ComputerDriverEnvironment uses itself"): + _define({name: lambda self: None}) + + +def test_a_drivers_own_private_helpers_and_the_hooks_are_its_own(): + Driver = _define({"_button": lambda self, d, b, down: None, "_move": lambda self: None, + "caller_identity": lambda self, context: "x", "buttons": ("left", "right")}) + assert Driver().caller_identity(None) == "x" diff --git a/packages/agentenv-protocol/tests/test_computer_extensions.py b/packages/agentenv-protocol/tests/test_computer_extensions.py new file mode 100644 index 00000000..a76b766f --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_extensions.py @@ -0,0 +1,196 @@ +"""The framework extensions: computer-display, computer-keyboard and computer-exec with the Linux default, which runs +commands in the session, caps their output, times them out and answers a repeated request_id with the first result.""" +from __future__ import annotations + +import asyncio +import sys +import time + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, MinimalDriver, http_client # noqa: E402 + +from agentenv_protocol.computer import ComputerSettings, ExecResult, linux, run_in_session, set_display # noqa: E402 + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the Linux defaults need a POSIX host") + + +def _run(**body) -> dict: + return {"argv": ["sh", "-c", "true"], "request_id": "r", **body} + + +@pytest.mark.asyncio +async def test_display_get_and_set_within_the_declared_modes(): + driver = FakeDriver() + async with http_client(driver) as http: + got = (await http.get("/agentenv/ext/computer-display/get")).json() + changed = await http.post("/agentenv/ext/computer-display/set", json={"width": 640, "height": 400, "scale": 2}) + refused = await http.post("/agentenv/ext/computer-display/set", json={"width": 800, "height": 600}) + assert got == {"width": 1280, "height": 800, "scale": 1.0, "modes": [[1280, 800], [1920, 1080], [640, 400]]} + assert changed.json()["width"] == 640 and changed.json()["scale"] == 2.0 + assert driver.screen.image.size == (1280, 800) + assert refused.status_code == 400 and "800×600 is not one of this computer's display modes" in refused.json()["error"]["message"] + + +@pytest.mark.asyncio +async def test_a_set_display_that_returns_nothing_answers_with_the_display_it_reads(): + class Quiet(FakeDriver): + @set_display + async def resize(self, width, height, scale): + self.screen.resize(width, height, scale) + + async with http_client(Quiet()) as http: + changed = (await http.post("/agentenv/ext/computer-display/set", json={"width": 1920, "height": 1080})).json() + assert (changed["width"], changed["height"]) == (1920, 1080) + + +@pytest.mark.asyncio +async def test_keyboard_settings_reach_the_driver_within_their_ranges(): + driver = FakeDriver() + async with http_client(driver) as http: + before = (await http.get("/agentenv/ext/computer-keyboard/get")).json() + after = (await http.post("/agentenv/ext/computer-keyboard/set", json={"tap_hold_ms": 80, "layout": "us"})).json() + too_long = await http.post("/agentenv/ext/computer-keyboard/set", json={"max_hold_ms": 600_001}) + layout = await http.post("/agentenv/ext/computer-keyboard/set", json={"layout": "de"}) + assert before == {"tap_hold_ms": 50, "type_delay_ms": 10, "layout": "us", "max_hold_ms": 10_000} + assert after["tap_hold_ms"] == 80 and driver.settings.tap_hold_ms == 80 + assert too_long.status_code == 400 and too_long.json()["error"]["code"] == "invalid_argument" + assert layout.status_code == 501 and layout.json()["error"]["code"] == "unsupported" + + +@pytest.mark.asyncio +async def test_a_command_runs_in_the_session_with_its_env_and_cwd(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + (tmp_path / "work").mkdir() + async with http_client(FakeDriver()) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", 'echo "$GREETING from $(pwd)"; echo oops >&2; exit 3'], cwd="work", + env={"GREETING": "hi"})) + assert r.json() == {"exit_code": 3, "stdout": f"hi from {(tmp_path / 'work').resolve()}\n", "stderr": "oops\n", + "truncated": False, "timed_out": False} + + +@pytest.mark.asyncio +async def test_a_command_past_its_timeout_is_killed_with_its_children(): + async with http_client(FakeDriver()) as http: + started = time.monotonic() + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", "sleep 30 & sleep 30; echo never"], timeout_s=0.3)) + assert r.json()["timed_out"] is True and r.json()["exit_code"] is None and r.json()["stdout"] == "" + assert time.monotonic() - started < 5 + + +@pytest.mark.asyncio +async def test_output_is_capped_per_stream(): + driver = FakeDriver(settings=ComputerSettings(exec_output_bytes=10)) + async with http_client(driver) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["sh", "-c", "yes | head -c 100000"])) + assert r.json()["stdout"] == "y\ny\ny\ny\ny\n" and r.json()["truncated"] is True + + +@pytest.mark.asyncio +async def test_a_background_app_holding_the_output_open_doesnt_hold_the_call(): + async with http_client(FakeDriver()) as http: + started = time.monotonic() + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", "echo started; (sleep 3; echo late) &"])) + elapsed = time.monotonic() - started + assert r.json()["exit_code"] == 0 and r.json()["stdout"] == "started\n" and elapsed < 2.5 + + +@pytest.mark.asyncio +async def test_a_repeated_request_id_returns_the_first_result_without_running_again(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + count = ["sh", "-c", "echo x >> runs; wc -l < runs"] + driver = FakeDriver() + async with http_client(driver) as http: + first = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id="a"))).json() + again = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id="a"))).json() + both = await asyncio.gather(*(http.post("/agentenv/ext/computer-exec/run", + json=_run(argv=["sh", "-c", "sleep 0.2; " + count[2]], request_id="b")) + for _ in range(3))) + other = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id="c"))).json() + assert first == again and first["stdout"].strip() == "1" + assert len({r.json()["stdout"] for r in both}) == 1 and both[0].json()["stdout"].strip() == "2" + assert other["stdout"].strip() == "3" + + +@pytest.mark.asyncio +async def test_a_failed_command_is_not_remembered(): + async with http_client(FakeDriver()) as http: + missing = await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["no-such-binary-xyz"])) + again = await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["no-such-binary-xyz"])) + assert missing.status_code == again.status_code == 400 + assert missing.json()["error"]["code"] == "invalid_argument" and "no-such-binary-xyz" in missing.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"argv": []}, {"argv": "ls -l"}, {"argv": ["ls", 1]}, {"request_id": ""}, {"env": {"A": 1}}, + {"timeout_s": 0}, {"timeout_s": 3601}, +]) +async def test_a_bad_command_request_is_refused(body): + async with http_client(FakeDriver()) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run(**body)) + assert r.status_code == 400 and r.json()["error"]["code"] in ("invalid_argument", "invalid_params") + + +@pytest.mark.asyncio +async def test_a_drivers_own_run_in_session_is_capped_and_checked(): + class Own(MinimalDriver): + output = ExecResult(0, b"a" * 50, b"", False) + + @run_in_session + async def run(self, argv, cwd, env, timeout_s): + return self.output + + driver = Own() + driver.settings = ComputerSettings(exec_output_bytes=8) + async with http_client(driver) as http: + capped = (await http.post("/agentenv/ext/computer-exec/run", json=_run())).json() + driver.output = "not a result" + broken = await http.post("/agentenv/ext/computer-exec/run", json=_run(request_id="other")) + assert capped == {"exit_code": 0, "stdout": "aaaaaaaa", "stderr": "", "truncated": True, "timed_out": False} + assert broken.status_code == 500 and broken.json()["error"]["code"] == "internal" + + +@pytest.mark.asyncio +async def test_a_background_apps_output_is_drained_after_the_call_returns(monkeypatch): + monkeypatch.setattr(linux, "DRAIN_S", 0.1) + async with http_client(FakeDriver()) as http: + r = await http.post("/agentenv/ext/computer-exec/run", json=_run( + argv=["sh", "-c", "echo now; (sleep 0.4; echo later) &"])) + draining = len(linux._draining) + await asyncio.sleep(1.0) + assert r.json()["stdout"] == "now\n" and draining >= 1 and not linux._draining + + +@pytest.mark.asyncio +async def test_only_the_latest_command_results_are_kept(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr("agentenv_protocol.computer.runtime.KEEP_EXEC_RESULTS", 1) + count = ["sh", "-c", "echo x >> runs; wc -l < runs"] + async with http_client(FakeDriver()) as http: + for request_id in ("a", "b", "a"): + last = (await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=count, request_id=request_id))).json() + assert last["stdout"].strip() == "3", "a's first result was dropped, so a ran again" + + +@pytest.mark.asyncio +async def test_a_running_command_is_joined_however_many_others_finish_meanwhile(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr("agentenv_protocol.computer.runtime.KEEP_EXEC_RESULTS", 1) + slow = _run(argv=["sh", "-c", "sleep 0.6; echo a >> runs"], request_id="a") + async with http_client(FakeDriver()) as http: + async def others_then_retry(): + await asyncio.sleep(0.1) + for request_id in ("b", "c"): + await http.post("/agentenv/ext/computer-exec/run", json=_run(argv=["true"], request_id=request_id)) + return await http.post("/agentenv/ext/computer-exec/run", json=slow) + + first, retried = await asyncio.gather(http.post("/agentenv/ext/computer-exec/run", json=slow), + others_then_retry()) + assert first.json() == retried.json() and (tmp_path / "runs").read_text() == "a\n" diff --git a/packages/agentenv-protocol/tests/test_computer_frames.py b/packages/agentenv-protocol/tests/test_computer_frames.py new file mode 100644 index 00000000..6c5eb9b7 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_frames.py @@ -0,0 +1,157 @@ +"""Frames: screenshots as frames by id, native crops that map back exactly, encodings, retention, staleness after a +display change, HiDPI mapping, and each caller's latest frame.""" +from __future__ import annotations + +import base64 +import io + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, FakeScreen, http_client, mcp_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol import ROLE_META_KEY # noqa: E402 +from agentenv_protocol.computer.frames import KEEP # noqa: E402 + + +def _image(result) -> PILImage.Image: + return PILImage.open(io.BytesIO(base64.b64decode(result.content[1].data))) + + +@pytest.mark.asyncio +async def test_a_screenshot_is_a_native_frame_with_its_geometry(): + async with mcp_session(FakeDriver(FakeScreen(1280, 800))) as session: + first = await session.call_tool("computer_screenshot", {}) + second = await session.call_tool("computer_screenshot", {"format": "jpeg", "quality": 40}) + data = first.structuredContent + assert data["frame_id"] == "f1" and (data["width"], data["height"]) == (1280, 800) + assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and "parent_frame_id" not in data + assert data["captured_at"].endswith("Z") and data["summary"] == "Captured f1: 1280×800 of the 1280×800 display" + assert first.content[0].text == data["summary"] and first.content[1].mimeType == "image/png" + assert _image(first).size == (1280, 800) + assert second.structuredContent["frame_id"] == "f2" and second.content[1].mimeType == "image/jpeg" + assert _image(second).format == "JPEG" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("args", [{"quality": 50}, {"format": "jpeg", "quality": 0}, {"format": "jpeg", "quality": 96}]) +async def test_quality_is_for_jpeg_within_range(args): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_screenshot", args) + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_a_region_is_a_native_crop_whose_points_map_back_exactly(): + driver = FakeDriver(FakeScreen(1280, 800)) + driver.screen.paint(700, 450, (1, 2, 3)) + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + crop = await session.call_tool("computer_screenshot", {"region": [600, 400, 900, 500]}) + nested = await session.call_tool("computer_screenshot", {"region": [100, 50, 200, 100], "frame_id": "f2"}) + clicked = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f3"}) + assert crop.structuredContent["parent_frame_id"] == "f1" and (crop.structuredContent["width"], + crop.structuredContent["height"]) == (300, 100) + assert crop.structuredContent["summary"] == "Cropped f2: 300×100 of f1 at (600, 400)" + assert crop.structuredContent["captured_at"] == (await _captured_at(driver, "f1")) + assert _image(crop).getpixel((100, 50))[:3] == (1, 2, 3) + assert nested.structuredContent["parent_frame_id"] == "f2" and _image(nested).getpixel((0, 0))[:3] == (1, 2, 3) + assert not clicked.isError and driver.screen.log[-1][1:3] == (701, 451) + + +async def _captured_at(driver: FakeDriver, frame_id: str) -> str: + return driver._computer.frames.get(frame_id, "default").result().captured_at + + +@pytest.mark.asyncio +@pytest.mark.parametrize("args, code", [ + ({"frame_id": "f1"}, "invalid_argument"), + ({"region": [0, 0, 10]}, "invalid_argument"), + ({"region": [10, 10, 10, 20]}, "out_of_bounds"), + ({"region": [0, 0, 1281, 10]}, "out_of_bounds"), + ({"region": [-1, 0, 10, 10]}, "out_of_bounds"), + ({"region": [0, 0, 10, 10], "frame_id": "f9"}, "stale_frame"), + ({"region": [0, 0, 1001, 10], "space": "norm"}, "out_of_bounds"), +]) +async def test_a_bad_region_is_refused(args, code): + async with mcp_session(FakeDriver()) as session: + await session.call_tool("computer_screenshot", {}) + refused = await session.call_tool("computer_screenshot", args) + assert refused.isError and refused.structuredContent["code"] == code + + +@pytest.mark.asyncio +async def test_a_region_in_norm_space_crops_the_same_box(): + async with mcp_session(FakeDriver(FakeScreen(1000, 500))) as session: + await session.call_tool("computer_screenshot", {}) + crop = await session.call_tool("computer_screenshot", {"region": [500, 500, 1000, 1000], "space": "norm"}) + assert (crop.structuredContent["width"], crop.structuredContent["height"]) == (500, 250) + + +@pytest.mark.asyncio +async def test_only_the_last_frames_are_kept(): + async with mcp_session(FakeDriver(FakeScreen(8, 8))) as session: + for _ in range(KEEP + 1): + await session.call_tool("computer_screenshot", {}) + evicted = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f1"}) + kept = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f2"}) + assert evicted.structuredContent["code"] == "stale_frame" and "no longer kept" in evicted.structuredContent["message"] + assert evicted.structuredContent["retryable"] is True and not kept.isError + + +@pytest.mark.asyncio +async def test_a_frame_goes_stale_when_the_display_changes(): + driver = FakeDriver() + async with mcp_session(driver) as session, http_client(driver) as http: + await session.call_tool("computer_screenshot", {}) + await http.post("/agentenv/ext/computer-display/set", json={"width": 1920, "height": 1080}) + stale = await session.call_tool("computer_click", {"x": 10, "y": 10}) + crop = await session.call_tool("computer_screenshot", {"region": [0, 0, 10, 10]}) + await session.call_tool("computer_screenshot", {}) + fresh = await session.call_tool("computer_click", {"x": 1900, "y": 10}) + assert stale.structuredContent["code"] == "stale_frame" + assert "captured at 1280×800 at scale 1 and the display is now 1920×1080 at scale 1" in stale.structuredContent["message"] + assert crop.structuredContent["code"] == "stale_frame" and not fresh.isError + + +@pytest.mark.asyncio +async def test_with_no_frame_yet_a_point_cannot_be_placed(): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_click", {"x": 1, "y": 1}) + assert refused.structuredContent["code"] == "stale_frame" + assert refused.structuredContent["message"] == "there is no frame yet: take a screenshot first" + + +@pytest.mark.asyncio +async def test_hidpi_frame_pixels_map_to_screen_pixels(): + driver = FakeDriver(FakeScreen(640, 400, scale=2.0)) + async with mcp_session(driver) as session: + shot = await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_click", {"x": 21, "y": 41}) + await session.call_tool("computer_click", {"x": 1279, "y": 799}) + position = await session.call_tool("computer_pointer_position", {}) + assert (shot.structuredContent["width"], shot.structuredContent["height"]) == (1280, 800) + assert shot.structuredContent["summary"] == "Captured f1: 1280×800 of the 640×400 at scale 2 display" + assert driver.screen.log[-2][1:3] == (10, 20) and driver.screen.log[-1][1:3] == (639, 399) + assert position.structuredContent["x"] == 1278 and position.structuredContent["screen"] == {"x": 639, "y": 399} + + +@pytest.mark.asyncio +async def test_each_caller_has_its_own_latest_frame(): + class ByRole(FakeDriver): + def caller_identity(self, context): + return context.caller.role or "default" + + driver = ByRole() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}, meta={ROLE_META_KEY: "alice"}) + await session.call_tool("computer_screenshot", {"region": [0, 0, 10, 10]}, meta={ROLE_META_KEY: "alice"}) + await session.call_tool("computer_screenshot", {}, meta={ROLE_META_KEY: "bob"}) + alice = await session.call_tool("computer_click", {"x": 9, "y": 9}, meta={ROLE_META_KEY: "alice"}) + bob = await session.call_tool("computer_click", {"x": 900, "y": 9}, meta={ROLE_META_KEY: "bob"}) + carol = await session.call_tool("computer_click", {"x": 1, "y": 1}, meta={ROLE_META_KEY: "carol"}) + assert alice.structuredContent["summary"].endswith("in f2") and bob.structuredContent["summary"].endswith("in f3") + assert carol.structuredContent["code"] == "stale_frame" diff --git a/packages/agentenv-protocol/tests/test_computer_queue.py b/packages/agentenv-protocol/tests/test_computer_queue.py new file mode 100644 index 00000000..93ca1f07 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_queue.py @@ -0,0 +1,477 @@ +"""Input: the input tools' parameters and refusals, the queue that runs input one call at a time in arrival order, +others' input since a frame, screenshots after acting, and the pointer and keyboard groups.""" +from __future__ import annotations + +import asyncio +import time + +import anyio +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import COLOURS, FakeDriver, FakeScreen, MinimalDriver, http_client, mcp_session # noqa: E402 + +from agentenv_protocol import ROLE_META_KEY # noqa: E402 +from agentenv_protocol.computer import ComputerSettings, pointer_position # noqa: E402 +from agentenv_protocol.computer.queue import InputQueue # noqa: E402 +from agentenv_protocol.tool_context import EMPTY # noqa: E402 + + +class ByRole(FakeDriver): + def caller_identity(self, context): + return context.caller.role or "default" + + +async def _shot(session, **meta) -> dict: + return (await session.call_tool("computer_screenshot", {}, meta=meta or None)).structuredContent + + +@pytest.mark.asyncio +async def test_a_click_reaches_the_screen_with_its_button_count_and_modifiers(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + result = await session.call_tool("computer_click", {"x": 5, "y": 6, "button": "right", "count": 2, + "modifiers": ["shift", "Control"]}) + assert result.structuredContent == { + "summary": "Clicked right ×2 at (5, 6) in f1", "applied": "yes", "delivery": "foreground", + "others_since_frame": 0, "effect": {"pointer_moved": True}} + assert driver.screen.log[-1] == ("click", 5, 6, "right", 2, ("Shift", "Control")) + assert driver.screen.colour_at(5, 6) == COLOURS["right"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("args, code, message", [ + ({"button": "back"}, "unsupported", "this computer has no back button; it has left, right, middle"), + ({"count": 4}, "invalid_argument", "count is from 1 to 3 on this computer, got 4"), + ({"count": 0}, "invalid_argument", "count is from 1 to 3"), + ({"modifiers": ["Hyperdrive"]}, "invalid_argument", "'Hyperdrive' is not a W3C key value"), + ({"x": 5, "y": None}, "invalid_argument", "give x and y, or neither to act at the pointer"), + ({"x": None, "y": None, "frame_id": "f1"}, "invalid_argument", "give x and y"), +]) +async def test_a_bad_click_is_refused_and_nothing_is_sent(args, code, message): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + refused = await session.call_tool("computer_click", {"x": 5, "y": 6, **args}) + assert refused.structuredContent["code"] == code and message in refused.structuredContent["message"] + assert refused.structuredContent["applied"] == "no" and driver.screen.log == [] + + +@pytest.mark.asyncio +async def test_without_the_pointer_group_a_click_needs_its_point(): + async with mcp_session(MinimalDriver()) as session: + await _shot(session) + refused = await session.call_tool("computer_click", {"x": 5}) + assert refused.isError and "give x and y" in refused.content[0].text + + +@pytest.mark.asyncio +async def test_a_drag_checks_every_point_before_sending_anything(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + dragged = await session.call_tool("computer_drag", {"path": [[1, 1], [50, 60], [99, 98]], "duration_ms": 300}) + out = await session.call_tool("computer_drag", {"path": [[1, 1], [1280, 5]]}) + short = await session.call_tool("computer_drag", {"path": [[1, 1]]}) + long = await session.call_tool("computer_drag", {"path": [[1, 1]] * 11}) + slow = await session.call_tool("computer_drag", {"path": [[1, 1], [2, 2]], "duration_ms": 10_001}) + bad = await session.call_tool("computer_drag", {"path": [[1, 1], [2, 2, 3]]}) + middle = await session.call_tool("computer_drag", {"path": [[1, 1], [2, 2]], "button": "middle"}) + assert dragged.structuredContent["summary"] == "Dragged left through 3 points of f1 in 300 ms" + assert driver.screen.log[0] == ("drag", ((1, 1), (50, 60), (99, 98)), "left", 300, ()) + assert out.structuredContent["code"] == "out_of_bounds" + assert short.structuredContent["code"] == long.structuredContent["code"] == "invalid_argument" + assert slow.structuredContent["code"] == bad.structuredContent["code"] == "invalid_argument" + assert not middle.isError and driver.screen.log[1] == ("drag", ((1, 1), (2, 2)), "middle", 500, ()) + assert len(driver.screen.log) == 2 + + +@pytest.mark.asyncio +async def test_scroll_moves_the_content_and_can_scroll_at_the_pointer(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + await session.call_tool("computer_scroll", {"x": 10, "y": 10, "direction": "down"}) + await session.call_tool("computer_pointer_move", {"x": 30, "y": 40}) + at_pointer = await session.call_tool("computer_scroll", {"direction": "left", "amount": 2}) + refused = await session.call_tool("computer_scroll", {"x": 1, "y": 1, "direction": "up", "amount": 101}) + assert driver.screen.scrolled == [-2, 3] + assert at_pointer.structuredContent["summary"] == "Scrolled left 2 at the pointer, screen (30, 40)" + assert driver.screen.log[-1] == ("scroll", 30, 40, "left", 2, ()) + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_a_click_without_a_point_clicks_at_the_pointer(): + driver = FakeDriver() + driver.screen.pointer = (70, 80) + async with mcp_session(driver) as session: + clicked = await session.call_tool("computer_click", {}) + assert clicked.structuredContent["others_since_frame"] == 0 and driver.screen.log[-1][1:3] == (70, 80) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text, code", [("", "invalid_argument"), ("a" * 10_001, "invalid_argument"), + ("bell\x07", "invalid_argument"), ("del\x7f", "invalid_argument")]) +async def test_text_that_cant_be_typed_whole_is_refused(text, code): + driver = FakeDriver() + async with mcp_session(driver) as session: + refused = await session.call_tool("computer_type", {"text": text}) + assert refused.structuredContent["code"] == code and driver.screen.log == [] + + +@pytest.mark.asyncio +async def test_unicode_newlines_and_tabs_are_typed(): + driver = FakeDriver() + async with mcp_session(driver) as session: + typed = await session.call_tool("computer_type", {"text": "Grüße 東京\n\tok", "screenshot": True}) + assert driver.screen.log == [("type", "Grüße 東京\n\tok")] + assert typed.structuredContent["summary"] == "Typed 12 characters, then captured f1" + assert typed.structuredContent["frame"]["frame_id"] == "f1" and typed.content[1].type == "image" + + +@pytest.mark.asyncio +async def test_a_key_chord_is_pressed_in_w3c_case_and_repeated_by_the_base_class(): + driver = FakeDriver(settings=ComputerSettings(tap_hold_ms=25)) + async with mcp_session(driver) as session: + pressed = await session.call_tool("computer_key", {"keys": ["control", "S"], "repeat": 3}) + single = await session.call_tool("computer_key", {"keys": [" "]}) + assert pressed.structuredContent["summary"] == "Pressed Control+S ×3" + assert driver.screen.log == [("press", ("Control", "S"), 25)] * 3 + [("press", (" ",), 25)] + assert single.structuredContent["applied"] == "yes" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("keys, repeat, code, hint", [ + (["ctrl", "s"], 1, "invalid_argument", "did you mean Control?"), + (["Control+s"], 1, "invalid_argument", "one key per entry: keys is a list, such as ['Control', 's']"), + (["Entr"], 1, "invalid_argument", "did you mean Enter?"), + (["Qwertyuiop"], 1, "invalid_argument", "named keys are W3C UI Events key values, such as Enter"), + ([], 1, "invalid_argument", None), + (["Enter"], 101, "invalid_argument", None), +]) +async def test_a_refused_key_says_what_was_meant(keys, repeat, code, hint): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_key", {"keys": keys, "repeat": repeat}) + assert refused.structuredContent["code"] == code and refused.structuredContent.get("hint") == hint + + +@pytest.mark.asyncio +async def test_a_key_that_isnt_a_string_is_refused_by_the_tools_schema(): + async with mcp_session(FakeDriver()) as session: + refused = await session.call_tool("computer_key", {"keys": [3]}) + assert refused.isError and refused.structuredContent is None + + +@pytest.mark.asyncio +async def test_a_driver_that_declares_its_keys_refuses_the_others(): + class Few(FakeDriver): + keys = ("Enter", "a", "Control") + + driver = Few() + async with mcp_session(driver) as session: + refused = await session.call_tool("computer_key", {"keys": ["Control", "Tab"]}) + allowed = await session.call_tool("computer_key", {"keys": ["control", "a"]}) + assert refused.structuredContent["code"] == "unsupported" and "['Tab']" in refused.structuredContent["message"] + assert not allowed.isError + + +@pytest.mark.asyncio +async def test_wait_waits_within_range_and_can_return_a_frame(): + async with mcp_session(FakeDriver()) as session: + started = time.monotonic() + waited = await session.call_tool("computer_wait", {"ms": 120}) + elapsed = time.monotonic() - started + shot = await session.call_tool("computer_wait", {"ms": 0, "screenshot": True}) + refused = await session.call_tool("computer_wait", {"ms": 60_001}) + assert waited.structuredContent == {"summary": "Waited 120 ms", "waited_ms": 120} + assert elapsed >= 0.1 + assert shot.structuredContent["frame"]["frame_id"] == "f1" and shot.content[1].type == "image" + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_input_runs_one_call_at_a_time_in_arrival_order(): + driver = FakeDriver() + driver.slow["click"] = 0.05 + async with mcp_session(driver) as session: + await _shot(session) + results = await asyncio.gather(*(session.call_tool("computer_click", {"x": i, "y": i}) for i in range(6))) + assert all(not r.isError for r in results) + assert [entry[1] for entry in driver.screen.log] == list(range(6)) + + +@pytest.mark.asyncio +async def test_a_call_waiting_past_the_limit_is_refused_input_busy(): + driver = FakeDriver(settings=ComputerSettings(queue_wait_s=0.1)) + driver.slow["click"] = 0.4 + async with mcp_session(driver) as session: + await _shot(session) + first, second = await asyncio.gather(session.call_tool("computer_click", {"x": 1, "y": 1}), + session.call_tool("computer_key", {"keys": ["a"]})) + assert not first.isError + assert second.structuredContent == {"code": "input_busy", "message": "waited 0.1 s for earlier input to finish", + "retryable": True, "applied": "no"} + + +@pytest.mark.asyncio +async def test_screenshots_and_waits_never_queue_behind_input(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + driver.slow["click"] = 0.5 + click = asyncio.ensure_future(session.call_tool("computer_click", {"x": 1, "y": 1})) + await asyncio.sleep(0.05) + started = time.monotonic() + await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_wait", {"ms": 0}) + observed = time.monotonic() - started + await click + assert observed < 0.4 + + +@pytest.mark.asyncio +async def test_others_input_since_a_frame_is_counted_and_ones_own_is_not(): + driver = ByRole() + async with mcp_session(driver) as session: + alice_frame = await _shot(session, **{ROLE_META_KEY: "alice"}) + await session.call_tool("computer_click", {"x": 1, "y": 1}, meta={ROLE_META_KEY: "alice"}) + await _shot(session, **{ROLE_META_KEY: "bob"}) + for _ in range(2): + await session.call_tool("computer_key", {"keys": ["a"]}, meta={ROLE_META_KEY: "bob"}) + alice = await session.call_tool("computer_click", {"x": 2, "y": 2, "frame_id": alice_frame["frame_id"]}, + meta={ROLE_META_KEY: "alice"}) + bob = await session.call_tool("computer_key", {"keys": ["b"]}, meta={ROLE_META_KEY: "bob"}) + assert alice.structuredContent["others_since_frame"] == 2 + assert bob.structuredContent["others_since_frame"] == 1 + + +@pytest.mark.asyncio +async def test_an_input_can_return_a_fresh_frame_that_shows_it(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + clicked = await session.call_tool("computer_click", {"x": 3, "y": 3, "screenshot": True}) + assert clicked.structuredContent["frame"]["frame_id"] == "f2" + assert clicked.structuredContent["summary"] == "Clicked left at (3, 3) in f1, then captured f2" + + +@pytest.mark.asyncio +async def test_the_pointer_tools(): + driver = FakeDriver() + async with mcp_session(driver) as session: + before = await session.call_tool("computer_pointer_position", {}) + await _shot(session) + moved = await session.call_tool("computer_pointer_move", {"x": 11, "y": 12}) + down = await session.call_tool("computer_pointer_down", {"button": "right", "x": 20, "y": 21}) + up = await session.call_tool("computer_pointer_up", {"button": "right"}) + inside = await session.call_tool("computer_pointer_position", {}) + await session.call_tool("computer_screenshot", {"region": [0, 0, 10, 10]}) + outside = await session.call_tool("computer_pointer_position", {}) + named = await session.call_tool("computer_pointer_position", {"frame_id": "f1"}) + half = await session.call_tool("computer_pointer_down", {"x": 1}) + listed = await session.call_tool("computer_pointer_list", {}) + assert before.structuredContent == {"summary": "The pointer is at screen (0, 0)", "screen": {"x": 0, "y": 0}} + assert moved.structuredContent["summary"] == "Moved the pointer at (11, 12) in f1" + assert down.structuredContent["summary"] == "Pressed right at (20, 21) in f1" + assert up.structuredContent["summary"] == "Released right" + assert driver.screen.log == [("move", 11, 12), ("move", 20, 21), ("down", "right"), ("up", "right")] + assert inside.structuredContent["inside_frame"] is True and inside.structuredContent["x"] == 20 + assert outside.structuredContent["inside_frame"] is False and "outside f2" in outside.structuredContent["summary"] + assert named.structuredContent["frame_id"] == "f1" + assert half.structuredContent["code"] == "invalid_argument" + assert listed.structuredContent["pointers"] == [ + {"id": "default", "kind": "foreground", "label": "The computer's pointer", "default": True}] + + +@pytest.mark.asyncio +async def test_a_bad_pointer_position_is_an_internal_error(): + class Lost(FakeDriver): + @pointer_position + async def where(self): + return "somewhere" + + async with mcp_session(Lost()) as session: + broken = await session.call_tool("computer_pointer_position", {}) + assert broken.structuredContent["code"] == "internal" + + +@pytest.mark.asyncio +async def test_keys_held_down_stay_down_until_released(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_keyboard_down", {"keys": ["shift"]}) + await session.call_tool("computer_keyboard_down", {"keys": ["Control"]}) + held = list(driver.screen.held) + partly = await session.call_tool("computer_keyboard_up", {"keys": ["Shift"]}) + rest = await session.call_tool("computer_keyboard_up", {}) + nothing = await session.call_tool("computer_keyboard_up", {}) + assert held == ["Shift", "Control"] and driver.screen.held == [] + assert partly.structuredContent["summary"] == "Released Shift" and rest.structuredContent["summary"] == "Released Control" + assert nothing.structuredContent["code"] == "invalid_argument" and driver._computer.held == {} + + +@pytest.mark.asyncio +async def test_a_hold_with_hold_ms_releases_in_reverse_order(): + driver = FakeDriver() + async with mcp_session(driver) as session: + held = await session.call_tool("computer_keyboard_down", {"keys": ["Shift", "a"], "hold_ms": 30}) + refused = await session.call_tool("computer_keyboard_down", {"keys": ["a"], "hold_ms": 10_001}) + assert held.structuredContent["summary"] == "Held Shift+a for 30 ms" + assert driver.screen.log == [("key_down", ("Shift", "a")), ("key_up", ("a", "Shift"))] + assert refused.structuredContent["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_keys_held_past_max_hold_ms_are_released(): + driver = FakeDriver(settings=ComputerSettings(max_hold_ms=50)) + async with mcp_session(driver) as session: + await session.call_tool("computer_keyboard_down", {"keys": ["Alt"]}) + with anyio.fail_after(5): + while driver.screen.held: + await asyncio.sleep(0.01) + assert driver.screen.log[-1] == ("key_up", ("Alt",)) and driver._computer.held == {} + + +@pytest.mark.asyncio +async def test_a_second_hold_restarts_the_release_timer(): + driver = FakeDriver(settings=ComputerSettings(max_hold_ms=150)) + async with mcp_session(driver) as session: + await session.call_tool("computer_keyboard_down", {"keys": ["Alt"]}) + await asyncio.sleep(0.1) + await session.call_tool("computer_keyboard_down", {"keys": ["Shift"]}) + await asyncio.sleep(0.1) + still = list(driver.screen.held) + with anyio.fail_after(5): + while driver.screen.held: + await asyncio.sleep(0.01) + assert still == ["Alt", "Shift"] and driver.screen.log[-1] == ("key_up", ("Shift", "Alt")) + + +@pytest.mark.asyncio +async def test_the_minimal_driver_serves_no_group_tools(): + driver = MinimalDriver(FakeScreen(64, 40)) + async with mcp_session(driver) as session: + await _shot(session) + clicked = await session.call_tool("computer_click", {"x": 3, "y": 4}) + missing = await session.call_tool("computer_pointer_move", {"x": 1, "y": 1}) + assert not clicked.isError and driver.screen.log == [("click", 3, 4, "left", 1, ())] + assert missing.isError and "Unknown tool" in missing.content[0].text + + +@pytest.mark.asyncio +async def test_a_caller_cancelled_while_waiting_never_leaves_the_queue_held(): + queue = InputQueue() + + async def hold(): + async with queue.turn(5): + await asyncio.sleep(0.2) + + async def wait(): + async with queue.turn(5): + pass + + holder = asyncio.ensure_future(hold()) + await asyncio.sleep(0.01) + waiter = asyncio.ensure_future(wait()) + await asyncio.sleep(0.01) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + await holder + async with queue.turn(0.5): + pass + + +@pytest.mark.asyncio +async def test_a_screenshot_that_fails_after_the_input_reports_the_input_as_done(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await _shot(session) + driver.fail["capture"] = RuntimeError("mss died") + clicked = await session.call_tool("computer_click", {"x": 4, "y": 4, "screenshot": True}) + driver.fail["capture"] = RuntimeError("mss died") + held = await session.call_tool("computer_keyboard_down", {"keys": ["Shift"], "screenshot": True}) + assert not clicked.isError and clicked.structuredContent["applied"] == "yes" and "frame" not in clicked.structuredContent + assert "the screenshot after it failed (internal: @capture failed: RuntimeError: mss died)" in clicked.content[0].text + assert [e[0] for e in driver.screen.log] == ["click", "key_down"] + assert not held.isError and driver._computer.held == {"default": ["Shift"]} + + +@pytest.mark.asyncio +async def test_the_automatic_release_waits_out_a_busy_queue(): + driver = FakeDriver(settings=ComputerSettings(max_hold_ms=50, queue_wait_s=0.05)) + async with mcp_session(driver) as session: + await _shot(session) + await session.call_tool("computer_keyboard_down", {"keys": ["Alt"]}) + driver.slow["drag"] = 0.4 + await session.call_tool("computer_drag", {"path": [[1, 1], [9, 9]]}) + with anyio.fail_after(5): + while driver.screen.held: + await asyncio.sleep(0.01) + kinds = [e[0] for e in driver.screen.log] + assert kinds == ["key_down", "drag", "key_up"] and driver._computer.held == {} + + +@pytest.mark.asyncio +async def test_a_point_queued_while_the_display_changes_is_refused_stale(): + driver = FakeDriver() + async with mcp_session(driver) as session, http_client(driver) as http: + await _shot(session) + driver.slow["click"] = 0.3 + first = asyncio.ensure_future(session.call_tool("computer_click", {"x": 1, "y": 1})) + await asyncio.sleep(0.05) + resize = asyncio.ensure_future(http.post("/agentenv/ext/computer-display/set", json={"width": 640, "height": 400})) + await asyncio.sleep(0.05) + queued = await session.call_tool("computer_click", {"x": 1270, "y": 10}) + await first + await resize + assert queued.structuredContent["code"] == "stale_frame" and queued.structuredContent["applied"] == "no" + assert [e[1:3] for e in driver.screen.log] == [(1, 1)] + + +@pytest.mark.asyncio +async def test_a_click_at_the_pointer_reads_the_pointer_when_its_turn_comes(): + driver = FakeDriver() + driver.slow["drag"] = 0.3 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_pointer_move", {"x": 5, "y": 5}) + dragging = asyncio.ensure_future(session.call_tool("computer_drag", {"path": [[10, 10], [100, 100]]})) + await asyncio.sleep(0.1) + clicked = await session.call_tool("computer_click", {}) + await dragging + assert clicked.structuredContent["summary"] == "Clicked left at the pointer, screen (100, 100)" + assert driver.screen.log[-1][:3] == ("click", 100, 100) + + +@pytest.mark.asyncio +async def test_input_that_lands_while_a_screenshot_is_taken_counts_as_after_it(): + driver = ByRole() + async with mcp_session(driver) as session: + await _shot(session, **{ROLE_META_KEY: "bob"}) + driver.slow["capture"] = 0.3 + shot = asyncio.ensure_future(_shot(session, **{ROLE_META_KEY: "alice"})) + await asyncio.sleep(0.1) + await session.call_tool("computer_click", {"x": 1, "y": 1}, meta={ROLE_META_KEY: "bob"}) + frame = await shot + del driver.slow["capture"] + clicked = await session.call_tool("computer_click", {"x": 2, "y": 2, "frame_id": frame["frame_id"]}, + meta={ROLE_META_KEY: "alice"}) + assert clicked.structuredContent["others_since_frame"] == 1 + + +@pytest.mark.asyncio +async def test_a_timed_hold_that_is_cancelled_still_lets_go_of_its_keys(): + driver = FakeDriver() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + holding = asyncio.ensure_future(driver.computer_keyboard_down(EMPTY, keys=["Shift"], hold_ms=5000)) + await asyncio.sleep(0.1) + holding.cancel() + with pytest.raises(asyncio.CancelledError): + await holding + assert driver.screen.log[-2:] == [("key_down", ("Shift",)), ("key_up", ("Shift",))] diff --git a/packages/agentenv-protocol/tests/test_computer_spaces.py b/packages/agentenv-protocol/tests/test_computer_spaces.py new file mode 100644 index 00000000..9352dbf1 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_spaces.py @@ -0,0 +1,119 @@ +"""Coordinate spaces: frame pixels and a 0..norm_scale grid, one mapping to the frame's pixels, refused and never +clamped outside the frame or the range, with the default space set per deploy.""" +from __future__ import annotations + +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") +fastmcp = pytest.importorskip("fastmcp") + +from computer_fake import FakeDriver, FakeScreen, http_client, mcp_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol import EnvironmentTool # noqa: E402 +from agentenv_protocol.computer import ComputerSettings, InvalidArgument, OutOfBounds # noqa: E402 +from agentenv_protocol.computer.frames import FrameStore # noqa: E402 +from agentenv_protocol.computer.operations import Display # noqa: E402 +from agentenv_protocol.computer.spaces import check_space, frame_point # noqa: E402 + + +def _frame(width: int, height: int): + return FrameStore().add(PILImage.new("RGB", (width, height)), Display(width, height), (0, {}), identity="default") + + +@pytest.mark.parametrize("width, height, x, y, expected", [ + (1280, 800, 0, 0, (0, 0)), + (1280, 800, 1000, 1000, (1279, 799)), + (1280, 800, 500, 500, (640, 400)), + (1279, 799, 500, 500, (639.5, 399.5)), + (3, 3, 999, 0, (2, 0)), + (3, 3, 666, 334, (1.998, 1.002)), +]) +def test_norm_points_land_on_the_frames_pixels(width, height, x, y, expected): + fx, fy = frame_point(_frame(width, height), x, y, "norm", 1000) + assert (round(fx, 3), round(fy, 3)) == expected + + +def test_frame_points_are_its_pixels_and_odd_pixels_stay_exact(): + frame = _frame(1279, 799) + assert frame_point(frame, 1278, 798, "frame", 1000) == (1278, 798) + assert frame.to_screen(1278.9, 0.5) == (1278, 0) + + +@pytest.mark.parametrize("x, y, space", [(1280, 0, "frame"), (0, -1, "frame"), (1000.5, 0, "norm"), (-0.1, 5, "norm")]) +def test_points_outside_the_frame_or_range_are_refused(x, y, space): + with pytest.raises(OutOfBounds): + frame_point(_frame(1280, 800), x, y, space, 1000) + + +@pytest.mark.parametrize("space, scale", [("grid", None), (None, 0), (None, 2_000_000), (None, 1.5)]) +def test_a_bad_space_or_scale_is_refused(space, scale): + with pytest.raises(InvalidArgument): + check_space(space, scale, "frame", 1000) + + +@pytest.mark.asyncio +async def test_the_default_space_and_scale_come_from_the_settings_and_a_call_can_override_them(): + driver = FakeDriver(FakeScreen(1280, 800), settings=ComputerSettings(default_space="norm", default_norm_scale=999)) + async with mcp_session(driver) as session: + tools = {t.name: t for t in (await session.list_tools()).tools} + await session.call_tool("computer_screenshot", {}) + await session.call_tool("computer_click", {"x": 999, "y": 0}) + await session.call_tool("computer_click", {"x": 7, "y": 9, "space": "frame"}) + await session.call_tool("computer_click", {"x": 1, "y": 1, "norm_scale": 1}) + assert "0 to 999 over the frame" in tools["computer_click"].description + assert "0 to 999 over the frame" in tools["computer_drag"].description + assert "default space" not in tools["computer_type"].description + assert [entry[1:3] for entry in driver.screen.log] == [(1279, 0), (7, 9), (1279, 799)] + + +@pytest.mark.asyncio +async def test_setting_the_default_space_rewrites_the_descriptions(): + driver = FakeDriver() + async with http_client(driver) as http, mcp_session(driver) as session: + before = {t.name: t.description for t in (await session.list_tools()).tools} + applied = (await http.post("/agentenv/ext/computer-spaces/set", + json={"default_space": "norm", "default_norm_scale": 1000})).json() + after = {t.name: t.description for t in (await session.list_tools()).tools} + card = (await http.get("/.well-known/agent-env.json")).json() + current = (await http.get("/agentenv/ext/computer-spaces/get")).json() + refused = await http.post("/agentenv/ext/computer-spaces/set", json={"default_space": "grid"}) + assert "frame space by default" in before["computer_click"] + assert applied == current == {"default_space": "norm", "default_norm_scale": 1000} + assert "norm space by default: 0 to 1000" in after["computer_click"] + card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click") + assert card_click["description"] == after["computer_click"] + assert refused.status_code == 400 and refused.json()["error"]["code"] == "invalid_argument" + + +@pytest.mark.asyncio +async def test_setting_the_default_space_rewrites_the_descriptions_on_fastmcp(): + app = fastmcp.FastMCP("desktop") + FakeDriver().mount(app) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app.http_app()), base_url="http://t") as http: + await http.post("/agentenv/ext/computer-spaces/set", json={"default_space": "norm", "default_norm_scale": 1000}) + card = (await http.get("/.well-known/agent-env.json")).json() + async with fastmcp.Client(app) as client: + listed = {t.name: t for t in await client.list_tools()} + await client.call_tool("computer_screenshot", {}) + clicked = await client.call_tool("computer_click", {"x": 500, "y": 500}) + card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click") + assert "norm space by default: 0 to 1000" in listed["computer_click"].description + assert card_click["description"] == listed["computer_click"].description + assert clicked.structured_content["summary"].startswith("Clicked left at (500, 500) (norm 1000) in f1") + + +@pytest.mark.asyncio +async def test_setting_the_default_space_rewrites_only_the_point_tools_the_driver_serves(): + class NoScroll(FakeDriver): + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + return [t for t in super().served_tools(tools) if t.name != "computer_scroll"] + + driver = NoScroll() + async with http_client(driver) as http, mcp_session(driver) as session: + applied = await http.post("/agentenv/ext/computer-spaces/set", json={"default_space": "norm"}) + listed = {t.name: t.description for t in (await session.list_tools()).tools} + assert applied.status_code == 200 and "computer_scroll" not in listed + assert "norm space by default" in listed["computer_click"] and "norm space by default" in listed["computer_drag"] diff --git a/packages/agentenv-protocol/tests/test_computer_x11_example.py b/packages/agentenv-protocol/tests/test_computer_x11_example.py new file mode 100644 index 00000000..2700e276 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_x11_example.py @@ -0,0 +1,290 @@ +"""The X11 example driver against fake mss and Xlib modules: it passes the definition checks, serves the whole +computer contract, and turns tool calls into the XTEST events they mean, without resending input, leaving keys or +buttons down, or letting go of keys it didn't press.""" +from __future__ import annotations + +import asyncio +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import LINUX, http_client, mcp_session # noqa: E402 + +from agentenv_protocol.computer import COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_POINTER # noqa: E402 + +DRIVER = Path(__file__).resolve().parents[1] / "examples" / "computer_x11" / "driver.py" +WIDTH, HEIGHT = 640, 400 +TOKYO = [0x01000000 | ord(ch) for ch in "東京"] +#: The fake keymap puts the named keys, such as Shift_L, from this keycode up; printable ASCII is its keysym + 8. +NAMED_KEYS = 200 + + +class ConnectionClosedError(Exception): + pass + + +class FakeX: + """An X display: its screen and pointer, a US-like keymap whose capital letters are the second level of their + keys, the keys down, the XTEST events sent, ``drops`` syncs and the ``fail_at``-th event that fail, and the + ``spare`` keycodes the keymap leaves empty.""" + + def __init__(self) -> None: + self.events: list = [] + self.down: set[int] = set() + self.remapped: list = [] + self.pointer = (0, 0) + self.opened = 0 + self.drops = 0 + self.fail_at = None + self.spare = {8} + self.display = types.SimpleNamespace(info=types.SimpleNamespace(min_keycode=8, max_keycode=255)) + + def screen(self): + query = lambda: types.SimpleNamespace(root_x=self.pointer[0], root_y=self.pointer[1]) # noqa: E731 + return types.SimpleNamespace(width_in_pixels=WIDTH, height_in_pixels=HEIGHT, + root=types.SimpleNamespace(query_pointer=query)) + + def sync(self) -> None: + if self.drops: + self.drops -= 1 + raise ConnectionClosedError("display") + + def has_extension(self, name: str) -> bool: + return name == "XTEST" + + def keysym_to_keycode(self, keysym: int) -> int: + if ord("A") <= keysym <= ord("Z"): + keysym += 0x20 + if 0xff00 <= keysym < 0xff38: + return NAMED_KEYS + keysym - 0xff00 + return keysym + 8 if 0x20 <= keysym < 0x7f else 0 + + def keycode_to_keysym(self, code: int, index: int) -> int: + sym = 0xff00 + code - NAMED_KEYS if code >= NAMED_KEYS else code - 8 + if index == 0: + return sym + return sym - 0x20 if index == 1 and ord("a") <= sym <= ord("z") else 0 + + def get_keyboard_mapping(self, first: int, count: int) -> list: + return [(0, 0) if first + i in self.spare else (1, 0) for i in range(count)] + + def change_keyboard_mapping(self, first: int, keysyms: list) -> None: + self.remapped.append((first, keysyms[0][0])) + + def query_keymap(self) -> list[int]: + return [sum(1 << bit for bit in range(8) if 8 * i + bit in self.down) for i in range(32)] + + +class FakeMSS: + monitors = [{"left": 0, "top": 0, "width": WIDTH, "height": HEIGHT}] + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + return None + + def grab(self, monitor: dict): + return types.SimpleNamespace(width=WIDTH, height=HEIGHT, bgra=bytes(WIDTH * HEIGHT * 4)) + + +@pytest.fixture +def x11(monkeypatch): + """The example's driver on a fake X display, and the keycodes the fake keymap gives named keys.""" + fake, keysyms = FakeX(), {} + + def open_display(): + fake.opened += 1 + return fake + + def fake_input(d, kind, detail=0, x=0, y=0): + if d.fail_at is not None: + d.fail_at -= 1 + if d.fail_at == 0: + d.fail_at = None + raise ConnectionClosedError("display") + d.events.append((kind, (x, y)) if kind == "move" else (kind, detail)) + if kind == "key-down": + d.down.add(detail) + elif kind == "key-up": + d.down.discard(detail) + + def module(name: str, **attrs) -> types.ModuleType: + m = types.ModuleType(name) + m.__dict__.update(attrs) + monkeypatch.setitem(sys.modules, name, m) + return m + + X = module("Xlib.X", MotionNotify="move", ButtonPress="button-down", ButtonRelease="button-up", + KeyPress="key-down", KeyRelease="key-up") + XK = module("Xlib.XK", string_to_keysym=lambda name: keysyms.setdefault(name, 0xff00 + len(keysyms))) + display = module("Xlib.display", Display=open_display) + module("Xlib.error", ConnectionClosedError=ConnectionClosedError, DisplayConnectionError=ConnectionClosedError) + xtest = module("Xlib.ext.xtest", fake_input=fake_input) + module("Xlib.ext", xtest=xtest) + module("Xlib", X=X, XK=XK, display=display) + module("mss", MSS=FakeMSS) + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) + monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + spec = importlib.util.spec_from_file_location("computer_x11_driver", DRIVER) + example = importlib.util.module_from_spec(spec) + spec.loader.exec_module(example) + return example.X11Driver(), fake, lambda name: fake.keysym_to_keycode(XK.string_to_keysym(name)) + + +async def _events(fake: FakeX, session, tool: str, args: dict) -> list: + fake.events.clear() + result = await session.call_tool(tool, args) + assert not result.isError, result.content[0].text + return list(fake.events) + + +@pytest.mark.asyncio +async def test_the_example_serves_the_whole_computer_contract(x11): + driver, _fake, _code = x11 + async with http_client(driver) as http, mcp_session(driver) as session: + card = (await http.get("/.well-known/agent-env.json")).json() + tools = {t.name for t in (await session.list_tools()).tools} + ready = (await http.get("/agentenv/ext/computer-ready/status")).json() + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + assert card["name"] == "x11-desktop" and len(tools) == 14 and "computer_keyboard_down" in tools + assert {COMPUTER_POINTER, COMPUTER_KEYBOARD, COMPUTER_EXEC} <= set(extensions) + assert extensions["urn:agentenv:computer/v1"]["params"]["max_click_count"] == 3 + assert ready["ready"] and [c["name"] for c in ready["checks"]] == ["display", "capture", "xtest"] + + +@pytest.mark.asyncio +async def test_tool_calls_become_the_xtest_events_they_mean(x11): + driver, fake, code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + clicked = await _events(fake, session, "computer_click", + {"x": 10, "y": 20, "button": "right", "modifiers": ["Shift"]}) + typed = await _events(fake, session, "computer_type", {"text": "aA\n"}) + dragged = await _events(fake, session, "computer_drag", {"path": [[0, 0], [40, 0]], "duration_ms": 0}) + chord = await _events(fake, session, "computer_key", {"keys": ["Control", "s"]}) + capital = await _events(fake, session, "computer_key", {"keys": ["A"]}) + shift, a = code("Shift_L"), ord("a") + 8 + assert clicked == [("key-down", shift), ("move", (10, 20)), ("button-down", 3), ("button-up", 3), ("key-up", shift)] + assert typed == [("key-down", a), ("key-up", a), ("key-down", shift), ("key-down", a), ("key-up", a), + ("key-up", shift), ("key-down", code("Return")), ("key-up", code("Return"))] + assert dragged == [("move", (0, 0)), ("button-down", 1), ("move", (20, 0)), ("move", (40, 0)), ("button-up", 1)] + assert chord == [("key-down", code("Control_L")), ("key-down", ord("s") + 8), ("key-up", ord("s") + 8), + ("key-up", code("Control_L"))] + assert capital == [("key-down", shift), ("key-down", a), ("key-up", a), ("key-up", shift)] + assert not fake.down + + +@pytest.mark.asyncio +async def test_keys_held_by_keyboard_down_stay_down_through_other_input(x11): + driver, fake, code = x11 + shift, a = code("Shift_L"), ord("a") + 8 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + held = await _events(fake, session, "computer_keyboard_down", {"keys": ["Shift"]}) + typed = await _events(fake, session, "computer_type", {"text": "A"}) + clicked = await _events(fake, session, "computer_click", {"x": 1, "y": 1, "modifiers": ["Shift"]}) + chord = await _events(fake, session, "computer_key", {"keys": ["Shift", "a"]}) + released = await _events(fake, session, "computer_keyboard_up", {}) + assert held == [("key-down", shift)] + assert typed == [("key-down", a), ("key-up", a)] and chord == [("key-down", a), ("key-up", a)] + assert clicked == [("move", (1, 1)), ("button-down", 1), ("button-up", 1)] + assert released == [("key-up", shift)] and not fake.down + + +@pytest.mark.asyncio +async def test_characters_off_the_keymap_share_the_spare_keycodes(x11): + driver, fake, _code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + repeated = await _events(fake, session, "computer_type", {"text": "東東"}) + mapped_once = list(fake.remapped) + recycled = await _events(fake, session, "computer_type", {"text": "京東"}) + assert repeated == [("key-down", 8), ("key-up", 8)] * 2 and mapped_once == [(8, TOKYO[0])] + assert recycled == [("key-down", 8), ("key-up", 8)] * 2 + assert fake.remapped == [(8, TOKYO[0]), (8, TOKYO[1]), (8, TOKYO[0])] + + +@pytest.mark.asyncio +async def test_a_held_characters_keycode_is_not_taken_by_another(x11): + driver, fake, _code = x11 + fake.spare = {8, 9} + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + held = await _events(fake, session, "computer_keyboard_down", {"keys": ["東"]}) + fake.events.clear() + chord = await session.call_tool("computer_key", {"keys": ["京", "日"]}) + held_too = await session.call_tool("computer_keyboard_down", {"keys": ["京", "日"]}) + refused = list(fake.events) + typed = await _events(fake, session, "computer_type", {"text": "京"}) + released = await _events(fake, session, "computer_keyboard_up", {}) + assert held == [("key-down", 9)] and typed == [("key-down", 8), ("key-up", 8)] + for call in (chord, held_too): + assert call.structuredContent["code"] == "unsupported" and call.structuredContent["applied"] == "no" + assert refused == [] and released == [("key-up", 9)] and not fake.down + + +@pytest.mark.asyncio +async def test_text_that_cant_be_typed_whole_is_refused_before_any_of_it(x11): + driver, fake, _code = x11 + fake.spare = set() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.events.clear() + refused = await session.call_tool("computer_type", {"text": "a東"}) + assert refused.structuredContent["code"] == "unsupported" and refused.structuredContent["applied"] == "no" + assert fake.events == [] + + +@pytest.mark.asyncio +async def test_input_is_not_sent_again_when_the_connection_drops_during_it(x11): + driver, fake, _code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.events.clear() + fake.fail_at = 2 + dropped = await session.call_tool("computer_click", {"x": 5, "y": 5}) + assert dropped.structuredContent["code"] == "delivery_failed" + assert dropped.structuredContent["applied"] == "unknown" + assert fake.events == [("move", (5, 5))] and fake.opened == 1 + + +@pytest.mark.asyncio +async def test_a_drag_that_fails_or_is_cancelled_lets_go_of_its_button_and_modifiers(x11): + driver, fake, code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.events.clear() + fake.fail_at = 4 + failed = await session.call_tool("computer_drag", {"path": [[0, 0], [40, 0]], "modifiers": ["Shift"]}) + assert failed.structuredContent["code"] == "delivery_failed" + assert fake.events[-2:] == [("button-up", 1), ("key-up", code("Shift_L"))] and not fake.down + fake.events.clear() + dragging = asyncio.ensure_future(driver.drag_along([[0, 0], [400, 0]], "left", 10_000, ["Control"])) + await asyncio.sleep(0.2) + dragging.cancel() + with pytest.raises(asyncio.CancelledError): + await dragging + assert fake.events[-2:] == [("button-up", 1), ("key-up", code("Control_L"))] and not fake.down + + +@pytest.mark.asyncio +async def test_a_dropped_x_connection_is_reopened_and_a_dead_one_is_device_unavailable(x11): + driver, fake, _code = x11 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + fake.drops = 1 + moved = await session.call_tool("computer_pointer_position", {}) + reopened = fake.opened + fake.drops = 2 + refused = await session.call_tool("computer_pointer_position", {}) + assert not moved.isError and reopened == 2 + assert refused.structuredContent["code"] == "device_unavailable" + assert "the X server is unavailable: ConnectionClosedError" in refused.structuredContent["message"] diff --git a/packages/agentenv-protocol/tests/test_errors.py b/packages/agentenv-protocol/tests/test_errors.py new file mode 100644 index 00000000..ec8123bf --- /dev/null +++ b/packages/agentenv-protocol/tests/test_errors.py @@ -0,0 +1,158 @@ +"""Structured errors: an AgentEnvError raised in an extension answers with its status and fields, raised in a tool +it answers isError with the fields as structured content on both FastMCPs, and the client parses them back.""" +from __future__ import annotations + +from unittest.mock import patch + +import anyio +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +fastmcp = pytest.importorskip("fastmcp") + +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 + +from agentenv_protocol import ( # noqa: E402 + AgentEnvEnvironment, + AgentEnvError, + ExtensionCallError, + ToolContext, + client, + environment_card, + extension, + tool, +) + +_URI = "urn:agentenv:frames/v1" + + +@environment_card(name="frames") +class _Env(AgentEnvEnvironment): + def __init__(self) -> None: + self.refuse_hook = False + + def on_tool_call(self, context: ToolContext): + if self.refuse_hook: + raise AgentEnvError("device_unavailable", "not ready", retryable=True, status=503) + return None + + @tool() + def click(self, x: int) -> str: + if x > 100: + raise AgentEnvError("out_of_bounds", f"({x}, 0) is outside frame f1 (100×100)", hint="x < 100") + return "ok" + + @tool() + def crash(self) -> str: + raise RuntimeError("kaboom") + + @extension(_URI) + async def stale(self, frame: str) -> dict: + raise AgentEnvError("stale_frame", f"frame {frame} is stale", retryable=True, status=409) + + @extension("urn:agentenv:partial/v1") + async def partial(self) -> dict: + raise AgentEnvError("delivery_failed", "typed 3 of 5 characters", applied="unknown", status=502) + + @extension("urn:agentenv:broken/v1") + async def broken(self) -> dict: + raise RuntimeError("kaboom") + + +def test_fields_carry_the_defaults_and_a_hint_only_when_given(): + assert AgentEnvError("invalid_argument", "bad").fields() == { + "code": "invalid_argument", "message": "bad", "retryable": False, "applied": "no"} + error = AgentEnvError("invalid_argument", "unknown key", hint="did you mean Return?") + assert error.fields()["hint"] == "did you mean Return?" and error.status == 400 and str(error) == "unknown key" + + +def _http(env: _Env) -> httpx.AsyncClient: + app = env.create_app().streamable_http_app() + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") + + +@pytest.mark.asyncio +async def test_an_extension_answers_with_the_errors_status_and_fields(): + async with _http(_Env()) as http: + stale = await http.post("/agentenv/ext/stale", json={"frame": "f3"}) + partial = await http.post("/agentenv/ext/partial", json={}) + assert stale.status_code == 409 + assert stale.json() == {"ok": False, "error": { + "code": "stale_frame", "message": "frame f3 is stale", "retryable": True, "applied": "no"}} + assert partial.status_code == 502 and partial.json()["error"]["applied"] == "unknown" + + +@pytest.mark.asyncio +async def test_any_other_extension_failure_is_extension_failed_with_applied_unknown(): + async with _http(_Env()) as http: + r = await http.post("/agentenv/ext/broken", json={}) + assert r.status_code == 500 + assert r.json()["error"] == {"code": "extension_failed", "message": "kaboom", "retryable": False, "applied": "unknown"} + + +@pytest.mark.asyncio +async def test_invalid_params_never_ran_so_applied_is_no(): + async with _http(_Env()) as http: + r = await http.post("/agentenv/ext/stale", json={"frame": "f1", "extra": 1}) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "invalid_params" and r.json()["error"]["applied"] == "no" + + +@pytest.mark.asyncio +async def test_a_tool_error_is_an_error_result_with_structured_fields_on_mcps_fastmcp(): + env = _Env() + with anyio.fail_after(30): + async with create_connected_server_and_client_session(env.create_app()) as session: + refused = await session.call_tool("click", {"x": 140}) + served = await session.call_tool("click", {"x": 4}) + crashed = await session.call_tool("crash", {}) + env.refuse_hook = True + hooked = await session.call_tool("click", {"x": 4}) + assert refused.isError is True + assert refused.content[0].text == "out_of_bounds: (140, 0) is outside frame f1 (100×100)" + assert refused.structuredContent == {"code": "out_of_bounds", "message": "(140, 0) is outside frame f1 (100×100)", + "retryable": False, "applied": "no", "hint": "x < 100"} + assert served.isError is False and served.content[0].text == "ok" + assert crashed.isError is True and "kaboom" in crashed.content[0].text and crashed.structuredContent is None + assert hooked.isError is True and hooked.structuredContent["code"] == "device_unavailable" + assert hooked.structuredContent["retryable"] is True + + +@pytest.mark.asyncio +async def test_a_tool_error_is_an_error_result_with_structured_fields_on_fastmcp(): + app = fastmcp.FastMCP("frames") + env = _Env() + env.mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as session: + refused = await session.call_tool("click", {"x": 140}, raise_on_error=False) + crashed = await session.call_tool("crash", {}, raise_on_error=False) + env.refuse_hook = True + hooked = await session.call_tool("click", {"x": 4}, raise_on_error=False) + assert refused.is_error is True + assert refused.content[0].text == "out_of_bounds: (140, 0) is outside frame f1 (100×100)" + assert refused.structured_content["code"] == "out_of_bounds" and refused.structured_content["hint"] == "x < 100" + assert crashed.is_error is True and "kaboom" in crashed.content[0].text and not crashed.structured_content + assert hooked.is_error is True and hooked.structured_content["code"] == "device_unavailable" + + +@pytest.mark.asyncio +async def test_the_client_raises_the_servers_fields_as_an_extension_call_error(): + env = _Env() + app = env.create_app().streamable_http_app() + card = await _card(app) + asgi = httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") + with patch("agentenv_protocol.client.httpx.AsyncClient", return_value=asgi): + with pytest.raises(ExtensionCallError) as raised: + await client.invoke_extension("http://t", card, _URI, {"frame": "f9"}) + error = raised.value + assert isinstance(error, httpx.HTTPStatusError) and error.response.status_code == 409 + assert (error.code, error.detail, error.retryable, error.applied, error.hint) == ( + "stale_frame", "frame f9 is stale", True, "no", None) + assert str(error) == "409 stale_frame: frame f9 is stale" + + +async def _card(app) -> dict: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as http: + return (await http.get("/.well-known/agent-env.json")).json() diff --git a/packages/agentenv-protocol/tests/test_extension_methods.py b/packages/agentenv-protocol/tests/test_extension_methods.py new file mode 100644 index 00000000..73d35aa8 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_extension_methods.py @@ -0,0 +1,236 @@ +"""Several methods per extension: handlers given ``op`` merge into one card entry with an endpoint per method, with +the environment's extension_params beside them; declaration-only extensions; served_extensions choosing what is +served; the refusals at mount.""" +from __future__ import annotations + +from unittest.mock import patch + +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") + +from agentenv_protocol import ( # noqa: E402 + AgentEnvEnvironment, + AgentEnvFastMCPApplication, + EnvironmentCapabilities, + EnvironmentCard, + EnvironmentExtension, + client, + environment_card, + extension, +) +from agentenv_protocol.agent_env_environment import _extension_slug # noqa: E402 + +_DISPLAY = "urn:agentenv:computer-display/v1" +_POINTER = "urn:agentenv:computer-pointer/v1" + + +@environment_card(name="display") +class _Display(AgentEnvEnvironment): + def __init__(self) -> None: + self.size = (1280, 800, 1.0) + + def extension_params(self, uri: str) -> dict: + return {"modes": [[1280, 800], [1920, 1080]]} if uri == _DISPLAY else {} + + def declared_extensions(self) -> list[EnvironmentExtension]: + return [EnvironmentExtension(uri=_POINTER, description="Pointer tools.", params={"buttons": ["left"]})] + + @extension(_DISPLAY, op="get", method="GET", description="Display resolution and scale.") + async def display_get(self) -> dict: + width, height, scale = self.size + return {"width": width, "height": height, "scale": scale} + + @extension(_DISPLAY, op="set") + async def display_set(self, width: int, height: int, scale: float = 1.0) -> dict: + self.size = (width, height, scale) + return await self.display_get() + + @extension("urn:agentenv:legacy/v1") + async def legacy(self) -> dict: + return {"legacy": True} + + +def _card(handler, card: EnvironmentCard | None = None) -> EnvironmentCard: + return AgentEnvFastMCPApplication(card or EnvironmentCard(name="t"), handler).environment_card + + +def _extensions(card: EnvironmentCard) -> dict: + return {e.uri: e for e in card.capabilities.extensions} + + +def _http(env: AgentEnvEnvironment) -> httpx.AsyncClient: + return _client(env.create_app().streamable_http_app()) + + +def _client(app) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") + + +@pytest.mark.parametrize("uri, slug", [ + ("urn:agentenv:computer-display/v1", "computer-display"), + ("urn:agentenv:computer-display/v12", "computer-display"), + ("urn:trycua:windows/v1", "trycua/windows"), + ("urn:agentenv:unversioned", "unversioned"), + ("urn:agentenv:mail/vnext", "mail/vnext"), +]) +def test_extension_slug_drops_the_agentenv_namespace_and_the_version(uri, slug): + assert _extension_slug(uri) == slug + + +def test_methods_sharing_a_uri_merge_into_one_card_entry_with_their_own_endpoints(): + extensions = _extensions(_card(_Display())) + display = extensions[_DISPLAY] + assert display.description == "Display resolution and scale." + assert display.params["modes"] == [[1280, 800], [1920, 1080]] and "endpoint" not in display.params + methods = display.params["methods"] + assert set(methods) == {"get", "set"} + assert methods["get"]["method"] == "GET" and methods["get"]["endpoint"] == "/agentenv/ext/computer-display/get" + assert methods["set"]["method"] == "POST" and methods["set"]["endpoint"] == "/agentenv/ext/computer-display/set" + assert methods["set"]["request"]["required"] == ["width", "height"] + assert extensions[_POINTER] == EnvironmentExtension(uri=_POINTER, description="Pointer tools.", params={"buttons": ["left"]}) + assert extensions["urn:agentenv:legacy/v1"].params["endpoint"] == "/agentenv/ext/legacy" + + +@pytest.mark.asyncio +async def test_each_method_is_served_at_its_endpoint_and_the_client_reaches_it_by_name(): + app = _Display().create_app().streamable_http_app() + async with _client(app) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + with patch("agentenv_protocol.client.httpx.AsyncClient", return_value=_client(app)): + changed = await client.invoke_extension("http://t", card, _DISPLAY, {"width": 1920, "height": 1080}, method="set") + got = await http.get("/agentenv/ext/computer-display/get") + legacy = await http.post("/agentenv/ext/legacy", json={}) + assert changed == {"width": 1920, "height": 1080, "scale": 1.0} + assert got.json() == {"width": 1920, "height": 1080, "scale": 1.0} and legacy.json() == {"legacy": True} + + +@pytest.mark.asyncio +async def test_an_explicit_path_is_the_methods_endpoint(): + class _Paths(AgentEnvEnvironment): + @extension(_DISPLAY, op="get", method="GET", path="/display") + async def get(self) -> dict: + return {"ok": True} + + env = _Paths() + assert _extensions(_card(env))[_DISPLAY].params["methods"]["get"]["endpoint"] == "/display" + async with _http(env) as http: + assert (await http.get("/display")).json() == {"ok": True} + + +def test_a_card_declared_extension_wins_over_the_merged_one(): + card = EnvironmentCard(name="d", capabilities=EnvironmentCapabilities(extensions=[EnvironmentExtension(uri=_DISPLAY)])) + assert [e for e in _card(_Display(), card).capabilities.extensions if e.uri == _DISPLAY] == [EnvironmentExtension(uri=_DISPLAY)] + + +def test_a_plain_handler_without_the_hooks_serves_methods_with_no_extra_params(): + class _Plain: + @extension(_DISPLAY, op="get") + async def get(self) -> dict: + return {} + + assert _extensions(_card(_Plain()))[_DISPLAY].params == { + "methods": {"get": {"method": "POST", "endpoint": "/agentenv/ext/computer-display/get", + "request": {"type": "object", "properties": {}}}}} + + +def test_params_are_refused_with_op(): + with pytest.raises(TypeError, match="extension_params"): + extension(_DISPLAY, op="get", params={"methods": {}}) + + +def test_two_handlers_for_one_method_are_refused(): + class _Dup: + @extension(_DISPLAY, op="get", path="/a") + async def a(self) -> dict: + return {} + + @extension(_DISPLAY, op="get", path="/b") + async def b(self) -> dict: + return {} + + with pytest.raises(ValueError, match="Multiple handlers for extension urn:agentenv:computer-display/v1 method get"): + _card(_Dup()) + + +@pytest.mark.parametrize("first, second", [("get", None), (None, "get"), (None, None)]) +def test_a_handler_without_op_shares_its_uri_with_no_other(first, second): + class _Mixed: + @extension(_DISPLAY, op=first, path="/a") + async def a(self) -> dict: + return {} + + @extension(_DISPLAY, op=second, path="/b") + async def b(self) -> dict: + return {} + + with pytest.raises(ValueError, match="Multiple handlers for extension urn:agentenv:computer-display/v1$"): + _card(_Mixed()) + + +def test_methods_disagreeing_on_description_are_refused(): + class _Described: + @extension(_DISPLAY, op="get", description="one") + async def a(self) -> dict: + return {} + + @extension(_DISPLAY, op="set", description="two") + async def b(self) -> dict: + return {} + + with pytest.raises(ValueError, match="declare different description values"): + _card(_Described()) + + +def test_an_extension_both_declared_and_served_is_refused(): + class _Both(AgentEnvEnvironment): + def declared_extensions(self) -> list[EnvironmentExtension]: + return [EnvironmentExtension(uri=_DISPLAY)] + + @extension(_DISPLAY, op="get") + async def a(self) -> dict: + return {} + + with pytest.raises(ValueError, match="both declared and served"): + _card(_Both()) + + +class _ReadOnly(_Display): + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + served = [] + for e in extensions: + if e.uri == _DISPLAY: + served.append(e.model_copy(update={"params": {**e.params, "methods": {"get": e.params["methods"]["get"]}}})) + elif e.uri != "urn:agentenv:legacy/v1": + served.append(e) + return served + + +@pytest.mark.asyncio +async def test_served_extensions_chooses_the_extensions_and_methods_served(): + env = _ReadOnly() + extensions = _extensions(_card(env)) + assert set(extensions[_DISPLAY].params["methods"]) == {"get"} and extensions[_DISPLAY].params["modes"] + assert "urn:agentenv:legacy/v1" not in extensions and _POINTER in extensions + async with _http(env) as http: + got = await http.get("/agentenv/ext/computer-display/get") + unset = await http.post("/agentenv/ext/computer-display/set", json={"width": 1920, "height": 1080}) + legacy = await http.post("/agentenv/ext/legacy", json={}) + assert got.json()["width"] == 1280 and unset.status_code == 404 and legacy.status_code == 404 + + +@pytest.mark.parametrize("chosen, error", [ + (lambda exts: [*exts, EnvironmentExtension(uri="urn:agentenv:ghost/v1")], + r"extensions with no @extension handler: \['urn:agentenv:ghost/v1'\]"), + (lambda exts: [e.model_copy(update={"params": {**e.params, "methods": {**e.params["methods"], "reset": {}}}}) + if e.uri == _DISPLAY else e for e in exts], + r"methods of urn:agentenv:computer-display/v1 with no @extension handler: \['reset'\]"), +]) +def test_served_extensions_naming_an_unhandled_extension_or_method_is_refused(chosen, error): + class _Bad(_Display): + def served_extensions(self, extensions): + return chosen(extensions) + + with pytest.raises(ValueError, match=error): + _card(_Bad()) diff --git a/packages/agentenv-protocol/tests/test_tool_access.py b/packages/agentenv-protocol/tests/test_tool_access.py new file mode 100644 index 00000000..ac04104a --- /dev/null +++ b/packages/agentenv-protocol/tests/test_tool_access.py @@ -0,0 +1,195 @@ +"""Per-role tool access with no gateway: tools/list filtered and tools/call refused per caller role on both FastMCPs, +disable-tool/v1 and enable-tool/v1 at the gateway's paths and bodies, and the rules' edge cases.""" +from __future__ import annotations + +import contextlib + +import anyio +import httpx +import pytest + +pytest.importorskip("mcp.server.fastmcp") +fastmcp = pytest.importorskip("fastmcp") + +from mcp import ClientSession # noqa: E402 +from mcp.client import streamable_http as _streamable_http # noqa: E402 +from mcp.server.fastmcp import FastMCP # noqa: E402 + +from agentenv_protocol import ( # noqa: E402 + ROLE_HEADER, + ROLE_META_KEY, + AgentEnvEnvironment, + AgentEnvFastMCPApplication, + EnvironmentCard, + environment_card, + tool, +) +from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI, ToolAccess # noqa: E402 + +pytestmark = pytest.mark.skipif(not hasattr(_streamable_http, "streamable_http_client"), + reason="mcp without streamable_http_client") + +_BASE = "http://127.0.0.1:8000" + + +@environment_card(name="desk") +class _Env(AgentEnvEnvironment): + tool_access = True + + @tool() + def read(self) -> str: + return "read" + + @tool() + def write(self) -> str: + return "write" + + @tool(hidden_by_default=True) + def grade(self) -> str: + return "grade" + + +@contextlib.asynccontextmanager +async def _served(): + """The env on mcp's FastMCP over streamable HTTP, and a way to open a session as a role.""" + app = FastMCP("desk", json_response=True) + _Env().mount(app) + asgi = app.streamable_http_app() + + @contextlib.asynccontextmanager + async def session(role: str | None = None): + headers = {ROLE_HEADER: role} if role else {} + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE, headers=headers) as http: + async with _streamable_http.streamable_http_client(f"{_BASE}/mcp", http_client=http) as (read, write, _): + async with ClientSession(read, write) as s: + await s.initialize() + yield s + + async with app.session_manager.run(): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE) as rest: + yield rest, session + + +async def _names(session) -> list: + return sorted(t.name for t in (await session.list_tools()).tools) + + +def test_the_card_advertises_both_extensions_at_the_gateways_paths(): + card = AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Env()).environment_card + extensions = {e.uri: e for e in card.capabilities.extensions} + assert extensions[DISABLE_TOOL_EXTENSION_URI].params["endpoint"] == "/tools/disable" + assert extensions[DISABLE_TOOL_EXTENSION_URI].params["methods"]["disable"]["method"] == "POST" + assert extensions[ENABLE_TOOL_EXTENSION_URI].params["methods"]["enable"]["request"]["required"] == ["role", "tools"] + + +def test_an_env_without_tool_access_advertises_neither(): + class _Plain(_Env): + tool_access = False + + card = AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Plain()).environment_card + assert not [e for e in (card.capabilities.extensions or []) if "tool/v1" in e.uri] + + +@pytest.mark.asyncio +async def test_roles_see_and_call_only_what_their_rules_allow(): + with anyio.fail_after(30): + async with _served() as (rest, session): + async with session() as anyone: + assert await _names(anyone) == ["read", "write"] + hidden = await anyone.call_tool("grade", {}) + changed = await rest.post("/tools/disable", json={"role": "agent", "tools": ["write"]}) + shown = await rest.post("/tools/enable", json={"role": "grader", "tools": ["grade"]}) + async with session("agent") as agent: + agent_tools = await _names(agent) + refused = await agent.call_tool("write", {}) + allowed = await agent.call_tool("read", {}) + forwarded = await agent.call_tool("grade", {}, meta={ROLE_META_KEY: "grader"}) + async with session("grader") as grader: + grader_tools = await _names(grader) + async with session("default") as default: + default_tools = await _names(default) + assert hidden.isError and hidden.structuredContent["code"] == "permission_denied" + assert changed.json() == {"role": "agent", "disabled": ["write"], "allowed": []} + assert shown.json() == {"role": "grader", "disabled": [], "allowed": ["grade"]} + assert agent_tools == ["read"] + assert refused.isError and refused.content[0].text == "permission_denied: Tool 'write' is disabled for role 'agent'" + assert refused.structuredContent["applied"] == "no" and not allowed.isError + assert not forwarded.isError and forwarded.content[0].text == "grade", "a gateway-stamped role wins over the header" + assert grader_tools == ["grade", "read", "write"] and default_tools == ["read", "write"] + + +@pytest.mark.asyncio +async def test_everything_off_then_one_tool_on_for_one_role(): + with anyio.fail_after(30): + async with _served() as (rest, session): + await rest.post("/tools/disable", json={"role": "*", "tools": "*"}) + await rest.post("/tools/enable", json={"role": "grader", "tools": ["read"]}) + async with session("agent") as agent: + agent_tools = await _names(agent) + async with session("grader") as grader: + grader_tools = await _names(grader) + assert agent_tools == [] and grader_tools == ["read"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"tools": ["read"]}, + {"role": "", "tools": ["read"]}, + {"role": "agent", "tools": "read"}, + {"role": "agent", "tools": ["read", 3]}, + ["not", "an", "object"], +]) +async def test_a_malformed_request_is_refused_and_changes_nothing(body): + with anyio.fail_after(30): + async with _served() as (rest, session): + r = await rest.post("/tools/disable", json=body) + garbled = await rest.post("/tools/disable", content=b"{not json") + async with session("agent") as agent: + tools = await _names(agent) + assert r.status_code == 400 and r.json()["error"]["code"] == "invalid_params" + assert garbled.status_code == 400 and tools == ["read", "write"] + + +@pytest.mark.asyncio +async def test_roles_are_honoured_on_fastmcp_too(): + app = fastmcp.FastMCP("desk") + _Env().mount(app) + asgi = app.http_app(path="/mcp", json_response=True) + with anyio.fail_after(30): + async with asgi.router.lifespan_context(asgi): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE) as rest: + await rest.post("/tools/disable", json={"role": "agent", "tools": ["write"]}) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url=_BASE, + headers={ROLE_HEADER: "agent"}) as http: + async with _streamable_http.streamable_http_client(f"{_BASE}/mcp", http_client=http) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + tools = await _names(session) + refused = await session.call_tool("write", {}) + assert tools == ["read"] + assert refused.isError and refused.structuredContent["code"] == "permission_denied" + + +def test_tool_access_needs_a_dispatch_to_read_the_caller_from(): + class _Shapeless: + def tool(self, name=None, description=None, **kwargs): + return lambda fn: fn + + def custom_route(self, path, methods): + return lambda fn: fn + + with pytest.raises(RuntimeError, match="tool_access would read an empty caller"): + AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Env()).add_routes_to_app(_Shapeless()) + + +def test_a_rule_for_every_role_reaches_roles_that_already_have_rules(): + access = ToolAccess(hidden_by_default=["grade"]) + access.apply("agent", "read", True) + access.apply("*", "write", True) + access.apply("*", "grade", False) + assert access.rules == {"agent": {"read": True, "write": True, "grade": False}, "*": {"write": True, "grade": False}} + assert access.is_disabled("agent", "grade") is False and access.is_disabled("other", "write") is True + access.apply("agent", "*", False) + assert access.rules["agent"] == {"*": False}, "a rule for every tool of a role replaces that role's named rules" + assert access.is_disabled("agent", "write") is False + assert access.is_disabled("agent", "grade") is False, "a hidden tool skips the role's '*' rule; the every-role rule naming it decides" diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py new file mode 100644 index 00000000..feeb1c58 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -0,0 +1,211 @@ +"""@tool annotations, structured output and _meta on both FastMCPs, served_tools choosing what a handler serves, and\nupdate_tool changing a served description.""" +from __future__ import annotations + +from typing import Annotated + +import anyio +import httpx +import pytest +from pydantic import BaseModel + +pytest.importorskip("mcp.server.fastmcp") +fastmcp = pytest.importorskip("fastmcp") + +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 +from mcp.types import CallToolResult, ImageContent, TextContent # noqa: E402 + +from agentenv_protocol import ( # noqa: E402 + HIDDEN_BY_DEFAULT_META_KEY, + AgentEnvEnvironment, + AgentEnvFastMCPApplication, + EnvironmentCard, + EnvironmentTool, + environment_card, + tool, +) + + +class _Frame(BaseModel): + frame_id: str + width: int + + +@environment_card(name="tools") +class _Env(AgentEnvEnvironment): + @tool(annotations={"readOnlyHint": True, "title": "Look"}, meta={"agentenv.io/group": "minimum"}) + def look(self) -> _Frame: + return _Frame(frame_id="f1", width=1280) + + @tool(structured_output=False) + def plain(self) -> dict: + return {"a": 1} + + @tool(hidden_by_default=True, meta={"agentenv.io/group": "admin"}) + def secret(self) -> str: + return "s" + + @tool() + def shot(self) -> Annotated[CallToolResult, _Frame]: + return CallToolResult( + content=[TextContent(type="text", text="Captured f2"), ImageContent(type="image", data="aGk=", mimeType="image/png")], + structuredContent={"frame_id": "f2", "width": 640}, + ) + + +async def _mcp_tools(env: AgentEnvEnvironment) -> dict: + with anyio.fail_after(30): + async with create_connected_server_and_client_session(env.create_app()) as session: + return {t.name: t for t in (await session.list_tools()).tools} + + +@pytest.mark.asyncio +async def test_annotations_meta_and_output_schemas_are_listed_on_mcps_fastmcp(): + tools = await _mcp_tools(_Env()) + look = tools["look"] + assert look.annotations.readOnlyHint is True and look.annotations.title == "Look" + assert look.meta == {"agentenv.io/group": "minimum"} + assert look.outputSchema["properties"]["frame_id"] == {"title": "Frame Id", "type": "string"} + assert tools["plain"].outputSchema is None + assert tools["secret"].meta == {"agentenv.io/group": "admin", HIDDEN_BY_DEFAULT_META_KEY: True} + assert tools["shot"].outputSchema["required"] == ["frame_id", "width"] + + +@pytest.mark.asyncio +async def test_structured_results_reach_the_caller_with_their_content_blocks(): + with anyio.fail_after(30): + async with create_connected_server_and_client_session(_Env().create_app()) as session: + look = await session.call_tool("look", {}) + shot = await session.call_tool("shot", {}) + assert look.structuredContent == {"frame_id": "f1", "width": 1280} + assert [c.type for c in shot.content] == ["text", "image"] and shot.structuredContent == {"frame_id": "f2", "width": 640} + + +@pytest.mark.asyncio +async def test_annotations_meta_and_output_schemas_are_listed_on_fastmcp(): + app = fastmcp.FastMCP("tools") + _Env().mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + tools = {t.name: t for t in await client.list_tools()} + look = await client.call_tool("look", {}) + shot = await client.call_tool("shot", {}) + assert tools["look"].annotations.readOnlyHint is True + assert tools["look"].meta["agentenv.io/group"] == "minimum" + assert tools["look"].outputSchema["properties"]["frame_id"]["type"] == "string" + assert tools["plain"].outputSchema is None + assert tools["secret"].meta[HIDDEN_BY_DEFAULT_META_KEY] is True + assert tools["shot"].outputSchema["required"] == ["frame_id", "width"] + assert look.structured_content == {"frame_id": "f1", "width": 1280} + assert [c.type for c in shot.content] == ["text", "image"] and shot.structured_content == {"frame_id": "f2", "width": 640} + + +@pytest.mark.asyncio +async def test_a_bare_call_tool_result_on_fastmcp_lists_no_output_schema(): + @environment_card(name="bare") + class _Bare(AgentEnvEnvironment): + @tool() + async def bare(self) -> CallToolResult: + return CallToolResult(content=[TextContent(type="text", text="done")]) + + app = fastmcp.FastMCP("bare") + _Bare().mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + listed = (await client.list_tools())[0] + result = await client.call_tool("bare", {}) + assert listed.outputSchema is None and result.content[0].text == "done" + + +class _Chooser(_Env): + def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: + return [t.model_copy(update={"description": "Take a look."}) if t.name == "look" else t + for t in tools if t.name != "secret"] + + +@pytest.mark.asyncio +async def test_served_tools_chooses_what_is_listed_and_advertised(): + env = _Chooser() + tools = await _mcp_tools(env) + assert set(tools) == {"look", "plain", "shot"} and tools["look"].description == "Take a look." + advertised = {t.name: t for t in AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Chooser()).environment_card.capabilities.tools} + assert set(advertised) == {"look", "plain", "shot"} and advertised["look"].description == "Take a look." + + +@pytest.mark.parametrize("chosen, error", [ + (lambda tools: [*tools, EnvironmentTool(name="ghost")], r"no @tool method: \['ghost'\]"), + (lambda tools: [*tools, tools[0]], "a tool twice"), +]) +def test_served_tools_naming_an_unmarked_tool_or_one_twice_is_refused(chosen, error): + class _Bad(_Env): + def served_tools(self, tools): + return chosen(tools) + + with pytest.raises(ValueError, match=error): + AgentEnvFastMCPApplication(EnvironmentCard(name="t"), _Bad()) + + +@pytest.mark.asyncio +async def test_structured_output_false_lists_no_schema_for_a_call_tool_result_on_fastmcp(): + @environment_card(name="opted-out") + class _OptedOut(AgentEnvEnvironment): + @tool(structured_output=False) + def shot(self) -> Annotated[CallToolResult, _Frame]: + return CallToolResult(content=[TextContent(type="text", text="ok")], structuredContent={"frame_id": "f1", "width": 1}) + + app = fastmcp.FastMCP("opted-out") + _OptedOut().mount(app) + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + listed = (await client.list_tools())[0] + result = await client.call_tool("shot", {}) + assert listed.outputSchema is None and result.content[0].text == "ok" + + +async def _card_tools(app) -> dict: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as http: + card = (await http.get("/.well-known/agent-env.json")).json() + return {t["name"]: t for t in card["capabilities"]["tools"]} + + +@pytest.mark.asyncio +async def test_update_tool_rewrites_a_description_on_the_card_and_in_tools_list_on_mcps_fastmcp(): + env = _Env() + app = env.create_app() + await env.update_tool("look", description="Look again.") + with anyio.fail_after(30): + async with create_connected_server_and_client_session(app) as session: + tools = {t.name: t for t in (await session.list_tools()).tools} + look = await session.call_tool("look", {}) + card = await _card_tools(app.streamable_http_app()) + assert tools["look"].description == card["look"]["description"] == "Look again." + assert set(card) == set(tools) == {"look", "plain", "secret", "shot"} + assert {t.name: t.description for t in env.mounted_card().capabilities.tools}["look"] == "Look again." + assert look.structuredContent == {"frame_id": "f1", "width": 1280} + + +@pytest.mark.asyncio +async def test_update_tool_rewrites_a_description_on_the_card_and_in_tools_list_on_fastmcp(): + env = _Env() + app = fastmcp.FastMCP("tools") + env.mount(app) + await env.update_tool("look", description="Look again.") + with anyio.fail_after(30): + async with fastmcp.Client(app) as client: + tools = {t.name: t for t in await client.list_tools()} + look = await client.call_tool("look", {}) + card = await _card_tools(app.http_app()) + assert tools["look"].description == card["look"]["description"] == "Look again." + assert tools["look"].annotations.readOnlyHint is True and tools["look"].meta["agentenv.io/group"] == "minimum" + assert look.structured_content == {"frame_id": "f1", "width": 1280} + + +@pytest.mark.asyncio +async def test_update_tool_needs_a_mounted_env_and_a_served_tool(): + with pytest.raises(RuntimeError, match="_Env is not mounted"): + await _Env().update_tool("look", description="x") + with pytest.raises(RuntimeError, match="_Env is not mounted"): + _Env().mounted_card() + env = _Chooser() + env.create_app() + with pytest.raises(ValueError, match="no served tool is named 'secret'"): + await env.update_tool("secret", description="x") diff --git a/pyproject.toml b/pyproject.toml index 2bfb4c35..da24f81d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ daytona = [ dev = [ "agentenv-framework[aws,boxd,daytona,gcp,sail,tensorlake,tenki,vercel]", # exercise portable optional providers and stores in CI "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x + "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs "griffelib==2.3.0", # the plugin API check; exact, since what it reports as a break changes between releases "tomli-w==1.2.0", # the Codex agent's config writer, which only its image installs; its tests import it "moto>=5.0.0", diff --git a/tst/unit/env/gateway/tool_access_parity_test.py b/tst/unit/env/gateway/tool_access_parity_test.py new file mode 100644 index 00000000..f8131b61 --- /dev/null +++ b/tst/unit/env/gateway/tool_access_parity_test.py @@ -0,0 +1,71 @@ +"""The gateway and an env serving tool access itself (agentenv_protocol.tool_access) keep separate code, since the +gateway's image installs no protocol; the same rule cases run against both so a task's tool-access steps mean the +same with or without a gateway in front.""" +from __future__ import annotations + +import random + +import pytest +from agentenv_protocol.tool_access import ToolAccess + +from agent_env.env.gateway.gateway import Gateway + +ROLES = ("default", "grader", "sim") +TOOLS = ("h", "n") +HIDDEN = {"h"} + + +def _both() -> tuple[Gateway, ToolAccess]: + gateway = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[]) + gateway._hidden_by_default = set(HIDDEN) + return gateway, ToolAccess(hidden_by_default=HIDDEN) + + +def _assert_same(gateway: Gateway, access: ToolAccess, steps: list) -> None: + assert access.rules == gateway._role_rules, steps + for role in ROLES + ("*", "never-mentioned"): + for tool in TOOLS + ("unlisted",): + assert access.is_disabled(role, tool) is gateway._is_disabled(role, tool), (steps, role, tool) + + +def test_random_step_sequences_leave_both_with_the_same_rules_and_answers(): + rng = random.Random(20261026) + for _ in range(5000): + gateway, access = _both() + steps = [(rng.choice(ROLES + ("*",)), rng.choice(TOOLS + ("*",)), rng.random() < 0.5) + for _ in range(rng.randint(0, 10))] + for step in steps: + gateway._apply_rule(*step) + access.apply(*step) + _assert_same(gateway, access, steps) + + +@pytest.mark.parametrize("steps", [ + [], + [("*", "*", False)], + [("*", "*", True), ("grader", "n", False)], + [("sim", "*", True), ("sim", "h", False)], + [("sim", "h", False), ("sim", "*", True)], + [("*", "h", False), ("default", "h", True)], + [("default", "n", True), ("*", "n", False)], + [("grader", "*", True), ("*", "*", False), ("grader", "h", False)], +]) +def test_the_steps_a_task_writes(steps): + gateway, access = _both() + for step in steps: + gateway._apply_rule(*step) + access.apply(*step) + _assert_same(gateway, access, steps) + + +@pytest.mark.parametrize("action, body", [ + ("disable", {"role": "sim", "tools": ["n", "h"]}), + ("enable", {"role": "sim", "tools": "*"}), + ("disable", {"role": "*", "tools": ["n"]}), +]) +def test_a_change_answers_with_the_gateways_body(action, body): + gateway, access = _both() + for tool in body["tools"] if isinstance(body["tools"], list) else ["*"]: + gateway._apply_rule(body["role"], tool, action == "disable") + expected = {"role": body["role"], **gateway._format_role_rules(gateway._role_rules.get(body["role"], {}))} + assert access.change(action, body) == expected diff --git a/uv.lock b/uv.lock index 12828ff8..82db1109 100644 --- a/uv.lock +++ b/uv.lock @@ -92,6 +92,7 @@ dev = [ { name = "griffelib" }, { name = "httpx2" }, { name = "moto" }, + { name = "pillow" }, { name = "psycopg2-binary" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -160,6 +161,7 @@ requires-dist = [ { name = "moto", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "openenv", specifier = ">=0.9,<0.11" }, { name = "packaging", specifier = ">=22" }, + { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, { name = "psycopg2-binary", marker = "extra == 'dev'", specifier = ">=2.9.0" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pymongo", specifier = ">=4.7" }, @@ -202,10 +204,16 @@ agent = [ { name = "regex" }, { name = "uvicorn" }, ] +computer = [ + { name = "mcp" }, + { name = "pillow" }, + { name = "uvicorn" }, +] dev = [ { name = "a2a-sdk", extra = ["http-server"] }, { name = "fastmcp" }, { name = "mcp" }, + { name = "pillow" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "python-multipart" }, @@ -219,7 +227,10 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'dev'", specifier = ">=3.4,<4" }, { name = "httpx", specifier = ">=0.27" }, { name = "mcp", marker = "extra == 'agent'", specifier = ">=1.25,<2" }, + { name = "mcp", marker = "extra == 'computer'", specifier = ">=1.25,<2" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.25,<2" }, + { name = "pillow", marker = "extra == 'computer'", specifier = ">=10.4" }, + { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, { name = "pydantic", specifier = ">=2.11" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, @@ -229,8 +240,9 @@ requires-dist = [ { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, { name = "starlette", specifier = ">=0.37" }, { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, + { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "dev"] +provides-extras = ["agent", "computer", "dev"] [[package]] name = "aiofile"