From 4e04198a793be7650d0ccf545a89019e48ad8f16 Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:12:45 +0800 Subject: [PATCH 1/3] feat(trajectory): expose captured LoopX tool context in research usage Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- .../goal-research-progress.tsx | 3 +- .../personal-workspace/goal-tool-context.tsx | 31 +++++ docs/reference/trajectory-assessment.md | 30 +++++ examples/personal-workspace-browser-smoke.mjs | 2 + .../research-history-backend.py | 3 + .../research-tool-context.mjs | 110 ++++++++++++++++ .../trajectory_assessment_store.py | 10 ++ .../trajectory_tool_context.py | 118 ++++++++++++++++++ loopx/cli_commands/benchmark_trajectory.py | 5 + .../trajectory_assessment_types.ts | 9 ++ .../capabilities/trajectory_codex_usage.ts | 3 + .../capabilities/trajectory_tool_context.ts | 72 +++++++++++ .../test_trajectory_input_capture.py | 24 +++- .../test_trajectory_tool_context.py | 71 +++++++++++ .../trajectory_codex_usage.test.ts | 7 ++ .../trajectory_tool_context.test.ts | 48 +++++++ 16 files changed, 544 insertions(+), 2 deletions(-) create mode 100644 apps/presentation/dashboard/src/features/personal-workspace/goal-tool-context.tsx create mode 100644 examples/personal-workspace-browser/research-tool-context.mjs create mode 100644 loopx/capabilities/benchmark_toolkit/trajectory_tool_context.py create mode 100644 loopx/control_plane/capabilities/trajectory_tool_context.ts create mode 100644 tests/capabilities/test_trajectory_tool_context.py create mode 100644 tests/control_plane_ts/trajectory_tool_context.test.ts diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx index ddcfccdaf3..80e117af02 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx @@ -5,6 +5,7 @@ import type {AssessmentCitation, AssessmentVerdict, TrajectoryAssessment, Trajec import type {TrajectoryNativeOutcomes} from "../../../../../../loopx/control_plane/capabilities/trajectory_native_outcomes.js"; import {ContextHelp} from "./context-help"; import {ResearchSynthesis} from "./goal-research-synthesis"; +import {ToolContextInventory} from "./goal-tool-context"; import "./goal-research-progress.css"; // Read models from benchmark-toolkit. This view formats existing counts and @@ -306,7 +307,7 @@ export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCi

{zh ? "其中输入" : "Of which input"} {incomplete.usage ? number(incomplete.usage.input_tokens) : "—"} · {zh ? "输出" : "output"} {incomplete.usage ? number(incomplete.usage.output_tokens) : "—"} tokens

: null} -
+
{usage ?

{zh ? "请求覆盖" : "Request coverage"}

{coverage?.status !== "verified" ? {coverage ? (zh ? "核对失败" : "Reconciliation failed") : (zh ? "未核对" : "Not reconciled")} : null} diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-tool-context.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-tool-context.tsx new file mode 100644 index 0000000000..d2ec0a010e --- /dev/null +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-tool-context.tsx @@ -0,0 +1,31 @@ +import type {TrajectoryToolContext} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js"; +import {ContextHelp} from "./context-help"; + +/** Formats the toolkit observation; never classifies commands or infers savings. */ +export function ToolContextInventory({value, zh}: {value: TrajectoryToolContext | undefined; zh: boolean}) { + if (!value) return null; + const number = (n: number) => n.toLocaleString(zh ? "zh-CN" : "en"); + const fraction = value.status === "observed" ? value.loopx_tool_output_fraction : null; + const percent = fraction === null ? "—" : fraction > 0 && fraction < 0.001 ? "<0.1%" + : fraction < 1 && fraction > 0.999 ? ">99.9%" + : new Intl.NumberFormat(zh ? "zh-CN" : "en", {style: "percent", maximumFractionDigits: 1}).format(fraction); + return
+

{zh ? "LoopX 工具返回 · 字节" : "LoopX tool returns · bytes"}

+ +

{zh ? "分母是捕获中追加的用户、系统、开发者文本及工具返回;分子仅识别简单 LoopX CLI 调用的返回。复合脚本、间接调用、文件读取和其他内容保持未归因,图片等非文本内容不计字节。" : "The denominator is appended user, system, developer text and tool returns. Only returns from simple LoopX CLI calls are assigned. Compound scripts, indirect calls, file reads and other content remain unassigned; non-text content such as images has no byte count."}

+

{zh ? "这是已记录内容的组成,不是逐请求重放的上下文或计费 token 占比。重复内容不等于无益重复,LoopX 返回也可能携带有用研究证据;不能据此推断净成本或提分。" : "This describes recorded content, not per-request replay or billable token share. Repeated content need not be waste; LoopX returns can carry useful research evidence. It does not establish net cost or score benefit."}

+
+ {value.status === "unavailable" ?

{zh ? "捕获未通过核对 · 占比未知" : "Capture could not be reconciled · share unknown"}

: <> +

{percent}{zh ? "占已记录输入文本" : "of recorded input text"}

+
{zh ? "返回明细" : "Return details"}
+
{zh ? "LoopX 工具返回" : "LoopX tool returns"}
{number(value.loopx_tool_output_bytes)} B
+
{zh ? "已记录输入文本" : "Recorded input text"}
{number(value.observed_input_text_bytes)} B
+
{zh ? "未归因文本" : "Unassigned text"}
{number(value.unassigned_input_text_bytes)} B
+
{zh ? "未计量内容块" : "Unmeasured content parts"}
{number(value.unmeasured_parts)}
+
{zh ? "重复相同返回" : "Repeated identical returns"}
{number(value.repeated_loopx_output_bytes)} B
+
{zh ? "已去重返回记录" : "Duplicate return records excluded"}
{number(value.duplicate_output_records)}
+ {value.operations.map(row =>
{row.operation}
{number(row.calls)} {zh ? "次" : "calls"} · {number(row.bytes)} B
)} +
+ } +
; +} diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index 2c45828d5f..b0e9b2c057 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -1064,6 +1064,36 @@ values with the existing warning and never borrow another pack's measurement. 不能反推 packet token;未采集、部分覆盖与真实零分开。比例是结构开销,不能单独 当作净效率损失、可避免成本或因果贡献。普通阅读不调用模型。 +### Captured tool context inventory + +Add `--include-tool-context` to `trajectory-pack-register` with the existing +`--usage-jsonl` and `--usage-session-id` to measure appended input text. The +Research workbench's existing Sources and usage disclosure displays the saved +inventory beside request-token attribution; browsing never launches a model. +Omitting the option preserves historical registration and projections. + +The numerator is the UTF-8 text returned by recognized simple LoopX CLI calls; +the denominator includes captured user/system/developer text and tool returns. +Native `response_item` records are used, not their event-message mirrors. +Duplicate output records for one call count once. Identical output from two +different calls is still two pieces of appended input; it is not automatically +redundant research. Operation counts distinguish context reads/projects from +other LoopX returns. No output or command argument text is sent to the browser. + +Only literal simple shell argv, Python `-m loopx[.entrypoint]`, and an exact +single literal `const/await tools.exec_command/text` wrapper are recognized. +Compound shell, arbitrary JavaScript, indirect commands, continuation tools and +file reads remain unassigned. Non-text content is counted as unmeasured parts. +The byte share covers the recorded component only: injected LoopX instructions +can remain unassigned, and useful domain evidence may be present in LoopX +returns. It is neither whole control-plane overhead nor per-request replay, +billable tokens, avoidable cost, or causal benefit. Keep the separate Turn-packet +estimate and provider counters; do not add their different units or denominators. + +中文:可选统计在原轨迹分析工作台中展示 LoopX 工具返回占已记录输入文本的字节 +比例,并拆出 context-read/project 等操作。未知和非文本内容分别保留;同内容的 +再次返回不等于无益重复。这不是全量控制面、逐请求重放或计费 token 的占比。 + ### Recorded Turn packet estimate The existing native `codex-cli` Turn host can record its actual injected packet diff --git a/examples/personal-workspace-browser-smoke.mjs b/examples/personal-workspace-browser-smoke.mjs index 27be8b805c..120acecbca 100644 --- a/examples/personal-workspace-browser-smoke.mjs +++ b/examples/personal-workspace-browser-smoke.mjs @@ -4,6 +4,7 @@ import {reportDownloadScenario} from "./personal-workspace-browser/report-downlo import { selectScenarioShard } from "./personal-workspace-browser/shard.mjs"; import {completedHistoryIsolationScenario} from "./personal-workspace-browser/completed-history-isolation.mjs"; import {researchWorkerReadbackScenario} from "./personal-workspace-browser/research-worker-readback.mjs"; +import {researchToolContextScenario} from "./personal-workspace-browser/research-tool-context.mjs"; import { zcodeGoalScenario } from "./personal-workspace-browser/zcode-goal.mjs"; import {goalWorkspaceCreateScenario} from "./personal-workspace-browser/goal-workspace-create.mjs"; import {agentModelSelectionScenario} from "./personal-workspace-browser/agent-model-selection.mjs"; @@ -132,6 +133,7 @@ scenarioCatalog.push(attentionUserActionScenario); scenarioCatalog.push(researchResultsScenario); scenarioCatalog.push(researchHistoryScenario); scenarioCatalog.push(researchMeasurementsScenario); +scenarioCatalog.push(researchToolContextScenario); scenarioCatalog.push(researchNativeReadingScenario); scenarioCatalog.push(researchFindingReaderScenario); scenarioCatalog.push(researchRecordSearchScenario); diff --git a/examples/personal-workspace-browser/research-history-backend.py b/examples/personal-workspace-browser/research-history-backend.py index e3661a0b65..b4ad045ff1 100644 --- a/examples/personal-workspace-browser/research-history-backend.py +++ b/examples/personal-workspace-browser/research-history-backend.py @@ -103,6 +103,9 @@ def main(): scope = dict(runtime_root=runtime, owner_goal_id=capture["goal_id"]) registered = register_pack(**scope, input_path=input_path, source_root=source_root, title=analysis["title"], config=analysis.get("config", fixture_config), + usage_jsonl=analysis.get("usage_jsonl"), + usage_session_id=analysis.get("usage_session_id"), + include_tool_context=analysis.get("include_tool_context", False), native_outcomes_json=analysis.get("native_outcomes_json")) for language, output in analysis["outputs"].items(): provider = write_python_host_fixture(root / f"analysis-provider-{index}-{language}", diff --git a/examples/personal-workspace-browser/research-tool-context.mjs b/examples/personal-workspace-browser/research-tool-context.mjs new file mode 100644 index 0000000000..22c6b3fd86 --- /dev/null +++ b/examples/personal-workspace-browser/research-tool-context.mjs @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import {execFileSync} from "node:child_process"; +import {mkdtemp, rm} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {resolve} from "node:path"; +import {resolveTestPython} from "../../scripts/test-python.mjs"; +import {outputDir, repoRoot} from "./fixture.mjs"; +import {startHistoryOwner} from "./research-history.mjs"; + +// Real source verification, TS reduction, pack registry, HTTP and packaged UI. +// The fixture has no model outputs: ordinary metric reading needs no analysis. +export const researchToolContextScenario = { + id: "research-tool-context", + async run({browser}) { + const temporary = await mkdtemp(resolve(tmpdir(), "loopx-tool-context-")); + let owner, page; + try { + const fixture = resolve(temporary, "capture.json"); + execFileSync(resolveTestPython(), ["-c", ` +import hashlib,json,sys +from pathlib import Path +from loopx.capabilities.explore.result_log import build_explore_node_event +goal='synthetic-context-reading' +text='Synthetic research observation; utility remains unknown.' +scope=dict(goal_id=goal,case_id='synthetic',run_id='fixed',revision='fixture',window='fixed capture') +pack=dict(schema_version='trajectory_assessment_input_v1',scope=scope,objective='Compare research outcomes and complete costs', + selection=dict(rule='one synthetic episode',population_episodes=1,exhaustive=True), + sources=[dict(id='facts',artifact='facts.txt',artifact_sha256=hashlib.sha256(text.encode()).hexdigest(),first_line=1,text=text)], + episodes=[dict(id='episode',question='Did new evidence change the next experiment?',source_ids=['facts'],eligible_metrics=['evidence_adoption'])]) +usage=dict(input_tokens=100,cached_input_tokens=60,output_tokens=5) +rows=[dict(type='session_meta',payload=dict(id='session')), + dict(type='event_msg',payload=dict(type='task_started',turn_id='turn')), + dict(type='response_item',payload=dict(type='message',role='user',content=[dict(type='input_text',text='x'*100)])), + dict(type='response_item',payload=dict(type='function_call',name='exec_command',call_id='control',arguments=json.dumps(dict(cmd='loopx quota context-read')))), + dict(type='response_item',payload=dict(type='function_call_output',call_id='control',output='x'*60)), + dict(type='response_item',payload=dict(type='function_call_output',call_id='task',output='x'*40)), + dict(type='event_msg',payload=dict(type='token_count',info=dict(total_token_usage=usage,last_token_usage=usage))), + dict(type='event_msg',payload=dict(type='task_complete',turn_id='turn'))] +for i,row in enumerate(rows):row['timestamp']=f'2026-01-01T00:00:0{i}Z' +analyses=[] +for enabled in [False,True]: + analyses.append(dict(title='Captured tool context' if enabled else 'Without tool inventory',input=pack, + artifacts={'facts.txt':text,'usage.jsonl':'\\n'.join(map(json.dumps,rows))},outputs={}, + usage_jsonl='usage.jsonl',usage_session_id='session',include_tool_context=enabled)) +broken=json.loads(json.dumps(analyses[-1])) +broken['title']='Conflicting tool capture' +conflicting=rows[:5]+[dict(type='response_item',payload=dict(type='function_call_output',call_id='control',output='different'))]+rows[5:] +broken['artifacts']['usage.jsonl']='\\n'.join(map(json.dumps,conflicting)) +analyses.append(broken) +Path(sys.argv[1]).write_text(json.dumps(dict(goal_id=goal,display_name='Synthetic context cost study', + events=[build_explore_node_event(goal_id=goal,node_id='question',title='Synthetic research question',recorded_at='2026-01-01T00:00:00Z')],analyses=analyses))) +`, fixture], {cwd: repoRoot}); + owner = await startHistoryOwner(fixture); + page = await browser.newPage({viewport: {width: 1440, height: 900}}); + const errors = []; + page.on("pageerror", error => errors.push(error.message)); + await page.goto(`${owner.base}/chat/?goalId=synthetic-context-reading&view=research&research_goal=synthetic-context-reading&statusUrl=%2Fstatus.json`, {waitUntil: "networkidle"}); + const progress = page.locator('.research-progress'); + const selector = progress.locator('select'); + await selector.selectOption({label: 'Without tool inventory'}); + await progress.locator('.research-usage').waitFor({state: 'attached'}); + await page.screenshot({path: resolve(outputDir, 'research-tool-context-before.png'), animations: 'disabled'}); + assert.equal(await progress.locator('[data-loopx-tool-context-share]').count(), 0); + await selector.selectOption({label: 'Captured tool context'}); + const basis = progress.locator('.research-progress-basis'); + await basis.locator(':scope > summary').focus(); await page.keyboard.press('Enter'); + const share = progress.locator('[data-loopx-tool-context-share]'); + await share.waitFor({state: 'visible'}); + assert.equal(await share.textContent(), '30%'); + const inventory = progress.getByRole('region', {name: 'LoopX 工具上下文', exact: true}); + const details = inventory.locator('details'); + await details.locator('summary').focus(); await page.keyboard.press('Enter'); + assert.match(await details.innerText(), /60 B/); + assert.match(await details.innerText(), /200 B/); + assert.match(await details.innerText(), /quota context-read/); + for (const width of [1440, 390]) { + await page.setViewportSize({width, height: 900}); + await inventory.scrollIntoViewIfNeeded(); + assert(await inventory.evaluate(node => node.scrollWidth <= node.clientWidth + 1)); + await page.screenshot({path: resolve(outputDir, `research-tool-context-${width}.png`), animations: 'disabled'}); + } + await page.evaluate(() => localStorage.setItem('loopx-pw-locale', 'en')); + await page.reload({waitUntil: 'networkidle'}); + const english = progress.getByRole('region', {name: 'LoopX tool context', exact: true}); + await share.waitFor({state: 'attached'}); + if (!await progress.locator('.research-progress-basis').evaluate(node => node.open)) { + await progress.locator('.research-progress-basis > summary').focus(); await page.keyboard.press('Enter'); + } + await english.waitFor({state: 'visible'}); + assert.equal(await share.textContent(), '30%'); + await selector.selectOption({label: 'Conflicting tool capture'}); + const unavailable = progress.getByRole('region', {name: 'LoopX tool context', exact: true, includeHidden: true}); + await unavailable.locator('[role="status"]').waitFor({state: 'attached'}); + if (!await progress.locator('.research-progress-basis').evaluate(node => node.open)) { + await progress.locator('.research-progress-basis > summary').focus(); await page.keyboard.press('Enter'); + } + assert.match(await unavailable.innerText(), /share unknown/); + assert.equal(await progress.locator('[data-loopx-tool-context-share]').count(), 0); + const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json(); + assert(readback.source_unchanged && readback.mutation_attempts === 0); + assert.deepEqual(errors, []); + return {coverageEntries: [], note: 'real-owner byte inventory, opt-out parity, unknown analysis, keyboard, desktop/phone and English; zero model calls'}; + } catch (error) { + if (page && !page.isClosed()) await page.screenshot({path: resolve(outputDir, 'research-tool-context-failed.png')}); + throw error; + } finally { + await page?.close(); await owner?.close(); await rm(temporary, {recursive: true, force: true}); + } + }, +}; diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py index 3abf690a69..6d41457d2e 100644 --- a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py +++ b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py @@ -136,17 +136,22 @@ def _usage_capture( session_id: str, *, include_incomplete: bool = False, + include_tool_context: bool = False, loopx_input_json: str | None = None, input_goal_id: str | None = None, ) -> dict: data = _source_path(source_root, artifact).read_bytes() events = [] + tool_context = [] # Transport only the provider's counter/lifecycle fields, not model text, # account metadata or tool outputs. TS owns all accounting decisions. for line_number, line in enumerate(data.decode("utf-8").splitlines(), 1): event = json.loads(line) if not isinstance(event, dict): raise ValueError("Usage capture contains a non-object event") + if include_tool_context: + from .trajectory_tool_context import project_tool_context + tool_context.extend(project_tool_context([event], first_line=line_number)) payload = event.get("payload") or {} if not isinstance(payload, dict): raise ValueError("Usage capture contains a non-object payload") @@ -229,6 +234,7 @@ def _usage_capture( "events": events, "session_id": session_id, "include_incomplete": include_incomplete, + **({"tool_context": tool_context} if include_tool_context else {}), **({"loopx_input_trace": observation, "input_goal_id": input_goal_id} if loopx_input_json is not None else {}), }, @@ -339,6 +345,7 @@ def register_pack( usage_jsonl: str | None = None, usage_session_id: str | None = None, include_incomplete_usage: bool = False, + include_tool_context: bool = False, loopx_input_json: str | None = None, native_outcomes_json: str | None = None, base_pack_id: str | None = None, @@ -374,6 +381,8 @@ def register_pack( record["base_pack_id"] = _opaque(base_pack_id) if include_incomplete_usage and usage_jsonl is None: raise ValueError("include-incomplete-usage requires a usage capture") + if include_tool_context and usage_jsonl is None: + raise ValueError("include-tool-context requires a usage capture") if loopx_input_json is not None and usage_jsonl is None: raise ValueError("LoopX input observation requires a usage capture") if (usage_jsonl is None) != (usage_session_id is None): @@ -384,6 +393,7 @@ def register_pack( record["solver_usage"] = _usage_capture( source_root, usage_jsonl, usage_session_id, include_incomplete=include_incomplete_usage, + include_tool_context=include_tool_context, loopx_input_json=loopx_input_json, input_goal_id=record["input"]["scope"]["goal_id"], ) diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_tool_context.py b/loopx/capabilities/benchmark_toolkit/trajectory_tool_context.py new file mode 100644 index 0000000000..5bbe4c6e06 --- /dev/null +++ b/loopx/capabilities/benchmark_toolkit/trajectory_tool_context.py @@ -0,0 +1,118 @@ +"""Codex capture IO: project text lengths and simple shell argv, never execute. + +The TS trajectory owner classifies commands and reduces observations. Compound +shell, Python/JS orchestration and tool continuations stay unassigned. This is an +inventory of appended input text, not the context replayed at every inference. +""" +from __future__ import annotations + +import hashlib +import json +import re +import shlex + + +def _argv(command: str, depth: int = 0) -> list[str] | None: + if depth > 2 or "\n" in command or "\r" in command: + return None + try: + lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|()<>") + lexer.whitespace_split = True + words = list(lexer) + except ValueError: + return None + if not words or any(word and set(word) <= set(";&|()<>") for word in words): + return None + # Expansion could select another executable or hide shell work. Quoted + # expansions are conservatively unassigned too; no string is evaluated. + if any("$" in word or "`" in word for word in words): + return None + if words[0].rsplit("/", 1)[-1] in {"bash", "sh"}: + return _argv(words[2], depth + 1) if len(words) == 3 and words[1] in {"-c", "-lc"} else None + return words + + +def _literal_exec_argv(script: object) -> list[str] | None: + """Recognize one complete literal exec wrapper, never arbitrary JavaScript. + + Codex's custom exec tool commonly wraps one exec_command in const/await/text. + Accept only JSON-valued object properties and one return; calls, spreads, + interpolation, multiple commands and computed properties remain unknown. + """ + if not isinstance(script, str): + return None + match = re.fullmatch( + r"\s*const\s+([A-Za-z_]\w*)\s*=\s*await\s+tools\.exec_command\(\s*\{(.*)\}\s*\)\s*;" + r"\s*text\(\s*\1(?:\.output)?\s*\)\s*;?\s*", script, re.S, + ) + if match is None: + return None + rest, values = match[2].strip(), {} + decoder = json.JSONDecoder() + while rest: + key = re.match(r'(?:"([A-Za-z_]\w*)"|([A-Za-z_]\w*))\s*:\s*', rest) + if key is None: + return None + name = key[1] or key[2] + if name in values: + return None + try: + value, end = decoder.raw_decode(rest[key.end():]) + except ValueError: + return None + if not isinstance(value, (str, int, float, bool, type(None))): + return None + values[name] = value + rest = rest[key.end() + end:].strip() + if rest and not rest.startswith(','): + return None + rest = rest[1:].strip() if rest else rest + command = values.get('cmd') + return _argv(command) if isinstance(command, str) else None + + +def project_tool_context(events: list[dict], *, first_line: int = 1) -> list[dict]: + """Keep only response_item text metadata; event_msg mirrors are excluded.""" + projected = [] + for number, event in enumerate(events, first_line): + if event.get("type") != "response_item": + continue + item = event.get("payload") + if not isinstance(item, dict): + raise ValueError("Invalid captured response item") + kind = item.get("type") + if kind in {"function_call", "custom_tool_call"}: + argv = None + if kind == "custom_tool_call" and item.get("name") in {"exec", "functions.exec"}: + argv = _literal_exec_argv(item.get("input")) + elif kind == "function_call" and item.get("name") in {"exec_command", "functions.exec_command", "shell_command", "functions.shell_command"}: + try: + args = json.loads(item.get("arguments", "")) + command = args.get("cmd", args.get("command")) if isinstance(args, dict) else None + argv = _argv(command) if isinstance(command, str) else None + except (ValueError, TypeError): + pass + projected.append({"line": number, "kind": "call", "call_id": item.get("call_id"), "argv": argv}) + elif kind in {"function_call_output", "custom_tool_call_output"}: + output = item.get("output") + if isinstance(output, str): + texts, missing = [output], 0 + elif isinstance(output, list): + texts = [part['text'] for part in output if isinstance(part, dict) and isinstance(part.get('text'), str)] + missing = len(output) - len(texts) + else: + projected.append({"line": number, "kind": "unmeasured"}) + continue + raw = ''.join(texts).encode('utf-8') + projected.append({"line": number, "kind": "output", "call_id": item.get("call_id"), + "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), + "unmeasured_parts": missing}) + elif kind == "message" and item.get("role") in {"user", "developer", "system"}: + content = item.get("content") + if not isinstance(content, list): + projected.append({"line": number, "kind": "unmeasured"}) + continue + texts = [part["text"] for part in content if isinstance(part, dict) and isinstance(part.get("text"), str)] + projected.append({"line": number, "kind": "message", "bytes": sum(len(t.encode("utf-8")) for t in texts), + "unmeasured_parts": len(content) - len(texts)}) + return projected diff --git a/loopx/cli_commands/benchmark_trajectory.py b/loopx/cli_commands/benchmark_trajectory.py index 5bc655724e..7f35c6f261 100644 --- a/loopx/cli_commands/benchmark_trajectory.py +++ b/loopx/cli_commands/benchmark_trajectory.py @@ -46,6 +46,10 @@ def register_benchmark_trajectory_commands(subparsers, add_format) -> None: "--loopx-input-json", help="Optional host-recorded LoopX input source beside its private native trace, relative to source-root; estimates recorded Turn packet text with the optional input-token-estimates provider. Requires usage capture; never starts a model.", ) + register.add_argument( + "--include-tool-context", action="store_true", + help="Inventory appended Codex input text and simple LoopX CLI tool outputs in the usage capture; bytes only, not replayed/provider tokens. No model call.", + ) register.add_argument( "--include-incomplete-usage", action="store_true", help="Observe persisted counters through an unfinished final Turn; requires usage capture. Does not establish complete cost or solver termination.", @@ -101,6 +105,7 @@ def _handle_benchmark_trajectory_command( usage_jsonl=args.usage_jsonl, usage_session_id=args.usage_session_id, include_incomplete_usage=args.include_incomplete_usage, + include_tool_context=args.include_tool_context, loopx_input_json=args.loopx_input_json, native_outcomes_json=args.native_outcomes_json, base_pack_id=args.base_pack_id, diff --git a/loopx/control_plane/capabilities/trajectory_assessment_types.ts b/loopx/control_plane/capabilities/trajectory_assessment_types.ts index bcbee7b1d9..c6e5f13819 100644 --- a/loopx/control_plane/capabilities/trajectory_assessment_types.ts +++ b/loopx/control_plane/capabilities/trajectory_assessment_types.ts @@ -98,8 +98,17 @@ export interface TrajectorySolverUsage { request_coverage?: TrajectoryRequestCoverage; /** Optional measured attribution or a labeled recorded-packet estimate; not a counter authority. */ loopx_input?: TrajectoryLoopXInput; + /** Appended captured text only, never provider-token or replay accounting. */ + tool_context?: TrajectoryToolContext; } +export type TrajectoryToolContext = {method: 'codex_loopx_tool_context_inventory_v1'} & ( + {status: 'observed'; observed_input_text_bytes: number; loopx_tool_output_bytes: number; + loopx_tool_output_fraction: number | null; unassigned_input_text_bytes: number; + unmeasured_parts: number; tool_outputs: number; duplicate_output_records: number; + repeated_loopx_output_bytes: number; operations: Array<{operation: string; calls: number; bytes: number}>} + | {status: 'unavailable'; reason: string}); + export type TrajectoryLoopXInput = { method: 'provider_input_token_attribution_v1' | 'tiktoken_o200k_base_input_estimate_v1'; estimate_basis?: {tokenizer: string; version: string; encoding: 'o200k_base'; diff --git a/loopx/control_plane/capabilities/trajectory_codex_usage.ts b/loopx/control_plane/capabilities/trajectory_codex_usage.ts index abe8b2959f..45a13dbc77 100644 --- a/loopx/control_plane/capabilities/trajectory_codex_usage.ts +++ b/loopx/control_plane/capabilities/trajectory_codex_usage.ts @@ -3,6 +3,7 @@ import {EffectRuntimeRequestError} from '../effect_runtime_errors.ts'; import type {TrajectorySolverUsage} from './trajectory_assessment_types.ts'; import {reconcileTrajectoryRequests, summarizeTrajectoryLoopXInput} from './trajectory_request_usage.ts'; import {estimateTrajectoryLoopXInput} from './trajectory_input_usage.ts'; +import {summarizeTrajectoryToolContext} from './trajectory_tool_context.ts'; const fail = (message: string): never => { throw new EffectRuntimeRequestError(`trajectory usage: ${message}`); }; const counterKeys = ['input_tokens', 'cached_input_tokens', 'output_tokens'] as const; @@ -79,9 +80,11 @@ export function summarizeTrajectoryCodexUsage(params: Record): const requestCoverage = reconcileTrajectoryRequests(rows, params.session_id); const loopxInput = summarizeTrajectoryLoopXInput(rows, params.session_id, requestCoverage) ?? estimateTrajectoryLoopXInput(params.loopx_input_trace, params.session_id, params.input_goal_id, rows, requestCoverage); + const toolContext = summarizeTrajectoryToolContext(params.tool_context); return {session_id: params.session_id, method: observed ? 'codex_observed_session_counters_v1' : 'codex_complete_session_counters_v1', ...(requestCoverage ? {request_coverage: requestCoverage} : {}), ...(loopxInput ? {loopx_input: loopxInput} : {}), + ...(toolContext ? {tool_context: toolContext} : {}), ...(observed ? {incomplete_turn: incomplete} : {}), start, end, completed_turns: turns.length, counter_observations: samples, input_tokens: previous.input_tokens, cached_input_tokens: previous.cached_input_tokens, diff --git a/loopx/control_plane/capabilities/trajectory_tool_context.ts b/loopx/control_plane/capabilities/trajectory_tool_context.ts new file mode 100644 index 0000000000..7395918764 --- /dev/null +++ b/loopx/control_plane/capabilities/trajectory_tool_context.ts @@ -0,0 +1,72 @@ +/** Read-only inventory, separate from request-token attribution and judgments. */ +import {BARE_SHA256_PATTERN} from '../content_digest.ts'; +import type {TrajectoryToolContext} from './trajectory_assessment_types.ts'; + +// Local observation buckets, not new execution or settlement vocabulary. +function operation(argv: unknown): string | null { + if (!Array.isArray(argv) || !argv.length || argv.some(v => typeof v !== 'string')) return null; + const args = [...argv] as string[]; + const executable = args.shift()!.split('/').at(-1); + if (executable !== 'loopx') { + if (!/^python(?:3(?:\.\d+)?)?$/.test(executable ?? '') || args.shift() !== '-m' + || !['loopx', 'loopx.entrypoint'].includes(args.shift() ?? '')) return null; + } + while (['--registry', '--runtime-root', '--format', '--cli-bin'].includes(args[0] ?? '')) { + if (args.length < 2) return null; + args.splice(0, 2); + } + const group = args[0]; + if (group === 'quota' && ['context-read', 'context-project', 'should-run', 'spend-slot', 'replan-context'].includes(args[1] ?? '')) return `quota ${args[1]}`; + return ['todo', 'refresh-state', 'explore', 'status', 'diagnose'].includes(group ?? '') ? group! : 'other loopx'; +} + +export function summarizeTrajectoryToolContext(value: unknown): TrajectoryToolContext | undefined { + if (value === undefined) return undefined; + const method = 'codex_loopx_tool_context_inventory_v1'; + try { + if (!Array.isArray(value)) throw new Error(); + const calls = new Map(), outputs = new Map(); + const content = new Set(), groups = new Map(); + let previous = 0, total = 0, loopx = 0, repeats = 0, unmeasured = 0, duplicates = 0, outputCount = 0; + const count = (n: unknown): number => { + if (typeof n !== 'number' || !Number.isSafeInteger(n) || n < 0) throw new Error(); + return n; + }; + for (const row of value) { + if (!row || typeof row !== 'object' || !Number.isSafeInteger(row.line) || row.line <= previous) throw new Error(); + previous = row.line; + if (row.kind === 'call') { + if (typeof row.call_id !== 'string' || !row.call_id || calls.has(row.call_id)) throw new Error(); + calls.set(row.call_id, operation(row.argv)); + } else if (row.kind === 'unmeasured') unmeasured++; + else if (row.kind === 'message') { + total = count(total + count(row.bytes)); unmeasured = count(unmeasured + count(row.unmeasured_parts)); + } else if (row.kind === 'output') { + if (typeof row.call_id !== 'string' || !row.call_id || typeof row.sha256 !== 'string' || !BARE_SHA256_PATTERN.test(row.sha256)) throw new Error(); + const bytes = count(row.bytes), missing = count(row.unmeasured_parts ?? 0); + const identity = `${row.sha256}:${bytes}:${missing}`; + const prior = outputs.get(row.call_id); + if (prior !== undefined) { + if (prior !== identity) throw new Error(); + duplicates++; continue; + } + outputs.set(row.call_id, identity); outputCount++; total = count(total + bytes); + unmeasured = count(unmeasured + missing); + const op = calls.get(row.call_id); + if (!op) continue; + loopx = count(loopx + bytes); + if (content.has(identity)) repeats = count(repeats + bytes); + content.add(identity); + const group = groups.get(op) ?? {operation: op, calls: 0, bytes: 0}; + group.calls++; group.bytes = count(group.bytes + bytes); groups.set(op, group); + } else throw new Error(); + } + return {method, status: 'observed', observed_input_text_bytes: total, loopx_tool_output_bytes: loopx, + loopx_tool_output_fraction: total > 0 ? loopx / total : null, + unassigned_input_text_bytes: total - loopx, unmeasured_parts: unmeasured, + tool_outputs: outputCount, duplicate_output_records: duplicates, repeated_loopx_output_bytes: repeats, + operations: [...groups.values()].sort((a, b) => b.bytes - a.bytes || a.operation.localeCompare(b.operation))}; + } catch { + return {method, status: 'unavailable', reason: 'Captured tool context could not be reconciled.'}; + } +} diff --git a/tests/capabilities/test_trajectory_input_capture.py b/tests/capabilities/test_trajectory_input_capture.py index b6d17e894d..3753ae217f 100644 --- a/tests/capabilities/test_trajectory_input_capture.py +++ b/tests/capabilities/test_trajectory_input_capture.py @@ -107,6 +107,27 @@ def test_invalid_optional_observation_cannot_erase_native_counters_or_escape_roo assert usage["projection"]["loopx_input"]["status"] == "unavailable" +def test_tool_inventory_is_opt_in_separate_and_retains_native_counter_identity(tmp_path): + _, rows = native_trace(tmp_path) + tool = [{'type': 'response_item', 'payload': {'type': 'custom_tool_call', + 'name': 'exec', 'call_id': 'tool', 'input': + 'const r = await tools.exec_command({cmd: "loopx quota context-read"}); text(r.output);'}}, + {'type': 'response_item', 'payload': {'type': 'custom_tool_call_output', + 'call_id': 'tool', 'output': '状态已保存'}}] + path = tmp_path / 'usage.jsonl' + path.write_text('\n'.join(map(json.dumps, rows[:2] + tool + rows[2:]))) + raw = path.read_bytes() + old = _usage_capture(tmp_path, 'usage.jsonl', 'session') + assert 'tool_context' not in old['projection'] + measured = _usage_capture(tmp_path, 'usage.jsonl', 'session', include_tool_context=True) + inventory = measured['projection'].pop('tool_context') + assert measured == old + assert inventory['status'] == 'observed' + assert inventory['loopx_tool_output_bytes'] == 15 + assert inventory['operations'] == [{'operation': 'quota context-read', 'calls': 1, 'bytes': 15}] + assert path.read_bytes() == raw + + def test_real_cli_registers_recorded_component_without_model_execution(tmp_path): from loopx.capabilities.benchmark_toolkit.trajectory_assessment_store import inspect_pack @@ -130,9 +151,10 @@ def test_real_cli_registers_recorded_component_without_model_execution(tmp_path) "--runtime-root", str(runtime), "--format", "json", "benchmark", "trajectory-pack-register", "--goal-id", "research", "--input-json", str(path), "--source-root", str(tmp_path), "--title", "Synthetic recorded input", "--usage-jsonl", "usage.jsonl", "--usage-session-id", "session", - "--loopx-input-json", str(source.relative_to(tmp_path))], capture_output=True, text=True, check=True) + "--loopx-input-json", str(source.relative_to(tmp_path)), "--include-tool-context"], capture_output=True, text=True, check=True) registered = json.loads(result.stdout) view = inspect_pack(runtime_root=runtime, owner_goal_id="research", pack_id=registered["pack_id"]) assert view["solver_usage"]["projection"]["loopx_input"]["input_fraction"] == 0.02 + assert view["solver_usage"]["projection"]["tool_context"]["loopx_tool_output_fraction"] is None assert view["job"] is None and view["result"] is None assert "hello world" not in json.dumps(view) diff --git a/tests/capabilities/test_trajectory_tool_context.py b/tests/capabilities/test_trajectory_tool_context.py new file mode 100644 index 0000000000..9cbf4b1bcb --- /dev/null +++ b/tests/capabilities/test_trajectory_tool_context.py @@ -0,0 +1,71 @@ +import hashlib +import json + +import pytest + +from loopx.capabilities.benchmark_toolkit.trajectory_tool_context import ( + _argv, _literal_exec_argv, project_tool_context, +) + + +def test_literal_shell_and_exec_wrapper_are_parsed_without_execution(tmp_path): + assert _argv('/venv/bin/loopx --format json quota context-read') == [ + '/venv/bin/loopx', '--format', 'json', 'quota', 'context-read'] + assert _argv("/bin/bash -lc 'loopx status'") == ['loopx', 'status'] + assert _argv('loopx todo --title "keep a; b"') == ['loopx', 'todo', '--title', 'keep a; b'] + wrapper = 'const r = await tools.exec_command({cmd: "loopx status", max_output_tokens: 100}); text(r.output);' + assert _literal_exec_argv(wrapper) == ['loopx', 'status'] + for script in [ + wrapper + 'doSomething();', wrapper.replace('"loopx status"', 'getCommand()'), + wrapper.replace('max_output_tokens: 100', '...options'), + wrapper.replace('cmd:', '["cmd"]:'), wrapper.replace('cmd:', 'cmd: "pytest", cmd:'), + 'const r = await tools.exec_command({cmd: "loopx status"}); text("fake");', + ]: + assert _literal_exec_argv(script) is None + marker = tmp_path / 'must-not-exist' + for command in [f'loopx status; touch {marker}', 'loopx status && pytest', + 'loopx status | cat', 'loopx status > out', 'loopx status\npytest', + '$(echo loopx) status', '`echo loopx` status', 'loopx "unterminated']: + assert _argv(command) is None + assert not marker.exists() + + +def test_projection_uses_native_response_items_not_mirrors_or_model_prose(): + def item(payload): + return {'type': 'response_item', 'payload': payload} + output = '状态已保存' + rows = [item({'type': 'function_call', 'name': 'exec_command', 'call_id': 'call', + 'arguments': json.dumps({'cmd': 'loopx status'})}), + item({'type': 'function_call_output', 'call_id': 'call', 'output': output}), + {'type': 'event_msg', 'payload': {'type': 'item_completed', 'output': output}}, + item({'type': 'message', 'role': 'user', 'content': [ + {'type': 'input_text', 'text': '任务'}, {'type': 'input_image'}]}), + item({'type': 'message', 'role': 'assistant', 'content': [{'text': 'loopx status'}]}), + item({'type': 'custom_tool_call', 'call_id': 'js', 'name': 'exec', + 'input': 'text(await tools.exec_command({cmd: "loopx status"}));'}), + item({'type': 'custom_tool_call_output', 'call_id': 'js', 'output': 'unassigned'})] + result = project_tool_context(rows) + assert result[0]['argv'] == ['loopx', 'status'] + assert result[1]['bytes'] == 15 + assert result[1]['sha256'] == hashlib.sha256(output.encode()).hexdigest() + assert result[2] == {'line': 4, 'kind': 'message', 'bytes': 6, 'unmeasured_parts': 1} + assert result[3]['argv'] is None + assert output not in json.dumps(result, ensure_ascii=False) + assert len(result) == 5 + + +def test_custom_tool_output_blocks_preserve_text_and_nontext_coverage(): + rows = [{'type': 'response_item', 'payload': {'type': 'custom_tool_call_output', + 'call_id': 'call', 'output': [{'type': 'input_text', 'text': '状态'}, + {'type': 'input_text', 'text': '已保存'}, {'type': 'input_image'}]}}] + value = project_tool_context(rows)[0] + assert value['bytes'] == 15 + assert value['unmeasured_parts'] == 1 + assert value['sha256'] == hashlib.sha256('状态已保存'.encode()).hexdigest() + + +@pytest.mark.parametrize('arguments', ['not json', '[]', '{}', '{"cmd": 42}']) +def test_unsupported_tool_arguments_remain_unassigned(arguments): + rows = [{'type': 'response_item', 'payload': {'type': 'function_call', + 'name': 'exec_command', 'call_id': 'call', 'arguments': arguments}}] + assert project_tool_context(rows)[0]['argv'] is None diff --git a/tests/control_plane_ts/trajectory_codex_usage.test.ts b/tests/control_plane_ts/trajectory_codex_usage.test.ts index b856de1b64..afcfda3cd2 100644 --- a/tests/control_plane_ts/trajectory_codex_usage.test.ts +++ b/tests/control_plane_ts/trajectory_codex_usage.test.ts @@ -24,6 +24,13 @@ test('duplicate counter observation does not charge twice',()=>{ const es=fixture();es.splice(3,0,structuredClone(es[2]));es.forEach((e,i)=>e.line=i+1); assert.equal(run(es).input_tokens,140); }); +test('invalid supplementary tool inventory cannot erase native counters', () => { + const before = run(); + const result = summarize({events: fixture(), session_id: 'session', tool_context: [{line: 0}]}); + assert.equal(result.tool_context?.status, 'unavailable'); + const {tool_context, ...native} = result; + assert.deepEqual(native, before); +}); test('unknown counters, resets, missing prefix, session mismatch and unfinished work reject',()=>{ for(const edit of [ (e:any[])=>{e[0].payload.id='other';}, diff --git a/tests/control_plane_ts/trajectory_tool_context.test.ts b/tests/control_plane_ts/trajectory_tool_context.test.ts new file mode 100644 index 0000000000..db36451df4 --- /dev/null +++ b/tests/control_plane_ts/trajectory_tool_context.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import {summarizeTrajectoryToolContext as summarize} from '../../loopx/control_plane/capabilities/trajectory_tool_context.ts'; +const sha = 'a'.repeat(64); +const call = (line: number, call_id: string, argv: unknown) => ({line, kind: 'call', call_id, argv}); +const output = (line: number, call_id: string, bytes: number, sha256 = sha) => ({line, kind: 'output', call_id, bytes, sha256}); + +test('inventory denominator includes unassigned input; byte share is not provider tokens', () => { + const result = summarize([{line: 1, kind: 'message', bytes: 100, unmeasured_parts: 1}, + call(2, 'a', ['/venv/bin/loopx', '--format', 'json', 'quota', 'context-read']), output(3, 'a', 60), + call(4, 'b', ['pytest']), output(5, 'b', 40)])!; + assert.equal(result.status, 'observed'); if (result.status !== 'observed') return; + assert.equal(result.observed_input_text_bytes, 200); assert.equal(result.loopx_tool_output_fraction, 0.3); + assert.equal(result.unassigned_input_text_bytes, 140); assert.equal(result.unmeasured_parts, 1); + assert.deepEqual(result.operations, [{operation: 'quota context-read', calls: 1, bytes: 60}]); + assert.equal('input_tokens' in result, false); +}); + +test('one mirrored output counts once; identical content from distinct calls is real repeated input', () => { + const result = summarize([call(1, 'a', ['loopx', 'status']), output(2, 'a', 60), output(3, 'a', 60), + call(4, 'b', ['loopx', 'status']), output(5, 'b', 60)])!; + assert.equal(result.status, 'observed'); if (result.status !== 'observed') return; + assert.equal(result.loopx_tool_output_bytes, 120); assert.equal(result.tool_outputs, 2); + assert.equal(result.duplicate_output_records, 1); assert.equal(result.repeated_loopx_output_bytes, 60); +}); + +test('Python module argv is recognized; arbitrary prose, scripts and quoted loopx are not', () => { + const yes = [['python3.13', '-m', 'loopx.entrypoint', 'quota', 'context-project'], ['python', '-m', 'loopx', 'todo']]; + const no = [null, ['echo', 'loopx', 'status'], ['python', '-c', 'import loopx'], ['not-loopx', 'status'], ['loopx-lookalike', 'status']]; + for (const argv of [...yes, ...no]) { + const r = summarize([call(1, 'a', argv), output(2, 'a', 50)])!; + assert.equal(r.status, 'observed'); if (r.status !== 'observed') continue; + assert.equal(r.loopx_tool_output_bytes, yes.includes(argv as string[]) ? 50 : 0); + } +}); + +test('conflicting mirrors, line order, counts and call identities withhold only the observation', () => { + for (const rows of [ + [call(1, 'a', ['loopx']), output(2, 'a', 50), output(3, 'a', 51)], + [call(1, 'a', ['loopx']), call(2, 'a', ['pytest'])], + [output(1, 'a', -1)], [output(1, 'a', 0.5)], [output(1, 'a', 50, 'bad')], + [{line: 1, kind: 'message', bytes: Number.MAX_SAFE_INTEGER, unmeasured_parts: 0}, output(2, 'a', 1)], + [output(1, 'a', 1), output(1, 'b', 1)], + ]) assert.equal(summarize(rows)?.status, 'unavailable'); + assert.equal(summarize(undefined), undefined); + const empty = summarize([])!; + assert.equal(empty.status, 'observed'); if (empty.status === 'observed') assert.equal(empty.loopx_tool_output_fraction, null); +}); From e773efcf745d5c110b8e8e13f4492a358bf8888b Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:23:41 +0800 Subject: [PATCH 2/3] fix(trajectory): distinguish task lease and Todo lifecycle context Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- docs/reference/trajectory-assessment.md | 8 +++++- .../capabilities/trajectory_tool_context.ts | 4 ++- .../trajectory_tool_context.test.ts | 27 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index b0e9b2c057..23e546952b 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -1079,8 +1079,12 @@ Duplicate output records for one call count once. Identical output from two different calls is still two pieces of appended input; it is not automatically redundant research. Operation counts distinguish context reads/projects from other LoopX returns. No output or command argument text is sent to the browser. +Task lease acquisition, inspection, renewal, release and transfer have separate +rows, as do common Todo operations such as claim and completion. These count +observed replies, not successful mutations or proved unnecessary recovery. -Only literal simple shell argv, Python `-m loopx[.entrypoint]`, and an exact +Only literal simple shell argv, Python `-m loopx`, `-m loopx.cli` or +`-m loopx.entrypoint`, and an exact single literal `const/await tools.exec_command/text` wrapper are recognized. Compound shell, arbitrary JavaScript, indirect commands, continuation tools and file reads remain unassigned. Non-text content is counted as unmeasured parts. @@ -1093,6 +1097,8 @@ estimate and provider counters; do not add their different units or denominators 中文:可选统计在原轨迹分析工作台中展示 LoopX 工具返回占已记录输入文本的字节 比例,并拆出 context-read/project 等操作。未知和非文本内容分别保留;同内容的 再次返回不等于无益重复。这不是全量控制面、逐请求重放或计费 token 的占比。 +租约获取、检查、续租、释放、转移及常见 Todo 操作分别列示;计数表示可识别的返回, +不证明状态写入成功或恢复操作可以省略。 ### Recorded Turn packet estimate diff --git a/loopx/control_plane/capabilities/trajectory_tool_context.ts b/loopx/control_plane/capabilities/trajectory_tool_context.ts index 7395918764..1fdce82dcb 100644 --- a/loopx/control_plane/capabilities/trajectory_tool_context.ts +++ b/loopx/control_plane/capabilities/trajectory_tool_context.ts @@ -9,7 +9,7 @@ function operation(argv: unknown): string | null { const executable = args.shift()!.split('/').at(-1); if (executable !== 'loopx') { if (!/^python(?:3(?:\.\d+)?)?$/.test(executable ?? '') || args.shift() !== '-m' - || !['loopx', 'loopx.entrypoint'].includes(args.shift() ?? '')) return null; + || !['loopx', 'loopx.cli', 'loopx.entrypoint'].includes(args.shift() ?? '')) return null; } while (['--registry', '--runtime-root', '--format', '--cli-bin'].includes(args[0] ?? '')) { if (args.length < 2) return null; @@ -17,6 +17,8 @@ function operation(argv: unknown): string | null { } const group = args[0]; if (group === 'quota' && ['context-read', 'context-project', 'should-run', 'spend-slot', 'replan-context'].includes(args[1] ?? '')) return `quota ${args[1]}`; + if (group === 'task-lease' && ['acquire', 'inspect', 'renew', 'release', 'transfer'].includes(args[1] ?? '')) return `task-lease ${args[1]}`; + if (group === 'todo' && ['add', 'list', 'claim', 'complete', 'update'].includes(args[1] ?? '')) return `todo ${args[1]}`; return ['todo', 'refresh-state', 'explore', 'status', 'diagnose'].includes(group ?? '') ? group! : 'other loopx'; } diff --git a/tests/control_plane_ts/trajectory_tool_context.test.ts b/tests/control_plane_ts/trajectory_tool_context.test.ts index db36451df4..6d7d77c1aa 100644 --- a/tests/control_plane_ts/trajectory_tool_context.test.ts +++ b/tests/control_plane_ts/trajectory_tool_context.test.ts @@ -25,8 +25,10 @@ test('one mirrored output counts once; identical content from distinct calls is }); test('Python module argv is recognized; arbitrary prose, scripts and quoted loopx are not', () => { - const yes = [['python3.13', '-m', 'loopx.entrypoint', 'quota', 'context-project'], ['python', '-m', 'loopx', 'todo']]; - const no = [null, ['echo', 'loopx', 'status'], ['python', '-c', 'import loopx'], ['not-loopx', 'status'], ['loopx-lookalike', 'status']]; + const yes = [['python3.13', '-m', 'loopx.entrypoint', 'quota', 'context-project'], ['python', '-m', 'loopx', 'todo'], + ['python3', '-m', 'loopx.cli', '--registry', '/synthetic/registry.json', '--format', 'json', 'task-lease', 'inspect']]; + const no = [null, ['echo', 'loopx', 'status'], ['python', '-c', 'import loopx'], ['not-loopx', 'status'], ['loopx-lookalike', 'status'], + ['python', '-m', 'loopx.cli_lookalike', 'task-lease', 'inspect']]; for (const argv of [...yes, ...no]) { const r = summarize([call(1, 'a', argv), output(2, 'a', 50)])!; assert.equal(r.status, 'observed'); if (r.status !== 'observed') continue; @@ -34,6 +36,27 @@ test('Python module argv is recognized; arbitrary prose, scripts and quoted loop } }); +test('lease preparation, proof reads and completion are separate observed costs', () => { + const ops = ['task-lease acquire', 'task-lease inspect', 'task-lease renew', 'task-lease release', 'task-lease transfer', + 'todo claim', 'todo complete', 'todo list', 'todo update', 'todo add']; + const rows = ops.flatMap((op, index) => [call(index * 2 + 1, `call-${index}`, ['loopx', ...op.split(' ')]), + output(index * 2 + 2, `call-${index}`, 10 + index)]); + const result = summarize(rows)!; + assert.equal(result.status, 'observed'); if (result.status !== 'observed') return; + assert.deepEqual(new Set(result.operations.map(row => row.operation)), new Set(ops)); + assert.equal(result.loopx_tool_output_bytes, 145); + assert.equal(result.observed_input_text_bytes, 145); + assert.equal(result.operations.reduce((total, row) => total + row.bytes, 0), 145); + assert.equal(result.operations.reduce((total, row) => total + row.calls, 0), 10); + // Categories name returned CLI text, not successful state transitions or + // avoidable recovery. Unknown subcommands stay in their existing bucket. + for (const [group, expected] of [['task-lease', 'other loopx'], ['todo', 'todo']]) { + const unknown = summarize([call(1, 'unknown', ['loopx', group, 'future-op']), output(2, 'unknown', 11)])!; + assert.equal(unknown.status, 'observed'); + if (unknown.status === 'observed') assert.equal(unknown.operations[0].operation, expected); + } +}); + test('conflicting mirrors, line order, counts and call identities withhold only the observation', () => { for (const rows of [ [call(1, 'a', ['loopx']), output(2, 'a', 50), output(3, 'a', 51)], From 5d3a5d1ebb3ab9f6050449ee0bed0ae091d1ce33 Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:55:34 +0800 Subject: [PATCH 3/3] test(trajectory): register tool context digest consumer Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- tests/control_plane_ts/content_digest_single_owner.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index b5e096e78a..7fdd69e9ae 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -106,6 +106,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/capabilities/goal_capability_organization.ts", "control_plane/capabilities/trajectory_assessment.ts", "control_plane/capabilities/trajectory_input_usage.ts", + "control_plane/capabilities/trajectory_tool_context.ts", "control_plane/collaboration/chat_mode.ts", "control_plane/collaboration/delegation.ts", "control_plane/collaboration/delegation_result_use.ts",