diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11fe459..5db668c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Test Repo Assist guards + run: python3 -m unittest discover -s tests -p 'test_repo_assist_*.py' - name: Install gh-aw run: gh extension install github/gh-aw env: diff --git a/docs/repo-assist.md b/docs/repo-assist.md index 1c7d2f2..359ae2c 100644 --- a/docs/repo-assist.md +++ b/docs/repo-assist.md @@ -118,6 +118,12 @@ Proactively moves the repository forward — considers the goals and aims of the Every non-command-mode run that performs work, whether scheduled or manually dispatched, updates a rolling monthly activity issue that gives maintainers a single place to see all activity and suggested actions. Command-mode and no-op runs do not update the issue. The action list is rebuilt from current repository state and memory, completed or closed items are removed, and run history is kept in reverse chronological order. +### Output Consistency + +Labels, comments, issue/PR changes, and PR-branch pushes all count as substantive work, even when no code changes. Such runs must not emit `noop`. Before applying any safe outputs, a deterministic check rejects noop/mutation combinations and requires substantive non-command runs to create or update the current month's activity issue with the current run's history link. A genuine noop (including a memory-only check) and command-mode reporting exemptions remain unchanged. + +If a required action or report is blocked, Repo Assist uses `report_incomplete` and identifies the failure and remaining obligations instead of claiming success. Final responses distinguish requested safe outputs from downstream delivery and memory validation from the later repo-memory commit; validation alone is not proof of persistence. + ### Persistent Memory Repo Assist uses one schema-validated `notes.json` file to carry concise, actionable state across runs. It records backlog cursors, unresolved issue interactions, fix attempts, the latest engineering checks, maintainer-completed monthly actions, and a short priority queue. The workflow verifies remembered facts against current repository state before acting, replaces superseded entries, and removes stale records rather than accumulating a run-by-run activity log. diff --git a/tests/test_repo_assist_output_consistency.py b/tests/test_repo_assist_output_consistency.py new file mode 100644 index 0000000..d64538e --- /dev/null +++ b/tests/test_repo_assist_output_consistency.py @@ -0,0 +1,181 @@ +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + + +WORKFLOW = (Path(__file__).parents[1] / "workflows/repo-assist.md").read_text() +VALIDATOR = textwrap.dedent( + WORKFLOW.split(" node <<'EOF'\n", 1)[1].split("\n EOF", 1)[0] +) +MONTH = datetime.now(timezone.utc).strftime("%Y-%m") +TITLE = f"[repo-assist] Monthly Activity {MONTH}" +RUN_URL = "https://github.com/example/project/actions/runs/123" +BODY = f"## Run History\n\n### Now - [Run]({RUN_URL})" + + +class OutputConsistencyTests(unittest.TestCase): + def check_outputs(self, items, command="", issues=None, api_failure=False): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output.json" + monthly = Path(directory) / "monthly.json" + gh = Path(directory) / "gh" + gh.write_text( + "#!/usr/bin/env python3\n" + "import os, pathlib, sys\n" + "if os.environ['API_FAILURE'] == 'true':\n" + " sys.exit('GitHub API unavailable')\n" + "print(pathlib.Path(os.environ['MONTHLY_ISSUES_FILE']).read_text())\n" + ) + gh.chmod(0o755) + output.write_text(json.dumps({"items": items})) + monthly.write_text("\n".join( + json.dumps([issue["number"], issue["title"]]) + for issue in ([{"number": 42, "title": TITLE}] if issues is None else issues) + )) + return subprocess.run( + ["node", "-e", VALIDATOR], + env=dict( + os.environ, OUTPUT_FILE=str(output), MONTHLY_ISSUES_FILE=str(monthly), + COMMAND_TEXT=command, GITHUB_SERVER_URL="https://github.com", + GITHUB_REPOSITORY="example/project", GITHUB_RUN_ID="123", + PATH=f"{directory}{os.pathsep}{os.environ['PATH']}", + API_FAILURE=str(api_failure).lower(), + ), + capture_output=True, text=True, check=False, + ) + + def assert_allowed(self, items, **kwargs): + result = self.check_outputs(items, **kwargs) + self.assertEqual(result.returncode, 0, result.stderr) + + def assert_rejected(self, items, message, **kwargs): + result = self.check_outputs(items, **kwargs) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + + def test_genuine_noop_and_memory_only_checks_are_allowed(self): + self.assert_allowed([{"type": "noop"}]) + self.assert_allowed([]) + + def test_noop_is_incompatible_with_every_enabled_mutation(self): + for kind in [ + "add_comment", "add_labels", "remove_labels", "create_issue", + "update_issue", "create_pull_request", "push_to_pull_request_branch", + ]: + with self.subTest(kind=kind): + self.assert_rejected( + [{"type": kind}, {"type": "noop"}], "cannot emit noop", + command="Even command mode must be consistent", + ) + + def test_incomplete_is_not_a_successful_noop(self): + self.assert_rejected( + [{"type": "report_incomplete"}, {"type": "noop"}], "cannot emit noop" + ) + + def test_labels_without_code_changes_still_require_reporting(self): + self.assert_rejected([{"type": "add_labels"}], "require a current-month activity") + + def test_audited_label_then_noop_regression(self): + self.assert_rejected([ + {"type": "add_labels", "item_number": 4461, "labels": [{"name": "bug"}]}, + {"type": "noop", "message": "Labelled issue; no further safe action"}, + ], "cannot emit noop") + + def test_audited_comment_without_monthly_report_regression(self): + self.assert_rejected([ + {"type": "add_comment", "item_number": 134, "body": "Verified implementation guidance"}, + ], "require a current-month activity") + + def test_current_month_creation_with_run_history_is_allowed(self): + self.assert_allowed([ + {"type": "add_comment"}, + {"type": "create_issue", "title": TITLE, "body": BODY}, + ]) + + def test_create_issue_prefix_is_normalized_like_safe_output_handler(self): + self.assert_allowed([{ + "type": "create_issue", "title": TITLE.removeprefix("[repo-assist] "), "body": BODY, + }]) + + def test_existing_monthly_issue_update_is_allowed(self): + self.assert_allowed([ + {"type": "push_to_pull_request_branch"}, + {"type": "update_issue", "issue_number": 42, "body": BODY}, + ]) + + def test_old_month_or_unrelated_issue_does_not_satisfy_reporting(self): + for title in ["[repo-assist] Monthly Activity 2000-01", "[repo-assist] Other issue"]: + with self.subTest(title=title): + self.assert_rejected( + [{"type": "update_issue", "issue_number": 42, "body": BODY}], + "require a current-month activity", issues=[{"number": 42, "title": title}], + ) + + def test_closing_the_tracker_does_not_satisfy_reporting(self): + self.assert_rejected([{ + "type": "update_issue", "issue_number": 42, "body": BODY, "status": "closed", + }], "require a current-month activity") + + def test_other_repository_tracker_does_not_satisfy_reporting(self): + self.assert_rejected([{ + "type": "update_issue", "issue_number": 42, "body": BODY, "repo": "other/project", + }], "require a current-month activity") + + def test_report_requires_history_and_current_run_link(self): + for body in [ + "", f"Only a link: {RUN_URL}", "## Run History\nOld run", + f"## Suggested Actions\n[Run]({RUN_URL})\n## Run History\nOld run", + f"## Run History\n- [Run]({RUN_URL})", + f"## Run History\nOld run\n## Other\n### Now - [Run]({RUN_URL})", + ]: + with self.subTest(body=body): + self.assert_rejected( + [{"type": "update_issue", "issue_number": 42, "body": body}], + "require a current-month activity", + ) + + def test_month_rollover_can_close_old_tracker_and_create_current_one(self): + self.assert_allowed([ + {"type": "update_issue", "issue_number": 41, "status": "closed"}, + {"type": "create_issue", "title": TITLE, "body": BODY}, + ], issues=[{"number": 41, "title": "[repo-assist] Monthly Activity 2000-01"}]) + + def test_command_mode_is_exempt_from_monthly_reporting(self): + self.assert_allowed([{"type": "add_comment"}], command="Investigate this issue") + + def test_blank_command_retains_non_command_reporting(self): + self.assert_rejected( + [{"type": "add_comment"}], "require a current-month activity", command=" \n" + ) + + def test_truthful_incomplete_run_can_preserve_partial_work(self): + self.assert_allowed([ + {"type": "add_comment"}, + {"type": "report_incomplete", "message": "Monthly report blocked"}, + ]) + + def test_diagnostics_alone_do_not_require_monthly_reporting(self): + self.assert_allowed([{"type": "missing_tool"}]) + self.assert_allowed([{"type": "report_incomplete"}]) + + def test_lookup_failures_are_explicit(self): + self.assert_rejected( + [{"type": "add_comment"}], "GitHub API unavailable", api_failure=True + ) + + def test_exempt_runs_do_not_depend_on_monthly_issue_lookup(self): + self.assert_allowed([{"type": "noop"}], api_failure=True) + self.assert_allowed([{"type": "add_comment"}], command="Investigate", api_failure=True) + self.assert_allowed( + [{"type": "add_comment"}, {"type": "report_incomplete"}], api_failure=True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflows/repo-assist.md b/workflows/repo-assist.md index bbbfae8..8929f38 100644 --- a/workflows/repo-assist.md +++ b/workflows/repo-assist.md @@ -77,7 +77,6 @@ checkout: fetch-depth: 0 # fetch full history tools: - web-fetch: github: toolsets: [all] min-integrity: none # This workflow is allowed to examine and comment on any issues or PRs @@ -188,7 +187,7 @@ safe-outputs: title-prefix: "[repo-assist] " labels: [automation, repo-assist] protected-files: - policy: request_review + policy: request-review exclude: - CHANGELOG.md - README.md @@ -219,6 +218,66 @@ safe-outputs: max: 5 target: "*" +jobs: + safe_outputs: + pre-steps: + - name: Download outputs for consistency check + uses: actions/download-artifact@v8 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: ${{ runner.temp }}/repo-assist-consistency + - name: Enforce Repo Assist output consistency + env: + GH_TOKEN: ${{ github.token }} + COMMAND_TEXT: ${{ needs.activation.outputs.text || inputs.command }} + OUTPUT_FILE: ${{ runner.temp }}/repo-assist-consistency/agent_output.json + run: | + node <<'EOF' + const fs = require("node:fs"); + const { execFileSync } = require("node:child_process"); + const outputs = JSON.parse(fs.readFileSync(process.env.OUTPUT_FILE, "utf8")); + const items = outputs.items; + const diagnostics = new Set(["noop", "report_incomplete", "missing_tool", "missing_data"]); + const mutations = items.filter(item => !diagnostics.has(item.type)); + const noop = items.some(item => item.type === "noop"); + const incomplete = items.some(item => item.type === "report_incomplete"); + if (noop && (mutations.length > 0 || incomplete)) { + throw new Error("Repo Assist cannot emit noop alongside a mutation or report_incomplete. Report the actual work or failure instead."); + } + + const commandMode = (process.env.COMMAND_TEXT || "").trim().length > 0; + if (!commandMode && mutations.length > 0 && !incomplete) { + const prefix = "[repo-assist] "; + const title = `${prefix}Monthly Activity ${new Date().toISOString().slice(0, 7)}`; + const response = execFileSync("gh", [ + "api", "--hostname", new URL(process.env.GITHUB_SERVER_URL).host, "--paginate", + `repos/${process.env.GITHUB_REPOSITORY}/issues?state=open&labels=repo-assist&per_page=100`, + "--jq", ".[] | [.number, .title] | @json", + ], { encoding: "utf8" }); + const titles = new Map(response.split(/\r?\n/).filter(Boolean) + .map(line => JSON.parse(line)).map(([number, title]) => [String(number), title])); + const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const hasReport = items.some(item => { + if (item.repo && item.repo.toLowerCase() !== process.env.GITHUB_REPOSITORY.toLowerCase()) return false; + let actualTitle; + if (item.type === "create_issue") { + actualTitle = item.title?.startsWith(prefix) ? item.title : prefix + item.title; + } else if (item.type === "update_issue" && item.status !== "closed") { + actualTitle = item.title || titles.get(String(item.issue_number)); + } + if (actualTitle !== title || typeof item.body !== "string") return false; + const history = item.body.split(/^## Run History[ \t]*\r?$/m)[1]; + return history?.split(/^## /m)[0].split(/\r?\n/) + .some(line => line.startsWith("### ") && line.includes(`[Run](${runUrl})`)) || false; + }); + if (!hasReport) { + throw new Error("Substantive non-command Repo Assist outputs require a current-month activity issue create/update containing this run's history link. Finish Task 11, or report_incomplete if reporting is blocked."); + } + } + console.log(`Repo Assist outputs consistent: ${mutations.length} mutation(s), command=${commandMode}, incomplete=${incomplete}. Memory persistence is reported separately by the repo-memory job.`); + EOF + steps: - name: Initialize Repo Assist memory env: @@ -373,6 +432,8 @@ Keep notes terse and current. Replace superseded entries, remove resolved issue After every change to `notes.json`, run `jq empty /tmp/gh-aw/repo-memory/default/notes.json`, then call `push_repo_memory`. A successful tool result is required before finishing the run. If either check reports an error, repair `notes.json` and retry both checks. Prefer `jq` with a temporary file and atomic rename over manual partial JSON edits. +In the final response, distinguish unchanged memory from changed memory validated by `push_repo_memory` for downstream persistence. Do not claim that validation alone proves the later repo-memory job committed the changes. If memory validation cannot be repaired, use `report_incomplete` and describe the outstanding state; do not claim full completion or silently discard actionable records to obtain success. + **Important**: Memory may not be 100% accurate. Issues may have been created, closed, or commented on; PRs may have been created, merged, commented on, or closed since the last run. Always verify memory against current repository state — reviewing recent activity since your last run is wise before acting on stale assumptions. **Memory backlog tracking**: Records in `issues`, `fixes`, and `priorities` are **action items for you**, not just informational notes. Each run, prioritise applicable work from those fields and remove or replace entries as their state changes. @@ -408,6 +469,15 @@ The weighting scheme naturally adapts to repo state: Always do Task 11 (Update Monthly Activity Summary Issue) after performing work in non-command mode, including when manually dispatched without a command. Command-mode and no-op runs do not update the issue. In all comments and PR descriptions, identify yourself as "Repo Assist". +### Output consistency (all modes) + +Before finishing, reconcile the safe outputs you requested with your final response and memory: + +- Use `noop` only when no GitHub mutation was requested. Labelling, comments, issue/PR creation or updates, and PR-branch pushes are substantive work even when no code changed; never follow them with `noop`. +- After any substantive non-command work, request the Task 11 activity-issue create/update with this run's history link. Command-mode runs remain exempt. A genuine noop or memory-only check does not require an activity update. +- If a required output, reporting step, or memory validation is blocked, use `report_incomplete` with the concrete failure and remaining obligations, not a success-shaped `noop`. This permits truthful partial work without claiming completion. +- In the final response, distinguish requested safe outputs from confirmed downstream delivery, and list any incomplete obligations. The safe-output job rejects contradictory noop/mutation batches and successful non-command batches missing their monthly report. + ### Task 1: Issue Labelling Process as many unlabelled issues and PRs as possible each run. Resume from memory's backlog cursor.