From d826bd580d190071ac816d1e688f12c9038f4d08 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Mon, 19 Oct 2026 15:00:18 +0530 Subject: [PATCH 1/5] fix(accessibility): count main issues only, matching the dashboard The issues export has a row for every occurrence. In a grouped report each repeated occurrence carries a "Duplicate Issue of" link to its main issue, and the dashboard leaves those rows out of its issue count. startAccessibilityScan and fetchAccessibilityIssues counted every row, so a scan the dashboard shows as 79 issues was reported as "We found 122 issues". The parser now returns main issues only, folds each duplicate into the issue its link points at (duplicate_count), and returns how many duplicates it left out; both tools state that number next to the total. A duplicate whose main issue cannot be identified is counted only there. Reports without the "Duplicate Issue of" column are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 36 +++- src/tools/accessiblity-utils/report-parser.ts | 64 ++++++- tests/tools/accessibilityReport.test.ts | 171 ++++++++++++++++++ tests/tools/accessibilityScanOutput.test.ts | 133 ++++++++++++++ 4 files changed, 395 insertions(+), 9 deletions(-) create mode 100644 tests/tools/accessibilityReport.test.ts create mode 100644 tests/tools/accessibilityScanOutput.test.ts diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 462c64dd..cca4ee56 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -165,6 +165,18 @@ async function executeFetchAccessibilityIssues( } } +/** How grouped reports' duplicates were handled, or nothing for an ungrouped report. */ +function duplicatesNote(duplicates: number | null, unlinked: number): string[] { + if (!duplicates) return []; + const unmatched = + unlinked > 0 + ? ` ${unlinked} of them could not be matched to a main issue and are counted only here.` + : ""; + return [ + `This report groups repeated occurrences: ${duplicates} duplicate occurrences are folded into their main issue (see duplicate_count) and are not included in the total.${unmatched}`, + ]; +} + async function fetchAccessibilityIssues( scanId: string, scanRunId: string, @@ -174,8 +186,14 @@ async function fetchAccessibilityIssues( const reportFetcher = await initializeReportFetcher(config); const reportLink = await reportFetcher.getReportLink(scanId, scanRunId); - const { records, page_length, total_issues, next_page } = - await parseAccessibilityReportFromCSV(reportLink, { nextPage: cursor }); + const { + records, + page_length, + total_issues, + next_page, + duplicate_issues, + unlinked_duplicates, + } = await parseAccessibilityReportFromCSV(reportLink, { nextPage: cursor }); const currentlyShown = cursor === 0 @@ -186,6 +204,7 @@ async function fetchAccessibilityIssues( const messages = [ `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, + ...duplicatesNote(duplicate_issues, unlinked_duplicates), `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify(records, null, 2))}`, ]; @@ -366,12 +385,14 @@ function createScanSuccessResponse( scanRunId: string, reportUrl: string, cursor: number | null, + duplicates: string[] = [], ): CallToolResult { const messages = [ `Accessibility scan "${name}" completed. check the BrowserStack dashboard for more details [https://scanner.browserstack.com/site-scanner/scan-details/${name}].`, `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, `You can also download the full report from the following link: ${reportUrl}`, `We found ${totalIssues} issues. Below are the details of the ${pageLength} most critical issues.`, + ...duplicates, `Scan results: ${wrapUntrusted("accessibility scan results", JSON.stringify(records, null, 2))}`, ]; @@ -413,8 +434,14 @@ async function runAccessibilityScan( const reportFetcher = await initializeReportFetcher(config); const reportLink = await reportFetcher.getReportLink(scanId, scanRunId); - const { records, page_length, total_issues, next_page } = - await parseAccessibilityReportFromCSV(reportLink); + const { + records, + page_length, + total_issues, + next_page, + duplicate_issues, + unlinked_duplicates, + } = await parseAccessibilityReportFromCSV(reportLink); return createScanSuccessResponse( name, @@ -425,6 +452,7 @@ async function runAccessibilityScan( scanRunId, reportLink, next_page, + duplicatesNote(duplicate_issues, unlinked_duplicates), ); } diff --git a/src/tools/accessiblity-utils/report-parser.ts b/src/tools/accessiblity-utils/report-parser.ts index 5128753f..68201463 100644 --- a/src/tools/accessiblity-utils/report-parser.ts +++ b/src/tools/accessiblity-utils/report-parser.ts @@ -9,6 +9,8 @@ type SimplifiedAccessibilityIssue = { HTML_snippet: string; how_to_fix: string; severity: string; + /** Grouped reports only: how many duplicate rows point at this issue. */ + duplicate_count?: number; }; type PaginationOptions = { @@ -22,10 +24,24 @@ type PaginatedResult = { records: SimplifiedAccessibilityIssue[]; /** Character offset for the next page, or null if done */ page_length: number; + /** Main issues only when the report is grouped, so it matches the run's issue count. */ total_issues: number; next_page: number | null; + /** Duplicate rows left out of `records`; null when the report is not grouped. */ + duplicate_issues: number | null; + /** Of those, the ones whose main issue could not be identified (counted only here). */ + unlinked_duplicates: number; }; +const DUPLICATE_COLUMN = "Duplicate Issue of"; +const ISSUE_LINK_COLUMN = "Issue link"; + +/** A grouped report marks a main issue `NA`; a duplicate carries its main issue's link. */ +function isMainIssue(row: Record): boolean { + const value = (row[DUPLICATE_COLUMN] ?? "").trim(); + return value === "" || value.toUpperCase() === "NA"; +} + export async function parseAccessibilityReportFromCSV( reportLink: string, { maxCharacterLength = 5000, nextPage = 0 }: PaginationOptions = {}, @@ -37,9 +53,13 @@ export async function parseAccessibilityReportFromCSV( typeof res.data === "string" ? res.data : JSON.stringify(res.data); let parsed: any[]; + let columns: string[] = []; try { parsed = parse(text, { - columns: true, + columns: (header: string[]) => { + columns = header; + return header; + }, skip_empty_lines: true, relax_column_count_less: true, }); @@ -51,18 +71,50 @@ export async function parseAccessibilityReportFromCSV( ); throw err; } - const all: SimplifiedAccessibilityIssue[] = parsed.map((row: any) => ({ + + // 2) Fold duplicates into their main issue (grouped reports only). + // + // The export writes a duplicate's "Duplicate Issue of" as the exact "Issue link" of the + // main issue it belongs to, so the two are matched verbatim. A value that matches no + // main issue, or matches more than one, is not guessed at: it is counted in + // `unlinked_duplicates` and the duplicate total, never against a specific issue. + const grouped = columns.includes(DUPLICATE_COLUMN); + const linkable = grouped && columns.includes(ISSUE_LINK_COLUMN); + const mainRows = grouped ? parsed.filter(isMainIssue) : parsed; + const duplicateRows = grouped + ? parsed.filter((row) => !isMainIssue(row)) + : []; + + const all: SimplifiedAccessibilityIssue[] = mainRows.map((row: any) => ({ issue_type: row["Issue type"], component: row["Component"], issue_description: row["Issue description"], HTML_snippet: row["HTML snippet"], how_to_fix: row["How to fix this issue"], severity: (row["Severity"] || "unknown").trim(), + ...(linkable ? { duplicate_count: 0 } : {}), })); + let unlinkedDuplicates = duplicateRows.length; + if (linkable && duplicateRows.length > 0) { + const byLink = new Map(); + mainRows.forEach((row: any, i: number) => { + const link = (row[ISSUE_LINK_COLUMN] ?? "").trim(); + if (!link) return; + // Two main issues sharing a link cannot tell their duplicates apart. + byLink.set(link, byLink.has(link) ? null : all[i]); + }); + unlinkedDuplicates = 0; + for (const row of duplicateRows) { + const main = byLink.get((row[DUPLICATE_COLUMN] ?? "").trim()); + if (main) main.duplicate_count = (main.duplicate_count ?? 0) + 1; + else unlinkedDuplicates++; + } + } + const totalIssues = all.length; - // 2) Sort by severity + // 3) Sort by severity const order: Record = { critical: 0, serious: 1, @@ -71,7 +123,7 @@ export async function parseAccessibilityReportFromCSV( }; all.sort((a, b) => (order[a.severity] ?? 99) - (order[b.severity] ?? 99)); - // 3) Walk to the starting offset + // 4) Walk to the starting offset let charCursor = 0; let idx = 0; for (; idx < all.length; idx++) { @@ -80,7 +132,7 @@ export async function parseAccessibilityReportFromCSV( charCursor += len; } - // 4) Collect up to maxCharacterLength + // 5) Collect up to maxCharacterLength const page: SimplifiedAccessibilityIssue[] = []; for (let i = idx; i < all.length; i++) { const recStr = JSON.stringify(all[i]); @@ -97,5 +149,7 @@ export async function parseAccessibilityReportFromCSV( next_page: hasMore ? charCursor : null, page_length: pageLength, total_issues: totalIssues, + duplicate_issues: grouped ? duplicateRows.length : null, + unlinked_duplicates: unlinkedDuplicates, }; } diff --git a/tests/tools/accessibilityReport.test.ts b/tests/tools/accessibilityReport.test.ts new file mode 100644 index 00000000..3034f124 --- /dev/null +++ b/tests/tools/accessibilityReport.test.ts @@ -0,0 +1,171 @@ +import { beforeEach, describe, expect, it, Mock, vi } from "vitest"; + +vi.mock("../../src/lib/apiClient.js", () => ({ + apiClient: { get: vi.fn(), post: vi.fn() }, +})); +vi.mock("../../src/logger.js", () => ({ + default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, +})); + +import { apiClient } from "../../src/lib/apiClient.js"; +import { parseAccessibilityReportFromCSV } from "../../src/tools/accessiblity-utils/report-parser.js"; + +const get = apiClient.get as Mock; + +const COLUMNS = [ + "Issue type", + "Issue name", + "Severity", + "Component", + "Behavior", + "Issue description", + "Affected page", + "Guideline(s)", + "HTML snippet", + "CSS selector", + "Needs review", + "Needs Review Status", + "Best practice", + "How to fix this issue", + "Issue link", + "Duplicate Issue of", + "Timestamp", + "Issue status", + "Page HTML Source", +]; + +const LINK_A = "https://dash.example/issue?activeIssueId=a"; +const LINK_B = "https://dash.example/issue?activeIssueId=b"; + +function row(fields: Record): Record { + return { + Behavior: "", + "Needs Review Status": "Confirmed", + "Best practice": "No", + Timestamp: "2026-10-01", + "Issue status": "Visible", + "Page HTML Source": "https://dash.example/html", + ...fields, + }; +} + +const contrast = { + "Issue type": "color-contrast", + "Issue name": "Elements must meet contrast", + Severity: "serious", + Component: "p.note", + "Issue description": "Low contrast", + "Guideline(s)": "WCAG 2.1 AA, 1.4.3", + "CSS selector": "p.note", + "Needs review": "No", + "How to fix this issue": "Raise contrast", +}; + +/** Two main issues; two duplicates of A, one duplicate whose main is not in the file. */ +const GROUPED_ROWS = [ + row({ + ...contrast, + "Affected page": "https://site.example/", + "HTML snippet": '

x

', + "Issue link": LINK_A, + "Duplicate Issue of": "NA", + }), + row({ + "Issue type": "image-alt", + "Issue name": "Images must have alt text", + Severity: "critical", + Component: "img.hero", + "Issue description": "Missing alt", + "Affected page": "https://site.example/", + "Guideline(s)": "WCAG 2.1 A", + "HTML snippet": "", + "CSS selector": "img.hero", + "Needs review": "Yes", + "How to fix this issue": "Add alt", + "Issue link": LINK_B, + "Duplicate Issue of": "NA", + }), + row({ + ...contrast, + "Affected page": "https://site.example/a", + "HTML snippet": "

y

", + "Issue link": "https://dash.example/issue?activeIssueId=c", + "Duplicate Issue of": LINK_A, + }), + row({ + ...contrast, + "Affected page": "https://site.example/b", + "HTML snippet": "

z

", + "Issue link": "https://dash.example/issue?activeIssueId=d", + "Duplicate Issue of": LINK_A, + }), + row({ + ...contrast, + "Issue type": "label", + "Affected page": "https://site.example/c", + "Issue link": "https://dash.example/issue?activeIssueId=e", + "Duplicate Issue of": "https://dash.example/issue?activeIssueId=gone", + }), +]; + +/** Quote every cell, CSV-style. */ +function toCsv(columns: string[], rows: Record[]): string { + const cell = (value = "") => `"${value.replace(/"/g, '""')}"`; + return [columns, ...rows.map((r) => columns.map((c) => r[c]))] + .map((cells) => cells.map(cell).join(",")) + .join("\n"); +} + +function csvResponse(text: string) { + return { ok: true, statusText: "OK", data: text }; +} + +describe("parseAccessibilityReportFromCSV", () => { + beforeEach(() => vi.clearAllMocks()); + + it("returns only main issues, with duplicates counted against the issue they point at", async () => { + get.mockResolvedValue(csvResponse(toCsv(COLUMNS, GROUPED_ROWS))); + const result = await parseAccessibilityReportFromCSV("https://csv"); + + expect(result.total_issues).toBe(2); + expect( + result.records.map((r) => [r.issue_type, r.duplicate_count]), + ).toEqual([ + ["image-alt", 0], + ["color-contrast", 2], + ]); + // The third duplicate's main issue is not in the file: counted in the total only. + expect(result.duplicate_issues).toBe(3); + expect(result.unlinked_duplicates).toBe(1); + expect(result.next_page).toBeNull(); + }); + + it("does not attribute duplicates when the report has no Issue link to match them by", async () => { + const withoutLinks = toCsv( + COLUMNS.filter((c) => c !== "Issue link"), + GROUPED_ROWS, + ); + get.mockResolvedValue(csvResponse(withoutLinks)); + const result = await parseAccessibilityReportFromCSV("https://csv"); + + expect(result.total_issues).toBe(2); + expect(result.records.every((r) => !("duplicate_count" in r))).toBe(true); + expect(result.duplicate_issues).toBe(3); + expect(result.unlinked_duplicates).toBe(3); + }); + + it("keeps every row of an ungrouped report and adds no duplicate fields", async () => { + const ungrouped = [ + "Issue type,Severity,Component,Issue description,HTML snippet,How to fix this issue,Issue link", + "image-alt,critical,img,Missing alt,,Add alt,https://d/1", + "image-alt,critical,img,Missing alt,,Add alt,https://d/2", + ].join("\n"); + get.mockResolvedValue(csvResponse(ungrouped)); + const result = await parseAccessibilityReportFromCSV("https://csv"); + + expect(result.total_issues).toBe(2); + expect(result.duplicate_issues).toBeNull(); + expect(result.unlinked_duplicates).toBe(0); + expect(result.records.every((r) => !("duplicate_count" in r))).toBe(true); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts new file mode 100644 index 00000000..b2516fbd --- /dev/null +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -0,0 +1,133 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// What startAccessibilityScan / fetchAccessibilityIssues tell the agent about the issue +// count: main issues only, matching the dashboard, with duplicates stated separately. + +const m = vi.hoisted(() => ({ + startScan: vi.fn(), + waitUntilComplete: vi.fn(), + getReportLink: vi.fn(), + parse: vi.fn(), +})); + +vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ + AccessibilityScanner: class { + setAuth = vi.fn(); + startScan = m.startScan; + waitUntilComplete = m.waitUntilComplete; + }, +})); +vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ + AccessibilityReportFetcher: class { + setAuth = vi.fn(); + getReportLink = m.getReportLink; + }, +})); +vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ + parseAccessibilityReportFromCSV: m.parse, +})); +vi.mock("../../src/lib/get-auth.js", () => ({ + getBrowserStackAuth: vi.fn().mockReturnValue("fake-user:fake-key"), +})); +vi.mock("../../src/logger.js", () => ({ + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); +vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); + +import addAccessibilityTools from "../../src/tools/accessibility.js"; + +const CONFIG = { + "browserstack-username": "fake-user", + "browserstack-access-key": "fake-key", +}; + +function register() { + const handlers: Record any> = {}; + const server: any = { + tool: vi.fn((...args: any[]) => { + handlers[args[0]] = args[args.length - 1]; + }), + server: { + getClientVersion: vi.fn().mockReturnValue({ version: "1.0" }), + getClientCapabilities: vi.fn().mockReturnValue({}), + }, + }; + addAccessibilityTools(server, CONFIG); + return handlers; +} + +const text = (result: any) => + result.content.map((c: { text: string }) => c.text).join("\n"); + +const CONTEXT = { sendNotification: vi.fn(), _meta: {} }; + +/** A grouped report: 79 main issues, 43 duplicates (all matched). */ +const GROUPED = { + records: [{ issue_type: "select-name", duplicate_count: 0 }], + page_length: 1, + total_issues: 79, + next_page: 140, + duplicate_issues: 43, + unlinked_duplicates: 0, +}; + +describe("accessibility issue counts", () => { + let handlers: Record any>; + + beforeEach(() => { + vi.clearAllMocks(); + m.startScan.mockResolvedValue({ + data: { id: "scan-42", scanRunId: "run-7" }, + }); + m.waitUntilComplete.mockResolvedValue("completed"); + m.getReportLink.mockResolvedValue("https://csv"); + handlers = register(); + }); + + it("startAccessibilityScan reports main issues as the total and states the duplicates separately", async () => { + m.parse.mockResolvedValue(GROUPED); + const result = await handlers.startAccessibilityScan( + { name: "dedupe", pageURL: "https://site.example" }, + CONTEXT, + ); + + expect(result.isError).toBeFalsy(); + const body = text(result); + expect(body).toContain("We found 79 issues."); + expect(body).toContain( + "43 duplicate occurrences are folded into their main issue", + ); + expect(body).not.toContain("could not be matched"); + }); + + it("fetchAccessibilityIssues reports main issues as the total and names unmatched duplicates", async () => { + m.parse.mockResolvedValue({ ...GROUPED, unlinked_duplicates: 2 }); + const result = await handlers.fetchAccessibilityIssues( + { scanId: "scan-42", scanRunId: "run-7" }, + CONTEXT, + ); + + const body = text(result); + expect(body).toContain("Retrieved 1 accessibility issues (Total: 79)"); + expect(body).toContain( + "43 duplicate occurrences are folded into their main issue", + ); + expect(body).toContain("2 of them could not be matched"); + }); + + it("adds no duplicates note for a report without duplicates", async () => { + m.parse.mockResolvedValue({ + ...GROUPED, + duplicate_issues: null, + next_page: null, + }); + const result = await handlers.startAccessibilityScan( + { name: "plain", pageURL: "https://site.example" }, + CONTEXT, + ); + + const body = text(result); + expect(body).toContain("We found 79 issues."); + expect(body).not.toContain("duplicate occurrences"); + }); +}); From c4df14ff0b9cb8d406651aa4b8425573120685af Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Mon, 19 Oct 2026 19:47:32 +0530 Subject: [PATCH 2/5] fix(accessibility): end each response block with a newline The scan and issue tools return one text block per message. A client that joins the blocks with no separator (Claude Code does) runs the report link into the next block's first word: "...&Key-Pair-Id=KWDR32L2KWU1BWe found 122 issues". The key pair id is opaque, so an agent copying the link takes the "W" with it and the download fails with 403 InvalidKey. Each block now ends with a newline. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 6 +++++- tests/tools/accessibilityScanOutput.test.ts | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index cca4ee56..46b82fc0 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -68,11 +68,15 @@ function createErrorResponse(message: string): CallToolResult { }; } +/** + * Each message ends with a newline: clients that join text blocks with no separator + * would otherwise run a link at the end of one block into the next block's first word. + */ function createSuccessResponse(messages: string[]): CallToolResult { return { content: messages.map((text) => ({ type: "text" as const, - text, + text: `${text}\n`, })), }; } diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index b2516fbd..b0e68f3f 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -130,4 +130,21 @@ describe("accessibility issue counts", () => { expect(body).toContain("We found 79 issues."); expect(body).not.toContain("duplicate occurrences"); }); + + it("keeps the report link intact when a client joins the blocks with no separator", async () => { + m.getReportLink.mockResolvedValue( + "https://reports.example/r.csv?Signature=abc&Key-Pair-Id=K1", + ); + m.parse.mockResolvedValue(GROUPED); + const result = await handlers.startAccessibilityScan( + { name: "joined", pageURL: "https://site.example" }, + CONTEXT, + ); + + const joined = result.content.map((c: { text: string }) => c.text).join(""); + expect(joined.match(/following link: (\S+)/)?.[1]).toBe( + "https://reports.example/r.csv?Signature=abc&Key-Pair-Id=K1", + ); + expect(joined).not.toMatch(/Scan Run ID: run-7\S/); + }); }); From ad61e1c7dd0d4167d4500ffa422e288c1ff401ec Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Mon, 19 Oct 2026 21:14:51 +0530 Subject: [PATCH 3/5] feat(accessibility): fetch issues of Automated Tests builds and Workflow Analyzer reports fetchAccessibilityIssues read only Website Scanner runs, so an agent that ran an SDK test with accessibility on, or was handed a Workflow Analyzer report, had no way to read its issues. The API already exports both the same way as a scan run: start a CSV export, poll the same route with the task id, download the link. Only the route and its parameter names differ. The tool now takes exactly one source: scanId with scanRunId, buildId (the Accessibility build id, thBuildId in the report link), or workflowReportId. The fetcher builds each source's export URLs; the parser, duplicate folding and paging are unchanged. A build the API does not find (the Automate build id is one) gets a message naming the id to pass instead, and the next-page hint repeats the source's own arguments. Rows now carry affected_page, since builds and Workflow Analyzer reports span several pages, and source_test when the export has a "Source test" column (Automated Tests builds). Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 91 ++++++++++++++---- .../accessiblity-utils/report-fetcher.ts | 76 ++++++++++++--- src/tools/accessiblity-utils/report-parser.ts | 9 ++ src/tools/tool-handoff.ts | 13 ++- tests/tools/accessibilityReport.test.ts | 24 +++++ .../tools/accessibilityReportFetcher.test.ts | 94 +++++++++++++++++++ tests/tools/accessibilityScanOutput.test.ts | 78 +++++++++++++++ 7 files changed, 354 insertions(+), 31 deletions(-) create mode 100644 tests/tools/accessibilityReportFetcher.test.ts diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 46b82fc0..f45e90ba 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -2,7 +2,10 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { AccessibilityScanner } from "./accessiblity-utils/scanner.js"; -import { AccessibilityReportFetcher } from "./accessiblity-utils/report-fetcher.js"; +import { + AccessibilityReportFetcher, + type IssueSource, +} from "./accessiblity-utils/report-fetcher.js"; import { AccessibilityAuthConfig, safeAuthConfigData, @@ -146,8 +149,48 @@ async function executeAccessibilityRAG( } } +interface FetchIssuesArgs { + scanId?: string; + scanRunId?: string; + buildId?: string; + workflowReportId?: string; + cursor?: number; +} + +/** The one source the arguments name, or why they do not name exactly one. */ +function issueSourceFrom(args: FetchIssuesArgs): IssueSource | string { + const named = [ + Boolean(args.scanId || args.scanRunId), + Boolean(args.buildId), + Boolean(args.workflowReportId), + ].filter(Boolean).length; + if (named !== 1) { + return "Pass exactly one source: scanId with scanRunId (a Website Scanner run), buildId (an Automated Tests build) or workflowReportId (a Workflow Analyzer report)."; + } + if (args.buildId) return { kind: "build", buildId: args.buildId }; + if (args.workflowReportId) { + return { kind: "workflow", reportId: args.workflowReportId }; + } + if (!args.scanId || !args.scanRunId) { + return "A Website Scanner run needs both scanId and scanRunId."; + } + return { kind: "scan", scanId: args.scanId, scanRunId: args.scanRunId }; +} + +/** The arguments that name the same source again, for the next-page hint. */ +function sourceArgs(source: IssueSource): string { + switch (source.kind) { + case "scan": + return `scanId: "${source.scanId}", scanRunId: "${source.scanRunId}"`; + case "build": + return `buildId: "${source.buildId}"`; + case "workflow": + return `workflowReportId: "${source.reportId}"`; + } +} + async function executeFetchAccessibilityIssues( - args: { scanId: string; scanRunId: string; cursor?: number }, + args: FetchIssuesArgs, server: McpServer, config: BrowserStackConfig, ): Promise { @@ -158,12 +201,9 @@ async function executeFetchAccessibilityIssues( undefined, config, ); - return await fetchAccessibilityIssues( - args.scanId, - args.scanRunId, - config, - args.cursor, - ); + const source = issueSourceFrom(args); + if (typeof source === "string") return createErrorResponse(source); + return await fetchAccessibilityIssues(source, config, args.cursor); } catch (error) { return handleMCPError("fetchAccessibilityIssues", server, config, error); } @@ -182,13 +222,12 @@ function duplicatesNote(duplicates: number | null, unlinked: number): string[] { } async function fetchAccessibilityIssues( - scanId: string, - scanRunId: string, + source: IssueSource, config: BrowserStackConfig, cursor = 0, ): Promise { const reportFetcher = await initializeReportFetcher(config); - const reportLink = await reportFetcher.getReportLink(scanId, scanRunId); + const reportLink = await reportFetcher.getReportLink(source); const { records, @@ -214,7 +253,7 @@ async function fetchAccessibilityIssues( if (next_page !== null) { messages.push( - `${remainingIssues} more issues available. Use fetchAccessibilityIssues with cursor: ${next_page} to get the next batch.`, + `${remainingIssues} more issues available. Use fetchAccessibilityIssues with ${sourceArgs(source)} and cursor: ${next_page} to get the next batch.`, ); } else { messages.push(`✅ All issues retrieved.`); @@ -436,7 +475,11 @@ async function runAccessibilityScan( } const reportFetcher = await initializeReportFetcher(config); - const reportLink = await reportFetcher.getReportLink(scanId, scanRunId); + const reportLink = await reportFetcher.getReportLink({ + kind: "scan", + scanId, + scanRunId, + }); const { records, @@ -628,15 +671,31 @@ export default function addAccessibilityTools( tools.fetchAccessibilityIssues = server.tool( "fetchAccessibilityIssues", - "Fetch accessibility issues from a completed scan with pagination support. Use cursor parameter to get subsequent pages of results." + + "Fetch accessibility issues with pagination support, from exactly one source: a completed Website Scanner run (scanId with scanRunId), an Automated Tests build from a BrowserStack SDK test run (buildId), or a Workflow Analyzer report (workflowReportId). Use cursor parameter to get subsequent pages of results." + NEEDS_A11Y_SCAN_ID, { scanId: z .string() - .describe("The scan ID from a completed accessibility scan"), + .optional() + .describe( + "Website Scanner: the scan ID from a completed accessibility scan", + ), scanRunId: z .string() - .describe("The scan run ID from a completed accessibility scan"), + .optional() + .describe( + "Website Scanner: the scan run ID from a completed accessibility scan", + ), + buildId: z + .string() + .optional() + .describe( + "Automated Tests: the Accessibility build id, shown as thBuildId in the build's accessibility report link (40 lowercase letters and digits). Not the Automate build id or the build name.", + ), + workflowReportId: z + .string() + .optional() + .describe("Workflow Analyzer: the report's numeric id, e.g. 123456"), cursor: z .number() .optional() diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index d81893da..1e1cbdb3 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -12,6 +12,63 @@ interface ReportResponse { error?: any; } +/** What to read issues from: a Website Scanner run, an Automated Tests build or a Workflow Analyzer report. */ +export type IssueSource = + | { kind: "scan"; scanId: string; scanRunId: string } + | { kind: "build"; buildId: string } + | { kind: "workflow"; reportId: string }; + +const API_BASE = "https://api-accessibility.browserstack.com/api"; + +/** + * Each source exports its issues the same way: one call starts a CSV export and returns a + * task id, and the same route polled with that task id returns the link. Only the route and + * its parameter names differ (the scanner's are snake_case, the other two camelCase). + */ +function exportUrls(source: IssueSource): { + start: string; + poll: (taskId: string) => string; +} { + const q = encodeURIComponent; + switch (source.kind) { + case "scan": { + const route = `${API_BASE}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/issues`; + return { + start: `${route}?scan_run_id=${q(source.scanRunId)}`, + poll: (taskId) => `${route}?task_id=${q(taskId)}`, + }; + } + case "build": { + const route = `${API_BASE}/automated-tests/v1/builds/issues`; + return { + start: `${route}?buildUuid=${q(source.buildId)}`, + poll: (taskId) => `${route}?taskId=${q(taskId)}`, + }; + } + case "workflow": { + const route = `${API_BASE}/workflow-analyzer/v1/reports/issues`; + return { + start: `${route}?reportId=${q(source.reportId)}`, + poll: (taskId) => `${route}?taskId=${q(taskId)}`, + }; + } + } +} + +/** Why an export did not start, in the API's words, and where a build's id comes from when it is not found. */ +function startFailure(source: IssueSource, status: number, body: any): string { + const reason = + [body?.error].flat().filter(Boolean).join("; ") || + body?.data?.message || + `HTTP ${status}`; + // The API takes only its own build id; the Automate build id is a 404 "Build not found". + const hint = + status === 404 && source.kind === "build" + ? ". Pass the Accessibility build id: the thBuildId in the build's accessibility report link (40 lowercase letters and digits), not the Automate build id or the build name" + : ""; + return `Failed to initiate report: ${reason}${hint}`; +} + export class AccessibilityReportFetcher { private auth: { username: string; password: string } | undefined; @@ -19,9 +76,8 @@ export class AccessibilityReportFetcher { this.auth = auth; } - async getReportLink(scanId: string, scanRunId: string): Promise { - // Initiate CSV link generation - const initUrl = `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${scanId}/scan_runs/issues?scan_run_id=${scanRunId}`; + async getReportLink(source: IssueSource): Promise { + const urls = exportUrls(source); let basicAuthHeader = undefined; if (this.auth) { @@ -29,22 +85,20 @@ export class AccessibilityReportFetcher { basicAuthHeader = "Basic " + Buffer.from(`${username}:${password}`).toString("base64"); } + // Initiate CSV link generation const initResp = await apiClient.get({ - url: initUrl, + url: urls.start, headers: basicAuthHeader ? { Authorization: basicAuthHeader } : undefined, + raise_error: false, }); const initData: ReportInitResponse = initResp.data; - if (!initData.success) { - throw new Error( - `Failed to initiate report: ${initData.error || initData.data.message}`, - ); + if (!initResp.ok || !initData?.success) { + throw new Error(startFailure(source, initResp.status, initData)); } const taskId = initData.data.task_id; // Poll for the generated CSV link (task is async, may take a few seconds) - const reportUrl = `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${scanId}/scan_runs/issues?task_id=${encodeURIComponent( - taskId, - )}`; + const reportUrl = urls.poll(taskId); const maxAttempts = 3; const pollIntervalMs = 2000; diff --git a/src/tools/accessiblity-utils/report-parser.ts b/src/tools/accessiblity-utils/report-parser.ts index 68201463..08e4d26a 100644 --- a/src/tools/accessiblity-utils/report-parser.ts +++ b/src/tools/accessiblity-utils/report-parser.ts @@ -9,10 +9,16 @@ type SimplifiedAccessibilityIssue = { HTML_snippet: string; how_to_fix: string; severity: string; + /** The page the issue is on; builds and Workflow Analyzer reports span several. */ + affected_page: string; + /** Automated Tests builds only: the test that found the issue. */ + source_test?: string; /** Grouped reports only: how many duplicate rows point at this issue. */ duplicate_count?: number; }; +const SOURCE_TEST_COLUMN = "Source test"; + type PaginationOptions = { /** How many JSON-chars max per “page” (default 10000) */ maxCharacterLength?: number; @@ -80,6 +86,7 @@ export async function parseAccessibilityReportFromCSV( // `unlinked_duplicates` and the duplicate total, never against a specific issue. const grouped = columns.includes(DUPLICATE_COLUMN); const linkable = grouped && columns.includes(ISSUE_LINK_COLUMN); + const hasSourceTest = columns.includes(SOURCE_TEST_COLUMN); const mainRows = grouped ? parsed.filter(isMainIssue) : parsed; const duplicateRows = grouped ? parsed.filter((row) => !isMainIssue(row)) @@ -92,6 +99,8 @@ export async function parseAccessibilityReportFromCSV( HTML_snippet: row["HTML snippet"], how_to_fix: row["How to fix this issue"], severity: (row["Severity"] || "unknown").trim(), + affected_page: row["Affected page"], + ...(hasSourceTest ? { source_test: row[SOURCE_TEST_COLUMN] } : {}), ...(linkable ? { duplicate_count: 0 } : {}), })); diff --git a/src/tools/tool-handoff.ts b/src/tools/tool-handoff.ts index ce41a2b3..cab2548f 100644 --- a/src/tools/tool-handoff.ts +++ b/src/tools/tool-handoff.ts @@ -88,11 +88,16 @@ export const NEEDS_SESSION_ID = " Requires a session id. listSessions lists them for a build; if you do not have the " + "build either, getBuildId or listBuildId resolves one from a project and build name."; -/** A completed scan's ids come from startAccessibilityScan, or from the agent. */ +/** + * A scan run's ids come from startAccessibilityScan. Nothing here lists Automated Tests builds + * or Workflow Analyzer reports, so their ids come from the user or the report's link. + */ export const NEEDS_A11Y_SCAN_ID = - " Requires the ids of a completed scan, which are returned by startAccessibilityScan. " + - "No tool here lists past scans, so if you do not have the ids, start a new scan rather " + - "than guessing."; + " Requires the id of a completed source: a Website Scanner run's ids, returned by " + + "startAccessibilityScan; an Automated Tests build's id, the thBuildId in its accessibility " + + "report link; or a Workflow Analyzer report's numeric id. No tool here lists past scans, " + + "builds or reports, so if you do not have the id, ask the user for the report link, or " + + "start a new scan, rather than guessing."; /** Auth-config ids are not listable by any tool here. */ export const NEEDS_A11Y_CONFIG_ID = diff --git a/tests/tools/accessibilityReport.test.ts b/tests/tools/accessibilityReport.test.ts index 3034f124..1ae6bbbf 100644 --- a/tests/tools/accessibilityReport.test.ts +++ b/tests/tools/accessibilityReport.test.ts @@ -168,4 +168,28 @@ describe("parseAccessibilityReportFromCSV", () => { expect(result.unlinked_duplicates).toBe(0); expect(result.records.every((r) => !("duplicate_count" in r))).toBe(true); }); + + it("says which page each issue is on and, for a build, which test found it", async () => { + const buildCsv = toCsv( + [...COLUMNS, "Source test"], + GROUPED_ROWS.map((r) => ({ ...r, "Source test": "test_checkout" })), + ); + get.mockResolvedValue(csvResponse(buildCsv)); + const { records } = await parseAccessibilityReportFromCSV("https://csv"); + + expect(records.map((r) => [r.affected_page, r.source_test])).toEqual([ + ["https://site.example/", "test_checkout"], + ["https://site.example/", "test_checkout"], + ]); + }); + + it("adds no source_test to a report without that column", async () => { + get.mockResolvedValue(csvResponse(toCsv(COLUMNS, GROUPED_ROWS))); + const { records } = await parseAccessibilityReportFromCSV("https://csv"); + + expect( + records.every((r) => r.affected_page === "https://site.example/"), + ).toBe(true); + expect(records.every((r) => !("source_test" in r))).toBe(true); + }); }); diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts new file mode 100644 index 00000000..a59ec334 --- /dev/null +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -0,0 +1,94 @@ +import { beforeEach, describe, expect, it, Mock, vi } from "vitest"; + +vi.mock("../../src/lib/apiClient.js", () => ({ + apiClient: { get: vi.fn() }, +})); + +import { apiClient } from "../../src/lib/apiClient.js"; +import { + AccessibilityReportFetcher, + IssueSource, +} from "../../src/tools/accessiblity-utils/report-fetcher.js"; + +const get = apiClient.get as Mock; +const API = "https://api-accessibility.browserstack.com/api"; +const BUILD_ID = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; + +const ok = (data: unknown) => ({ + ok: true, + status: 200, + data: { success: true, data }, +}); +const failed = (status: number, error: unknown) => ({ + ok: false, + status, + data: { success: false, data: null, error }, +}); + +function fetcher() { + const f = new AccessibilityReportFetcher(); + f.setAuth({ username: "u", password: "p" }); + return f; +} + +describe("AccessibilityReportFetcher.getReportLink", () => { + beforeEach(() => vi.clearAllMocks()); + + it.each<[string, IssueSource, string, string]>([ + [ + "a Website Scanner run", + { kind: "scan", scanId: "1234", scanRunId: "5678" }, + `${API}/website-scanner/v1/scans/1234/scan_runs/issues?scan_run_id=5678`, + `${API}/website-scanner/v1/scans/1234/scan_runs/issues?task_id=t%3A1`, + ], + [ + "an Automated Tests build", + { kind: "build", buildId: BUILD_ID }, + `${API}/automated-tests/v1/builds/issues?buildUuid=${BUILD_ID}`, + `${API}/automated-tests/v1/builds/issues?taskId=t%3A1`, + ], + [ + "a Workflow Analyzer report", + { kind: "workflow", reportId: "123456" }, + `${API}/workflow-analyzer/v1/reports/issues?reportId=123456`, + `${API}/workflow-analyzer/v1/reports/issues?taskId=t%3A1`, + ], + ])( + "exports the issues of %s on its own route", + async (_label, source, startUrl, pollUrl) => { + get + .mockResolvedValueOnce(ok({ task_id: "t:1", message: "started" })) + .mockResolvedValueOnce(ok({ reportLink: "https://csv" })); + + await expect(fetcher().getReportLink(source)).resolves.toBe( + "https://csv", + ); + expect(get.mock.calls.map(([options]) => options.url)).toEqual([ + startUrl, + pollUrl, + ]); + expect(get.mock.calls[0][0].headers).toEqual({ + Authorization: `Basic ${Buffer.from("u:p").toString("base64")}`, + }); + }, + ); + + it("says which id a build needs when the API does not find it", async () => { + get.mockResolvedValueOnce(failed(404, ["Build not found"])); + + await expect( + fetcher().getReportLink({ + kind: "build", + buildId: "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c", + }), + ).rejects.toThrow(/Build not found.*thBuildId.*not the Automate build id/s); + }); + + it("passes on the API's reason when it rejects a report id", async () => { + get.mockResolvedValueOnce(failed(400, ["Report id must be an Integer"])); + + await expect( + fetcher().getReportLink({ kind: "workflow", reportId: "abc" }), + ).rejects.toThrow("Report id must be an Integer"); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index b0e68f3f..91f03a1c 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -148,3 +148,81 @@ describe("accessibility issue counts", () => { expect(joined).not.toMatch(/Scan Run ID: run-7\S/); }); }); + +describe("fetchAccessibilityIssues sources", () => { + let handlers: Record any>; + const BUILD_ID = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; + + beforeEach(() => { + vi.clearAllMocks(); + m.startScan.mockResolvedValue({ + data: { id: "scan-42", scanRunId: "run-7" }, + }); + m.waitUntilComplete.mockResolvedValue("completed"); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue(GROUPED); + handlers = register(); + }); + + it.each([ + [ + "a Website Scanner run", + { scanId: "scan-42", scanRunId: "run-7" }, + { kind: "scan", scanId: "scan-42", scanRunId: "run-7" }, + 'scanId: "scan-42", scanRunId: "run-7"', + ], + [ + "an Automated Tests build", + { buildId: BUILD_ID }, + { kind: "build", buildId: BUILD_ID }, + `buildId: "${BUILD_ID}"`, + ], + [ + "a Workflow Analyzer report", + { workflowReportId: "123456" }, + { kind: "workflow", reportId: "123456" }, + 'workflowReportId: "123456"', + ], + ])( + "reads the issues of %s and names it again for the next page", + async (_label, args, source, nextPageArgs) => { + const result = await handlers.fetchAccessibilityIssues(args, CONTEXT); + + expect(result.isError).toBeFalsy(); + expect(m.getReportLink).toHaveBeenCalledWith(source); + const body = text(result); + expect(body).toContain("Retrieved 1 accessibility issues (Total: 79)"); + expect(body).toContain( + `Use fetchAccessibilityIssues with ${nextPageArgs} and cursor: 140`, + ); + }, + ); + + it("startAccessibilityScan reads the run it just started", async () => { + await handlers.startAccessibilityScan( + { name: "run", pageURL: "https://site.example" }, + CONTEXT, + ); + expect(m.getReportLink).toHaveBeenCalledWith({ + kind: "scan", + scanId: "scan-42", + scanRunId: "run-7", + }); + }); + + it.each([ + ["no source", {}, /exactly one/], + [ + "two sources", + { scanId: "scan-42", scanRunId: "run-7", buildId: BUILD_ID }, + /exactly one/, + ], + ["a scan without its run", { scanId: "scan-42" }, /scanRunId/], + ])("refuses %s without calling the API", async (_label, args, message) => { + const result = await handlers.fetchAccessibilityIssues(args, CONTEXT); + + expect(result.isError).toBe(true); + expect(text(result)).toMatch(message); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); +}); From 1e04e5b142289d541d5f2a586fb76d32947396b8 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Wed, 21 Oct 2026 10:50:04 +0530 Subject: [PATCH 4/5] feat(accessibility): take a report's dashboard link and return its CSV link An agent handed a dashboard report link had to find the ids itself, and fetchAccessibilityIssues returned parsed issues but not the CSV. Both now go through the tool. reportUrl takes a Website Scanner or Automated Tests report link. An Automated Tests link carries its build id. A Website Scanner link carries one run's thBuildId, which only the scan list can map to ids: each scan there carries its latest run's link, so the newest scans are walked (15 pages) for a match. A link to an older run is reported as such, with the scan's id, rather than guessed at. The list is walked at its default page size: at limit=50 or 100 its cursor skips scans. Every answer now includes the CSV's download link, as startAccessibilityScan already does. When a link was resolved the answer says to what, and the next-page hint uses the ids, so later pages skip the lookup. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 40 +++++- .../accessiblity-utils/report-fetcher.ts | 123 ++++++++++++++++-- src/tools/tool-handoff.ts | 13 +- .../tools/accessibilityReportFetcher.test.ts | 76 +++++++++++ tests/tools/accessibilityScanOutput.test.ts | 44 +++++++ 5 files changed, 276 insertions(+), 20 deletions(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index f45e90ba..7705ee91 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -154,19 +154,25 @@ interface FetchIssuesArgs { scanRunId?: string; buildId?: string; workflowReportId?: string; + reportUrl?: string; cursor?: number; } +/** A source named by its ids, or a dashboard link still to be resolved to them. */ +type RequestedSource = IssueSource | { kind: "link"; reportUrl: string }; + /** The one source the arguments name, or why they do not name exactly one. */ -function issueSourceFrom(args: FetchIssuesArgs): IssueSource | string { +function issueSourceFrom(args: FetchIssuesArgs): RequestedSource | string { const named = [ Boolean(args.scanId || args.scanRunId), Boolean(args.buildId), Boolean(args.workflowReportId), + Boolean(args.reportUrl), ].filter(Boolean).length; if (named !== 1) { - return "Pass exactly one source: scanId with scanRunId (a Website Scanner run), buildId (an Automated Tests build) or workflowReportId (a Workflow Analyzer report)."; + return "Pass exactly one source: scanId with scanRunId (a Website Scanner run), buildId (an Automated Tests build), workflowReportId (a Workflow Analyzer report) or reportUrl (a report's dashboard link)."; } + if (args.reportUrl) return { kind: "link", reportUrl: args.reportUrl }; if (args.buildId) return { kind: "build", buildId: args.buildId }; if (args.workflowReportId) { return { kind: "workflow", reportId: args.workflowReportId }; @@ -189,6 +195,18 @@ function sourceArgs(source: IssueSource): string { } } +/** What a dashboard link resolved to, in words. */ +function describeSource(source: IssueSource): string { + switch (source.kind) { + case "scan": + return `Website Scanner scan ${source.scanId}, run ${source.scanRunId}`; + case "build": + return `Automated Tests build ${source.buildId}`; + case "workflow": + return `Workflow Analyzer report ${source.reportId}`; + } +} + async function executeFetchAccessibilityIssues( args: FetchIssuesArgs, server: McpServer, @@ -222,11 +240,15 @@ function duplicatesNote(duplicates: number | null, unlinked: number): string[] { } async function fetchAccessibilityIssues( - source: IssueSource, + requested: RequestedSource, config: BrowserStackConfig, cursor = 0, ): Promise { const reportFetcher = await initializeReportFetcher(config); + const source = + requested.kind === "link" + ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) + : requested; const reportLink = await reportFetcher.getReportLink(source); const { @@ -246,8 +268,12 @@ async function fetchAccessibilityIssues( const remainingIssues = total_issues - currentlyShown; const messages = [ + ...(requested.kind === "link" + ? [`Resolved the report link to ${describeSource(source)}.`] + : []), `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, ...duplicatesNote(duplicate_issues, unlinked_duplicates), + `You can also download the full report (CSV) from the following link, valid for about 12 hours: ${reportLink}`, `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify(records, null, 2))}`, ]; @@ -671,7 +697,7 @@ export default function addAccessibilityTools( tools.fetchAccessibilityIssues = server.tool( "fetchAccessibilityIssues", - "Fetch accessibility issues with pagination support, from exactly one source: a completed Website Scanner run (scanId with scanRunId), an Automated Tests build from a BrowserStack SDK test run (buildId), or a Workflow Analyzer report (workflowReportId). Use cursor parameter to get subsequent pages of results." + + "Fetch accessibility issues with pagination support, from exactly one source: a completed Website Scanner run (scanId with scanRunId), an Automated Tests build from a BrowserStack SDK test run (buildId), a Workflow Analyzer report (workflowReportId), or a Website Scanner or Automated Tests report's dashboard link (reportUrl). Also returns a link to download the full report as a CSV. Use cursor parameter to get subsequent pages of results." + NEEDS_A11Y_SCAN_ID, { scanId: z @@ -696,6 +722,12 @@ export default function addAccessibilityTools( .string() .optional() .describe("Workflow Analyzer: the report's numeric id, e.g. 123456"), + reportUrl: z + .string() + .optional() + .describe( + "A report's Accessibility dashboard link, as the user pasted it: a Website Scanner or Automated Tests report link carrying thBuildId. A Website Scanner link resolves only to its scan's latest run. Workflow Analyzer links are not supported; use workflowReportId.", + ), cursor: z .number() .optional() diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index 1e1cbdb3..646937f4 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -20,6 +20,33 @@ export type IssueSource = const API_BASE = "https://api-accessibility.browserstack.com/api"; +/** + * How far a report link's scan is searched for. The scan list has no name or link filter and + * spans the whole organisation, so the walk covers only the newest scans, which is where a + * link someone just opened will be. It runs at the list's default page size (~1.2 s a page): + * at limit=50 or 100 the cursor skips scans, so a larger page cannot be used to go faster. + */ +export const MAX_SCAN_LIST_PAGES = 15; + +interface ScanListPage { + scans?: { + id: number; + name: string; + scanUrl?: string; + lastScanDetails?: { scanRunId?: number }; + }[]; + pagination?: { hasNext?: boolean; nextPage?: string }; +} + +/** The thBuildId a dashboard report link carries, if any. */ +function thBuildIdOf(link: string | undefined): string | null { + try { + return link ? new URL(link).searchParams.get("thBuildId") : null; + } catch { + return null; + } +} + /** * Each source exports its issues the same way: one call starts a CSV export and returns a * task id, and the same route polled with that task id returns the link. Only the route and @@ -76,19 +103,97 @@ export class AccessibilityReportFetcher { this.auth = auth; } + private authHeaders(): Record | undefined { + if (!this.auth) return undefined; + const { username, password } = this.auth; + return { + Authorization: + "Basic " + Buffer.from(`${username}:${password}`).toString("base64"), + }; + } + + /** + * The source a dashboard report link points at. An Automated Tests link carries the build + * id itself. A Website Scanner link carries the thBuildId of one run, which only the scan + * list can map to ids: each scan there carries its latest run's link. A link to an older + * run therefore matches no scan, and is reported as such rather than guessed at. + */ + async sourceFromReportUrl(reportUrl: string): Promise { + let url: URL | undefined; + try { + url = new URL(reportUrl); + } catch { + url = undefined; + } + const thBuildId = url?.searchParams.get("thBuildId"); + if (url && thBuildId && url.pathname.includes("/automated-tests/")) { + return { kind: "build", buildId: thBuildId }; + } + const scanName = url?.pathname.match( + /\/site-scanner\/projects\/([^/]+)\/builds/, + )?.[1]; + if (thBuildId && scanName) { + return this.findScanRun(thBuildId, decodeURIComponent(scanName)); + } + throw new Error( + "This link does not name a report the API can look up. Pass an Automated Tests or Website Scanner report link from the Accessibility dashboard (both carry thBuildId), or the ids: scanId with scanRunId, buildId, or workflowReportId", + ); + } + + /** Walk the newest scans for the one whose own link carries this thBuildId. */ + private async findScanRun( + thBuildId: string, + scanName: string, + ): Promise { + let nextPage: string | undefined; + let sameName: number | undefined; + for (let page = 0; page < MAX_SCAN_LIST_PAGES; page++) { + const resp = await apiClient.get({ + url: `${API_BASE}/website-scanner/v1/scans${ + nextPage ? `?nextPage=${encodeURIComponent(nextPage)}` : "" + }`, + headers: this.authHeaders(), + }); + const data: ScanListPage | undefined = resp.data?.data; + for (const scan of data?.scans ?? []) { + if (thBuildIdOf(scan.scanUrl) === thBuildId) { + return { + kind: "scan", + scanId: String(scan.id), + scanRunId: String(scan.lastScanDetails?.scanRunId), + }; + } + if (sameName === undefined && scan.name === scanName) { + sameName = scan.id; + } + } + if (!data?.pagination?.hasNext || !data.pagination.nextPage) { + nextPage = undefined; + break; + } + nextPage = data.pagination.nextPage; + } + if (sameName !== undefined) { + throw new Error( + `This link is to an earlier run of scan "${scanName}" (scanId ${sameName}). A link resolves only to a scan's latest run; ask the user for that run's scanRunId and pass it with scanId ${sameName}`, + ); + } + // nextPage is still set only when the walk stopped at the page cap, not at the end. + const searched = nextPage + ? ` among the most recent scans (${MAX_SCAN_LIST_PAGES} pages)` + : ""; + throw new Error( + `No Website Scanner scan${searched} matches this link. Pass scanId and scanRunId instead`, + ); + } + async getReportLink(source: IssueSource): Promise { const urls = exportUrls(source); - let basicAuthHeader = undefined; - if (this.auth) { - const { username, password } = this.auth; - basicAuthHeader = - "Basic " + Buffer.from(`${username}:${password}`).toString("base64"); - } // Initiate CSV link generation const initResp = await apiClient.get({ url: urls.start, - headers: basicAuthHeader ? { Authorization: basicAuthHeader } : undefined, + headers: this.authHeaders(), raise_error: false, }); const initData: ReportInitResponse = initResp.data; @@ -105,9 +210,7 @@ export class AccessibilityReportFetcher { for (let attempt = 1; attempt <= maxAttempts; attempt++) { const reportResp = await apiClient.get({ url: reportUrl, - headers: basicAuthHeader - ? { Authorization: basicAuthHeader } - : undefined, + headers: this.authHeaders(), }); const reportData: ReportResponse = reportResp.data; if (!reportData.success) { diff --git a/src/tools/tool-handoff.ts b/src/tools/tool-handoff.ts index cab2548f..885e55db 100644 --- a/src/tools/tool-handoff.ts +++ b/src/tools/tool-handoff.ts @@ -89,15 +89,16 @@ export const NEEDS_SESSION_ID = "build either, getBuildId or listBuildId resolves one from a project and build name."; /** - * A scan run's ids come from startAccessibilityScan. Nothing here lists Automated Tests builds - * or Workflow Analyzer reports, so their ids come from the user or the report's link. + * A scan run's ids come from startAccessibilityScan. Nothing here lists builds or reports, + * but the tool takes a report's dashboard link and looks the ids up itself. */ export const NEEDS_A11Y_SCAN_ID = - " Requires the id of a completed source: a Website Scanner run's ids, returned by " + + " Requires a completed source: a Website Scanner run's ids, returned by " + "startAccessibilityScan; an Automated Tests build's id, the thBuildId in its accessibility " + - "report link; or a Workflow Analyzer report's numeric id. No tool here lists past scans, " + - "builds or reports, so if you do not have the id, ask the user for the report link, or " + - "start a new scan, rather than guessing."; + "report link; a Workflow Analyzer report's numeric id; or a Website Scanner or Automated " + + "Tests report's dashboard link, passed as reportUrl. No tool here lists past scans, builds " + + "or reports, so if you have none of these, ask the user for the report link, or start a " + + "new scan, rather than guessing."; /** Auth-config ids are not listable by any tool here. */ export const NEEDS_A11Y_CONFIG_ID = diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index a59ec334..6adf91b9 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -8,6 +8,7 @@ import { apiClient } from "../../src/lib/apiClient.js"; import { AccessibilityReportFetcher, IssueSource, + MAX_SCAN_LIST_PAGES, } from "../../src/tools/accessiblity-utils/report-fetcher.js"; const get = apiClient.get as Mock; @@ -92,3 +93,78 @@ describe("AccessibilityReportFetcher.getReportLink", () => { ).rejects.toThrow("Report id must be an Integer"); }); }); + +describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { + const DASHBOARD = "https://accessibility.browserstack.com"; + const scannerLink = (name: string, thBuildId: string) => + `${DASHBOARD}/site-scanner/projects/${encodeURIComponent(name)}/builds?thBuildId=${thBuildId}&recurring=false&tunnel_vision=false`; + const scan = ( + id: number, + name: string, + thBuildId: string, + scanRunId: number, + ) => ({ + id, + name, + scanUrl: scannerLink(name, thBuildId), + lastScanDetails: { scanRunId }, + }); + const scansPage = (scans: unknown[], nextPage?: string) => + ok({ scans, pagination: { hasNext: Boolean(nextPage), nextPage } }); + + beforeEach(() => vi.clearAllMocks()); + + it("takes an Automated Tests link's build id without calling the API", async () => { + await expect( + fetcher().sourceFromReportUrl( + `${DASHBOARD}/automated-tests/projects/p/builds/b/1?thBuildId=${BUILD_ID}`, + ), + ).resolves.toEqual({ kind: "build", buildId: BUILD_ID }); + expect(get).not.toHaveBeenCalled(); + }); + + it("finds a Website Scanner link's scan and latest run by paging the scan list", async () => { + get + .mockResolvedValueOnce(scansPage([scan(1, "other", "aaa", 11)], "tok2")) + .mockResolvedValueOnce( + scansPage([scan(1234, "checkout fix", "bbb", 5678)]), + ); + + await expect( + fetcher().sourceFromReportUrl(scannerLink("checkout fix", "bbb")), + ).resolves.toEqual({ kind: "scan", scanId: "1234", scanRunId: "5678" }); + // No limit: larger pages make the list skip scans, so it is walked at its default size. + expect(get.mock.calls.map(([options]) => options.url)).toEqual([ + `${API}/website-scanner/v1/scans`, + `${API}/website-scanner/v1/scans?nextPage=tok2`, + ]); + }); + + it("names the scan when the link is to one of its earlier runs", async () => { + get.mockResolvedValueOnce( + scansPage([scan(1234, "checkout fix", "newer", 5678)]), + ); + + await expect( + fetcher().sourceFromReportUrl(scannerLink("checkout fix", "older")), + ).rejects.toThrow(/earlier run of scan "checkout fix" \(scanId 1234\)/); + }); + + it("stops after the newest scans and asks for the ids", async () => { + get.mockResolvedValue(scansPage([scan(1, "other", "aaa", 11)], "more")); + + await expect( + fetcher().sourceFromReportUrl(scannerLink("checkout fix", "bbb")), + ).rejects.toThrow(/most recent scans.*scanId and scanRunId/s); + expect(get).toHaveBeenCalledTimes(MAX_SCAN_LIST_PAGES); + }); + + it("refuses a link that carries no report id", async () => { + await expect( + fetcher().sourceFromReportUrl( + "https://scanner.browserstack.com/site-scanner/scan-details/checkout", + ), + ).rejects.toThrow(/thBuildId/); + expect(get).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 91f03a1c..1e4b1fdd 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -7,6 +7,7 @@ const m = vi.hoisted(() => ({ startScan: vi.fn(), waitUntilComplete: vi.fn(), getReportLink: vi.fn(), + sourceFromReportUrl: vi.fn(), parse: vi.fn(), })); @@ -21,6 +22,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ AccessibilityReportFetcher: class { setAuth = vi.fn(); getReportLink = m.getReportLink; + sourceFromReportUrl = m.sourceFromReportUrl; }, })); vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ @@ -195,9 +197,42 @@ describe("fetchAccessibilityIssues sources", () => { expect(body).toContain( `Use fetchAccessibilityIssues with ${nextPageArgs} and cursor: 140`, ); + expect(body).toContain( + "download the full report (CSV) from the following link", + ); + expect(body).toContain("https://csv\n"); }, ); + it("resolves a dashboard report link and pages by the ids it found", async () => { + const reportUrl = + "https://accessibility.browserstack.com/site-scanner/projects/x/builds?thBuildId=abc"; + m.sourceFromReportUrl.mockResolvedValue({ + kind: "scan", + scanId: "scan-42", + scanRunId: "run-7", + }); + const result = await handlers.fetchAccessibilityIssues( + { reportUrl }, + CONTEXT, + ); + + expect(result.isError).toBeFalsy(); + expect(m.sourceFromReportUrl).toHaveBeenCalledWith(reportUrl); + expect(m.getReportLink).toHaveBeenCalledWith({ + kind: "scan", + scanId: "scan-42", + scanRunId: "run-7", + }); + const body = text(result); + expect(body).toContain( + "Resolved the report link to Website Scanner scan scan-42, run run-7.", + ); + expect(body).toContain( + 'Use fetchAccessibilityIssues with scanId: "scan-42", scanRunId: "run-7" and cursor: 140', + ); + }); + it("startAccessibilityScan reads the run it just started", async () => { await handlers.startAccessibilityScan( { name: "run", pageURL: "https://site.example" }, @@ -218,11 +253,20 @@ describe("fetchAccessibilityIssues sources", () => { /exactly one/, ], ["a scan without its run", { scanId: "scan-42" }, /scanRunId/], + [ + "a report link with another source", + { + reportUrl: "https://accessibility.browserstack.com/x", + buildId: BUILD_ID, + }, + /exactly one/, + ], ])("refuses %s without calling the API", async (_label, args, message) => { const result = await handlers.fetchAccessibilityIssues(args, CONTEXT); expect(result.isError).toBe(true); expect(text(result)).toMatch(message); expect(m.getReportLink).not.toHaveBeenCalled(); + expect(m.sourceFromReportUrl).not.toHaveBeenCalled(); }); }); From 1643f4a3488f3700052e65438a03b5b260d9ab30 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Wed, 21 Oct 2026 15:44:35 +0530 Subject: [PATCH 5/5] feat(accessibility): wait for a report to finish processing before reading its issues An agent that fetched an SDK build's issues right after the test run got 14 issues, called final ("All issues retrieved"); the build completed minutes later with 27. After the tests end a build stays "finalizing" (advanced issue processing, for an advanced scan) and the export serves whatever exists so far. fetchAccessibilityIssues now reads the report's status first: a scan run's status endpoint, a Workflow Analyzer report's summary, and for a build its project's build list, since the build overview carries no status. While it is queued, running, processing or finalizing, a first read waits, polling every 10 s for up to 5 min with progress notifications, then reads the issues and says it waited. If processing outlasts the wait, the answer says the issues are only those found so far and to fetch again, instead of "All issues retrieved". A later page does not wait again, and a status that cannot be read is not waited on. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 113 +++++++++++++++++- .../accessiblity-utils/report-fetcher.ts | 64 ++++++++++ .../tools/accessibilityReportFetcher.test.ts | 66 ++++++++++ tests/tools/accessibilityScanOutput.test.ts | 109 ++++++++++++++++- 4 files changed, 347 insertions(+), 5 deletions(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 7705ee91..9b3b42b8 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -209,6 +209,7 @@ function describeSource(source: IssueSource): string { async function executeFetchAccessibilityIssues( args: FetchIssuesArgs, + context: ScanProgressContext | undefined, server: McpServer, config: BrowserStackConfig, ): Promise { @@ -221,7 +222,7 @@ async function executeFetchAccessibilityIssues( ); const source = issueSourceFrom(args); if (typeof source === "string") return createErrorResponse(source); - return await fetchAccessibilityIssues(source, config, args.cursor); + return await fetchAccessibilityIssues(source, config, args.cursor, context); } catch (error) { return handleMCPError("fetchAccessibilityIssues", server, config, error); } @@ -239,16 +240,112 @@ function duplicatesNote(duplicates: number | null, unlinked: number): string[] { ]; } +/** How often, and for how long, a first read waits on a report that is still processing. */ +export const REPORT_STATUS_POLL_MS = 10_000; +export const REPORT_STATUS_WAIT_MS = 5 * 60 * 1000; + +/** Statuses during which a report's issues are still being added. */ +const PROCESSING_STATUSES = new Set([ + "queued", + "running", + "processing", + "finalizing", +]); + +interface ProcessingState { + /** The status when the issues were read; undefined when it could not be read. */ + status?: string; + /** The status first seen, before any waiting. */ + firstStatus?: string; + waitedMs: number; +} + +/** + * Where a report is in its processing before its issues are read. Issues keep being added + * until it completes (an advanced scan adds its advanced-rule issues while "finalizing", for + * minutes after the tests end), so a first read waits for it, reporting progress, for up to + * REPORT_STATUS_WAIT_MS. A later page does not wait again, and a status that cannot be read + * is not waited on. + */ +async function waitForProcessing( + reportFetcher: AccessibilityReportFetcher, + source: IssueSource, + wait: boolean, + context: ScanProgressContext | undefined, +): Promise { + const started = Date.now(); + const firstStatus = await reportFetcher.reportStatus(source); + let status = firstStatus; + while ( + wait && + status && + PROCESSING_STATUSES.has(status) && + Date.now() - started < REPORT_STATUS_WAIT_MS + ) { + const waited = Date.now() - started; + if (context) { + await notifyScanProgress( + context, + `The report is still ${status}; waiting for it to complete (${formatDuration(waited)} of up to ${formatDuration(REPORT_STATUS_WAIT_MS)}).`, + Math.min(99, Math.round((waited / REPORT_STATUS_WAIT_MS) * 100)), + ); + } + await new Promise((resolve) => setTimeout(resolve, REPORT_STATUS_POLL_MS)); + status = await reportFetcher.reportStatus(source); + } + return { status, firstStatus, waitedMs: Date.now() - started }; +} + +function formatDuration(ms: number): string { + const seconds = Math.round(ms / 1000); + return seconds < 60 + ? `${seconds}s` + : `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`; +} + +/** What the answer says about the report's processing, if anything. */ +function processingNote({ + status, + firstStatus, + waitedMs, +}: ProcessingState): string[] { + if (status && PROCESSING_STATUSES.has(status)) { + const meaning = + status === "finalizing" + ? ", which on an advanced scan means advanced issue processing" + : ""; + return [ + `⚠️ This report is still processing (status: ${status}${meaning}). The issues below are only those found so far: more will be added and the total will grow. Call fetchAccessibilityIssues again in a few minutes for the complete list.`, + ]; + } + if (firstStatus && PROCESSING_STATUSES.has(firstStatus)) { + return [ + `The report was still processing (status: ${firstStatus}); waited ${formatDuration(waitedMs)} for it to complete before reading its issues.`, + ]; + } + return []; +} + async function fetchAccessibilityIssues( requested: RequestedSource, config: BrowserStackConfig, cursor = 0, + context?: ScanProgressContext, ): Promise { const reportFetcher = await initializeReportFetcher(config); const source = requested.kind === "link" ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) : requested; + const processing = await waitForProcessing( + reportFetcher, + source, + cursor === 0, + context, + ); + const stillProcessing = + processing.status !== undefined && + PROCESSING_STATUSES.has(processing.status); const reportLink = await reportFetcher.getReportLink(source); const { @@ -271,6 +368,7 @@ async function fetchAccessibilityIssues( ...(requested.kind === "link" ? [`Resolved the report link to ${describeSource(source)}.`] : []), + ...processingNote(processing), `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, ...duplicatesNote(duplicate_issues, unlinked_duplicates), `You can also download the full report (CSV) from the following link, valid for about 12 hours: ${reportLink}`, @@ -281,6 +379,10 @@ async function fetchAccessibilityIssues( messages.push( `${remainingIssues} more issues available. Use fetchAccessibilityIssues with ${sourceArgs(source)} and cursor: ${next_page} to get the next batch.`, ); + } else if (stillProcessing) { + messages.push( + `All issues found so far retrieved; the report is still processing.`, + ); } else { messages.push(`✅ All issues retrieved.`); } @@ -740,8 +842,13 @@ export default function addAccessibilityTools( destructiveHint: false, idempotentHint: true, }, - async (args) => { - return await executeFetchAccessibilityIssues(args, server, config); + async (args, context) => { + return await executeFetchAccessibilityIssues( + args, + context as ScanProgressContext, + server, + config, + ); }, ); diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index 646937f4..7a0b038b 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -28,6 +28,9 @@ const API_BASE = "https://api-accessibility.browserstack.com/api"; */ export const MAX_SCAN_LIST_PAGES = 15; +/** A build whose status is checked is one just run, so it sits at the top of its project's list. */ +const MAX_BUILD_LIST_PAGES = 3; + interface ScanListPage { scans?: { id: number; @@ -227,4 +230,65 @@ export class AccessibilityReportFetcher { `Report link was not available after ${maxAttempts} attempts. The CSV generation may still be in progress.`, ); } + + /** + * Where the report is in its processing, as the report's own status reads it: `completed`, + * or `queued` / `running` / `processing` / `finalizing` while issues are still being added. + * Undefined when it cannot be read. A build's overview carries no status, so a build's comes + * from its project's build list (newest first), found by id. + */ + async reportStatus(source: IssueSource): Promise { + const q = encodeURIComponent; + try { + switch (source.kind) { + case "scan": + return ( + await this.getData( + `${API_BASE}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/${q(source.scanRunId)}/status`, + ) + )?.status; + case "workflow": + return ( + await this.getData( + `${API_BASE}/workflow-analyzer/v1/reports/${q(source.reportId)}`, + ) + )?.summary?.status; + case "build": + return await this.buildStatus(source.buildId); + } + } catch { + return undefined; + } + } + + private async buildStatus(buildId: string): Promise { + const q = encodeURIComponent; + const project = ( + await this.getData(`${API_BASE}/automated-tests/v1/builds/${q(buildId)}`) + )?.projectName; + if (!project) return undefined; + let nextPage: string | undefined; + for (let page = 0; page < MAX_BUILD_LIST_PAGES; page++) { + const data = await this.getData( + `${API_BASE}/automated-tests/v1/builds?project_name=${q(project)}&limit=100${ + nextPage ? `&next_page=${q(nextPage)}` : "" + }`, + ); + const row = data?.builds?.find((build: any) => build.id === buildId); + if (row) return row.status; + if (!data?.pagination?.has_next || !data.pagination.next_page) break; + nextPage = data.pagination.next_page; + } + return undefined; + } + + /** The `data` of a successful API response, or undefined. */ + private async getData(url: string): Promise { + const resp = await apiClient.get({ + url, + headers: this.authHeaders(), + raise_error: false, + }); + return resp.ok ? resp.data?.data : undefined; + } } diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index 6adf91b9..6c6d6f4a 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -168,3 +168,69 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { expect(get).not.toHaveBeenCalled(); }); }); + +describe("AccessibilityReportFetcher.reportStatus", () => { + const urls = () => get.mock.calls.map(([options]) => options.url); + + beforeEach(() => vi.clearAllMocks()); + + it("reads a scan run's status from its status endpoint", async () => { + get.mockResolvedValueOnce( + ok({ scan_id: "1234", scan_run_id: "5678", status: "finalizing" }), + ); + + await expect( + fetcher().reportStatus({ + kind: "scan", + scanId: "1234", + scanRunId: "5678", + }), + ).resolves.toBe("finalizing"); + expect(urls()).toEqual([ + `${API}/website-scanner/v1/scans/1234/scan_runs/5678/status`, + ]); + }); + + it("reads a Workflow Analyzer report's status from its summary", async () => { + get.mockResolvedValueOnce(ok({ summary: { status: "completed" } })); + + await expect( + fetcher().reportStatus({ kind: "workflow", reportId: "123456" }), + ).resolves.toBe("completed"); + expect(urls()).toEqual([`${API}/workflow-analyzer/v1/reports/123456`]); + }); + + it("reads a build's status from its project's build list, as the build overview has none", async () => { + get + .mockResolvedValueOnce(ok({ id: BUILD_ID, projectName: "Checkout" })) + .mockResolvedValueOnce( + ok({ + builds: [{ id: "newer", status: "running" }], + pagination: { has_next: true, next_page: "p2" }, + }), + ) + .mockResolvedValueOnce( + ok({ + builds: [{ id: BUILD_ID, status: "finalizing" }], + pagination: { has_next: false }, + }), + ); + + await expect( + fetcher().reportStatus({ kind: "build", buildId: BUILD_ID }), + ).resolves.toBe("finalizing"); + expect(urls()).toEqual([ + `${API}/automated-tests/v1/builds/${BUILD_ID}`, + `${API}/automated-tests/v1/builds?project_name=Checkout&limit=100`, + `${API}/automated-tests/v1/builds?project_name=Checkout&limit=100&next_page=p2`, + ]); + }); + + it("returns nothing when the status cannot be read", async () => { + get.mockResolvedValueOnce(failed(500, ["Internal server Error"])); + + await expect( + fetcher().reportStatus({ kind: "build", buildId: BUILD_ID }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 1e4b1fdd..817c1a7e 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // What startAccessibilityScan / fetchAccessibilityIssues tell the agent about the issue // count: main issues only, matching the dashboard, with duplicates stated separately. @@ -8,6 +8,7 @@ const m = vi.hoisted(() => ({ waitUntilComplete: vi.fn(), getReportLink: vi.fn(), sourceFromReportUrl: vi.fn(), + reportStatus: vi.fn(), parse: vi.fn(), })); @@ -23,6 +24,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ setAuth = vi.fn(); getReportLink = m.getReportLink; sourceFromReportUrl = m.sourceFromReportUrl; + reportStatus = m.reportStatus; }, })); vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ @@ -36,7 +38,10 @@ vi.mock("../../src/logger.js", () => ({ })); vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); -import addAccessibilityTools from "../../src/tools/accessibility.js"; +import addAccessibilityTools, { + REPORT_STATUS_POLL_MS, + REPORT_STATUS_WAIT_MS, +} from "../../src/tools/accessibility.js"; const CONFIG = { "browserstack-username": "fake-user", @@ -270,3 +275,103 @@ describe("fetchAccessibilityIssues sources", () => { expect(m.sourceFromReportUrl).not.toHaveBeenCalled(); }); }); + +describe("fetchAccessibilityIssues and a report still processing", () => { + const BUILD_ID = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; + const sendNotification = vi.fn(); + const context = { sendNotification, _meta: { progressToken: "tok" } }; + let handlers: Record any>; + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue({ ...GROUPED, next_page: null }); + handlers = register(); + }); + + afterEach(() => vi.useRealTimers()); + + it("reads the issues straight away when the report is complete", async () => { + m.reportStatus.mockResolvedValue("completed"); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + const body = text(result); + expect(body).toContain("✅ All issues retrieved."); + expect(body).not.toContain("still processing"); + }); + + it("waits while the report is finalizing, then reads the complete issues", async () => { + m.reportStatus + .mockResolvedValueOnce("finalizing") + .mockResolvedValueOnce("finalizing") + .mockResolvedValueOnce("completed"); + const pending = handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + await vi.advanceTimersByTimeAsync(REPORT_STATUS_POLL_MS * 2); + const result = await pending; + + expect(m.reportStatus).toHaveBeenCalledTimes(3); + expect(m.getReportLink).toHaveBeenCalledTimes(1); + expect(sendNotification).toHaveBeenCalledWith( + expect.objectContaining({ + method: "notifications/progress", + params: expect.objectContaining({ + message: expect.stringContaining("still finalizing"), + }), + }), + ); + const body = text(result); + expect(body).toContain( + "The report was still processing (status: finalizing); waited 20s for it to complete", + ); + expect(body).toContain("✅ All issues retrieved."); + }); + + it("says the issues are only those found so far when processing outlasts the wait", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const pending = handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + await vi.advanceTimersByTimeAsync( + REPORT_STATUS_WAIT_MS + REPORT_STATUS_POLL_MS, + ); + const result = await pending; + + const body = text(result); + expect(body).toContain( + "This report is still processing (status: finalizing", + ); + expect(body).toContain("only those found so far"); + expect(body).not.toContain("✅ All issues retrieved."); + }); + + it("does not wait again for a later page, but still says the report is processing", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID, cursor: 4400 }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(text(result)).toContain("This report is still processing"); + }); + + it("does not wait when the status cannot be read", async () => { + m.reportStatus.mockResolvedValue(undefined); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(text(result)).toContain("✅ All issues retrieved."); + }); +});