diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index d35338a3..2d6497b1 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -209,6 +209,7 @@ function describeSource(source: IssueSource): string { async function executeFetchAccessibilityIssues( args: FetchIssuesArgs, + context: ScanProgressContext | undefined, server: McpServer, config: BrowserStackConfig, ): Promise { @@ -221,7 +222,7 @@ async function executeFetchAccessibilityIssues( ); const source = issueSourceFrom(args); if (typeof source === "string") return createErrorResponse(source); - return await fetchAccessibilityIssues(source, config, args.cursor); + return await fetchAccessibilityIssues(source, config, args.cursor, context); } catch (error) { return handleMCPError("fetchAccessibilityIssues", server, config, error); } @@ -257,16 +258,119 @@ function excludedIssuesNote( return notes; } +/** How often, and for how long, a first read waits on a report that is still processing. */ +export const REPORT_STATUS_POLL_MS = 10_000; +export const REPORT_STATUS_WAIT_MS = 5 * 60 * 1000; + +/** Statuses during which a report's issues are still being added. */ +const PROCESSING_STATUSES = new Set([ + "queued", + "running", + "processing", + "finalizing", +]); + +/** + * Of those, the one worth waiting on: the advanced-issue processing after the run, which + * ends within minutes. A report whose scan or tests are still running is read as it is, + * with a note, since that can last as long as the suite does. + */ +const WAIT_STATUSES = new Set(["finalizing"]); + +interface ProcessingState { + /** The status when the issues were read; undefined when it could not be read. */ + status?: string; + /** The status first seen, before any waiting. */ + firstStatus?: string; + waitedMs: number; +} + +/** + * Where a report is in its processing before its issues are read. Issues keep being added + * until it completes; an advanced scan adds its advanced-rule issues while "finalizing", for + * minutes after the tests end, so a first read waits for that, reporting progress, for up to + * REPORT_STATUS_WAIT_MS. A run still in progress is not waited on (it can take as long as + * the suite), nor is a later page, nor a status that cannot be read. + */ +async function waitForProcessing( + reportFetcher: AccessibilityReportFetcher, + source: IssueSource, + wait: boolean, + context: ScanProgressContext | undefined, +): Promise { + const started = Date.now(); + const firstStatus = await reportFetcher.reportStatus(source); + let status = firstStatus; + while ( + wait && + status && + WAIT_STATUSES.has(status) && + Date.now() - started < REPORT_STATUS_WAIT_MS + ) { + const waited = Date.now() - started; + if (context) { + await notifyScanProgress( + context, + `The report is still ${status}; waiting for it to complete (${formatDuration(waited)} of up to ${formatDuration(REPORT_STATUS_WAIT_MS)}).`, + Math.min(99, Math.round((waited / REPORT_STATUS_WAIT_MS) * 100)), + ); + } + await new Promise((resolve) => setTimeout(resolve, REPORT_STATUS_POLL_MS)); + status = await reportFetcher.reportStatus(source); + } + return { status, firstStatus, waitedMs: Date.now() - started }; +} + +function formatDuration(ms: number): string { + const seconds = Math.round(ms / 1000); + return seconds < 60 + ? `${seconds}s` + : `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`; +} + +/** What the answer says about the report's processing, if anything. */ +function processingNote({ + status, + firstStatus, + waitedMs, +}: ProcessingState): string[] { + if (status && PROCESSING_STATUSES.has(status)) { + const meaning = + status === "finalizing" + ? ", which on an advanced scan means advanced issue processing" + : ""; + return [ + `⚠️ This report is still processing (status: ${status}${meaning}). The issues below are only those found so far: more will be added and the total will grow. Call fetchAccessibilityIssues again in a few minutes for the complete list.`, + ]; + } + if (firstStatus && PROCESSING_STATUSES.has(firstStatus)) { + return [ + `The report was still processing (status: ${firstStatus}); waited ${formatDuration(waitedMs)} for it to complete before reading its issues.`, + ]; + } + return []; +} + async function fetchAccessibilityIssues( requested: RequestedSource, config: BrowserStackConfig, cursor = 0, + context?: ScanProgressContext, ): Promise { const reportFetcher = await initializeReportFetcher(config); const source = requested.kind === "link" ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) : requested; + const processing = await waitForProcessing( + reportFetcher, + source, + cursor === 0, + context, + ); + const stillProcessing = + processing.status !== undefined && + PROCESSING_STATUSES.has(processing.status); const reportLink = await reportFetcher.getReportLink(source); const { @@ -290,6 +394,7 @@ async function fetchAccessibilityIssues( ...(requested.kind === "link" ? [`Resolved the report link to ${describeSource(source)}.`] : []), + ...processingNote(processing), `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, ...excludedIssuesNote(duplicate_issues, unlinked_duplicates, hidden_issues), `You can also download the full report (CSV) from the following link, valid for about 12 hours: ${reportLink}`, @@ -300,6 +405,10 @@ async function fetchAccessibilityIssues( messages.push( `${remainingIssues} more issues available. Use fetchAccessibilityIssues with ${sourceArgs(source)} and cursor: ${next_page} to get the next batch.`, ); + } else if (stillProcessing) { + messages.push( + `All issues found so far retrieved; the report is still processing.`, + ); } else { messages.push(`✅ All issues retrieved.`); } @@ -759,8 +868,13 @@ export default function addAccessibilityTools( destructiveHint: false, idempotentHint: true, }, - async (args) => { - return await executeFetchAccessibilityIssues(args, server, config); + async (args, context) => { + return await executeFetchAccessibilityIssues( + args, + context as ScanProgressContext, + server, + config, + ); }, ); diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index f0cfb22c..bf12fd24 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -28,6 +28,9 @@ const API_BASE = "https://api-accessibility.browserstack.com/api"; */ export const MAX_SCAN_LIST_PAGES = 15; +/** A build whose status is checked is one just run, so it sits at the top of its project's list. */ +const MAX_BUILD_LIST_PAGES = 3; + interface ScanListPage { scans?: { id: number; @@ -233,4 +236,65 @@ export class AccessibilityReportFetcher { `Report link was not available after ${maxAttempts} attempts. The CSV generation may still be in progress.`, ); } + + /** + * Where the report is in its processing, as the report's own status reads it: `completed`, + * or `queued` / `running` / `processing` / `finalizing` while issues are still being added. + * Undefined when it cannot be read. A build's overview carries no status, so a build's comes + * from its project's build list (newest first), found by id. + */ + async reportStatus(source: IssueSource): Promise { + const q = encodeURIComponent; + try { + switch (source.kind) { + case "scan": + return ( + await this.getData( + `${API_BASE}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/${q(source.scanRunId)}/status`, + ) + )?.status; + case "workflow": + return ( + await this.getData( + `${API_BASE}/workflow-analyzer/v1/reports/${q(source.reportId)}`, + ) + )?.summary?.status; + case "build": + return await this.buildStatus(source.buildId); + } + } catch { + return undefined; + } + } + + private async buildStatus(buildId: string): Promise { + const q = encodeURIComponent; + const project = ( + await this.getData(`${API_BASE}/automated-tests/v1/builds/${q(buildId)}`) + )?.projectName; + if (!project) return undefined; + let nextPage: string | undefined; + for (let page = 0; page < MAX_BUILD_LIST_PAGES; page++) { + const data = await this.getData( + `${API_BASE}/automated-tests/v1/builds?project_name=${q(project)}&limit=100${ + nextPage ? `&next_page=${q(nextPage)}` : "" + }`, + ); + const row = data?.builds?.find((build: any) => build.id === buildId); + if (row) return row.status; + if (!data?.pagination?.has_next || !data.pagination.next_page) break; + nextPage = data.pagination.next_page; + } + return undefined; + } + + /** The `data` of a successful API response, or undefined. */ + private async getData(url: string): Promise { + const resp = await apiClient.get({ + url, + headers: this.authHeaders(), + raise_error: false, + }); + return resp.ok ? resp.data?.data : undefined; + } } diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index de99cbdb..0321d264 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -185,3 +185,69 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl and a scan without a ru ); }); }); + +describe("AccessibilityReportFetcher.reportStatus", () => { + const urls = () => get.mock.calls.map(([options]) => options.url); + + beforeEach(() => vi.clearAllMocks()); + + it("reads a scan run's status from its status endpoint", async () => { + get.mockResolvedValueOnce( + ok({ scan_id: "1234", scan_run_id: "5678", status: "finalizing" }), + ); + + await expect( + fetcher().reportStatus({ + kind: "scan", + scanId: "1234", + scanRunId: "5678", + }), + ).resolves.toBe("finalizing"); + expect(urls()).toEqual([ + `${API}/website-scanner/v1/scans/1234/scan_runs/5678/status`, + ]); + }); + + it("reads a Workflow Analyzer report's status from its summary", async () => { + get.mockResolvedValueOnce(ok({ summary: { status: "completed" } })); + + await expect( + fetcher().reportStatus({ kind: "workflow", reportId: "123456" }), + ).resolves.toBe("completed"); + expect(urls()).toEqual([`${API}/workflow-analyzer/v1/reports/123456`]); + }); + + it("reads a build's status from its project's build list, as the build overview has none", async () => { + get + .mockResolvedValueOnce(ok({ id: BUILD_ID, projectName: "Checkout" })) + .mockResolvedValueOnce( + ok({ + builds: [{ id: "newer", status: "running" }], + pagination: { has_next: true, next_page: "p2" }, + }), + ) + .mockResolvedValueOnce( + ok({ + builds: [{ id: BUILD_ID, status: "finalizing" }], + pagination: { has_next: false }, + }), + ); + + await expect( + fetcher().reportStatus({ kind: "build", buildId: BUILD_ID }), + ).resolves.toBe("finalizing"); + expect(urls()).toEqual([ + `${API}/automated-tests/v1/builds/${BUILD_ID}`, + `${API}/automated-tests/v1/builds?project_name=Checkout&limit=100`, + `${API}/automated-tests/v1/builds?project_name=Checkout&limit=100&next_page=p2`, + ]); + }); + + it("returns nothing when the status cannot be read", async () => { + get.mockResolvedValueOnce(failed(500, ["Internal server Error"])); + + await expect( + fetcher().reportStatus({ kind: "build", buildId: BUILD_ID }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 536560c2..a440d123 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // What startAccessibilityScan / fetchAccessibilityIssues tell the agent about the issue // count: main issues only, matching the dashboard, with duplicates stated separately. @@ -8,6 +8,7 @@ const m = vi.hoisted(() => ({ waitUntilComplete: vi.fn(), getReportLink: vi.fn(), sourceFromReportUrl: vi.fn(), + reportStatus: vi.fn(), parse: vi.fn(), })); @@ -23,6 +24,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ setAuth = vi.fn(); getReportLink = m.getReportLink; sourceFromReportUrl = m.sourceFromReportUrl; + reportStatus = m.reportStatus; }, })); vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ @@ -36,7 +38,10 @@ vi.mock("../../src/logger.js", () => ({ })); vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); -import addAccessibilityTools from "../../src/tools/accessibility.js"; +import addAccessibilityTools, { + REPORT_STATUS_POLL_MS, + REPORT_STATUS_WAIT_MS, +} from "../../src/tools/accessibility.js"; const CONFIG = { "browserstack-username": "fake-user", @@ -303,3 +308,120 @@ describe("fetchAccessibilityIssues sources", () => { expect(m.sourceFromReportUrl).not.toHaveBeenCalled(); }); }); + +describe("fetchAccessibilityIssues and a report still processing", () => { + const BUILD_ID = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; + const sendNotification = vi.fn(); + const context = { sendNotification, _meta: { progressToken: "tok" } }; + let handlers: Record any>; + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue({ ...GROUPED, next_page: null }); + handlers = register(); + }); + + afterEach(() => vi.useRealTimers()); + + it("reads the issues straight away when the report is complete", async () => { + m.reportStatus.mockResolvedValue("completed"); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + const body = text(result); + expect(body).toContain("✅ All issues retrieved."); + expect(body).not.toContain("still processing"); + }); + + it("waits while the report is finalizing, then reads the complete issues", async () => { + m.reportStatus + .mockResolvedValueOnce("finalizing") + .mockResolvedValueOnce("finalizing") + .mockResolvedValueOnce("completed"); + const pending = handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + await vi.advanceTimersByTimeAsync(REPORT_STATUS_POLL_MS * 2); + const result = await pending; + + expect(m.reportStatus).toHaveBeenCalledTimes(3); + expect(m.getReportLink).toHaveBeenCalledTimes(1); + expect(sendNotification).toHaveBeenCalledWith( + expect.objectContaining({ + method: "notifications/progress", + params: expect.objectContaining({ + message: expect.stringContaining("still finalizing"), + }), + }), + ); + const body = text(result); + expect(body).toContain( + "The report was still processing (status: finalizing); waited 20s for it to complete", + ); + expect(body).toContain("✅ All issues retrieved."); + }); + + it("says the issues are only those found so far when processing outlasts the wait", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const pending = handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + await vi.advanceTimersByTimeAsync( + REPORT_STATUS_WAIT_MS + REPORT_STATUS_POLL_MS, + ); + const result = await pending; + + const body = text(result); + expect(body).toContain( + "This report is still processing (status: finalizing", + ); + expect(body).toContain("only those found so far"); + expect(body).not.toContain("✅ All issues retrieved."); + }); + + it("does not wait again for a later page, but still says the report is processing", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID, cursor: 4400 }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(text(result)).toContain("This report is still processing"); + }); + + it("does not wait while the tests are still running, and says the issues are those found so far", async () => { + // A long suite can run for an hour; blocking the first read for five minutes and then + // returning partial results anyway serves nobody. Only post-run processing is waited on. + m.reportStatus.mockResolvedValue("running"); + + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(sendNotification).not.toHaveBeenCalled(); + const body = text(result); + expect(body).toContain("still processing (status: running"); + expect(body).toContain("only those found so far"); + }); + + it("does not wait when the status cannot be read", async () => { + m.reportStatus.mockResolvedValue(undefined); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(text(result)).toContain("✅ All issues retrieved."); + }); +});