From 2c5b1317cf5601967ef75382ec15b58117f1ea3d Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Wed, 21 Oct 2026 15:44:35 +0530 Subject: [PATCH 1/2] feat(accessibility): wait for a report to finish processing before reading its issues An agent that fetched an SDK build's issues right after the test run got 14 issues, called final ("All issues retrieved"); the build completed minutes later with 27. After the tests end a build stays "finalizing" (advanced issue processing, for an advanced scan) and the export serves whatever exists so far. fetchAccessibilityIssues now reads the report's status first: a scan run's status endpoint, a Workflow Analyzer report's summary, and for a build its project's build list, since the build overview carries no status. While it is queued, running, processing or finalizing, a first read waits, polling every 10 s for up to 5 min with progress notifications, then reads the issues and says it waited. If processing outlasts the wait, the answer says the issues are only those found so far and to fetch again, instead of "All issues retrieved". A later page does not wait again, and a status that cannot be read is not waited on. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 113 +++++++++++++++++- .../accessiblity-utils/report-fetcher.ts | 64 ++++++++++ .../tools/accessibilityReportFetcher.test.ts | 66 ++++++++++ tests/tools/accessibilityScanOutput.test.ts | 109 ++++++++++++++++- 4 files changed, 347 insertions(+), 5 deletions(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index d35338a3..eca32d64 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -209,6 +209,7 @@ function describeSource(source: IssueSource): string { async function executeFetchAccessibilityIssues( args: FetchIssuesArgs, + context: ScanProgressContext | undefined, server: McpServer, config: BrowserStackConfig, ): Promise { @@ -221,7 +222,7 @@ async function executeFetchAccessibilityIssues( ); const source = issueSourceFrom(args); if (typeof source === "string") return createErrorResponse(source); - return await fetchAccessibilityIssues(source, config, args.cursor); + return await fetchAccessibilityIssues(source, config, args.cursor, context); } catch (error) { return handleMCPError("fetchAccessibilityIssues", server, config, error); } @@ -257,16 +258,112 @@ function excludedIssuesNote( return notes; } +/** How often, and for how long, a first read waits on a report that is still processing. */ +export const REPORT_STATUS_POLL_MS = 10_000; +export const REPORT_STATUS_WAIT_MS = 5 * 60 * 1000; + +/** Statuses during which a report's issues are still being added. */ +const PROCESSING_STATUSES = new Set([ + "queued", + "running", + "processing", + "finalizing", +]); + +interface ProcessingState { + /** The status when the issues were read; undefined when it could not be read. */ + status?: string; + /** The status first seen, before any waiting. */ + firstStatus?: string; + waitedMs: number; +} + +/** + * Where a report is in its processing before its issues are read. Issues keep being added + * until it completes (an advanced scan adds its advanced-rule issues while "finalizing", for + * minutes after the tests end), so a first read waits for it, reporting progress, for up to + * REPORT_STATUS_WAIT_MS. A later page does not wait again, and a status that cannot be read + * is not waited on. + */ +async function waitForProcessing( + reportFetcher: AccessibilityReportFetcher, + source: IssueSource, + wait: boolean, + context: ScanProgressContext | undefined, +): Promise { + const started = Date.now(); + const firstStatus = await reportFetcher.reportStatus(source); + let status = firstStatus; + while ( + wait && + status && + PROCESSING_STATUSES.has(status) && + Date.now() - started < REPORT_STATUS_WAIT_MS + ) { + const waited = Date.now() - started; + if (context) { + await notifyScanProgress( + context, + `The report is still ${status}; waiting for it to complete (${formatDuration(waited)} of up to ${formatDuration(REPORT_STATUS_WAIT_MS)}).`, + Math.min(99, Math.round((waited / REPORT_STATUS_WAIT_MS) * 100)), + ); + } + await new Promise((resolve) => setTimeout(resolve, REPORT_STATUS_POLL_MS)); + status = await reportFetcher.reportStatus(source); + } + return { status, firstStatus, waitedMs: Date.now() - started }; +} + +function formatDuration(ms: number): string { + const seconds = Math.round(ms / 1000); + return seconds < 60 + ? `${seconds}s` + : `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`; +} + +/** What the answer says about the report's processing, if anything. */ +function processingNote({ + status, + firstStatus, + waitedMs, +}: ProcessingState): string[] { + if (status && PROCESSING_STATUSES.has(status)) { + const meaning = + status === "finalizing" + ? ", which on an advanced scan means advanced issue processing" + : ""; + return [ + `⚠️ This report is still processing (status: ${status}${meaning}). The issues below are only those found so far: more will be added and the total will grow. Call fetchAccessibilityIssues again in a few minutes for the complete list.`, + ]; + } + if (firstStatus && PROCESSING_STATUSES.has(firstStatus)) { + return [ + `The report was still processing (status: ${firstStatus}); waited ${formatDuration(waitedMs)} for it to complete before reading its issues.`, + ]; + } + return []; +} + async function fetchAccessibilityIssues( requested: RequestedSource, config: BrowserStackConfig, cursor = 0, + context?: ScanProgressContext, ): Promise { const reportFetcher = await initializeReportFetcher(config); const source = requested.kind === "link" ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) : requested; + const processing = await waitForProcessing( + reportFetcher, + source, + cursor === 0, + context, + ); + const stillProcessing = + processing.status !== undefined && + PROCESSING_STATUSES.has(processing.status); const reportLink = await reportFetcher.getReportLink(source); const { @@ -290,6 +387,7 @@ async function fetchAccessibilityIssues( ...(requested.kind === "link" ? [`Resolved the report link to ${describeSource(source)}.`] : []), + ...processingNote(processing), `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, ...excludedIssuesNote(duplicate_issues, unlinked_duplicates, hidden_issues), `You can also download the full report (CSV) from the following link, valid for about 12 hours: ${reportLink}`, @@ -300,6 +398,10 @@ async function fetchAccessibilityIssues( messages.push( `${remainingIssues} more issues available. Use fetchAccessibilityIssues with ${sourceArgs(source)} and cursor: ${next_page} to get the next batch.`, ); + } else if (stillProcessing) { + messages.push( + `All issues found so far retrieved; the report is still processing.`, + ); } else { messages.push(`✅ All issues retrieved.`); } @@ -759,8 +861,13 @@ export default function addAccessibilityTools( destructiveHint: false, idempotentHint: true, }, - async (args) => { - return await executeFetchAccessibilityIssues(args, server, config); + async (args, context) => { + return await executeFetchAccessibilityIssues( + args, + context as ScanProgressContext, + server, + config, + ); }, ); diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index f0cfb22c..bf12fd24 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -28,6 +28,9 @@ const API_BASE = "https://api-accessibility.browserstack.com/api"; */ export const MAX_SCAN_LIST_PAGES = 15; +/** A build whose status is checked is one just run, so it sits at the top of its project's list. */ +const MAX_BUILD_LIST_PAGES = 3; + interface ScanListPage { scans?: { id: number; @@ -233,4 +236,65 @@ export class AccessibilityReportFetcher { `Report link was not available after ${maxAttempts} attempts. The CSV generation may still be in progress.`, ); } + + /** + * Where the report is in its processing, as the report's own status reads it: `completed`, + * or `queued` / `running` / `processing` / `finalizing` while issues are still being added. + * Undefined when it cannot be read. A build's overview carries no status, so a build's comes + * from its project's build list (newest first), found by id. + */ + async reportStatus(source: IssueSource): Promise { + const q = encodeURIComponent; + try { + switch (source.kind) { + case "scan": + return ( + await this.getData( + `${API_BASE}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/${q(source.scanRunId)}/status`, + ) + )?.status; + case "workflow": + return ( + await this.getData( + `${API_BASE}/workflow-analyzer/v1/reports/${q(source.reportId)}`, + ) + )?.summary?.status; + case "build": + return await this.buildStatus(source.buildId); + } + } catch { + return undefined; + } + } + + private async buildStatus(buildId: string): Promise { + const q = encodeURIComponent; + const project = ( + await this.getData(`${API_BASE}/automated-tests/v1/builds/${q(buildId)}`) + )?.projectName; + if (!project) return undefined; + let nextPage: string | undefined; + for (let page = 0; page < MAX_BUILD_LIST_PAGES; page++) { + const data = await this.getData( + `${API_BASE}/automated-tests/v1/builds?project_name=${q(project)}&limit=100${ + nextPage ? `&next_page=${q(nextPage)}` : "" + }`, + ); + const row = data?.builds?.find((build: any) => build.id === buildId); + if (row) return row.status; + if (!data?.pagination?.has_next || !data.pagination.next_page) break; + nextPage = data.pagination.next_page; + } + return undefined; + } + + /** The `data` of a successful API response, or undefined. */ + private async getData(url: string): Promise { + const resp = await apiClient.get({ + url, + headers: this.authHeaders(), + raise_error: false, + }); + return resp.ok ? resp.data?.data : undefined; + } } diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index de99cbdb..0321d264 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -185,3 +185,69 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl and a scan without a ru ); }); }); + +describe("AccessibilityReportFetcher.reportStatus", () => { + const urls = () => get.mock.calls.map(([options]) => options.url); + + beforeEach(() => vi.clearAllMocks()); + + it("reads a scan run's status from its status endpoint", async () => { + get.mockResolvedValueOnce( + ok({ scan_id: "1234", scan_run_id: "5678", status: "finalizing" }), + ); + + await expect( + fetcher().reportStatus({ + kind: "scan", + scanId: "1234", + scanRunId: "5678", + }), + ).resolves.toBe("finalizing"); + expect(urls()).toEqual([ + `${API}/website-scanner/v1/scans/1234/scan_runs/5678/status`, + ]); + }); + + it("reads a Workflow Analyzer report's status from its summary", async () => { + get.mockResolvedValueOnce(ok({ summary: { status: "completed" } })); + + await expect( + fetcher().reportStatus({ kind: "workflow", reportId: "123456" }), + ).resolves.toBe("completed"); + expect(urls()).toEqual([`${API}/workflow-analyzer/v1/reports/123456`]); + }); + + it("reads a build's status from its project's build list, as the build overview has none", async () => { + get + .mockResolvedValueOnce(ok({ id: BUILD_ID, projectName: "Checkout" })) + .mockResolvedValueOnce( + ok({ + builds: [{ id: "newer", status: "running" }], + pagination: { has_next: true, next_page: "p2" }, + }), + ) + .mockResolvedValueOnce( + ok({ + builds: [{ id: BUILD_ID, status: "finalizing" }], + pagination: { has_next: false }, + }), + ); + + await expect( + fetcher().reportStatus({ kind: "build", buildId: BUILD_ID }), + ).resolves.toBe("finalizing"); + expect(urls()).toEqual([ + `${API}/automated-tests/v1/builds/${BUILD_ID}`, + `${API}/automated-tests/v1/builds?project_name=Checkout&limit=100`, + `${API}/automated-tests/v1/builds?project_name=Checkout&limit=100&next_page=p2`, + ]); + }); + + it("returns nothing when the status cannot be read", async () => { + get.mockResolvedValueOnce(failed(500, ["Internal server Error"])); + + await expect( + fetcher().reportStatus({ kind: "build", buildId: BUILD_ID }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 536560c2..5c607cf0 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // What startAccessibilityScan / fetchAccessibilityIssues tell the agent about the issue // count: main issues only, matching the dashboard, with duplicates stated separately. @@ -8,6 +8,7 @@ const m = vi.hoisted(() => ({ waitUntilComplete: vi.fn(), getReportLink: vi.fn(), sourceFromReportUrl: vi.fn(), + reportStatus: vi.fn(), parse: vi.fn(), })); @@ -23,6 +24,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ setAuth = vi.fn(); getReportLink = m.getReportLink; sourceFromReportUrl = m.sourceFromReportUrl; + reportStatus = m.reportStatus; }, })); vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ @@ -36,7 +38,10 @@ vi.mock("../../src/logger.js", () => ({ })); vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); -import addAccessibilityTools from "../../src/tools/accessibility.js"; +import addAccessibilityTools, { + REPORT_STATUS_POLL_MS, + REPORT_STATUS_WAIT_MS, +} from "../../src/tools/accessibility.js"; const CONFIG = { "browserstack-username": "fake-user", @@ -303,3 +308,103 @@ describe("fetchAccessibilityIssues sources", () => { expect(m.sourceFromReportUrl).not.toHaveBeenCalled(); }); }); + +describe("fetchAccessibilityIssues and a report still processing", () => { + const BUILD_ID = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; + const sendNotification = vi.fn(); + const context = { sendNotification, _meta: { progressToken: "tok" } }; + let handlers: Record any>; + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue({ ...GROUPED, next_page: null }); + handlers = register(); + }); + + afterEach(() => vi.useRealTimers()); + + it("reads the issues straight away when the report is complete", async () => { + m.reportStatus.mockResolvedValue("completed"); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + const body = text(result); + expect(body).toContain("✅ All issues retrieved."); + expect(body).not.toContain("still processing"); + }); + + it("waits while the report is finalizing, then reads the complete issues", async () => { + m.reportStatus + .mockResolvedValueOnce("finalizing") + .mockResolvedValueOnce("finalizing") + .mockResolvedValueOnce("completed"); + const pending = handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + await vi.advanceTimersByTimeAsync(REPORT_STATUS_POLL_MS * 2); + const result = await pending; + + expect(m.reportStatus).toHaveBeenCalledTimes(3); + expect(m.getReportLink).toHaveBeenCalledTimes(1); + expect(sendNotification).toHaveBeenCalledWith( + expect.objectContaining({ + method: "notifications/progress", + params: expect.objectContaining({ + message: expect.stringContaining("still finalizing"), + }), + }), + ); + const body = text(result); + expect(body).toContain( + "The report was still processing (status: finalizing); waited 20s for it to complete", + ); + expect(body).toContain("✅ All issues retrieved."); + }); + + it("says the issues are only those found so far when processing outlasts the wait", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const pending = handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + await vi.advanceTimersByTimeAsync( + REPORT_STATUS_WAIT_MS + REPORT_STATUS_POLL_MS, + ); + const result = await pending; + + const body = text(result); + expect(body).toContain( + "This report is still processing (status: finalizing", + ); + expect(body).toContain("only those found so far"); + expect(body).not.toContain("✅ All issues retrieved."); + }); + + it("does not wait again for a later page, but still says the report is processing", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID, cursor: 4400 }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(text(result)).toContain("This report is still processing"); + }); + + it("does not wait when the status cannot be read", async () => { + m.reportStatus.mockResolvedValue(undefined); + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(text(result)).toContain("✅ All issues retrieved."); + }); +}); From 7a1e5a565b44bb75c190a2f66195ec3b2172fa57 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Thu, 22 Oct 2026 11:12:45 +0530 Subject: [PATCH 2/2] fix(accessibility): wait only while a report is finalizing, not while its tests run A build whose suite is still running can take as long as the suite; blocking the first read for five minutes and then returning partial results anyway serves nobody. The wait now covers only the post-run advanced-issue processing ("finalizing"), which ends within minutes. A run in progress is read as it is, with the note that the issues are those found so far. Co-Authored-By: Claude Fable 5.1 --- src/tools/accessibility.ts | 17 ++++++++++++----- tests/tools/accessibilityScanOutput.test.ts | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index eca32d64..2d6497b1 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -270,6 +270,13 @@ const PROCESSING_STATUSES = new Set([ "finalizing", ]); +/** + * Of those, the one worth waiting on: the advanced-issue processing after the run, which + * ends within minutes. A report whose scan or tests are still running is read as it is, + * with a note, since that can last as long as the suite does. + */ +const WAIT_STATUSES = new Set(["finalizing"]); + interface ProcessingState { /** The status when the issues were read; undefined when it could not be read. */ status?: string; @@ -280,10 +287,10 @@ interface ProcessingState { /** * Where a report is in its processing before its issues are read. Issues keep being added - * until it completes (an advanced scan adds its advanced-rule issues while "finalizing", for - * minutes after the tests end), so a first read waits for it, reporting progress, for up to - * REPORT_STATUS_WAIT_MS. A later page does not wait again, and a status that cannot be read - * is not waited on. + * until it completes; an advanced scan adds its advanced-rule issues while "finalizing", for + * minutes after the tests end, so a first read waits for that, reporting progress, for up to + * REPORT_STATUS_WAIT_MS. A run still in progress is not waited on (it can take as long as + * the suite), nor is a later page, nor a status that cannot be read. */ async function waitForProcessing( reportFetcher: AccessibilityReportFetcher, @@ -297,7 +304,7 @@ async function waitForProcessing( while ( wait && status && - PROCESSING_STATUSES.has(status) && + WAIT_STATUSES.has(status) && Date.now() - started < REPORT_STATUS_WAIT_MS ) { const waited = Date.now() - started; diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 5c607cf0..a440d123 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -397,6 +397,23 @@ describe("fetchAccessibilityIssues and a report still processing", () => { expect(text(result)).toContain("This report is still processing"); }); + it("does not wait while the tests are still running, and says the issues are those found so far", async () => { + // A long suite can run for an hour; blocking the first read for five minutes and then + // returning partial results anyway serves nobody. Only post-run processing is waited on. + m.reportStatus.mockResolvedValue("running"); + + const result = await handlers.fetchAccessibilityIssues( + { buildId: BUILD_ID }, + context, + ); + + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(sendNotification).not.toHaveBeenCalled(); + const body = text(result); + expect(body).toContain("still processing (status: running"); + expect(body).toContain("only those found so far"); + }); + it("does not wait when the status cannot be read", async () => { m.reportStatus.mockResolvedValue(undefined); const result = await handlers.fetchAccessibilityIssues(