From 396a89e40a1591d8ae0ebb0316036e62898be183 Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Fri, 16 Oct 2026 19:17:59 +0100 Subject: [PATCH 1/7] Add RNTester visual test flows Adds the first two visual tests as Maestro flows under `packages/rn-tester/visual-tests/flows/`. Each flow deep-links to an RNTester example through `helpers/open-example.yml`, waits for it to render and takes a screenshot cropped to the example's `testID`. The flows only capture, and never decide whether a test passed. The last line clipping example gains a `testID` to crop to. Changelog: [Internal] --- .../js/examples/Text/TextSharedExamples.js | 2 +- .../flows/text-last-line-clipping.yml | 15 +++++++++++++++ .../visual-tests/flows/text-width-mode.yml | 18 ++++++++++++++++++ .../visual-tests/helpers/open-example.yml | 15 +++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 packages/rn-tester/visual-tests/flows/text-last-line-clipping.yml create mode 100644 packages/rn-tester/visual-tests/flows/text-width-mode.yml create mode 100644 packages/rn-tester/visual-tests/helpers/open-example.yml diff --git a/packages/rn-tester/js/examples/Text/TextSharedExamples.js b/packages/rn-tester/js/examples/Text/TextSharedExamples.js index 00ed6c2e27bb..f10cacc936fd 100644 --- a/packages/rn-tester/js/examples/Text/TextSharedExamples.js +++ b/packages/rn-tester/js/examples/Text/TextSharedExamples.js @@ -249,7 +249,7 @@ function LastLineClippingExample(): React.Node { // clipped by one physical pixel after layout rounding (issue #53450). The // border makes any clipping of the last line visible. return ( - + {[11, 12, 13, 14, 15, 16, 17].map(fontSize => ( Date: Fri, 16 Oct 2026 19:18:00 +0100 Subject: [PATCH 2/7] Capture visual tests in Test All Adds two Test All jobs that run the visual flows against the RNTester Release builds Test All already makes: `test_visual_android_rntester` after `build_android` and `test_visual_ios_rntester` after `test_ios_rntester`. They call the reusable `visual-capture-rntester.yml`, which runs every flow in one Maestro invocation on one canonical device per platform (a Pixel 7 API 35 emulator and an iPhone 17 Pro on iOS 26.2), retrying only failed flows. Each job uploads the screenshots plus a manifest as a `visual-capture-` artifact. The manifest records the commit, the base commit and the device environment, hashed into an environment key so that baselines from a different device setup are never compared. The emulator version is pinned and part of the key, and the capture dismisses a Pixel Launcher ANR dialog that can appear at boot. These jobs run PR code and have no secrets. Nothing consumes the artifacts yet. Changelog: [Internal] --- .../__tests__/__fixtures__/visual.js | 27 ++ .../__tests__/visual-capture-test.js | 145 +++++++++ .github/workflow-scripts/visual/capture.js | 205 +++++++++++++ .github/workflow-scripts/visual/manifest.js | 282 ++++++++++++++++++ .github/workflow-scripts/visual/png.js | 30 ++ .github/workflows/test-all.yml | 14 + .github/workflows/visual-capture-rntester.yml | 166 +++++++++++ 7 files changed, 869 insertions(+) create mode 100644 .github/workflow-scripts/__tests__/__fixtures__/visual.js create mode 100644 .github/workflow-scripts/__tests__/visual-capture-test.js create mode 100644 .github/workflow-scripts/visual/capture.js create mode 100644 .github/workflow-scripts/visual/manifest.js create mode 100644 .github/workflow-scripts/visual/png.js create mode 100644 .github/workflows/visual-capture-rntester.yml diff --git a/.github/workflow-scripts/__tests__/__fixtures__/visual.js b/.github/workflow-scripts/__tests__/__fixtures__/visual.js new file mode 100644 index 000000000000..0d07ae0fcf66 --- /dev/null +++ b/.github/workflow-scripts/__tests__/__fixtures__/visual.js @@ -0,0 +1,27 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const SHA_A = 'a'.repeat(40); +const SHA_B = 'b'.repeat(40); + +const ANDROID_ENVIRONMENT = { + apiLevel: 35, + abi: 'x86_64', + nativeBridge: 'libndk_translation.so', + systemImage: + 'google/sdk_gphone64_x86_64/emu64xa:15/AE3A/1:userdebug/dev-keys', + screenSize: '1080x2400', + density: 420, + fontScale: '1.0', + locale: 'en-US', + animatorDurationScale: '0.0', + emulatorVersion: '37.2.12', +}; + +module.exports = {ANDROID_ENVIRONMENT, SHA_A, SHA_B}; diff --git a/.github/workflow-scripts/__tests__/visual-capture-test.js b/.github/workflow-scripts/__tests__/visual-capture-test.js new file mode 100644 index 000000000000..a5c870d5ce0e --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-capture-test.js @@ -0,0 +1,145 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + collectFlows, + failedFlowsFromJUnit, + maestroArgs, +} = require('../visual/capture'); +const {environmentKey} = require('../visual/manifest'); +const {ANDROID_ENVIRONMENT} = require('./__fixtures__/visual'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +describe('visual capture', () => { + let directory; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-test-')); + }); + + afterEach(() => { + fs.rmSync(directory, {recursive: true, force: true}); + }); + + describe('environmentKey', () => { + test('is readable and stable', () => { + expect(environmentKey('android', ANDROID_ENVIRONMENT)).toMatch( + /^android-api35-x86-64-1080x2400-420dpi-[0-9a-f]{8}$/, + ); + expect(environmentKey('android', {...ANDROID_ENVIRONMENT})).toBe( + environmentKey('android', ANDROID_ENVIRONMENT), + ); + }); + + test.each([ + ['fontScale', '1.15'], + ['emulatorVersion', '37.3.1'], + ])('changes when %s drifts', (property, value) => { + expect( + environmentKey('android', {...ANDROID_ENVIRONMENT, [property]: value}), + ).not.toBe(environmentKey('android', ANDROID_ENVIRONMENT)); + }); + + test('ignores properties that are only recorded', () => { + expect( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + appearance: 'light', + contentSizeCategory: 'large', + locale: 'en_US', + xcode: 'Xcode 26.2', + }), + ).toBe( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + appearance: 'light', + contentSizeCategory: 'large', + locale: 'en_US', + xcode: 'Xcode 26.3', + }), + ); + }); + test('names iOS keys by runtime and device', () => { + expect( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + }), + ).toMatch(/^ios-26-2-iphone-17-pro-[0-9a-f]{8}$/); + }); + }); + + describe('capture', () => { + test('collects top-level flows in order and ignores other files', () => { + fs.writeFileSync(path.join(directory, 'b.yml'), ''); + fs.writeFileSync(path.join(directory, 'a.yaml'), ''); + fs.writeFileSync(path.join(directory, 'notes.md'), ''); + fs.mkdirSync(path.join(directory, 'helpers')); + fs.writeFileSync(path.join(directory, 'helpers/open.yml'), ''); + expect(collectFlows(directory)).toEqual([ + path.join(directory, 'a.yaml'), + path.join(directory, 'b.yml'), + ]); + }); + + test('rejects flows Maestro would report under the same name', () => { + fs.writeFileSync(path.join(directory, 'a.yml'), ''); + fs.writeFileSync(path.join(directory, 'a.yaml'), ''); + expect(() => collectFlows(directory)).toThrow("both report as 'a'"); + }); + + test('reads failed flows from a Maestro JUnit report', () => { + const flows = ['/flows/text-a.yml', '/flows/text-b.yml', '/flows/c.yml']; + const report = [ + '', + '', + '', + ].join('\n'); + expect(failedFlowsFromJUnit(report, flows)).toEqual([ + '/flows/text-b.yml', + '/flows/c.yml', + ]); + expect(() => + failedFlowsFromJUnit('', flows), + ).toThrow("unknown flow 'other'"); + expect(failedFlowsFromJUnit('', flows)).toEqual([]); + }); + + test.each([ + [1, undefined, false], + [2, 'UDID', true], + ])('builds Maestro arguments for attempt %d', (attempt, udid, reuse) => { + const args = maestroArgs({ + flows: ['/f/a.yml', '/f/b.yml'], + appId: 'com.example', + outputDir: '/out', + reportPath: '/r.xml', + udid, + attempt, + }); + expect( + args.slice(args.indexOf('test'), args.indexOf('test') + 3), + ).toEqual(['test', '/f/a.yml', '/f/b.yml']); + expect(args).toEqual( + expect.arrayContaining([ + '--format=junit', + '--output=/r.xml', + 'APP_ID=com.example', + 'VISUAL_OUTPUT_DIR=/out', + ]), + ); + expect(args.includes('--no-reinstall-driver')).toBe(reuse); + expect(args.includes('--udid=UDID')).toBe(udid != null); + }); + }); +}); diff --git a/.github/workflow-scripts/visual/capture.js b/.github/workflow-scripts/visual/capture.js new file mode 100644 index 000000000000..b19e08dad98b --- /dev/null +++ b/.github/workflow-scripts/visual/capture.js @@ -0,0 +1,205 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Captures every visual flow in one Maestro invocation, so the JVM and + * driver startup (around two minutes on an iOS simulator) is paid once per + * job rather than once per flow. Flows that fail are rerun, and only those, + * on later attempts. + */ + +const childProcess = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const MAESTRO = path.join(process.env.HOME ?? '', '.maestro/bin/maestro'); +const DEFAULT_ATTEMPTS = 3; +const ATTEMPT_TIMEOUT_MS = 30 * 60 * 1000; + +const usage = ` +node capture.js [ios_udid] + +Installs the app, then runs every flow in in one Maestro +invocation, writing screenshots under . Failed flows are retried +up to ${DEFAULT_ATTEMPTS} attempts in total (VISUAL_CAPTURE_ATTEMPTS). +`; + +/** The flows directly inside `flowsDir`, sorted. Helpers live elsewhere. */ +function collectFlows(flowsDir) { + const flows = fs + .readdirSync(flowsDir, {withFileTypes: true}) + .filter(entry => entry.isFile() && /\.ya?ml$/.test(entry.name)) + .map(entry => path.join(flowsDir, entry.name)) + .sort(); + flowsByReportName(flows); + return flows; +} + +/** + * Maps Maestro's report names to flow paths. Maestro reports a flow under its + * file name without the extension, so names must be unique. + */ +function flowsByReportName(flows) { + const byName = new Map(); + for (const flow of flows) { + const name = path.basename(flow).replace(/\.ya?ml$/, ''); + const existing = byName.get(name); + if (existing != null) { + throw new Error( + `${existing} and ${flow} would both report as '${name}'. Rename one.`, + ); + } + byName.set(name, flow); + } + return byName; +} + +/** + * The flows a Maestro JUnit report marks as anything but SUCCESS, in report + * order, matched by the report's file path or, failing that, its name. Throws + * on a report entry that matches none of `flows`. + */ +function failedFlowsFromJUnit(report, flows) { + const byName = flowsByReportName(flows); + const failed = []; + for (const [, attributes] of report.matchAll(/]*)>/g)) { + const name = /\bname="([^"]*)"/.exec(attributes)?.[1]; + const file = /\bfile="([^"]*)"/.exec(attributes)?.[1]; + const status = /\bstatus="([^"]*)"/.exec(attributes)?.[1]; + if (name == null || status === 'SUCCESS') { + continue; + } + const flow = + flows.find(candidate => path.resolve(candidate) === file) ?? + byName.get(name); + if (flow == null) { + throw new Error(`The Maestro report has an unknown flow '${name}'`); + } + failed.push(flow); + } + return failed; +} + +function maestroArgs({flows, appId, outputDir, reportPath, udid, attempt}) { + return [ + ...(udid != null ? [`--udid=${udid}`] : []), + 'test', + ...flows, + '--format=junit', + `--output=${reportPath}`, + '-e', + `APP_ID=${appId}`, + '-e', + `VISUAL_OUTPUT_DIR=${outputDir}`, + // The first attempt installs the driver. Retries reuse it. + ...(attempt > 1 ? ['--no-reinstall-driver'] : []), + ]; +} + +function installApp(platform, appPath, udid) { + if (platform === 'android') { + childProcess.execFileSync('adb', ['install', '-r', appPath], { + stdio: 'inherit', + }); + } else { + childProcess.execFileSync('xcrun', ['simctl', 'install', udid, appPath], { + stdio: 'inherit', + }); + } +} + +function runAttempt(args) { + const result = childProcess.spawnSync(MAESTRO, args, { + stdio: 'inherit', + timeout: ATTEMPT_TIMEOUT_MS, + env: { + ...process.env, + MAESTRO_CLI_NO_ANALYTICS: '1', + MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED: 'true', + MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000', + }, + }); + if (result.error != null) { + throw result.error; + } + return result.status; +} + +function writeSummary(lines) { + const text = `${lines.join('\n')}\n`; + console.info(text); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); + } +} + +function main(args = process.argv.slice(2)) { + const [platform, appPath, appId, flowsDir, outputDir, udid] = args; + if ( + !['android', 'ios'].includes(platform) || + outputDir == null || + (platform === 'ios' && udid == null) + ) { + throw new Error(usage); + } + const attempts = Number( + process.env.VISUAL_CAPTURE_ATTEMPTS ?? DEFAULT_ATTEMPTS, + ); + const allFlows = collectFlows(flowsDir); + fs.mkdirSync(outputDir, {recursive: true}); + installApp(platform, appPath, udid); + + const summary = [`### Visual capture (${platform})`, '']; + let pending = allFlows; + for (let attempt = 1; attempt <= attempts && pending.length > 0; attempt++) { + const reportPath = path.resolve(`visual-capture-report-${attempt}.xml`); + const started = Date.now(); + console.info( + `Attempt ${attempt}/${attempts}: ${pending.length} flow(s) in one Maestro run`, + ); + const status = runAttempt( + maestroArgs({ + flows: pending, + appId, + outputDir, + reportPath, + udid, + attempt, + }), + ); + const report = fs.existsSync(reportPath) + ? fs.readFileSync(reportPath, 'utf8') + : ''; + const failed = failedFlowsFromJUnit(report, pending); + if (status !== 0 && failed.length === 0) { + // Maestro failed without attributing it to a flow, for example because + // it could not reach the device. Retrying won't help. + throw new Error(`Maestro exited with ${status} without a failed flow`); + } + summary.push( + `Attempt ${attempt}: ${pending.length - failed.length} of ${pending.length} flow(s) passed in ${Math.round((Date.now() - started) / 1000)} s`, + ); + pending = failed; + } + + if (pending.length > 0) { + summary.push('', `Failed after ${attempts} attempt(s):`); + summary.push(...pending.map(flow => `- \`${flow}\``)); + } + writeSummary(summary); + if (pending.length > 0) { + process.exitCode = 1; + } +} + +if (require.main === module) { + main(); +} + +module.exports = {collectFlows, failedFlowsFromJUnit, maestroArgs}; diff --git a/.github/workflow-scripts/visual/manifest.js b/.github/workflow-scripts/visual/manifest.js new file mode 100644 index 000000000000..6b7832ede2c4 --- /dev/null +++ b/.github/workflow-scripts/visual/manifest.js @@ -0,0 +1,282 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {readPngDimensions} = require('./png'); +const childProcess = require('child_process'); +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const MANIFEST_VERSION = 1; +const PLATFORMS = ['android', 'ios']; + +// //, each segment lowercase kebab-case. +const TEST_ID_PATTERN = + /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*){2}$/; +const SHA_PATTERN = /^[0-9a-f]{40}$/; + +// Environment properties that select a baseline. Anything else in the +// environment is recorded for diagnosis but does not change the key. +const KEYED_PROPERTIES = { + android: [ + 'apiLevel', + 'abi', + 'nativeBridge', + 'systemImage', + 'screenSize', + 'density', + 'fontScale', + 'locale', + 'animatorDurationScale', + 'emulatorVersion', + ], + ios: ['runtime', 'deviceType', 'appearance', 'contentSizeCategory', 'locale'], +}; + +function sha256(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex'); +} + +function canonicalJson(value) { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value != null && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function slug(value) { + return String(value) + .toLowerCase() + .replace(/[^a-z0-9.]+/g, '-') + .replace(/^-|-$/g, ''); +} + +/** + * A readable prefix for the properties people look for first, followed by a + * hash of every keyed property so that drift in any of them selects a + * different baseline instead of producing a spurious diff. + */ +function environmentKey(platform, environment) { + const keyed = {}; + for (const property of KEYED_PROPERTIES[platform]) { + keyed[property] = environment[property] ?? null; + } + const hash = sha256(Buffer.from(canonicalJson({platform, keyed}))).slice( + 0, + 8, + ); + const readable = + platform === 'android' + ? [ + 'android', + `api${environment.apiLevel}`, + environment.abi, + environment.screenSize, + `${environment.density}dpi`, + ] + : [ + 'ios', + String(environment.runtime).replace(/^iOS-/, ''), + environment.deviceType, + ]; + return [...readable.map(slug), hash].join('-'); +} + +function collectImages(captureDir) { + const images = []; + const visit = directory => { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(entryPath); + } else if (entry.isFile() && entry.name.endsWith('.png')) { + images.push(entryPath); + } + } + }; + visit(captureDir); + return images.sort(); +} + +function buildManifest({captureDir, platform, environment, git}) { + const images = collectImages(captureDir).map(imagePath => { + const relativePath = path + .relative(captureDir, imagePath) + .split(path.sep) + .join('/'); + const buffer = fs.readFileSync(imagePath); + const dimensions = readPngDimensions(buffer); + if (dimensions == null) { + throw new Error(`Not a PNG: ${relativePath}`); + } + return { + id: relativePath.replace(/\.png$/, ''), + path: relativePath, + ...dimensions, + bytes: buffer.length, + sha256: sha256(buffer), + }; + }); + + return { + version: MANIFEST_VERSION, + platform, + environmentKey: environmentKey(platform, environment), + environment, + git, + images, + }; +} + +function run(command, args) { + return childProcess + .execFileSync(command, args, {encoding: 'utf8', timeout: 30000}) + .trim(); +} + +/** The emulator package's version, from the Android SDK. */ +function readEmulatorVersion() { + const sdk = process.env.ANDROID_HOME ?? process.env.ANDROID_SDK_ROOT; + if (sdk == null) { + return null; + } + const properties = fs.readFileSync( + path.join(sdk, 'emulator', 'source.properties'), + 'utf8', + ); + return /^Pkg\.Revision=(\S+)$/m.exec(properties)?.[1] ?? null; +} + +function readAndroidEnvironment() { + const getprop = name => run('adb', ['shell', 'getprop', name]); + const setting = (namespace, name) => + run('adb', ['shell', 'settings', 'get', namespace, name]); + const wm = name => { + const output = run('adb', ['shell', 'wm', name]); + const override = /Override \w+: (\S+)/.exec(output); + return (override ?? /Physical \w+: (\S+)/.exec(output))?.[1] ?? output; + }; + return { + apiLevel: Number(getprop('ro.build.version.sdk')), + abi: getprop('ro.product.cpu.abi'), + nativeBridge: getprop('ro.dalvik.vm.native.bridge'), + systemImage: getprop('ro.build.fingerprint'), + screenSize: wm('size'), + density: Number(wm('density')), + fontScale: setting('system', 'font_scale'), + locale: getprop('persist.sys.locale') || getprop('ro.product.locale'), + animatorDurationScale: setting('global', 'animator_duration_scale'), + emulatorVersion: readEmulatorVersion(), + }; +} + +function readIosEnvironment(udid) { + const devices = JSON.parse( + run('xcrun', ['simctl', 'list', 'devices', '-j']), + ).devices; + let device = null; + let runtime = null; + for (const [runtimeId, runtimeDevices] of Object.entries(devices)) { + const match = runtimeDevices.find(candidate => candidate.udid === udid); + if (match != null) { + device = match; + runtime = runtimeId.replace('com.apple.CoreSimulator.SimRuntime.', ''); + } + } + if (device == null) { + throw new Error(`Simulator ${udid} not found`); + } + const ui = name => run('xcrun', ['simctl', 'ui', udid, name]); + return { + runtime, + deviceType: device.deviceTypeIdentifier.replace( + 'com.apple.CoreSimulator.SimDeviceType.', + '', + ), + appearance: ui('appearance'), + contentSizeCategory: ui('content_size'), + locale: run('xcrun', [ + 'simctl', + 'spawn', + udid, + 'defaults', + 'read', + '-g', + 'AppleLocale', + ]), + xcode: run('xcodebuild', ['-version']).split('\n').join(' '), + }; +} + +function readGitInfo() { + const git = args => run('git', args); + return { + // For pull_request events actions/checkout tests the merge commit, whose + // first parent is the base branch commit the PR was tested against. + testedSha: git(['rev-parse', 'HEAD']), + baseSha: git(['rev-parse', 'HEAD^1']), + headSha: process.env.VISUAL_HEAD_SHA || git(['rev-parse', 'HEAD']), + event: process.env.GITHUB_EVENT_NAME ?? null, + ref: process.env.GITHUB_REF ?? null, + }; +} + +const usage = ` +node manifest.js [ios_udid] + +Writes a manifest for the PNGs under , reading the environment +from the running emulator (adb) or the simulator with . +`; + +function main(args = process.argv.slice(2)) { + const [platform, captureDir, outputPath, udid] = args; + if ( + !PLATFORMS.includes(platform) || + captureDir == null || + outputPath == null + ) { + throw new Error(usage); + } + const environment = + platform === 'android' + ? readAndroidEnvironment() + : readIosEnvironment(udid); + const manifest = buildManifest({ + captureDir, + platform, + environment, + git: readGitInfo(), + }); + fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`); + console.info( + `Wrote ${manifest.images.length} image(s) for ${manifest.environmentKey} to ${outputPath}`, + ); +} + +if (require.main === module) { + main(); +} + +module.exports = { + KEYED_PROPERTIES, + MANIFEST_VERSION, + PLATFORMS, + SHA_PATTERN, + TEST_ID_PATTERN, + buildManifest, + canonicalJson, + environmentKey, + sha256, +}; diff --git a/.github/workflow-scripts/visual/png.js b/.github/workflow-scripts/visual/png.js new file mode 100644 index 000000000000..73d9e5df978b --- /dev/null +++ b/.github/workflow-scripts/visual/png.js @@ -0,0 +1,30 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); + +/** + * Reads width and height from a PNG's IHDR chunk without decoding pixels, so + * that untrusted files can be checked before any image library sees them. + * Returns null if the buffer does not start with a well-formed PNG header. + */ +function readPngDimensions(buffer) { + if ( + buffer.length < 24 || + !buffer.subarray(0, 8).equals(PNG_SIGNATURE) || + buffer.toString('ascii', 12, 16) !== 'IHDR' + ) { + return null; + } + return {width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20)}; +} + +module.exports = {readPngDimensions}; diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index fe4f2a56633b..3f4d780d2b2e 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -209,6 +209,13 @@ jobs: fail-on-error: true secrets: inherit + test_visual_ios_rntester: + needs: test_ios_rntester + if: ${{ needs.test_ios_rntester.result == 'success' }} + uses: ./.github/workflows/visual-capture-rntester.yml + with: + platform: ios + test_e2e_ios_templateapp: needs: [build_npm_package, prebuild_apple_dependencies, check_code_changes] if: needs.check_code_changes.outputs.should_test_ios == 'true' @@ -406,6 +413,13 @@ jobs: retry-attempt: 2 secrets: inherit + test_visual_android_rntester: + needs: build_android + if: ${{ needs.build_android.result == 'success' }} + uses: ./.github/workflows/visual-capture-rntester.yml + with: + platform: android + build_npm_package: runs-on: 8-core-ubuntu needs: diff --git a/.github/workflows/visual-capture-rntester.yml b/.github/workflows/visual-capture-rntester.yml new file mode 100644 index 000000000000..cde456a4b221 --- /dev/null +++ b/.github/workflows/visual-capture-rntester.yml @@ -0,0 +1,166 @@ +# Untrusted half of RNTester visual regression testing. Called from Test All +# once the RNTester Release build for a platform is ready, it captures the +# visual flows on one canonical device and uploads the PNGs plus a manifest as +# an artifact. It has no secrets and no storage access: comparison and +# publication happen in visual-compare-rntester.yml, which runs from the +# default branch when Test All completes. +name: Visual Capture RNTester + +on: + workflow_call: + inputs: + platform: + description: 'The platform to capture, either android or ios' + required: true + type: string + +permissions: + contents: read + +env: + MAESTRO_VERSION: 2.6.1 + VISUAL_FLOWS: packages/rn-tester/visual-tests/flows + VISUAL_OUTPUT_DIR: /tmp/visual-capture + VISUAL_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + +jobs: + capture_android: + if: inputs.platform == 'android' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The manifest records HEAD^1, the commit this capture is based on. + fetch-depth: 2 + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'zulu' + - name: Install Maestro + run: curl -Ls "https://get.maestro.mobile.dev" | bash + - name: Enable KVM group perms + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Download RNTester APK + uses: actions/download-artifact@v7 + with: + name: rntester-release + path: /tmp/rntester-android + - name: Capture + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 35 + target: google_apis + arch: x86_64 + # Pins screen size and density, which the action otherwise leaves + # to the AVD defaults. + profile: pixel_7 + # Emulator 37.2.12. Its version can change rendering, so it's + # pinned and is part of the environment key. + emulator-build: '16428233' + ram-size: '8192M' + heap-size: '4096M' + disk-size: '10G' + cores: '4' + disable-animations: true + avd-name: visual_emulator + script: | + adb shell settings put system font_scale 1.0 + # Pixel Launcher often stops responding during boot, and its dialog + # covers the app and anything captured beneath it. The setting + # suppresses new dialogs, and stopping the launcher dismisses one + # that's already showing. + adb shell settings put global hide_error_dialogs 1 + adb shell am force-stop "$(adb shell cmd package resolve-activity --brief -a android.intent.action.MAIN -c android.intent.category.HOME | tail -1 | cut -d/ -f1 | tr -d '\r')" + mkdir -p "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/capture.js android /tmp/rntester-android/app-arm64-v8a-release.apk com.facebook.react.uiapp "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/manifest.js android "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" + - name: Upload captures + uses: ./.github/actions/upload-artifact + with: + name: visual-capture-android + path: ${{ env.VISUAL_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 14 + - name: Upload Maestro logs + if: failure() + uses: ./.github/actions/upload-artifact + with: + name: visual-maestro-logs-android + path: | + ~/.maestro/tests + visual-capture-report-*.xml + if-no-files-found: ignore + + capture_ios: + if: inputs.platform == 'ios' + runs-on: macos-26 + env: + DEVICE_MODEL: iPhone-17-Pro + DEVICE_OS: iOS-26-2 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The manifest records HEAD^1, the commit this capture is based on. + fetch-depth: 2 + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'zulu' + - name: Install Maestro + run: | + curl -Ls "https://get.maestro.mobile.dev" | bash + brew tap facebook/fb + brew install facebook/fb/idb-companion + - name: Download RNTester app + uses: actions/download-artifact@v7 + with: + name: RNTesterApp-NewArch-Release + path: /tmp/RNTesterBuild/RNTester.app + - name: Boot and configure simulator + id: simulator + run: | + UDID=$(xcrun simctl list devices available -j | jq -r --arg runtime "com.apple.CoreSimulator.SimRuntime.$DEVICE_OS" --arg name "${DEVICE_MODEL//-/ }" '.devices[$runtime][] | select(.name == $name) | .udid' | head -1) + if [[ -z "$UDID" ]]; then + echo "::error::No $DEVICE_MODEL simulator on $DEVICE_OS" + xcrun simctl list devices available + exit 1 + fi + xcrun simctl boot "$UDID" + xcrun simctl bootstatus "$UDID" + xcrun simctl ui "$UDID" appearance light + xcrun simctl ui "$UDID" content_size large + xcrun simctl spawn "$UDID" defaults write -g AppleLocale en_US + xcrun simctl spawn "$UDID" defaults write -g AppleLanguages -array en + echo "udid=$UDID" >> "$GITHUB_OUTPUT" + - name: Capture + run: | + mkdir -p "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/capture.js ios /tmp/RNTesterBuild/RNTester.app com.meta.RNTester.localDevelopment "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" "${{ steps.simulator.outputs.udid }}" + node .github/workflow-scripts/visual/manifest.js ios "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" "${{ steps.simulator.outputs.udid }}" + - name: Upload captures + uses: ./.github/actions/upload-artifact + with: + name: visual-capture-ios + path: ${{ env.VISUAL_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 14 + - name: Upload Maestro logs + if: failure() + uses: ./.github/actions/upload-artifact + with: + name: visual-maestro-logs-ios + path: | + ~/.maestro/tests + visual-capture-report-*.xml + if-no-files-found: ignore From 5f68b233aca8c7702d383b54368b20ffcb64b903 Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Fri, 16 Oct 2026 19:18:01 +0100 Subject: [PATCH 3/7] Validate and compare visual captures Adds the comparison core, as pure functions over a capture directory and a baseline. `validate.js` treats the capture artifact as untrusted: it rejects path traversal, symlinks, unlisted or duplicate files, hash mismatches and oversized images, reading PNG dimensions from the header before any image library decodes them. `compare.js` compares each image with pixelmatch under the per-test policy in `visual-tests/config.json`, which is strict by default. Adds `pixelmatch` and `pngjs` as dev dependencies. Changelog: [Internal] --- .../__tests__/visual-compare-test.js | 261 ++++++++++++++++++ .github/workflow-scripts/visual/compare.js | 152 ++++++++++ .github/workflow-scripts/visual/validate.js | 183 ++++++++++++ package.json | 2 + packages/rn-tester/visual-tests/config.json | 8 + yarn.lock | 17 ++ 6 files changed, 623 insertions(+) create mode 100644 .github/workflow-scripts/__tests__/visual-compare-test.js create mode 100644 .github/workflow-scripts/visual/compare.js create mode 100644 .github/workflow-scripts/visual/validate.js create mode 100644 packages/rn-tester/visual-tests/config.json diff --git a/.github/workflow-scripts/__tests__/visual-compare-test.js b/.github/workflow-scripts/__tests__/visual-compare-test.js new file mode 100644 index 000000000000..170673ca4a96 --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-compare-test.js @@ -0,0 +1,261 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {compareCapture, compareImages, policyFor} = require('../visual/compare'); +const {buildManifest, environmentKey, sha256} = require('../visual/manifest'); +const {readPngDimensions} = require('../visual/png'); +const {validateCapture} = require('../visual/validate'); +const {ANDROID_ENVIRONMENT, SHA_A, SHA_B} = require('./__fixtures__/visual'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const {PNG} = require('pngjs'); + +function makePng(width, height, paint = () => [255, 255, 255, 255]) { + const png = new PNG({width, height}); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const offset = (y * width + x) * 4; + const [r, g, b, a] = paint(x, y); + png.data[offset] = r; + png.data[offset + 1] = g; + png.data[offset + 2] = b; + png.data[offset + 3] = a; + } + } + return PNG.sync.write(png); +} + +function writeCapture(directory, images, overrides = {}) { + for (const [id, buffer] of Object.entries(images)) { + const filePath = path.join(directory, `${id}.png`); + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + fs.writeFileSync(filePath, buffer); + } + const manifest = buildManifest({ + captureDir: directory, + platform: 'android', + environment: ANDROID_ENVIRONMENT, + git: {testedSha: SHA_A, baseSha: SHA_B, headSha: SHA_A}, + }); + const finalManifest = {...manifest, ...overrides}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(finalManifest), + ); + return finalManifest; +} + +describe('visual comparison', () => { + let directory; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-test-')); + }); + + afterEach(() => { + fs.rmSync(directory, {recursive: true, force: true}); + }); + + describe('readPngDimensions', () => { + test('reads the IHDR dimensions', () => { + expect(readPngDimensions(makePng(3, 2))).toEqual({width: 3, height: 2}); + }); + + test.each([ + [Buffer.from('not a png at all, but long enough')], + [Buffer.alloc(0)], + ])('rejects non-PNG data', buffer => { + expect(readPngDimensions(buffer)).toBeNull(); + }); + }); + + describe('validateCapture', () => { + test('accepts a well-formed capture', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + expect(validateCapture(directory)).toEqual(manifest); + }); + + test.each([ + ['../escape/a/b'], + ['/absolute/a/b'], + ['Text/Upper/case'], + ['too/few'], + ['a/b/c/d'], + ])('rejects test ID %s', id => { + writeCapture(directory, {'text/width-mode/default': makePng(4, 4)}); + const manifest = JSON.parse( + fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'), + ); + manifest.images[0] = {...manifest.images[0], id, path: `${id}.png`}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + expect(() => validateCapture(directory)).toThrow('Invalid test ID'); + }); + + test('rejects a path that does not match the ID', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + manifest.images[0].path = '../../etc/passwd'; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + expect(() => validateCapture(directory)).toThrow('path must be'); + }); + + test('rejects duplicates, mismatched hashes and unlisted files', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + manifest.images.push({...manifest.images[0]}); + manifest.images[0] = {...manifest.images[0], sha256: '0'.repeat(64)}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + fs.writeFileSync(path.join(directory, 'extra.sh'), 'echo hi'); + let message = ''; + try { + validateCapture(directory); + } catch (error) { + message = error.message; + } + expect(message).toContain('Duplicate test ID'); + expect(message).toContain('does not match its manifest entry'); + expect(message).toContain('Files not in the manifest: extra.sh'); + }); + + test('rejects non-PNG content and oversized images', () => { + writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + 'text/width-mode/large': makePng(20, 4), + }); + fs.writeFileSync( + path.join(directory, 'text/width-mode/default.png'), + 'not a png', + ); + expect(() => + validateCapture(directory, { + maxImages: 10, + maxImageBytes: 1024 * 1024, + maxTotalBytes: 1024 * 1024, + maxDimension: 10, + maxManifestBytes: 1024 * 1024, + }), + ).toThrow(/default: not a PNG[\s\S]*large: 20x4 is outside/); + }); + + test('rejects symlinks', () => { + writeCapture(directory, {'text/width-mode/default': makePng(4, 4)}); + fs.rmSync(path.join(directory, 'text/width-mode/default.png')); + fs.symlinkSync( + '/etc/hosts', + path.join(directory, 'text/width-mode/default.png'), + ); + expect(() => validateCapture(directory)).toThrow( + 'file is missing or not a regular file', + ); + }); + + test('rejects invalid SHAs and environment keys', () => { + writeCapture( + directory, + {'text/width-mode/default': makePng(4, 4)}, + { + environmentKey: 'ios-../../x', + git: {testedSha: 'HEAD', baseSha: SHA_B, headSha: SHA_A}, + }, + ); + expect(() => validateCapture(directory)).toThrow( + /Invalid environment key[\s\S]*Invalid git.testedSha/, + ); + }); + }); + + describe('compareImages', () => { + const strict = policyFor('a/b/c', null); + + test('identical images are unchanged', () => { + expect(compareImages(makePng(8, 8), makePng(8, 8), strict)).toMatchObject( + {status: 'unchanged', diffPixels: 0, diff: null}, + ); + }); + + test('a single changed pixel fails the strict policy', () => { + const changed = makePng(8, 8, (x, y) => + x === 3 && y === 3 ? [254, 255, 255, 255] : [255, 255, 255, 255], + ); + const result = compareImages(makePng(8, 8), changed, strict); + expect(result).toMatchObject({status: 'changed', diffPixels: 1}); + expect(readPngDimensions(result.diff)).toEqual({width: 8, height: 8}); + }); + + test('a per-test tolerance can absorb a known amount of noise', () => { + const changed = makePng(8, 8, (x, y) => + x === 3 && y === 3 ? [0, 0, 0, 255] : [255, 255, 255, 255], + ); + const policy = policyFor('a/b/c', {tests: {'a/b/c': {maxDiffPixels: 1}}}); + expect(compareImages(makePng(8, 8), changed, policy).status).toBe( + 'unchanged', + ); + expect( + policyFor('a/b/d', {tests: {'a/b/c': {maxDiffPixels: 1}}}), + ).toEqual(strict); + }); + + test('different dimensions are reported, not compared', () => { + expect(compareImages(makePng(8, 8), makePng(8, 9), strict)).toMatchObject( + { + status: 'dimension-mismatch', + expectedSize: {width: 8, height: 8}, + capturedSize: {width: 8, height: 9}, + }, + ); + }); + }); + + describe('compareCapture', () => { + test('classifies unchanged, changed, added and removed tests', async () => { + const white = makePng(4, 4); + const black = makePng(4, 4, () => [0, 0, 0, 255]); + const image = (id, buffer) => ({id, sha256: sha256(buffer)}); + const results = await compareCapture({ + manifest: { + images: [ + image('a/a/same', white), + image('a/a/changed', black), + image('a/a/added', white), + ], + }, + baselineManifest: { + images: [ + image('a/a/same', white), + image('a/a/changed', white), + image('a/a/removed', white), + ], + }, + readExpected: async () => white, + readCaptured: async () => black, + config: null, + }); + expect(results.map(result => [result.id, result.status])).toEqual([ + ['a/a/added', 'added'], + ['a/a/changed', 'changed'], + ['a/a/removed', 'removed'], + ['a/a/same', 'unchanged'], + ]); + }); + }); +}); diff --git a/.github/workflow-scripts/visual/compare.js b/.github/workflow-scripts/visual/compare.js new file mode 100644 index 000000000000..86694966dc4b --- /dev/null +++ b/.github/workflow-scripts/visual/compare.js @@ -0,0 +1,152 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const pixelmatch = require('pixelmatch'); +const {PNG} = require('pngjs'); + +// Strict by default: any changed pixel fails. Loosen per test in +// packages/rn-tester/visual-tests/config.json, with evidence. +const DEFAULT_POLICY = { + // pixelmatch's per-pixel color distance threshold, from 0 to 1. + threshold: 0, + // Count anti-aliased pixels as differences. + includeAA: true, + maxDiffPixels: 0, + // Optional: also fail when this share of the image changes. + maxDiffRatio: null, +}; + +function policyFor(testId, config) { + return {...DEFAULT_POLICY, ...config?.defaults, ...config?.tests?.[testId]}; +} + +/** + * Compares two PNG buffers. Returns the outcome and, when the images have + * the same dimensions and differ, a diff image highlighting changed pixels. + */ +function compareImages(expectedBuffer, capturedBuffer, policy) { + const expected = PNG.sync.read(expectedBuffer); + const captured = PNG.sync.read(capturedBuffer); + if ( + expected.width !== captured.width || + expected.height !== captured.height + ) { + return { + status: 'dimension-mismatch', + expectedSize: {width: expected.width, height: expected.height}, + capturedSize: {width: captured.width, height: captured.height}, + diffPixels: null, + diffRatio: null, + diff: null, + }; + } + + const {width, height} = expected; + const diff = new PNG({width, height}); + const diffPixels = pixelmatch( + expected.data, + captured.data, + diff.data, + width, + height, + {threshold: policy.threshold, includeAA: policy.includeAA}, + ); + const diffRatio = diffPixels / (width * height); + const changed = + diffPixels > policy.maxDiffPixels || + (policy.maxDiffRatio != null && diffRatio > policy.maxDiffRatio); + return { + status: changed ? 'changed' : 'unchanged', + diffPixels, + diffRatio, + diff: diffPixels > 0 ? PNG.sync.write(diff) : null, + }; +} + +/** + * Compares every captured image with the baseline. `readExpected(id)` and + * `readCaptured(id)` return PNG buffers. Results are sorted by test ID. + */ +async function compareCapture({ + manifest, + baselineManifest, + readExpected, + readCaptured, + config, +}) { + const expectedById = new Map( + (baselineManifest?.images ?? []).map(image => [image.id, image]), + ); + const results = []; + + for (const image of manifest.images) { + const expected = expectedById.get(image.id); + expectedById.delete(image.id); + if (expected == null) { + results.push({id: image.id, status: 'added', captured: image}); + continue; + } + if (expected.sha256 === image.sha256) { + results.push({ + id: image.id, + status: 'unchanged', + diffPixels: 0, + diffRatio: 0, + captured: image, + expected, + }); + continue; + } + const policy = policyFor(image.id, config); + const {diff, ...outcome} = compareImages( + await readExpected(image.id), + await readCaptured(image.id), + policy, + ); + results.push({ + id: image.id, + ...outcome, + policy, + captured: image, + expected, + diffImage: diff, + }); + } + + for (const expected of expectedById.values()) { + results.push({id: expected.id, status: 'removed', expected}); + } + + return results.sort((a, b) => a.id.localeCompare(b.id)); +} + +function summarize(results) { + const counts = { + unchanged: 0, + changed: 0, + 'dimension-mismatch': 0, + added: 0, + removed: 0, + }; + for (const result of results) { + counts[result.status] += 1; + } + return { + counts, + hasDifferences: results.some(result => result.status !== 'unchanged'), + }; +} + +module.exports = { + DEFAULT_POLICY, + compareCapture, + compareImages, + policyFor, + summarize, +}; diff --git a/.github/workflow-scripts/visual/validate.js b/.github/workflow-scripts/visual/validate.js new file mode 100644 index 000000000000..0b6361d420e5 --- /dev/null +++ b/.github/workflow-scripts/visual/validate.js @@ -0,0 +1,183 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + MANIFEST_VERSION, + PLATFORMS, + SHA_PATTERN, + TEST_ID_PATTERN, + sha256, +} = require('./manifest'); +const {readPngDimensions} = require('./png'); +const fs = require('fs'); +const path = require('path'); + +const LIMITS = { + maxImages: 200, + maxImageBytes: 5 * 1024 * 1024, + maxTotalBytes: 100 * 1024 * 1024, + maxDimension: 4096, + maxManifestBytes: 1024 * 1024, +}; + +const ENVIRONMENT_KEY_PATTERN = /^(android|ios)-[a-z0-9.-]{1,120}-[0-9a-f]{8}$/; + +/** + * Validates a capture artifact produced by untrusted pull request code. The + * artifact is data only: nothing in it is executed, and every path is checked + * before it is read. Returns the parsed manifest, or throws an Error listing + * every problem found. + */ +function validateCapture(captureDir, limits = LIMITS) { + const errors = []; + const manifestPath = path.join(captureDir, 'manifest.json'); + const manifestStat = lstatOrNull(manifestPath); + if (manifestStat == null || !manifestStat.isFile()) { + throw new Error('manifest.json is missing or not a regular file'); + } + if (manifestStat.size > limits.maxManifestBytes) { + throw new Error( + `manifest.json is larger than ${limits.maxManifestBytes} bytes`, + ); + } + + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (error) { + throw new Error(`manifest.json is not valid JSON: ${error.message}`); + } + + if (manifest?.version !== MANIFEST_VERSION) { + errors.push(`Unsupported manifest version: ${String(manifest?.version)}`); + } + if (!PLATFORMS.includes(manifest?.platform)) { + errors.push(`Unknown platform: ${String(manifest?.platform)}`); + } + if ( + typeof manifest?.environmentKey !== 'string' || + !ENVIRONMENT_KEY_PATTERN.test(manifest.environmentKey) || + !manifest.environmentKey.startsWith(`${manifest.platform}-`) + ) { + errors.push(`Invalid environment key: ${String(manifest?.environmentKey)}`); + } + for (const field of ['testedSha', 'baseSha', 'headSha']) { + if (!SHA_PATTERN.test(String(manifest?.git?.[field]))) { + errors.push(`Invalid git.${field}`); + } + } + + const images = Array.isArray(manifest?.images) ? manifest.images : null; + if (images == null) { + errors.push('images must be an array'); + } else if (images.length > limits.maxImages) { + errors.push(`Too many images: ${images.length} > ${limits.maxImages}`); + } else { + const seen = new Set(); + let totalBytes = 0; + for (const image of images) { + const id = String(image?.id); + if (!TEST_ID_PATTERN.test(id)) { + errors.push(`Invalid test ID: ${JSON.stringify(image?.id)}`); + continue; + } + if (seen.has(id)) { + errors.push(`Duplicate test ID: ${id}`); + continue; + } + seen.add(id); + // The path is derived from the validated ID, never taken from the + // manifest, so it cannot escape the capture directory. + if (image.path !== `${id}.png`) { + errors.push(`${id}: path must be ${id}.png`); + continue; + } + const imagePath = path.join(captureDir, ...id.split('/')) + '.png'; + const stat = lstatOrNull(imagePath); + if (stat == null || !stat.isFile()) { + errors.push(`${id}: file is missing or not a regular file`); + continue; + } + if (stat.size > limits.maxImageBytes) { + errors.push( + `${id}: ${stat.size} bytes exceeds ${limits.maxImageBytes}`, + ); + continue; + } + totalBytes += stat.size; + const buffer = fs.readFileSync(imagePath); + const dimensions = readPngDimensions(buffer); + if (dimensions == null) { + errors.push(`${id}: not a PNG`); + continue; + } + if ( + dimensions.width === 0 || + dimensions.height === 0 || + dimensions.width > limits.maxDimension || + dimensions.height > limits.maxDimension + ) { + errors.push( + `${id}: ${dimensions.width}x${dimensions.height} is outside 1-${limits.maxDimension} px`, + ); + continue; + } + if ( + image.width !== dimensions.width || + image.height !== dimensions.height || + image.bytes !== stat.size || + image.sha256 !== sha256(buffer) + ) { + errors.push(`${id}: file does not match its manifest entry`); + } + } + if (totalBytes > limits.maxTotalBytes) { + errors.push(`Total size ${totalBytes} exceeds ${limits.maxTotalBytes}`); + } + const unlisted = listFiles(captureDir).filter( + file => file !== 'manifest.json' && !seen.has(file.replace(/\.png$/, '')), + ); + if (unlisted.length > 0) { + errors.push( + `Files not in the manifest: ${unlisted.slice(0, 10).join(', ')}`, + ); + } + } + + if (errors.length > 0) { + throw new Error(`Invalid capture artifact:\n- ${errors.join('\n- ')}`); + } + return manifest; +} + +function lstatOrNull(filePath) { + try { + return fs.lstatSync(filePath); + } catch { + return null; + } +} + +function listFiles(root) { + const files = []; + const visit = (directory, prefix) => { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const relative = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) { + visit(path.join(directory, entry.name), relative); + } else { + files.push(relative); + } + } + }; + visit(root, ''); + return files; +} + +module.exports = {LIMITS, validateCapture}; diff --git a/package.json b/package.json index 1fa53cb0e539..a1743c2892cd 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,8 @@ "micromatch": "^4.0.4", "node-fetch": "^2.2.0", "nullthrows": "^1.1.1", + "pixelmatch": "5.3.0", + "pngjs": "7.0.0", "prettier": "3.9.4", "react": "19.3.0", "react-test-renderer": "19.3.0", diff --git a/packages/rn-tester/visual-tests/config.json b/packages/rn-tester/visual-tests/config.json new file mode 100644 index 000000000000..e1e08c5047c6 --- /dev/null +++ b/packages/rn-tester/visual-tests/config.json @@ -0,0 +1,8 @@ +{ + "defaults": { + "threshold": 0, + "includeAA": true, + "maxDiffPixels": 0 + }, + "tests": {} +} diff --git a/yarn.lock b/yarn.lock index 24979eb64692..50389318dcaf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7774,6 +7774,13 @@ pirates@^4.0.1, pirates@^4.0.4, pirates@^4.0.6: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== +pixelmatch@5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a" + integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q== + dependencies: + pngjs "^6.0.0" + pkg-dir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" @@ -7809,6 +7816,16 @@ plist@^3.0.5, plist@^3.1.0: base64-js "^1.5.1" xmlbuilder "^15.1.1" +pngjs@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26" + integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow== + +pngjs@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821" + integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg== + possible-typed-array-names@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" From ad4196c3511b1a8e227451f072ad633c676e48ab Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Fri, 16 Oct 2026 19:18:02 +0100 Subject: [PATCH 4/7] Add storage keys, reports and the approval gate for visual tests Adds the remaining building blocks for the trusted workflow. `storage.js` defines the bucket layout (write-once baselines keyed by environment and commit, reports per PR head) and finds the nearest baseline in a commit's ancestry. `report.js` renders the check run output and the PR comment. `github.js` is a minimal REST client. `approval.js` fingerprints the set of differences, so an approval label only covers the differences it was applied to. Changelog: [Internal] --- .../__tests__/visual-report-test.js | 243 ++++++++++++++++++ .github/workflow-scripts/visual/approval.js | 84 ++++++ .github/workflow-scripts/visual/github.js | 145 +++++++++++ .github/workflow-scripts/visual/report.js | 217 ++++++++++++++++ .github/workflow-scripts/visual/storage.js | 140 ++++++++++ 5 files changed, 829 insertions(+) create mode 100644 .github/workflow-scripts/__tests__/visual-report-test.js create mode 100644 .github/workflow-scripts/visual/approval.js create mode 100644 .github/workflow-scripts/visual/github.js create mode 100644 .github/workflow-scripts/visual/report.js create mode 100644 .github/workflow-scripts/visual/storage.js diff --git a/.github/workflow-scripts/__tests__/visual-report-test.js b/.github/workflow-scripts/__tests__/visual-report-test.js new file mode 100644 index 000000000000..28c2ffc9d4bc --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-report-test.js @@ -0,0 +1,243 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + APPROVAL_LABEL, + differenceFingerprint, + gateDecision, +} = require('../visual/approval'); +const {environmentKey, sha256} = require('../visual/manifest'); +const { + COMMENT_MARKER, + renderCheckOutput, + renderPullRequestComment, +} = require('../visual/report'); +const { + approvalKey, + baselinePrefix, + publicUrl, + reportPrefix, +} = require('../visual/storage'); +const {SHA_A, SHA_B} = require('./__fixtures__/visual'); + +describe('visual reports and approval', () => { + describe('storage keys', () => { + test('builds write-once keys', () => { + expect(baselinePrefix('ios-x-0123abcd', SHA_A)).toBe( + `baselines/ios-x-0123abcd/${SHA_A}`, + ); + expect( + reportPrefix({prNumber: 12, headSha: SHA_A, runId: 34, runAttempt: 2}), + ).toBe(`reports/pull/12/${SHA_A}/34-2`); + expect( + reportPrefix({ + prNumber: null, + headSha: SHA_A, + runId: 34, + runAttempt: 1, + }), + ).toBe(`reports/main/${SHA_A}/34-1`); + expect(approvalKey(12, '0123456789abcdef')).toBe( + 'approvals/pull/12/0123456789abcdef.json', + ); + }); + + test.each([ + [() => baselinePrefix('k', 'main')], + [ + () => + reportPrefix({prNumber: -1, headSha: SHA_A, runId: 1, runAttempt: 1}), + ], + [ + () => + reportPrefix({ + prNumber: 1, + headSha: SHA_A, + runId: 1.5, + runAttempt: 1, + }), + ], + [() => approvalKey(1, '../x')], + ])('rejects invalid input', build => { + expect(build).toThrow('Invalid'); + }); + + test('encodes each path segment of public URLs', () => { + expect(publicUrl('https://cdn.example/bucket/', 'a b/c#d.png')).toBe( + 'https://cdn.example/bucket/a%20b/c%23d.png', + ); + }); + }); + + describe('approval', () => { + const platform = results => ({ + platform: 'ios', + environmentKey: 'ios-x-0123abcd', + baselineSha: SHA_B, + results, + }); + const changed = sha => ({ + id: 'a/b/c', + status: 'changed', + expected: {sha256: 'e'}, + captured: {sha256: sha}, + }); + + test('has no fingerprint without differences', () => { + expect( + differenceFingerprint([platform([{id: 'a/b/c', status: 'unchanged'}])]), + ).toBeNull(); + }); + + test('fingerprints the exact captured images', () => { + const first = differenceFingerprint([platform([changed('1')])]); + expect(first).toMatch(/^[0-9a-f]{16}$/); + expect(differenceFingerprint([platform([changed('1')])])).toBe(first); + expect(differenceFingerprint([platform([changed('2')])])).not.toBe(first); + }); + + test.each([ + [null, false, null, 'success', false], + ['f', false, null, 'failure', false], + ['f', true, {fingerprint: 'f', approver: 'x'}, 'success', false], + ['f', true, {fingerprint: 'g', approver: 'x'}, 'failure', true], + ['f', true, null, 'failure', true], + ])( + 'gate for fingerprint %s, label %s, approval %j', + (fingerprint, labelPresent, approval, conclusion, staleLabel) => { + const decision = gateDecision({fingerprint, labelPresent, approval}); + expect(decision.conclusion).toBe(conclusion); + expect(decision.staleLabel ?? false).toBe(staleLabel); + if (conclusion === 'failure' && !staleLabel) { + expect(decision.summary).toContain(APPROVAL_LABEL); + } + }, + ); + }); + + describe('renderPullRequestComment', () => { + const changed = id => ({ + id, + status: 'changed', + diffPixels: 3, + diffRatio: 0.03, + expected: {}, + captured: {}, + hasDiffImage: true, + }); + const platform = (name, results, error = null) => ({ + platform: name, + environmentKey: `${name}-x-0123abcd`, + baseline: {sha: SHA_B, distance: 0}, + results, + error, + checkUrl: `https://github.example/checks/${name}`, + imageUrl: (kind, id) => `https://cdn.example/${name}/${kind}/${id}.png`, + }); + + test('shows each diff with expected and captured collapsed', () => { + const body = renderPullRequestComment({ + platforms: [ + platform('android', [ + {id: 'a/b/same', status: 'unchanged'}, + changed('a/b/changed'), + ]), + platform('ios', [{id: 'a/b/new', status: 'added', captured: {}}]), + ], + headSha: SHA_A, + gateSummary: 'Needs approval.', + }); + expect(body.startsWith(COMMENT_MARKER)).toBe(true); + expect(body).toContain( + '**2 visual difference(s)** at `aaaaaaaaaaaa`. Needs approval.', + ); + expect(body).toContain( + '### android `a/b/changed`: Changed, 3 px (3.000%)', + ); + expect(body).toContain( + 'diff', + ); + expect(body).toContain( + '
Expected and Captured', + ); + expect(body).toContain('### ios `a/b/new`: New'); + expect(body).toContain('
Captured'); + expect(body).not.toContain('a/b/same'); + }); + + test('lists tests beyond the limit by name', () => { + const results = Array.from({length: 23}, (_, index) => + changed(`a/b/test-${String(index).padStart(2, '0')}`), + ); + const body = renderPullRequestComment({ + platforms: [platform('android', results)], + headSha: SHA_A, + gateSummary: '', + }); + expect(body).toContain('3 more:'); + expect(body).toContain( + '- android `a/b/test-22`: Changed, 3 px (3.000%), [check run](https://github.example/checks/android)', + ); + expect(body).not.toContain('android/diff/a/b/test-20.png'); + }); + + test('reports errors and the absence of differences', () => { + const body = renderPullRequestComment({ + platforms: [ + platform('android', [{id: 'a/b/same', status: 'unchanged'}]), + platform('ios', [], {title: 'no baseline for this environment'}), + ], + headSha: SHA_A, + gateSummary: 'No visual differences.', + }); + expect(body).toContain('No visual differences at `aaaaaaaaaaaa`.'); + expect(body).toContain( + '**ios:** [no baseline for this environment](https://github.example/checks/ios)', + ); + }); + }); + + describe('renderCheckOutput', () => { + test('links expected, captured and diff images for each difference', () => { + const output = renderCheckOutput({ + platform: 'android', + environmentKey: 'android-x-0123abcd', + baseline: {sha: SHA_B, distance: 2}, + results: [ + {id: 'a/b/same', status: 'unchanged'}, + { + id: 'a/b/changed', + status: 'changed', + diffPixels: 3, + diffRatio: 0.03, + expected: {}, + captured: {}, + hasDiffImage: true, + }, + ], + imageUrl: (kind, id) => `https://cdn.example/${kind}/${id}.png`, + error: null, + }); + expect(output.title).toBe('android: 1 visual difference(s)'); + expect(output.summary).toContain('2 commit(s) earlier'); + expect(output.text).toContain( + 'https://cdn.example/expected/a/b/changed.png', + ); + expect(output.text).toContain( + 'https://cdn.example/captured/a/b/changed.png', + ); + expect(output.text).toContain('3 px (3.000%)'); + expect(output.images).toEqual([ + expect.objectContaining({ + image_url: 'https://cdn.example/diff/a/b/changed.png', + }), + ]); + }); + }); +}); diff --git a/.github/workflow-scripts/visual/approval.js b/.github/workflow-scripts/visual/approval.js new file mode 100644 index 000000000000..c9569332f384 --- /dev/null +++ b/.github/workflow-scripts/visual/approval.js @@ -0,0 +1,84 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {sha256} = require('./manifest'); + +const APPROVAL_LABEL = 'visual-change-approved'; +const CHECK_NAME_PREFIX = 'Visual regression'; +const GATE_CHECK_NAME = 'Visual regression approval'; +const APPROVER_PERMISSIONS = ['admin', 'maintain', 'write']; + +/** + * Identifies a set of visual differences: which tests differ, how, and the + * exact expected and captured images. An approval applies to one fingerprint, + * so a push that changes any difference needs a new approval. Returns null + * when there are no differences. + */ +function differenceFingerprint(platforms) { + const lines = []; + for (const {platform, environmentKey, baselineSha, results} of platforms) { + for (const result of results) { + if (result.status === 'unchanged') { + continue; + } + lines.push( + [ + platform, + environmentKey, + baselineSha ?? '', + result.id, + result.status, + result.expected?.sha256 ?? '', + result.captured?.sha256 ?? '', + ].join('\t'), + ); + } + } + if (lines.length === 0) { + return null; + } + return sha256(Buffer.from(lines.sort().join('\n'))).slice(0, 16); +} + +/** + * The approval gate passes when there is nothing to approve, or when the + * approval label is present and an approval was recorded for this exact + * fingerprint. + */ +function gateDecision({fingerprint, labelPresent, approval}) { + if (fingerprint == null) { + return {conclusion: 'success', summary: 'No visual differences.'}; + } + if (labelPresent && approval?.fingerprint === fingerprint) { + return { + conclusion: 'success', + summary: `Visual differences approved by @${approval.approver}.`, + }; + } + if (labelPresent) { + return { + conclusion: 'failure', + summary: `The visual differences changed after approval. A maintainer must review the new report and re-apply \`${APPROVAL_LABEL}\`.`, + staleLabel: true, + }; + } + return { + conclusion: 'failure', + summary: `Visual differences need maintainer approval. Review the report, then apply \`${APPROVAL_LABEL}\`.`, + }; +} + +module.exports = { + APPROVAL_LABEL, + APPROVER_PERMISSIONS, + CHECK_NAME_PREFIX, + GATE_CHECK_NAME, + differenceFingerprint, + gateDecision, +}; diff --git a/.github/workflow-scripts/visual/github.js b/.github/workflow-scripts/visual/github.js new file mode 100644 index 000000000000..9dd1645207f9 --- /dev/null +++ b/.github/workflow-scripts/visual/github.js @@ -0,0 +1,145 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +function createGitHubClient({ + token = process.env.GITHUB_TOKEN, + repository = process.env.GITHUB_REPOSITORY, + apiUrl = process.env.GITHUB_API_URL ?? 'https://api.github.com', +} = {}) { + if (!token || !repository) { + throw new Error('GITHUB_TOKEN and GITHUB_REPOSITORY are required'); + } + + async function request(method, route, body) { + const response = await fetch(`${apiUrl}/repos/${repository}${route}`, { + method, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + ...(body != null ? {'Content-Type': 'application/json'} : null), + }, + body: body != null ? JSON.stringify(body) : undefined, + }); + if (response.status === 404 && method === 'GET') { + return null; + } + if (!response.ok) { + throw new Error( + `${method} ${route} failed with ${response.status}: ${await response.text()}`, + ); + } + return response.status === 204 ? null : response.json(); + } + + return { + repository, + + /** Finds the open PR whose head is `headSha` in `headRepository`. */ + async findPullRequest(headSha, headRepository) { + for (let page = 1; page <= 10; page++) { + const pulls = await request( + 'GET', + `/pulls?state=open&sort=updated&direction=desc&per_page=100&page=${page}`, + ); + const match = pulls.find( + pull => + pull.head.sha === headSha && + pull.head.repo?.full_name === headRepository, + ); + if (match != null || pulls.length < 100) { + return match ?? null; + } + } + return null; + }, + + /** Whether `sha` is an ancestor of, or equal to, `branch`. */ + async isOnBranch(sha, branch) { + const comparison = await request( + 'GET', + `/compare/${sha}...${encodeURIComponent(branch)}`, + ); + return ( + comparison != null && + (comparison.status === 'ahead' || comparison.status === 'identical') + ); + }, + + /** `sha` and up to `count - 1` of its ancestors, newest first. */ + async ancestry(sha, count) { + const commits = await request( + 'GET', + `/commits?sha=${sha}&per_page=${count}`, + ); + return (commits ?? []).map(commit => commit.sha); + }, + + async labels(prNumber) { + const labels = await request( + 'GET', + `/issues/${prNumber}/labels?per_page=100`, + ); + return (labels ?? []).map(label => label.name); + }, + + removeLabel(prNumber, label) { + return request( + 'DELETE', + `/issues/${prNumber}/labels/${encodeURIComponent(label)}`, + ); + }, + + comment(prNumber, body) { + return request('POST', `/issues/${prNumber}/comments`, {body}); + }, + + /** The first comment by `author` on the PR that contains `marker`. */ + async findComment(prNumber, marker, author) { + for (let page = 1; ; page++) { + const comments = await request( + 'GET', + `/issues/${prNumber}/comments?per_page=100&page=${page}`, + ); + const match = (comments ?? []).find( + comment => + comment.user?.login === author && comment.body?.includes(marker), + ); + if (match != null || (comments ?? []).length < 100) { + return match ?? null; + } + } + }, + + updateComment(commentId, body) { + return request('PATCH', `/issues/comments/${commentId}`, {body}); + }, + + async permission(login) { + const result = await request( + 'GET', + `/collaborators/${encodeURIComponent(login)}/permission`, + ); + return result?.permission ?? 'none'; + }, + + createCheckRun({name, headSha, conclusion, output, detailsUrl}) { + return request('POST', '/check-runs', { + name, + head_sha: headSha, + status: 'completed', + conclusion, + output, + details_url: detailsUrl, + }); + }, + }; +} + +module.exports = {createGitHubClient}; diff --git a/.github/workflow-scripts/visual/report.js b/.github/workflow-scripts/visual/report.js new file mode 100644 index 000000000000..7063847c2add --- /dev/null +++ b/.github/workflow-scripts/visual/report.js @@ -0,0 +1,217 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {summarize} = require('./compare'); + +// GitHub limits check run output text to 65535 characters and 50 images. +const MAX_TEXT_LENGTH = 60000; +const MAX_IMAGES = 50; + +const STATUS_LABELS = { + changed: 'Changed', + 'dimension-mismatch': 'Size changed', + added: 'New', + removed: 'Removed', + unchanged: 'Unchanged', +}; + +// GitHub limits comment bodies to 65536 characters. Each changed test costs +// about 1 KB of Markdown, so later tests are listed by name only. +const MAX_COMMENT_TESTS = 20; + +const COMMENT_MARKER = ''; + +function formatPercent(ratio) { + return ratio == null ? '' : `${(ratio * 100).toFixed(3)}%`; +} + +/** How a differing result changed, for example "9633 px (17.502%)". */ +function describeChange(result) { + if (result.status === 'dimension-mismatch') { + return `${result.expectedSize.width}×${result.expectedSize.height} → ${result.capturedSize.width}×${result.capturedSize.height}`; + } + return result.diffPixels != null + ? `${result.diffPixels} px (${formatPercent(result.diffRatio)})` + : ''; +} + +function heading(result) { + const change = describeChange(result); + return `\`${result.id}\`: ${STATUS_LABELS[result.status]}${change ? `, ${change}` : ''}`; +} + +/** + * Renders one platform's comparison as GitHub check run output. `imageUrl` + * maps (kind, test ID) to the public URL of an uploaded expected, captured + * or diff image. + */ +function renderCheckOutput({ + platform, + environmentKey, + baseline, + results, + imageUrl, + error, +}) { + if (error != null) { + return { + title: `${platform}: ${error.title}`, + summary: error.summary, + text: '', + images: [], + }; + } + + const {counts, hasDifferences} = summarize(results); + const differing = results.filter(result => result.status !== 'unchanged'); + const baselineLine = + baseline.distance === 0 + ? `Compared with the baseline for base commit \`${baseline.sha.slice(0, 12)}\`.` + : `No baseline was published for the base commit, so this compares with the nearest ancestor that has one, \`${baseline.sha.slice(0, 12)}\` (${baseline.distance} commit(s) earlier). Differences can come from commits in between.`; + + const summary = [ + hasDifferences + ? `${differing.length} of ${results.length} visual test(s) differ.` + : `All ${results.length} visual test(s) match.`, + '', + baselineLine, + '', + `Environment: \`${environmentKey}\``, + '', + Object.entries(counts) + .filter(([, count]) => count > 0) + .map(([status, count]) => `${STATUS_LABELS[status]}: ${count}`) + .join(' · '), + ].join('\n'); + + const sections = []; + const images = []; + for (const result of differing) { + const row = []; + const cell = kind => ``; + if (result.expected != null) { + row.push(['Expected', cell('expected')]); + } + if (result.captured != null) { + row.push(['Captured', cell('captured')]); + } + if (result.hasDiffImage) { + row.push(['Diff', cell('diff')]); + } + const details = describeChange(result); + sections.push( + [ + `### ${heading(result)}`, + '', + `| ${row.map(([label]) => label).join(' | ')} |`, + `|${row.map(() => '---').join('|')}|`, + `| ${row.map(([, html]) => html).join(' | ')} |`, + ].join('\n'), + ); + if (result.hasDiffImage && images.length < MAX_IMAGES) { + images.push({ + alt: `${result.id} diff`, + image_url: imageUrl('diff', result.id), + caption: `${result.id}: ${details}`, + }); + } + } + + let text = sections.join('\n\n'); + if (text.length > MAX_TEXT_LENGTH) { + text = `${text.slice(0, MAX_TEXT_LENGTH)}\n\n…truncated. See results.json for the full list.`; + } + + return { + title: hasDifferences + ? `${platform}: ${differing.length} visual difference(s)` + : `${platform}: no visual differences`, + summary, + text, + images, + }; +} + +/** + * Renders the pull request comment for all platforms. Each changed test + * shows its diff, with the expected and captured images in a collapsed + * section. `platforms` carry the same fields as `renderCheckOutput` takes, + * plus `checkUrl`, the platform's check run. + */ +function renderPullRequestComment({platforms, headSha, gateSummary}) { + const differing = platforms.flatMap(platform => + platform.error != null + ? [] + : platform.results + .filter(result => result.status !== 'unchanged') + .map(result => ({platform, result})), + ); + const lines = [COMMENT_MARKER, '## Visual regression', '']; + lines.push( + differing.length > 0 + ? `**${differing.length} visual difference(s)** at \`${headSha.slice(0, 12)}\`. ${gateSummary}` + : `No visual differences at \`${headSha.slice(0, 12)}\`.`, + '', + ); + for (const platform of platforms) { + if (platform.error != null) { + lines.push( + `**${platform.platform}:** [${platform.error.title}](${platform.checkUrl})`, + '', + ); + } + } + + for (const [index, {platform, result}] of differing.entries()) { + if (index === MAX_COMMENT_TESTS) { + lines.push( + `${differing.length - index} more:`, + '', + ...differing + .slice(index) + .map( + ({platform: more, result: rest}) => + `- ${more.platform} ${heading(rest)}, [check run](${more.checkUrl})`, + ), + '', + ); + break; + } + const image = (kind, width) => + `${kind}`; + lines.push(`### ${platform.platform} ${heading(result)}`, ''); + if (result.hasDiffImage) { + lines.push(image('diff', 360), ''); + } + const panels = []; + if (result.expected != null) { + panels.push(['Expected', image('expected', 250)]); + } + if (result.captured != null) { + panels.push(['Captured', image('captured', 250)]); + } + if (panels.length > 0) { + lines.push( + `${panels + .map(([label]) => label) + .join(' and ')}`, + '', + `| ${panels.map(([label]) => label).join(' | ')} |`, + `|${panels.map(() => '---').join('|')}|`, + `| ${panels.map(([, html]) => html).join(' | ')} |`, + '', + '
', + '', + ); + } + } + return lines.join('\n'); +} + +module.exports = {COMMENT_MARKER, renderCheckOutput, renderPullRequestComment}; diff --git a/.github/workflow-scripts/visual/storage.js b/.github/workflow-scripts/visual/storage.js new file mode 100644 index 000000000000..e84fd76e5da0 --- /dev/null +++ b/.github/workflow-scripts/visual/storage.js @@ -0,0 +1,140 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {SHA_PATTERN} = require('./manifest'); + +// Object layout. Every key is write-once: reports and approvals include the +// run that produced them, and a baseline is only ever written for a new SHA. +// +// baselines///manifest.json +// baselines///.png +// reports/pull///-/results.json +// reports/pull///-//{expected,captured,diff}/.png +// reports/main//-/... +// approvals/pull//.json + +function assertSha(sha) { + if (!SHA_PATTERN.test(sha)) { + throw new Error(`Invalid SHA: ${String(sha)}`); + } +} + +function assertPositiveInteger(value, name) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`Invalid ${name}: ${String(value)}`); + } +} + +function baselinePrefix(environmentKey, sha) { + assertSha(sha); + return `baselines/${environmentKey}/${sha}`; +} + +function reportPrefix({prNumber, headSha, runId, runAttempt}) { + assertSha(headSha); + assertPositiveInteger(runId, 'run ID'); + assertPositiveInteger(runAttempt, 'run attempt'); + if (prNumber == null) { + return `reports/main/${headSha}/${runId}-${runAttempt}`; + } + assertPositiveInteger(prNumber, 'PR number'); + return `reports/pull/${prNumber}/${headSha}/${runId}-${runAttempt}`; +} + +function approvalKey(prNumber, fingerprint) { + assertPositiveInteger(prNumber, 'PR number'); + if (!/^[0-9a-f]{16}$/.test(fingerprint)) { + throw new Error(`Invalid fingerprint: ${String(fingerprint)}`); + } + return `approvals/pull/${prNumber}/${fingerprint}.json`; +} + +function publicUrl(publicBaseUrl, key) { + return `${publicBaseUrl.replace(/\/$/, '')}/${key + .split('/') + .map(encodeURIComponent) + .join('/')}`; +} + +async function fetchOrNull(url) { + const response = await fetch(url, {cache: 'no-store'}); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`GET ${url} failed with ${response.status}`); + } + return response; +} + +async function fetchJsonOrNull(url) { + const response = await fetchOrNull(url); + return response == null ? null : response.json(); +} + +async function fetchBuffer(url) { + const response = await fetchOrNull(url); + if (response == null) { + throw new Error(`GET ${url} returned 404`); + } + return Buffer.from(await response.arrayBuffer()); +} + +/** + * Lists the immediate "subdirectories" under a prefix, using the public GCS + * JSON API. + */ +async function listPrefixes(bucket, prefix) { + const prefixes = []; + let pageToken = null; + do { + const url = new URL( + `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(bucket)}/o`, + ); + url.searchParams.set('prefix', prefix); + url.searchParams.set('delimiter', '/'); + if (pageToken != null) { + url.searchParams.set('pageToken', pageToken); + } + const page = await fetchJsonOrNull(url.toString()); + prefixes.push(...(page?.prefixes ?? [])); + pageToken = page?.nextPageToken ?? null; + } while (pageToken != null); + return prefixes; +} + +/** + * Returns the first of `candidateShas` (newest first) that has a published + * baseline for `environmentKey`, with its manifest, or null. + */ +async function findBaseline({publicBaseUrl, environmentKey, candidateShas}) { + for (const [distance, sha] of candidateShas.entries()) { + const manifest = await fetchJsonOrNull( + publicUrl( + publicBaseUrl, + `${baselinePrefix(environmentKey, sha)}/manifest.json`, + ), + ); + if (manifest != null) { + return {sha, distance, manifest}; + } + } + return null; +} + +module.exports = { + approvalKey, + baselinePrefix, + fetchBuffer, + fetchJsonOrNull, + findBaseline, + listPrefixes, + publicUrl, + reportPrefix, +}; From 8cf5c813ad28f433c8fe5820cfd8a133a8ab0f73 Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Fri, 16 Oct 2026 19:18:04 +0100 Subject: [PATCH 5/7] Compare visual captures in a trusted workflow Adds Visual Compare RNTester, which runs from the default branch on `workflow_run` when Test All completes, including when unrelated jobs fail. It downloads each platform's capture, validates it, compares it with the baseline for the PR's base commit and uploads the report through GitHub OIDC, so no secret is stored. It posts a check run per platform, the approval gate check and a single PR comment. On pushes to `main`, it publishes the captures as a new baseline. The bucket and identity provider are configured through the `VISUAL_*` repository variables. Changelog: [Internal] --- .../workflow-scripts/visual/compare-run.js | 465 ++++++++++++++++++ .github/workflows/visual-compare-rntester.yml | 135 +++++ 2 files changed, 600 insertions(+) create mode 100644 .github/workflow-scripts/visual/compare-run.js create mode 100644 .github/workflows/visual-compare-rntester.yml diff --git a/.github/workflow-scripts/visual/compare-run.js b/.github/workflow-scripts/visual/compare-run.js new file mode 100644 index 000000000000..5027ff729538 --- /dev/null +++ b/.github/workflow-scripts/visual/compare-run.js @@ -0,0 +1,465 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Trusted half of the visual regression pipeline. Runs from the default + * branch on `workflow_run`, after the untrusted capture workflow finishes. + * + * node compare-run.js prepare Validate captures, compare with baselines, + * and lay out files to upload under OUT_DIR. + * node compare-run.js report After the upload, create check runs and + * evaluate the approval gate. + */ + +const { + APPROVAL_LABEL, + CHECK_NAME_PREFIX, + GATE_CHECK_NAME, + differenceFingerprint, + gateDecision, +} = require('./approval'); +const {compareCapture, summarize} = require('./compare'); +const {createGitHubClient} = require('./github'); +const { + COMMENT_MARKER, + renderCheckOutput, + renderPullRequestComment, +} = require('./report'); +const { + approvalKey, + baselinePrefix, + fetchBuffer, + fetchJsonOrNull, + findBaseline, + publicUrl, + reportPrefix, +} = require('./storage'); +const {validateCapture} = require('./validate'); +const fs = require('fs'); +const path = require('path'); + +const BASELINE_SEARCH_DEPTH = 20; +const PLATFORMS = ['android', 'ios']; +// The default Actions token posts as this bot. Only its comments are edited. +const COMMENT_AUTHOR = 'github-actions[bot]'; + +function requireEnv(name) { + const value = process.env[name]; + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + return value; +} + +function writeFile(filePath, contents) { + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + fs.writeFileSync(filePath, contents); +} + +async function resolveContext(github) { + const event = requireEnv('CAPTURE_EVENT'); + const headSha = requireEnv('CAPTURE_HEAD_SHA'); + const headBranch = requireEnv('CAPTURE_HEAD_BRANCH'); + const headRepository = requireEnv('CAPTURE_HEAD_REPOSITORY'); + const defaultBranch = requireEnv('DEFAULT_BRANCH'); + const runId = Number(requireEnv('CAPTURE_RUN_ID')); + const runAttempt = Number(requireEnv('CAPTURE_RUN_ATTEMPT')); + + if (event === 'pull_request') { + const pull = await github.findPullRequest(headSha, headRepository); + if (pull == null) { + return null; + } + return { + kind: 'pull', + prNumber: pull.number, + headSha, + runId, + runAttempt, + defaultBranch, + }; + } + if ( + event === 'push' && + headBranch === defaultBranch && + headRepository === github.repository + ) { + return { + kind: 'main', + prNumber: null, + headSha, + runId, + runAttempt, + defaultBranch, + }; + } + return null; +} + +async function comparePlatform({ + github, + context, + captureDir, + config, + publicBaseUrl, + outDir, + prefix, +}) { + const manifest = validateCapture(captureDir); + const {platform, environmentKey, git} = manifest; + if (git.headSha !== context.headSha) { + throw new Error( + `${platform}: manifest head ${git.headSha} does not match run head ${context.headSha}`, + ); + } + if (!(await github.isOnBranch(git.baseSha, context.defaultBranch))) { + return { + platform, + environmentKey, + manifest, + baseline: null, + results: [], + error: { + title: 'base commit is not on the default branch', + summary: `The capture's base commit \`${git.baseSha}\` is not on \`${context.defaultBranch}\`, so there is no baseline to compare with. Rebase or merge \`${context.defaultBranch}\` and push again.`, + }, + }; + } + + const candidateShas = await github.ancestry( + git.baseSha, + BASELINE_SEARCH_DEPTH, + ); + const baseline = await findBaseline({ + publicBaseUrl, + environmentKey, + candidateShas, + }); + if (baseline == null) { + return { + platform, + environmentKey, + manifest, + baseline: null, + results: [], + error: { + title: 'no baseline for this environment', + summary: `No baseline for environment \`${environmentKey}\` was published for \`${git.baseSha.slice(0, 12)}\` or its ${BASELINE_SEARCH_DEPTH - 1} preceding commits on \`${context.defaultBranch}\`. If the environment key is new, the CI device image changed: the next push to \`${context.defaultBranch}\` publishes a baseline for it. Otherwise check that baseline publication on \`${context.defaultBranch}\` is succeeding.`, + }, + }; + } + + const expectedUrl = id => + publicUrl( + publicBaseUrl, + `${baselinePrefix(environmentKey, baseline.sha)}/${id}.png`, + ); + const results = await compareCapture({ + manifest, + baselineManifest: baseline.manifest, + readExpected: id => fetchBuffer(expectedUrl(id)), + readCaptured: id => + fs.promises.readFile(path.join(captureDir, `${id}.png`)), + config, + }); + + for (const result of results) { + if (result.status === 'unchanged') { + continue; + } + const platformPrefix = `${prefix}/${platform}`; + if (result.captured != null) { + writeFile( + path.join( + outDir, + 'upload', + platformPrefix, + 'captured', + `${result.id}.png`, + ), + fs.readFileSync(path.join(captureDir, `${result.id}.png`)), + ); + } + if (result.diffImage != null) { + writeFile( + path.join(outDir, 'upload', platformPrefix, 'diff', `${result.id}.png`), + result.diffImage, + ); + } + } + + return { + platform, + environmentKey, + manifest, + baseline: {sha: baseline.sha, distance: baseline.distance}, + results, + error: null, + }; +} + +function missingCapture(platform, context) { + return { + platform, + environmentKey: null, + manifest: null, + baseline: null, + results: [], + error: { + title: 'no capture', + summary: `Test All run ${context.runId} has no ${platform} capture, so there is nothing to compare. Its capture job failed or didn't run: check the run for a build or capture failure.`, + }, + }; +} + +function stageBaseline({outDir, captureDir, manifest, sha}) { + const prefix = baselinePrefix(manifest.environmentKey, sha); + for (const image of manifest.images) { + writeFile( + path.join(outDir, 'baseline-images', prefix, `${image.id}.png`), + fs.readFileSync(path.join(captureDir, `${image.id}.png`)), + ); + } + // Uploaded after the images: a baseline exists once its manifest does. + writeFile( + path.join(outDir, 'baseline-manifests', prefix, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + ); +} + +function serializableResults(results) { + return results.map(({diffImage, ...result}) => ({ + ...result, + hasDiffImage: diffImage != null, + })); +} + +async function prepare() { + const github = createGitHubClient(); + const artifactsDir = requireEnv('VISUAL_ARTIFACTS_DIR'); + const outDir = requireEnv('OUT_DIR'); + const publicBaseUrl = requireEnv('VISUAL_PUBLIC_BASE_URL'); + const config = JSON.parse( + fs.readFileSync(requireEnv('VISUAL_CONFIG'), 'utf8'), + ); + + const context = await resolveContext(github); + if (context == null) { + console.info('Not an open pull request or a push to the default branch.'); + writeFile(path.join(outDir, 'state.json'), 'null\n'); + return; + } + const prefix = reportPrefix(context); + + const platforms = []; + for (const expectedPlatform of PLATFORMS) { + const captureDir = path.join( + artifactsDir, + `visual-capture-${expectedPlatform}`, + ); + if (!fs.existsSync(captureDir)) { + platforms.push(missingCapture(expectedPlatform, context)); + continue; + } + const result = await comparePlatform({ + github, + context, + captureDir, + config, + publicBaseUrl, + outDir, + prefix, + }); + if (result.platform !== expectedPlatform) { + throw new Error( + `visual-capture-${expectedPlatform}: manifest is for ${result.platform}`, + ); + } + if (context.kind === 'main') { + stageBaseline({ + outDir, + captureDir, + manifest: result.manifest, + sha: context.headSha, + }); + } + platforms.push(result); + } + + const fingerprint = differenceFingerprint( + platforms.map(platform => ({ + platform: platform.platform, + environmentKey: platform.environmentKey, + baselineSha: platform.baseline?.sha, + results: platform.results, + })), + ); + const state = { + context, + reportPrefix: prefix, + fingerprint, + platforms: platforms.map(({manifest, results, ...platform}) => ({ + ...platform, + git: manifest?.git ?? null, + results: serializableResults(results), + })), + }; + writeFile( + path.join(outDir, 'upload', prefix, 'results.json'), + `${JSON.stringify(state, null, 2)}\n`, + ); + writeFile(path.join(outDir, 'state.json'), `${JSON.stringify(state)}\n`); + + for (const platform of state.platforms) { + const {counts} = summarize(platform.results); + console.info( + `${platform.platform} (${platform.environmentKey}): ${ + platform.error?.title ?? JSON.stringify(counts) + }`, + ); + } +} + +async function report() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const publicBaseUrl = requireEnv('VISUAL_PUBLIC_BASE_URL'); + const state = JSON.parse( + fs.readFileSync(path.join(outDir, 'state.json'), 'utf8'), + ); + if (state == null) { + return; + } + const {context, reportPrefix: prefix, fingerprint} = state; + const detailsUrl = publicUrl(publicBaseUrl, `${prefix}/results.json`); + + const summaryLines = []; + const commentPlatforms = []; + for (const platform of state.platforms) { + const imageUrl = (kind, id) => + kind === 'expected' + ? publicUrl( + publicBaseUrl, + `${baselinePrefix(platform.environmentKey, platform.baseline.sha)}/${id}.png`, + ) + : publicUrl( + publicBaseUrl, + `${prefix}/${platform.platform}/${kind}/${id}.png`, + ); + const output = renderCheckOutput({...platform, imageUrl}); + const {hasDifferences} = summarize(platform.results); + const checkRun = await github.createCheckRun({ + name: `${CHECK_NAME_PREFIX} (${platform.platform})`, + headSha: context.headSha, + conclusion: + platform.error != null + ? 'neutral' + : hasDifferences + ? 'failure' + : 'success', + output, + detailsUrl, + }); + commentPlatforms.push({...platform, imageUrl, checkUrl: checkRun.html_url}); + summaryLines.push( + `## ${output.title}`, + '', + output.summary, + '', + output.text, + '', + ); + } + + if (context.kind === 'pull') { + const labels = await github.labels(context.prNumber); + const approval = + fingerprint == null + ? null + : await fetchJsonOrNull( + publicUrl( + publicBaseUrl, + approvalKey(context.prNumber, fingerprint), + ), + ); + const decision = gateDecision({ + fingerprint, + labelPresent: labels.includes(APPROVAL_LABEL), + approval, + }); + if (decision.staleLabel) { + await github.removeLabel(context.prNumber, APPROVAL_LABEL); + await github.comment( + context.prNumber, + `The visual differences changed in \`${context.headSha.slice(0, 12)}\`, so \`${APPROVAL_LABEL}\` was removed. Review the new report and apply it again to approve.`, + ); + } + await github.createCheckRun({ + name: GATE_CHECK_NAME, + headSha: context.headSha, + conclusion: decision.conclusion, + output: {title: decision.summary, summary: decision.summary}, + detailsUrl, + }); + summaryLines.push(`## Approval`, '', decision.summary, ''); + await upsertComment({ + github, + prNumber: context.prNumber, + body: renderPullRequestComment({ + platforms: commentPlatforms, + headSha: context.headSha, + gateSummary: decision.summary, + }), + hasContent: + fingerprint != null || + commentPlatforms.some(platform => platform.error != null), + }); + } + + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryLines.join('\n')); + } +} + +/** + * Keeps one report comment per PR up to date. A PR that never had + * differences gets no comment. Once it has one, later runs edit it, including + * to say the differences are gone. + */ +async function upsertComment({github, prNumber, body, hasContent}) { + const existing = await github.findComment( + prNumber, + COMMENT_MARKER, + COMMENT_AUTHOR, + ); + if (existing != null) { + await github.updateComment(existing.id, body); + } else if (hasContent) { + await github.comment(prNumber, body); + } +} + +async function main(command = process.argv[2]) { + if (command === 'prepare') { + await prepare(); + } else if (command === 'report') { + await report(); + } else { + throw new Error('Usage: node compare-run.js prepare|report'); + } +} + +if (require.main === module) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = {resolveContext}; diff --git a/.github/workflows/visual-compare-rntester.yml b/.github/workflows/visual-compare-rntester.yml new file mode 100644 index 000000000000..f17981de34f5 --- /dev/null +++ b/.github/workflows/visual-compare-rntester.yml @@ -0,0 +1,135 @@ +# Trusted half of RNTester visual regression testing. Runs from the default +# branch after Test All completes, including for pull requests from forks, and +# compares whatever visual-capture-rntester.yml captured in that run. The +# capture artifacts are treated as inert data: they are validated before any +# image is decoded, and nothing in them is executed. +# +# Pull requests: compare with the baseline for the base commit (or its +# nearest ancestor with one), upload a report, and set check runs. +# Pushes to the default branch: as above, then publish the captures as the +# immutable baseline for that commit. +name: Visual Compare RNTester + +on: + workflow_run: + workflows: [Test All] + types: [completed] + +permissions: + contents: read + +concurrency: + # Serializes baseline publication on the default branch. + group: ${{ github.workflow }}-${{ github.event.workflow_run.event == 'push' && 'main' || github.event.workflow_run.id }} + cancel-in-progress: false + +env: + OUT_DIR: /tmp/visual-out + VISUAL_ARTIFACTS_DIR: /tmp/visual-artifacts + VISUAL_BUCKET: ${{ vars.VISUAL_BUCKET }} + VISUAL_PUBLIC_BASE_URL: ${{ vars.VISUAL_PUBLIC_BASE_URL }} + VISUAL_CONFIG: packages/rn-tester/visual-tests/config.json + +jobs: + find_captures: + # Test All can fail for reasons unrelated to rendering, so a failed run is + # still compared. A platform with no capture is reported as such. + if: contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-latest + permissions: + actions: read + outputs: + found: ${{ steps.find.outputs.found }} + captures: ${{ steps.find.outputs.captures }} + steps: + - name: Find captures + id: find + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + CAPTURES=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" \ + --jq '.artifacts[] | select(.name | startswith("visual-capture-")) | .name' | tr '\n' ' ') + echo "Captures: ${CAPTURES:-none}" + echo "found=$([[ -n "$CAPTURES" ]] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "captures=$CAPTURES" >> "$GITHUB_OUTPUT" + + compare: + needs: find_captures + if: needs.find_captures.outputs.found == 'true' + runs-on: ubuntu-latest + permissions: + actions: read + checks: write + contents: read + id-token: write + issues: write + pull-requests: write + steps: + - name: Checkout default branch + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Install node dependencies + uses: ./.github/actions/yarn-install + # Downloaded by name: a pattern that matches one artifact extracts it + # without the per-artifact directory that compare-run.js reads. + - name: Download Android capture + if: contains(needs.find_captures.outputs.captures, 'visual-capture-android') + uses: actions/download-artifact@v7 + with: + name: visual-capture-android + path: ${{ env.VISUAL_ARTIFACTS_DIR }}/visual-capture-android + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Download iOS capture + if: contains(needs.find_captures.outputs.captures, 'visual-capture-ios') + uses: actions/download-artifact@v7 + with: + name: visual-capture-ios + path: ${{ env.VISUAL_ARTIFACTS_DIR }}/visual-capture-ios + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Compare + env: + GITHUB_TOKEN: ${{ github.token }} + CAPTURE_EVENT: ${{ github.event.workflow_run.event }} + CAPTURE_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + CAPTURE_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + CAPTURE_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} + CAPTURE_RUN_ID: ${{ github.event.workflow_run.id }} + CAPTURE_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node .github/workflow-scripts/visual/compare-run.js prepare + - name: Check for files to upload + id: staged + run: | + if [[ -d "$OUT_DIR/upload" ]]; then + echo "upload=true" >> "$GITHUB_OUTPUT" + fi + - name: Authenticate to Google Cloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Upload report and baseline + if: steps.staged.outputs.upload == 'true' + run: | + # Objects are write-once; --no-clobber makes a rerun skip what an + # earlier attempt already uploaded. + gcloud storage cp --recursive --no-clobber "$OUT_DIR/upload/*" "gs://$VISUAL_BUCKET/" + if [[ -d "$OUT_DIR/baseline-images" ]]; then + gcloud storage cp --recursive --no-clobber "$OUT_DIR/baseline-images/*" "gs://$VISUAL_BUCKET/" + # A baseline exists once its manifest does, so the manifest goes last. + gcloud storage cp --recursive --no-clobber "$OUT_DIR/baseline-manifests/*" "gs://$VISUAL_BUCKET/" + fi + - name: Report + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/compare-run.js report From 34fb6d08f313629032964d8b7bfb612622786976 Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Fri, 16 Oct 2026 19:18:05 +0100 Subject: [PATCH 6/7] Approve visual changes with a label Adds Visual Approve RNTester, which runs on `pull_request_target` when a maintainer applies `visual-change-approved`. It records an approval for the current fingerprint of differences and re-evaluates the gate check. A later push with different differences removes the label and needs a new review. Changelog: [Internal] --- .../workflow-scripts/visual/approve-run.js | 182 ++++++++++++++++++ .github/workflows/visual-approve-rntester.yml | 67 +++++++ 2 files changed, 249 insertions(+) create mode 100644 .github/workflow-scripts/visual/approve-run.js create mode 100644 .github/workflows/visual-approve-rntester.yml diff --git a/.github/workflow-scripts/visual/approve-run.js b/.github/workflow-scripts/visual/approve-run.js new file mode 100644 index 000000000000..81a80f18b092 --- /dev/null +++ b/.github/workflow-scripts/visual/approve-run.js @@ -0,0 +1,182 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Records a maintainer's approval of a PR's visual differences and updates + * the approval gate. Runs from the default branch on `pull_request_target` + * when the approval label is added or removed. It never checks out or runs + * pull request code. + * + * node approve-run.js prepare Decide, and stage an approval record to + * upload under OUT_DIR. + * node approve-run.js report After the upload, update the gate check. + */ + +const { + APPROVAL_LABEL, + APPROVER_PERMISSIONS, + GATE_CHECK_NAME, + gateDecision, +} = require('./approval'); +const {createGitHubClient} = require('./github'); +const { + approvalKey, + fetchJsonOrNull, + listPrefixes, + publicUrl, +} = require('./storage'); +const fs = require('fs'); +const path = require('path'); + +function requireEnv(name) { + const value = process.env[name]; + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + return value; +} + +/** The results of the most recent comparison for a PR head, or null. */ +async function latestResults({bucket, publicBaseUrl, prNumber, headSha}) { + const runs = await listPrefixes( + bucket, + `reports/pull/${prNumber}/${headSha}/`, + ); + const latest = runs + .map(prefix => { + const match = /\/(\d+)-(\d+)\/$/.exec(prefix); + return match == null + ? null + : {prefix, runId: Number(match[1]), attempt: Number(match[2])}; + }) + .filter(Boolean) + .sort((a, b) => b.runId - a.runId || b.attempt - a.attempt)[0]; + return latest == null + ? null + : fetchJsonOrNull(publicUrl(publicBaseUrl, `${latest.prefix}results.json`)); +} + +async function prepare() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const event = JSON.parse( + fs.readFileSync(requireEnv('GITHUB_EVENT_PATH'), 'utf8'), + ); + const writeState = state => + fs.writeFileSync( + path.join(outDir, 'state.json'), + `${JSON.stringify(state)}\n`, + ); + fs.mkdirSync(outDir, {recursive: true}); + + if (event.label?.name !== APPROVAL_LABEL) { + writeState(null); + return; + } + + const prNumber = event.pull_request.number; + const headSha = event.pull_request.head.sha; + const results = await latestResults({ + bucket: requireEnv('VISUAL_BUCKET'), + publicBaseUrl: requireEnv('VISUAL_PUBLIC_BASE_URL'), + prNumber, + headSha, + }); + const fingerprint = results?.fingerprint ?? null; + + if (event.action === 'labeled') { + const approver = event.sender.login; + const permission = await github.permission(approver); + if (!APPROVER_PERMISSIONS.includes(permission)) { + await github.removeLabel(prNumber, APPROVAL_LABEL); + await github.comment( + prNumber, + `@${approver} \`${APPROVAL_LABEL}\` needs write access to this repository, so it was removed.`, + ); + writeState(null); + return; + } + if (results == null) { + await github.removeLabel(prNumber, APPROVAL_LABEL); + await github.comment( + prNumber, + `There is no visual report for \`${headSha.slice(0, 12)}\` yet, so \`${APPROVAL_LABEL}\` was removed. Apply it again once the report is posted.`, + ); + writeState(null); + return; + } + if (fingerprint != null) { + const key = approvalKey(prNumber, fingerprint); + const record = { + fingerprint, + approver, + approvedAt: new Date().toISOString(), + headSha, + }; + const recordPath = path.join(outDir, 'upload', key); + fs.mkdirSync(path.dirname(recordPath), {recursive: true}); + fs.writeFileSync(recordPath, `${JSON.stringify(record, null, 2)}\n`); + } + } + + writeState({ + prNumber, + headSha, + fingerprint, + labelPresent: event.action === 'labeled', + }); +} + +async function report() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const state = JSON.parse( + fs.readFileSync(path.join(outDir, 'state.json'), 'utf8'), + ); + if (state == null) { + return; + } + const {prNumber, headSha, fingerprint, labelPresent} = state; + const approval = + fingerprint == null + ? null + : await fetchJsonOrNull( + publicUrl( + requireEnv('VISUAL_PUBLIC_BASE_URL'), + approvalKey(prNumber, fingerprint), + ), + ); + const decision = gateDecision({fingerprint, labelPresent, approval}); + await github.createCheckRun({ + name: GATE_CHECK_NAME, + headSha, + conclusion: decision.conclusion, + output: {title: decision.summary, summary: decision.summary}, + }); + console.info(decision.summary); +} + +async function main(command = process.argv[2]) { + if (command === 'prepare') { + await prepare(); + } else if (command === 'report') { + await report(); + } else { + throw new Error('Usage: node approve-run.js prepare|report'); + } +} + +if (require.main === module) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = {latestResults}; diff --git a/.github/workflows/visual-approve-rntester.yml b/.github/workflows/visual-approve-rntester.yml new file mode 100644 index 000000000000..9c6f05f7e1e6 --- /dev/null +++ b/.github/workflows/visual-approve-rntester.yml @@ -0,0 +1,67 @@ +# Records a maintainer's approval of a pull request's visual differences when +# the visual-change-approved label is added, and updates the approval gate +# when it is added or removed. Runs from the default branch on +# pull_request_target and never checks out pull request code. +name: Visual Approve RNTester + +on: + pull_request_target: + types: [labeled, unlabeled] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + +env: + OUT_DIR: /tmp/visual-approval + VISUAL_BUCKET: ${{ vars.VISUAL_BUCKET }} + VISUAL_PUBLIC_BASE_URL: ${{ vars.VISUAL_PUBLIC_BASE_URL }} + +jobs: + approve: + if: github.event.label.name == 'visual-change-approved' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + id-token: write + issues: write + pull-requests: write + steps: + - name: Checkout default branch + uses: actions/checkout@v6 + with: + # Never the pull request: this workflow has write permissions. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Decide + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/approve-run.js prepare + - name: Check for an approval to record + id: staged + run: | + if [[ -d "$OUT_DIR/upload" ]]; then + echo "upload=true" >> "$GITHUB_OUTPUT" + fi + - name: Authenticate to Google Cloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Record approval + if: steps.staged.outputs.upload == 'true' + run: gcloud storage cp --recursive --no-clobber "$OUT_DIR/upload/*" "gs://$VISUAL_BUCKET/" + - name: Update gate + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/approve-run.js report From 856441771890c151556176b1a719b53a212ce42a Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Fri, 16 Oct 2026 19:18:06 +0100 Subject: [PATCH 7/7] Document RNTester visual tests Adds `packages/rn-tester/visual-tests/README.md` on how the tests run in CI, adding a test, running flows locally, handling a failed check and tolerances, and links it from RNTester's README. Changelog: [Internal] --- packages/rn-tester/README.md | 4 ++ packages/rn-tester/visual-tests/README.md | 72 +++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 packages/rn-tester/visual-tests/README.md diff --git a/packages/rn-tester/README.md b/packages/rn-tester/README.md index b0b3cdb367e7..47c6cc4508d1 100644 --- a/packages/rn-tester/README.md +++ b/packages/rn-tester/README.md @@ -67,3 +67,7 @@ This is different from apps created using `react-native init` which have a depen ## Running tests (iOS) RNTester provides both integration and unit tests. You can run them all by hitting `Cmd+U` in Xcode. Alternatively, it is possible to run just unit tests or integration tests using the keybinding after selecting the appropriate scheme (either `RNTesterUnitTests` or `RNTesterIntegrationTests`). + +## Visual tests + +CI captures screenshots of selected RNTester examples on Android and iOS and compares them with `main`. See [visual-tests/README.md](visual-tests/README.md) to add a test or review a failure. diff --git a/packages/rn-tester/visual-tests/README.md b/packages/rn-tester/visual-tests/README.md new file mode 100644 index 000000000000..bc068b744e52 --- /dev/null +++ b/packages/rn-tester/visual-tests/README.md @@ -0,0 +1,72 @@ +# RNTester visual tests + +Visual tests capture screenshots of RNTester examples in CI and compare them with the screenshots of the same tests on `main`. They catch rendering changes, particularly in text layout, that functional E2E tests don't. + +Each test is a [Maestro](https://maestro.dev) flow in `flows/`. Maestro only opens the example and takes the screenshot - comparison happens afterwards in CI, so a flow never decides whether it passed. + +## How it runs in CI + +1. Test All builds RNTester Release for Android and iOS. Once a build is ready, the `test_visual_android_rntester` and `test_visual_ios_rntester` jobs (`.github/workflows/visual-capture-rntester.yml`) run every flow in this directory on one canonical device per platform: an API 35 Pixel 7 emulator and an iPhone 17 Pro simulator on iOS 26.2. They upload the screenshots and a manifest of the device environment as `visual-capture-android` and `visual-capture-ios` artifacts. +2. When Test All completes, Visual Compare RNTester (`.github/workflows/visual-compare-rntester.yml`) compares the screenshots with the baseline and reports a `Visual regression (android)` and `Visual regression (ios)` check and a `Visual regression approval` gate, plus a single PR comment with the expected, captured and diff images for anything that changed. +3. On pushes to `main`, the captures become the baseline for that commit. + +The baseline for a PR is the one for the commit it's based on, or the nearest earlier commit on `main` that has one. Baselines are keyed by device environment too (OS image, screen size, density, font scale, locale and so on), so a change to the CI device starts a fresh baseline rather than failing every PR. + +## Adding a test + +1. Pick or add an RNTester example, and give its root view a `testID`. Keep the example's content fixed: no dates, random values, network images or animations that are still running when the screenshot is taken. +2. Add a flow to `flows/`. File names must be unique, and each `takeScreenshot` path becomes the test ID, so make it descriptive: + + ```yaml + appId: ${APP_ID} + --- + - runFlow: + file: ../helpers/open-example.yml + env: + EXAMPLE_URL: rntester://example/TextExample/textWidthMode + - extendedWaitUntil: + visible: + id: 'text-width-mode-example' + timeout: 15000 + - takeScreenshot: + path: ${VISUAL_OUTPUT_DIR}/text/width-mode/default + cropOn: + id: 'text-width-mode-example' + ``` + + `open-example.yml` deep-links to `rntester://example//`. Wait for something that's only visible once the example has rendered, and crop to the example's `testID` so that the status bar and surrounding UI aren't part of the screenshot. + +3. Open a PR. The new test has no baseline yet, so the report lists it as new, and like any other difference it needs a maintainer's approval (see below). Once the PR lands, `main` is captured with it and later PRs are compared against that. + +To try a flow locally, build and install RNTester Release on a simulator or emulator and run it with Maestro: + +```sh +maestro test \ + -e APP_ID=com.meta.RNTester.localDevelopment \ + -e VISUAL_OUTPUT_DIR=/tmp/visual \ + packages/rn-tester/visual-tests/flows/text-width-mode.yml +``` + +Use `com.facebook.react.uiapp` as the `APP_ID` on Android. Local screenshots are only useful for checking the flow and the crop - they won't match CI's pixel for pixel unless you use the same device and OS image. + +## When a check fails + +Open the PR comment or the check run and compare the expected, captured and diff images. + +- If the change is unintended, fix it and push. +- If it's intended, a maintainer applies the `visual-change-approved` label. The approval covers the exact set of differences in the report: if a later push changes them, the label is removed and the PR needs a new review. The baseline itself only changes once the PR lands and `main` is captured again. + +## Tolerances + +Comparison is strict by default: any changed pixel is a difference. If a test is genuinely nondeterministic, loosen it for that test in `config.json`, keyed by test ID, and say why in the PR: + +```json +{ + "defaults": {"threshold": 0, "includeAA": true, "maxDiffPixels": 0}, + "tests": { + "text/width-mode/default": {"maxDiffPixels": 20} + } +} +``` + +`threshold` is pixelmatch's per-pixel color distance from 0 to 1, `includeAA` counts anti-aliased pixels, and `maxDiffPixels` and `maxDiffRatio` set how many changed pixels (or what share of the image) can pass. Prefer making the example deterministic over adding a tolerance.