diff --git a/apps/desktop/build/entitlements.mac.plist b/apps/desktop/build/entitlements.mac.plist index 51ec6a97a57..edf7bcaf4b9 100644 --- a/apps/desktop/build/entitlements.mac.plist +++ b/apps/desktop/build/entitlements.mac.plist @@ -14,10 +14,5 @@ Hardened Runtime denies the camera no matter what the user allowed. --> com.apple.security.device.camera - - com.apple.security.device.bluetooth - diff --git a/apps/desktop/e2e/browser-sign-in-prompts.spec.ts b/apps/desktop/e2e/browser-sign-in-prompts.spec.ts new file mode 100644 index 00000000000..d52d13cd97d --- /dev/null +++ b/apps/desktop/e2e/browser-sign-in-prompts.spec.ts @@ -0,0 +1,560 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { type ElectronApplication, _electron as electron, expect, test } from '@playwright/test' +import type { + BrowserAppLaunchNotice, + BrowserPagePrompt, + BrowserToolName, +} from '@sim/browser-protocol' +import type { SimDesktopApi } from '@sim/desktop-bridge' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { build } from 'esbuild' +import postcss from 'postcss' +import loadPostcssConfig from 'postcss-load-config' + +const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) +const SCOPE = 'browser-sign-in-prompts-e2e' +const SIM_DIR = fileURLToPath(new URL('../../sim/', import.meta.url)) +const SHELL_FIXTURE = + 'Sim fixture

Browser sign-in fixture

' +const CREDENTIALS = `Basic ${Buffer.from('alice:s3cret').toString('base64')}` +/** Long enough for a refused request to have surfaced if it were going to. */ +const QUIET_MS = 1_500 + +type Bridge = typeof globalThis & { + simDesktop: SimDesktopApi + pagePrompt?: BrowserPagePrompt | null + launchNotice?: BrowserAppLaunchNotice | null + boundsTimer?: number +} + +type MainGlobals = typeof globalThis & { openedExternal?: string[] } + +/** + * A browser tab's sign-in challenges and links to other apps belong to the + * user: Sim shows the question for the tab that asked, only a click in Sim + * answers it, and nothing is asked of an older renderer that cannot show it. + * These checks drive the real shell, native tab views, and Sim's prompt UI. + */ +test('sign-in and app-launch prompts wait for the user on the page that asked', async () => { + test.setTimeout(180_000) + const reportPath = + process.env.DESKTOP_BROWSER_SIGNIN_REPORT_PATH ?? + test.info().outputPath('browser-sign-in-prompts.json') + const checks: { + name: string + status: 'passed' | 'failed' + durationMs: number + error?: string + }[] = [] + const check = async (name: string, run: () => Promise) => { + const started = Date.now() + try { + await test.step(name, run) + checks.push({ name, status: 'passed', durationMs: Date.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: Date.now() - started, + error: getErrorMessage(error), + }) + throw error + } + } + const browserComponents = join( + SIM_DIR, + 'app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session' + ) + const bundle = await build({ + stdin: { + contents: `import { createElement } from 'react'; +import { createRoot } from 'react-dom/client'; +import { useBrowserSessionStore as store } from ${JSON.stringify(join(SIM_DIR, 'stores/browser-session/store.ts'))}; +import { BrowserAppLaunchNoticeBar, BrowserPagePromptModal } from ${JSON.stringify(join(browserComponents, 'browser-page-prompt.tsx'))}; +import { useBrowserPanelOcclusion } from ${JSON.stringify(join(browserComponents, 'browser-panel-occlusion.ts'))}; +const scope = ${JSON.stringify(SCOPE)}; +const api = globalThis.simDesktop.browserAgent; +api.onPageState(state => store.getState().setPageState(state)); +store.subscribe(state => { + globalThis.pagePrompt = state.sessions[scope]?.pageState?.prompt ?? null; + globalThis.launchNotice = state.sessions[scope]?.pageState?.appLaunchNotice ?? null; +}); +function Fixture() { + const page = store(state => state.sessions[scope]?.pageState); + useBrowserPanelOcclusion(scope, page?.tabId ?? null, true, () => document.getElementById('native-panel')?.getBoundingClientRect() ?? null); + const respond = (requestId, response) => api.panelAction({ action: 'respond-prompt', requestId, response }, scope); + return createElement('div', null, + page?.appLaunchNotice ? createElement(BrowserAppLaunchNoticeBar, { key: page.appLaunchNotice.requestId, notice: page.appLaunchNotice, onRespond: respond }) : null, + createElement(BrowserPagePromptModal, { prompt: page?.prompt, open: Boolean(page?.prompt), onRespond: respond })); +} +createRoot(document.getElementById('root')).render(createElement(Fixture));`, + resolveDir: SIM_DIR, + loader: 'tsx', + }, + bundle: true, + write: false, + outfile: test.info().outputPath('fixture.js'), + external: ['node:async_hooks'], + banner: { js: 'var process={env:{NODE_ENV:"development"},browser:true};' }, + format: 'iife', + platform: 'browser', + tsconfig: join(SIM_DIR, 'tsconfig.json'), + define: { 'process.env.NODE_ENV': '"development"' }, + }) + const config = await loadPostcssConfig({}, SIM_DIR) + const stylesheet = join(SIM_DIR, 'app/_styles/globals.css') + const css = await postcss(config.plugins).process(readFileSync(stylesheet, 'utf8'), { + from: stylesheet, + }) + const calls = new Map() + const challenges: string[] = [] + let otherPort = 0 + const handleSite = (request: IncomingMessage, response: ServerResponse, port: number) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname + if (path === '/basic') { + challenges.push(`${port}${path}`) + if (request.headers.authorization === CREDENTIALS) { + response.writeHead(200, { 'Content-Type': 'text/html' }) + response.end('secret') + return + } + response.writeHead(401, { + 'Content-Type': 'text/html', + 'WWW-Authenticate': 'Basic realm="Fixture realm"', + }) + response.end('denied') + return + } + response.writeHead(200, { 'Content-Type': 'text/html' }) + if (path === '/subresource') { + response.end( + `sub` + ) + return + } + if (path === '/links') { + response.end(`links +

Join meeting

+

Diagnostics

+

Unhandled

+

Mail

+`) + return + } + if (path === '/auto') { + response.end( + `auto` + ) + return + } + response.end('next') + } + const server: Server = createServer(async (request, response) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname + if (path === '/fixture.js' || path === '/fixture.css') { + response.setHeader('Content-Type', path.endsWith('.js') ? 'text/javascript' : 'text/css') + response.end( + path.endsWith('.js') + ? bundle.outputFiles.find((file) => file.path.endsWith('.js'))?.text + : css.css + ) + return + } + if (path === '/api/desktop/tool/authorize') { + let body = '' + for await (const chunk of request) body += chunk.toString() + const authorization = calls.get(JSON.parse(body).toolCallId) + response.writeHead(authorization ? 200 : 403, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(authorization ?? {})) + return + } + // The app origin serves the shell; the same server on localhost is the web. + if (request.headers.host?.startsWith('localhost')) { + const address = server.address() + handleSite(request, response, typeof address === 'object' && address ? address.port : 0) + return + } + response.writeHead(200, { + 'Content-Type': 'text/html', + 'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; Path=/', + }) + response.end(SHELL_FIXTURE) + }) + const otherSite: Server = createServer((request, response) => + handleSite(request, response, otherPort) + ) + const userData = mkdtempSync(join(tmpdir(), 'sim-browser-signin-e2e-')) + let app: ElectronApplication | undefined + let passed = false + try { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + await new Promise((resolve) => otherSite.listen(0, '127.0.0.1', resolve)) + const address = server.address() + const otherAddress = otherSite.address() + if ( + !address || + typeof address === 'string' || + !otherAddress || + typeof otherAddress === 'string' + ) + throw new Error('Missing fixture address') + otherPort = otherAddress.port + const origin = `http://127.0.0.1:${address.port}` + const site = `http://localhost:${address.port}` + const crossSite = `http://localhost:${otherPort}` + const shellApp = await electron.launch({ + args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], + cwd: DESKTOP_DIR, + env: { ...process.env, SIM_DESKTOP_ORIGIN: origin, SIM_DESKTOP_USER_DATA: userData }, + }) + app = shellApp + // Launching real apps is out of scope: record what the shell would hand the OS, + // and register a fixture handler for the schemes the page links to. + await shellApp.evaluate(({ app, shell }) => { + const main = globalThis as MainGlobals + main.openedExternal = [] + shell.openExternal = async (url: string) => { + main.openedExternal?.push(url) + } + app.getApplicationNameForProtocol = (url: string) => + /^(simfixture|ms-msdt|mailto):/i.test(url) ? 'Fixture Meetings' : '' + }) + const openedExternal = () => + shellApp.evaluate(() => [...((globalThis as MainGlobals).openedExternal ?? [])]) + const shell = await shellApp.firstWindow() + await shellApp.evaluate(({ app, BrowserWindow }) => { + const host = BrowserWindow.getAllWindows()[0] + if (!host) throw new Error('Missing host window') + app.focus({ steal: true }) + host.focus() + }) + await expect(shell.getByRole('heading')).toHaveText('Browser sign-in fixture') + await shell.evaluate(async (scope) => { + const bridge = globalThis as Bridge + const api = bridge.simDesktop.browserAgent + await api.activateScope(scope) + const updateBounds = () => + api.setPanelBounds( + { x: 0, y: 120, width: innerWidth, height: innerHeight - 120 }, + null, + scope + ) + updateBounds() + bridge.boundsTimer = window.setInterval(updateBounds, 200) + }, SCOPE) + + let callCount = 0 + const startTool = (tool: BrowserToolName, args: Record) => { + const callId = `browser-signin-${++callCount}` + calls.set(callId, { chatId: SCOPE, toolName: tool, args }) + return shell.evaluate( + ({ callId, tool, args, scope }) => + (globalThis as Bridge).simDesktop.browserAgent.executeTool(callId, tool, args, scope), + { callId, tool, args, scope: SCOPE } + ) + } + const execute = async (tool: BrowserToolName, args: Record) => { + const result = await startTool(tool, args) + expect(result.ok, result.error).toBe(true) + return result.result + } + /** Browser-chrome actions are user gestures, so each follows a real click in Sim. */ + const panelAction = async (action: Record) => { + await shell.getByRole('heading').click() + await shell.evaluate( + ({ action, scope }) => + (globalThis as Bridge).simDesktop.browserAgent.panelAction( + action as unknown as Parameters[0], + scope + ), + { action, scope: SCOPE } + ) + } + const pagePrompt = () => shell.evaluate(() => (globalThis as Bridge).pagePrompt ?? null) + const launchNotice = () => shell.evaluate(() => (globalThis as Bridge).launchNotice ?? null) + /** The page currently at `url`; `read` is `title` or a script evaluated in it. */ + const pageAt = (url: string, read: string) => + shellApp.evaluate( + async ({ webContents }, { url, read }) => { + const contents = webContents + .getAllWebContents() + .find((candidate) => !candidate.isDestroyed() && candidate.getURL() === url) + if (!contents) return null + if (read === 'title') return contents.getTitle() as T + return (await contents.executeJavaScript(read, true)) as T + }, + { url, read } + ) + const titleAt = (url: string) => () => pageAt(url, 'title') + const navigate = async (url: string) => { + await panelAction({ action: 'navigate', url }) + await expect.poll(() => pageAt(url, 'title')).not.toBeNull() + } + /** Trusted input into the page, the way the user's own click arrives and focuses it. */ + const userClick = (url: string, selector: string) => + shellApp.evaluate( + async ({ webContents }, { url, selector }) => { + const contents = webContents + .getAllWebContents() + .find((candidate) => !candidate.isDestroyed() && candidate.getURL() === url) + if (!contents) throw new Error(`No page at ${url}`) + const rect: { x: number; y: number } = await contents.executeJavaScript( + `(() => { const { x, y } = document.querySelector(${JSON.stringify(selector)}).getBoundingClientRect(); return { x, y } })()` + ) + const point = { x: Math.round(rect.x + 3), y: Math.round(rect.y + 3) } + contents.focus() + contents.sendInputEvent({ type: 'mouseDown', ...point, button: 'left', clickCount: 1 }) + contents.sendInputEvent({ type: 'mouseUp', ...point, button: 'left', clickCount: 1 }) + }, + { url, selector } + ) + /** Types into the visible sign-in prompt the way the user does: the password field unlocks on focus. */ + const signInAs = async (username: string, password: string) => { + const dialog = shell.getByRole('dialog') + await expect(dialog).toHaveCount(1) + await expect(async () => { + await dialog.getByRole('textbox', { name: 'Username' }).fill(username) + await dialog.getByRole('textbox', { name: 'Password' }).click() + await shell.keyboard.press('ControlOrMeta+A') + await shell.keyboard.type(password) + await expect(dialog.getByRole('textbox', { name: 'Username' })).toHaveValue(username, { + timeout: 1_000, + }) + await expect(dialog.getByRole('textbox', { name: 'Password' })).toHaveValue(password, { + timeout: 1_000, + }) + }).toPass({ timeout: 10_000 }) + } + const expectQuiet = async (openedBefore: string[]) => { + await sleep(QUIET_MS) + expect(await pagePrompt()).toBeNull() + expect(await launchNotice()).toBeNull() + expect(await openedExternal()).toEqual(openedBefore) + } + + await execute('browser_open_url', { url: `${site}/next` }) + await panelAction({ action: 'switch-tab', tabId: '1' }) + await expect.poll(titleAt(`${site}/next`)).toBe('next') + + await check('an older renderer leaves sign-in challenges and app links refused', async () => { + await panelAction({ action: 'enable-page-dialogs' }) + await navigate(`${site}/basic`) + await expect.poll(titleAt(`${site}/basic`)).toBe('denied') + expect(await pagePrompt()).toBeNull() + await navigate(`${site}/links`) + await expect.poll(titleAt(`${site}/links`)).toBe('links') + await userClick(`${site}/links`, '#app') + await expectQuiet([]) + }) + + await panelAction({ action: 'enable-page-dialogs', pagePrompts: true }) + + await check( + 'basic auth asks for the page, re-asks after a wrong password, then signs in', + async () => { + await panelAction({ action: 'navigate', url: `${site}/basic` }) + await expect.poll(pagePrompt).toMatchObject({ + kind: 'http-auth', + origin: site, + proxy: false, + insecure: true, + retry: false, + }) + await expect(shell.getByText('Your connection to this site is not private.')).toBeVisible() + await signInAs('alice', 'wrong') + await shell.getByRole('button', { name: 'Sign in', exact: true }).click() + await expect.poll(pagePrompt).toMatchObject({ kind: 'http-auth', retry: true }) + await expect(shell.getByText('Incorrect username or password.')).toBeVisible() + await signInAs('alice', 's3cret') + await shell.getByRole('button', { name: 'Sign in', exact: true }).click() + await expect.poll(titleAt(`${site}/basic`)).toBe('secret') + expect(await pagePrompt()).toBeNull() + } + ) + + await check('a cross-origin subresource challenge never asks', async () => { + const before = challenges.length + await navigate(`${site}/subresource`) + await expect.poll(titleAt(`${site}/subresource`)).toBe('img-failed') + expect(challenges.slice(before)).toContain(`${otherPort}/basic`) + expect(await pagePrompt()).toBeNull() + }) + + await check( + "the agent's sign-in challenge waits for the user instead of being answered", + async () => { + const navigation = startTool('browser_navigate', { url: `${crossSite}/basic` }) + await expect.poll(pagePrompt).toMatchObject({ kind: 'http-auth', origin: crossSite }) + await sleep(QUIET_MS) + expect(await pagePrompt()).toMatchObject({ kind: 'http-auth', retry: false }) + await signInAs('alice', 's3cret') + await shell.getByRole('button', { name: 'Sign in', exact: true }).click() + const result = await navigation + expect(result.ok, result.error).toBe(true) + await expect.poll(titleAt(`${crossSite}/basic`)).toBe('secret') + } + ) + + await check("the agent's clicks never open another app", async () => { + await navigate(`${site}/links`) + const opened = await openedExternal() + const snapshot = await execute('browser_snapshot', {}) + const ref = /link "Join meeting" \[ref=(\d+)\]/.exec( + String((snapshot as { outline?: string }).outline) + )?.[1] + expect(ref, 'snapshot lists the Join meeting link').toBeTruthy() + const result = await execute('browser_click', { elementId: Number(ref) }) + await expectQuiet(opened) + const notices = [ + ...((result as { notices?: string[] }).notices ?? []), + ...(((await execute('browser_snapshot', {})) as { notices?: string[] }).notices ?? []), + ] + expect(notices.join('\n')).toContain('never open from agent actions') + }) + + await check('a link to another app asks after a click and opens only on Open', async () => { + await userClick(`${site}/links`, '#app') + await expect.poll(pagePrompt).toMatchObject({ + kind: 'external-app', + origin: site, + appName: 'Fixture Meetings', + canAlwaysAllow: false, + }) + const open = shell.getByRole('button', { name: 'Open', exact: true }) + await expect(open).toBeVisible() + expect(await openedExternal()).toEqual([]) + await expect(open).toBeEnabled() + await open.click() + await expect.poll(openedExternal).toEqual(['simfixture://join?id=7']) + expect(await pagePrompt()).toBeNull() + }) + + await check('without a click, a page only offers to open the app', async () => { + await navigate(`${site}/auto`) + await expect.poll(launchNotice).toMatchObject({ appName: 'Fixture Meetings', origin: site }) + expect(await pagePrompt()).toBeNull() + expect(await openedExternal()).toEqual(['simfixture://join?id=7']) + await shell.getByRole('button', { name: 'Open', exact: true }).click() + await expect.poll(openedExternal).toEqual(['simfixture://join?id=7', 'simfixture://auto']) + await expect.poll(launchNotice).toBeNull() + }) + + await check('a page cannot keep asking to open apps', async () => { + await navigate(`${site}/links`) + await userClick(`${site}/links`, '#app') + await expectQuiet(['simfixture://join?id=7', 'simfixture://auto']) + }) + + await check( + 'denied and unhandled schemes never open, mail links open after a click', + async () => { + await shell.evaluate( + (scope) => (globalThis as Bridge).simDesktop.browserAgent.openTab?.(scope), + SCOPE + ) + const opened = await openedExternal() + // A fresh document per link: one page's refused launch must not mask the next. + let links = '' + for (const [index, selector] of ['#denied', '#unhandled'].entries()) { + links = `${site}/links?tab=2&link=${index}` + await navigate(links) + await userClick(links, selector) + await expectQuiet(opened) + } + await userClick(links, '#mail') + await expect.poll(openedExternal).toEqual([...opened, 'mailto:a@example.com?subject=Hi']) + expect(await pagePrompt()).toBeNull() + } + ) + + await check('a passkey sign-in lets the user choose the account', async () => { + const passkeyPage = `${site}/next?passkey` + await navigate(passkeyPage) + await shellApp.evaluate(async ({ webContents }) => { + const contents = webContents + .getAllWebContents() + .find((candidate) => candidate.getURL().includes('/next?passkey')) + if (!contents) throw new Error('No passkey page') + if (!contents.debugger.isAttached()) contents.debugger.attach('1.3') + await contents.debugger.sendCommand('WebAuthn.enable', { enableUI: false }) + await contents.debugger.sendCommand('WebAuthn.addVirtualAuthenticator', { + options: { + protocol: 'ctap2', + transport: 'internal', + hasResidentKey: true, + hasUserVerification: true, + isUserVerified: true, + automaticPresenceSimulation: true, + }, + }) + }) + const created = await pageAt( + passkeyPage, + `(async () => { + const make = (id, name) => navigator.credentials.create({ publicKey: { + challenge: new Uint8Array(16), rp: { id: 'localhost', name: 'Fixture' }, + user: { id: new TextEncoder().encode(id), name, displayName: name }, + pubKeyCredParams: [{ type: 'public-key', alg: -7 }], + authenticatorSelection: { residentKey: 'required', userVerification: 'preferred' }, + } }) + await make('user-ada', 'ada@example.com') + await make('user-grace', 'grace@example.com') + globalThis.passkeyOutcome = 'pending' + navigator.credentials.get({ publicKey: { + challenge: new Uint8Array(16), rpId: 'localhost', userVerification: 'preferred', + } }).then((credential) => new TextDecoder().decode(credential.response.userHandle), + (error) => 'error:' + error.name).then((outcome) => { globalThis.passkeyOutcome = outcome }) + return 'created' + })()` + ) + expect(created).toBe('created') + await expect.poll(pagePrompt).toMatchObject({ + kind: 'passkey-account', + relyingPartyId: 'localhost', + }) + const prompt = await pagePrompt() + expect( + prompt?.kind === 'passkey-account' && prompt.accounts.map((account) => account.name).sort() + ).toEqual(['ada@example.com', 'grace@example.com']) + // A stray Enter must never send an identity: nothing is preselected and Enter does not + // continue. It dismisses when the prompt has keyboard focus and is ignored when it does + // not (as on a CI runner without window focus); either way, refuse and ask again. + const passkeyOutcome = () => pageAt(passkeyPage, 'globalThis.passkeyOutcome') + await shell.keyboard.press('Enter') + await sleep(QUIET_MS) + expect(['pending', 'error:NotAllowedError']).toContain(await passkeyOutcome()) + if ((await passkeyOutcome()) === 'pending') { + await shell.getByRole('button', { name: 'Cancel', exact: true }).click() + } + await expect.poll(passkeyOutcome).toBe('error:NotAllowedError') + await pageAt( + passkeyPage, + `globalThis.passkeyOutcome = 'pending' + navigator.credentials.get({ publicKey: { + challenge: new Uint8Array(16), rpId: 'localhost', userVerification: 'preferred', + } }).then((credential) => new TextDecoder().decode(credential.response.userHandle), + (error) => 'error:' + error.name).then((outcome) => { globalThis.passkeyOutcome = outcome }); 'asked'` + ) + await expect.poll(pagePrompt).toMatchObject({ kind: 'passkey-account' }) + const dialog = shell.getByRole('dialog') + await dialog.getByRole('button').filter({ hasText: 'Choose an account' }).click() + await shell.getByRole('menuitemradio', { name: 'grace@example.com' }).click() + await shell.getByRole('button', { name: 'Continue', exact: true }).click() + await expect.poll(passkeyOutcome).toBe('user-grace') + expect(await pagePrompt()).toBeNull() + }) + passed = true + } finally { + mkdirSync(dirname(reportPath), { recursive: true }) + writeFileSync(reportPath, JSON.stringify({ passed, checks }, null, 2)) + await app?.close() + await new Promise((resolve) => server.close(() => resolve())) + await new Promise((resolve) => otherSite.close(() => resolve())) + rmSync(userData, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index c9219a003db..a52cc14977e 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -72,7 +72,6 @@ mac: NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions. NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser. NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet. - NSBluetoothAlwaysUsageDescription: Sim uses Bluetooth to complete passkey sign-ins with a nearby phone in the built-in browser. entitlements: build/entitlements.mac.plist entitlementsInherit: build/entitlements.mac.plist notarize: true diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 846b05e5e99..d0ada80bf5f 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -81,6 +81,11 @@ import { setFocusedInputValue, typeIntoElement, } from '@/main/browser-agent/page-functions' +import { + appLaunchNoticeForContents, + pagePromptForContents, + respondToPagePrompt, +} from '@/main/browser-agent/page-prompts' import { isPanelVisible, panelWindow } from '@/main/browser-agent/panel' import { withFailedPostActionObservation, @@ -560,6 +565,8 @@ function pageStateFor(contents: WebContents, tabId: string): BrowserPageState { const issue = session.pageIssueForContents(contents) const mediaPermissionRequest = session.mediaPermissionRequestForContents(contents) const dialog = session.pageDialogForContents(contents) + const prompt = pagePromptForContents(contents) + const appLaunchNotice = appLaunchNoticeForContents(contents) return { scopeId: session.getBrowserScopeId(), tabId, @@ -571,6 +578,8 @@ function pageStateFor(contents: WebContents, tabId: string): BrowserPageState { ...(issue ? { issue } : {}), ...(mediaPermissionRequest ? { mediaPermissionRequest } : {}), ...(dialog ? { dialog } : {}), + ...(prompt ? { prompt } : {}), + ...(appLaunchNotice ? { appLaunchNotice } : {}), } } @@ -809,6 +818,7 @@ export function initDriver( }) }, onDownloadsChanged: (state) => callbacks.onDownloadsChanged?.(state), + onAgentNotice: recordNotice, }, getMainWindow, persistence, @@ -5505,6 +5515,11 @@ export async function handlePanelAction( } if (action.action === 'enable-page-dialogs') { session.enablePageDialogs() + if (action.pagePrompts === true) session.enablePagePrompts() + return + } + if (action.action === 'respond-prompt') { + respondToPagePrompt(session.getBrowserScopeId(), action) return } if (action.action === 'respond-dialog') { diff --git a/apps/desktop/src/main/browser-agent/page-prompts.test.ts b/apps/desktop/src/main/browser-agent/page-prompts.test.ts new file mode 100644 index 00000000000..68ab5f06532 --- /dev/null +++ b/apps/desktop/src/main/browser-agent/page-prompts.test.ts @@ -0,0 +1,414 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => import('@/test/electron-mock')) + +import type { BrowserPagePrompt, BrowserPagePromptResponse } from '@sim/browser-protocol' +import type { + AuthenticationResponseDetails, + AuthInfo, + Certificate, + Session, + WebContents, + WebFrameMain, +} from 'electron' +import { app, webContents as electronWebContents, shell } from 'electron' +import { + appLaunchNoticeForContents, + handleBrowserLogin, + handleSelectClientCertificate, + handleSelectPasskeyAccount, + httpAuthChallenge, + noteMainFrameNavigation, + pagePromptForContents, + requestExternalLaunch, + resetPagePromptDecisions, + respondToPagePrompt, + setPagePromptHost, +} from '@/main/browser-agent/page-prompts' +import { registerAgentWebContents } from '@/main/browser-agent/registry' + +const PAGE = 'https://meet.example' + +interface Context { + promptsEnabled: boolean + onScreen: boolean + userActivated: boolean + agentActedLast: boolean + agentDrives: boolean +} + +let context: Context + +function agentContents(session: Session = {} as Session): WebContents { + const contents = { + isDestroyed: () => false, + getURL: () => `${PAGE}/j/1`, + mainFrame: { frameTreeNodeId: 1 }, + session, + } as WebContents + registerAgentWebContents(contents) + return contents +} + +function shownPrompt(contents: WebContents): BrowserPagePrompt { + const prompt = pagePromptForContents(contents) + if (!prompt) throw new Error('No prompt shown') + return prompt +} + +function answer(contents: WebContents, response: BrowserPagePromptResponse, scopeId = 'chat') { + respondToPagePrompt(scopeId, { + action: 'respond-prompt', + requestId: shownPrompt(contents).requestId, + response, + }) +} + +beforeEach(() => { + vi.useFakeTimers() + context = { + promptsEnabled: true, + onScreen: true, + userActivated: false, + agentActedLast: false, + agentDrives: false, + } + setPagePromptHost({ + tabContext: () => ({ scopeId: 'chat', ...context }), + publish: () => {}, + focus: () => {}, + noticeForAgent: () => {}, + }) +}) + +afterEach(() => { + setPagePromptHost(null) + resetPagePromptDecisions() + vi.useRealTimers() +}) + +describe('httpAuthChallenge', () => { + const url = 'https://intranet.example:8443/reports' + const page = { url, isRequestForNavigation: true } + const server = { isProxy: false, host: 'intranet.example', port: 8443 } + + it("asks about the tab's own top-level navigation, challenged by the host it requested", () => { + expect(httpAuthChallenge(page, server, url)).toEqual({ + origin: 'https://intranet.example:8443', + proxy: false, + insecure: false, + }) + expect( + httpAuthChallenge( + { url: 'http://router.local/', isRequestForNavigation: true }, + { isProxy: false, host: 'router.local', port: 80 }, + 'http://router.local/' + ) + ).toMatchObject({ insecure: true }) + }) + + it('cancels frame, subresource, and cross-origin challenges', () => { + expect(httpAuthChallenge(page, server, 'https://intranet.example:8443/')).toBeNull() + expect(httpAuthChallenge(page, server, undefined)).toBeNull() + expect(httpAuthChallenge({ ...page, isRequestForNavigation: false }, server, url)).toBeNull() + expect(httpAuthChallenge(page, { ...server, host: 'evil.example' }, url)).toBeNull() + expect(httpAuthChallenge(page, { ...server, port: 443 }, url)).toBeNull() + }) + + it('asks about a proxy for any request', () => { + expect( + httpAuthChallenge( + { url: 'https://example.com/pixel.png', isRequestForNavigation: false }, + { isProxy: true, host: 'proxy.corp', port: 3128 }, + undefined + ) + ).toEqual({ origin: 'proxy.corp:3128', proxy: true, insecure: false }) + }) +}) + +describe('HTTP sign-in', () => { + const url = 'https://intranet.example/' + const login = (contents: WebContents, firstAuthAttempt = true) => { + const callback = vi.fn() + handleBrowserLogin( + { preventDefault: () => {} }, + contents, + { url, isRequestForNavigation: true, firstAuthAttempt } as AuthenticationResponseDetails, + { + isProxy: false, + scheme: 'basic', + host: 'intranet.example', + port: 443, + realm: '', + } as AuthInfo, + callback + ) + return callback + } + + it('stops asking a page that keeps reloading into the same challenge', () => { + const contents = agentContents() + for (let attempt = 0; attempt < 3; attempt++) { + noteMainFrameNavigation(contents, url) + const callback = login(contents) + expect(pagePromptForContents(contents)).toMatchObject({ kind: 'http-auth' }) + answer(contents, { allowed: false }) + expect(callback).toHaveBeenCalledExactlyOnceWith() + } + noteMainFrameNavigation(contents, url) + expect(login(contents)).toHaveBeenCalledExactlyOnceWith() + expect(pagePromptForContents(contents)).toBeUndefined() + + vi.advanceTimersByTime(60_000) + login(contents) + expect(pagePromptForContents(contents)).toMatchObject({ kind: 'http-auth' }) + }) + + it('passes on only the credentials of an accepted answer', () => { + const contents = agentContents() + noteMainFrameNavigation(contents, url) + const callback = login(contents) + answer(contents, { allowed: true, kind: 'http-auth', username: 'ada', password: 'pw' }) + expect(callback).toHaveBeenCalledExactlyOnceWith('ada', 'pw') + }) +}) + +describe('client certificates', () => { + const certificate = (fingerprint: string, subjectName: string) => + ({ fingerprint, subjectName, issuerName: 'Corp CA', validExpiry: 2_000_000_000 }) as Certificate + const first = certificate('sha256/first', 'Personal') + const work = certificate('sha256/work', 'Work badge') + + function request(contents: WebContents | null, list: Certificate[], url = 'https://mtls.corp') { + const event = { preventDefault: vi.fn() } + const callback = vi.fn() + handleSelectClientCertificate(event, contents, url, list, callback) + return { event, callback } + } + + it("leaves the main window on Electron's default", () => { + const { event, callback } = request({ isDestroyed: () => false } as WebContents, [first]) + expect(event.preventDefault).not.toHaveBeenCalled() + expect(callback).not.toHaveBeenCalled() + }) + + it('sends no certificate when there is none, or no renderer to ask', () => { + const contents = agentContents() + const empty = request(contents, []) + expect(empty.event.preventDefault).toHaveBeenCalled() + expect(empty.callback).toHaveBeenCalledExactlyOnceWith() + + context.promptsEnabled = false + expect(request(contents, [first, work]).callback).toHaveBeenCalledExactlyOnceWith() + }) + + it('sends only the picked certificate, remembers it per host, and forgets it on reset', () => { + const session = {} as Session + const contents = agentContents(session) + const { callback } = request(contents, [first, work]) + expect(shownPrompt(contents)).toMatchObject({ kind: 'client-certificate', host: 'mtls.corp' }) + answer(contents, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/work' }) + expect(callback).toHaveBeenCalledExactlyOnceWith(work) + + expect(request(agentContents(session), [first, work]).callback).toHaveBeenCalledWith(work) + expect( + request(agentContents(session), [first, work], 'https://other.corp').callback + ).not.toHaveBeenCalled() + + const otherSession = {} as Session + const otherContents = agentContents(otherSession) + request(otherContents, [first]) + answer(otherContents, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/first' }) + + resetPagePromptDecisions(session) + expect(request(agentContents(session), [first, work]).callback).not.toHaveBeenCalled() + expect(request(agentContents(otherSession), [first]).callback).toHaveBeenCalledWith(first) + + resetPagePromptDecisions() + expect(request(agentContents(otherSession), [first]).callback).not.toHaveBeenCalled() + }) + + it("remembers Don't send, but not a prompt nobody answered", () => { + const declined = agentContents() + const refused = request(declined, [first]) + answer(declined, { allowed: false }) + expect(refused.callback).toHaveBeenCalledExactlyOnceWith() + expect(request(agentContents(declined.session), [first]).callback).toHaveBeenCalledWith() + + const ignored = agentContents() + const unanswered = request(ignored, [first]) + vi.advanceTimersByTime(120_000) + expect(unanswered.callback).toHaveBeenCalledExactlyOnceWith() + expect(request(agentContents(ignored.session), [first]).callback).not.toHaveBeenCalled() + }) + + it('sends nothing for an unknown choice, an off-screen answer, or another chat', () => { + const forged = agentContents() + const unknown = request(forged, [first]) + answer(forged, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/forged' }) + expect(unknown.callback).toHaveBeenCalledExactlyOnceWith() + + const elsewhere = agentContents() + const otherChat = request(elsewhere, [first]) + answer(elsewhere, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/first' }, 'x') + expect(otherChat.callback).not.toHaveBeenCalled() + + context.onScreen = false + const hidden = agentContents() + const offScreen = request(hidden, [first]) + answer(hidden, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/first' }) + expect(offScreen.callback).toHaveBeenCalledExactlyOnceWith() + }) +}) + +describe('app links', () => { + const topFrame = (origin = PAGE) => ({ parent: null, frameTreeNodeId: 1, origin }) as WebFrameMain + + beforeEach(() => { + app.getApplicationNameForProtocol = vi.fn(() => 'Zoom') + vi.mocked(shell.openExternal).mockClear() + context.userActivated = true + }) + + it('refuses a launch a frame initiated, even one targeting the top document', () => { + const contents = agentContents() + const frame = { + parent: topFrame(), + frameTreeNodeId: 2, + origin: 'https://ads.example', + } as WebFrameMain + requestExternalLaunch(contents, 'zoommtg://zoom.us/join', frame) + requestExternalLaunch(contents, 'zoommtg://zoom.us/join', topFrame('https://ads.example')) + requestExternalLaunch(contents, 'zoommtg://zoom.us/join', null) + expect(pagePromptForContents(contents)).toBeUndefined() + expect(appLaunchNoticeForContents(contents)).toBeUndefined() + }) + + it('asks after the top document is clicked, and otherwise only offers passively', () => { + const clicked = agentContents() + requestExternalLaunch(clicked, 'zoommtg://zoom.us/join', topFrame()) + expect(pagePromptForContents(clicked)).toMatchObject({ kind: 'external-app', appName: 'Zoom' }) + + context.userActivated = false + const unclicked = agentContents() + requestExternalLaunch(unclicked, 'zoommtg://zoom.us/join', topFrame()) + expect(pagePromptForContents(unclicked)).toBeUndefined() + expect(appLaunchNoticeForContents(unclicked)).toMatchObject({ origin: PAGE, appName: 'Zoom' }) + }) + + it('opens nothing, not even a clicked mail link, without a renderer that shows prompts', async () => { + context.promptsEnabled = false + requestExternalLaunch(agentContents(), 'mailto:ada@example.com', topFrame()) + await vi.advanceTimersByTimeAsync(0) + expect(shell.openExternal).not.toHaveBeenCalled() + }) + + it('opens only the first link while its prompt is pending', async () => { + const contents = agentContents() + requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=1', topFrame()) + requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=2', topFrame()) + expect(appLaunchNoticeForContents(contents)).toBeUndefined() + answer(contents, { allowed: true, kind: 'external-app', alwaysAllow: false }) + await vi.advanceTimersByTimeAsync(0) + expect(shell.openExternal).toHaveBeenCalledExactlyOnceWith('zoommtg://zoom.us/join?id=1') + }) + + it('forgets "Always allow" on reset', async () => { + const contents = agentContents() + requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=1', topFrame()) + answer(contents, { allowed: true, kind: 'external-app', alwaysAllow: true }) + resetPagePromptDecisions() + vi.advanceTimersByTime(20_000) + requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=2', topFrame()) + await vi.advanceTimersByTimeAsync(0) + expect(pagePromptForContents(contents)).toMatchObject({ kind: 'external-app' }) + expect(shell.openExternal).toHaveBeenCalledExactlyOnceWith('zoommtg://zoom.us/join?id=1') + }) +}) + +describe('accepting an answer', () => { + const topFrame = { parent: null, frameTreeNodeId: 1, origin: PAGE } as WebFrameMain + + /** What a renderer could send over IPC without `allowed: true`, which the gesture gate keys on. */ + function withoutAllowed(response: Record): BrowserPagePromptResponse { + // double-cast-allowed: an IPC payload is untyped; this is the shape a hostile renderer could send + return response as unknown as BrowserPagePromptResponse + } + + it.each([ + [ + 'client-certificate', + { kind: 'client-certificate', choiceId: 'sha256/first' }, + (contents: WebContents, callback: () => void) => + handleSelectClientCertificate( + { preventDefault: () => {} }, + contents, + 'https://mtls.corp', + [{ fingerprint: 'sha256/first', subjectName: 'Personal' } as Certificate], + callback + ), + ], + [ + 'passkey-account', + { kind: 'passkey-account', choiceId: 'credential-1' }, + (contents: WebContents, callback: () => void) => { + vi.mocked(electronWebContents.fromFrame).mockReturnValue(contents) + handleSelectPasskeyAccount( + {}, + { + relyingPartyId: 'meet.example', + accounts: [{ credentialId: 'credential-1', name: 'ada', userHandle: 'u1' }], + frame: topFrame, + }, + callback + ) + }, + ], + [ + 'http-auth', + { kind: 'http-auth', username: 'ada', password: 'pw' }, + (contents: WebContents, callback: () => void) => { + noteMainFrameNavigation(contents, 'https://intranet.example/') + handleBrowserLogin( + { preventDefault: () => {} }, + contents, + { + url: 'https://intranet.example/', + isRequestForNavigation: true, + firstAuthAttempt: true, + } as AuthenticationResponseDetails, + { isProxy: false, scheme: 'basic', host: 'intranet.example', port: 443 } as AuthInfo, + callback + ) + }, + ], + ])('sends nothing for a %s answer without allowed: true', (_kind, response, ask) => { + const contents = agentContents() + const callback = vi.fn() + ask(contents, callback) + answer(contents, withoutAllowed(response)) + expect(callback).toHaveBeenCalledExactlyOnceWith() + }) + + it.each([ + ['an app prompt', { kind: 'external-app', alwaysAllow: true }, 'prompt'], + ['an app notice', { kind: 'app-launch-notice' }, 'notice'], + ])('opens nothing for %s answer without allowed: true', async (_label, response, slot) => { + app.getApplicationNameForProtocol = vi.fn(() => 'Zoom') + vi.mocked(shell.openExternal).mockClear() + context.userActivated = slot === 'prompt' + const contents = agentContents() + requestExternalLaunch(contents, 'zoommtg://zoom.us/join', topFrame) + const requestId = + slot === 'prompt' + ? shownPrompt(contents).requestId + : appLaunchNoticeForContents(contents)?.requestId + respondToPagePrompt('chat', { + action: 'respond-prompt', + requestId, + response: withoutAllowed(response), + }) + await vi.advanceTimersByTimeAsync(0) + expect(shell.openExternal).not.toHaveBeenCalled() + }) +}) diff --git a/apps/desktop/src/main/browser-agent/page-prompts.ts b/apps/desktop/src/main/browser-agent/page-prompts.ts new file mode 100644 index 00000000000..2a96b040571 --- /dev/null +++ b/apps/desktop/src/main/browser-agent/page-prompts.ts @@ -0,0 +1,600 @@ +/** + * Sign-in and app-launch decisions a browser tab's page waits on: HTTP + * authentication, passkey account choice, client certificates, and links to + * other apps. Sim's renderer shows each one for the exact tab that asked; only + * a user-activated renderer response answers it, and every request is refused + * when the renderer cannot show it, does not answer in time, or the page moves + * on to another document. + */ +import type { + BrowserAppLaunchNotice, + BrowserPagePrompt, + BrowserPagePromptResponse, + BrowserPanelAction, +} from '@sim/browser-protocol' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { truncate } from '@sim/utils/string' +import type { + AuthenticationResponseDetails, + AuthInfo, + Certificate, + Session, + WebAuthnAccount, + WebContents, + WebFrameMain, +} from 'electron' +import { app, webContents as electronWebContents } from 'electron' +import { isAgentWebContents } from '@/main/browser-agent/registry' +import { appLinkTarget, openExternalSafe, parseHttpUrl } from '@/main/navigation' + +const logger = createLogger('BrowserPagePrompts') + +/** Long enough to type a password; a request never waits on a renderer beyond it. */ +const PAGE_PROMPT_TIMEOUT_MS = 120_000 +const APP_LAUNCH_NOTICE_TIMEOUT_MS = 60_000 +/** Prompts per tab and origin per window, across reloads as well as rejected attempts. */ +const MAX_HTTP_AUTH_PROMPTS = 3 +const HTTP_AUTH_PROMPT_WINDOW_MS = 60_000 +const MAX_CREDENTIAL_LENGTH = 1_024 +const MAX_DISPLAY_TEXT_LENGTH = 120 +const MAX_PROMPT_CHOICES = 50 +const LAUNCH_RATE_WINDOW_MS = 20_000 +const MAX_LAUNCHES_PER_WINDOW = 2 +const MAX_REMEMBERED_DECISIONS = 200 +/** Controls and bidi overrides that let untrusted text break or reorder a line. */ +const UNTRUSTED_TEXT_CONTROLS = /[\u0000-\u001f\u007f-\u009f‎‏‪-‮⁦-⁩]/g + +interface PageTabContext { + scopeId: string + /** The chat's renderer declared that it shows page prompts. */ + promptsEnabled: boolean + /** The visible page of the visible chat. */ + onScreen: boolean + /** The user's real click or keypress reached this document moments ago, after any agent input. */ + userActivated: boolean + /** The agent's input is being delivered, or was the latest input in this tab. */ + agentActedLast: boolean + /** The agent drives this tab and the user has not claimed it. */ + agentDrives: boolean +} + +/** The session's side of a tab; injected to keep the dependency one-way. */ +interface PagePromptHost { + tabContext(contents: WebContents): PageTabContext | null + publish(contents: WebContents): void + focus(contents: WebContents): void + noticeForAgent(contents: WebContents, notice: string): void +} + +type RequestKind = BrowserPagePrompt['kind'] | 'app-launch-notice' +/** `null` is a cancellation: timeout, navigation, or an answer the user could not have given. */ +type RequestAnswer = (response: BrowserPagePromptResponse | null) => void +type WithoutRequestId = View extends unknown ? Omit : never + +interface PendingRequest { + id: string + contents: WebContents + kind: RequestKind + view: BrowserPagePrompt | BrowserAppLaunchNotice + key: string + answers: RequestAnswer[] + timer: ReturnType +} + +interface TabRequests { + prompt?: PendingRequest + notice?: PendingRequest +} + +let host: PagePromptHost | null = null +const pendingById = new Map() +const pendingByContents = new WeakMap() +const mainFrameNavigations = new WeakMap() +const recentAttemptsByContents = new WeakMap>() +/** `null` remembers an explicit "no certificate" for that host. */ +let certificateDecisions = new WeakMap>() +const alwaysAllowedLaunches = new Map() + +export function setPagePromptHost(next: PagePromptHost | null): void { + host = next + if (next) return + for (const entry of [...pendingById.values()]) settle(entry, null) +} + +/** Forgets remembered certificate and app-launch choices, as a sign-out or data wipe must. */ +export function resetPagePromptDecisions(session?: Session): void { + if (session) certificateDecisions.delete(session) + else certificateDecisions = new WeakMap() + alwaysAllowedLaunches.clear() +} + +function withinWindow(at: number, windowMs: number): boolean { + const elapsed = Date.now() - at + return elapsed >= 0 && elapsed < windowMs +} + +function rememberBounded(map: Map, key: string, value: Value): void { + map.delete(key) + map.set(key, value) + while (map.size > MAX_REMEMBERED_DECISIONS) { + const oldest = map.keys().next().value + if (oldest === undefined) break + map.delete(oldest) + } +} + +/** The live list of this tab's recent attempts under `key`, pruned to the window. */ +function recentAttempts(contents: WebContents, key: string, windowMs: number): number[] { + const byKey = recentAttemptsByContents.get(contents) ?? new Map() + recentAttemptsByContents.set(contents, byKey) + const recent = (byKey.get(key) ?? []).filter((at) => withinWindow(at, windowMs)) + rememberBounded(byKey, key, recent) + return recent +} + +/** Untrusted page or server text, reduced to one bounded display line. */ +function displayText(value: unknown, maxLength = MAX_DISPLAY_TEXT_LENGTH): string { + if (typeof value !== 'string') return '' + return truncate( + value.replace(UNTRUSTED_TEXT_CONTROLS, ' ').replace(/\s+/g, ' ').trim(), + maxLength + ) +} + +/** Distinct, bounded choices keyed by a non-empty string id. */ +function uniqueChoices( + items: readonly Item[], + idOf: (item: Item) => unknown, + toChoice: (id: string, item: Item) => Choice +): Choice[] { + const seen = new Set() + const choices: Choice[] = [] + for (const item of items) { + const id = idOf(item) + if (typeof id !== 'string' || !id || id.length > MAX_CREDENTIAL_LENGTH || seen.has(id)) continue + seen.add(id) + choices.push(toChoice(id, item)) + if (choices.length >= MAX_PROMPT_CHOICES) break + } + return choices +} + +/** + * Answers Electron exactly once. `ask` returns true once it owns the answer; + * otherwise, and on any error, the request is refused. + */ +function withPrompt( + label: string, + callback: (...args: Args | []) => void, + ask: (answer: (...args: Args | []) => void) => boolean +): void { + let answered = false + const answer = (...args: Args | []) => { + if (answered) return + answered = true + try { + callback(...args) + } catch (error) { + logger.warn(`Could not answer ${label}`, { error: getErrorMessage(error) }) + } + } + let owned = false + try { + owned = ask(answer) + } catch (error) { + logger.warn(`Could not ask about ${label}`, { error: getErrorMessage(error) }) + } finally { + if (!owned) answer() + } +} + +/** + * Holds a request for the user. A tab shows one prompt and one notice at a + * time: a new notice replaces the old one, and a prompt only joins an identical + * pending challenge when `joinable`. False when it cannot be shown. + */ +function hold( + contents: WebContents, + kind: RequestKind, + view: WithoutRequestId | WithoutRequestId, + key: string, + answer: RequestAnswer, + { joinable = false, focus = false }: { joinable?: boolean; focus?: boolean } = {} +): 'held' | 'joined' | false { + if (!host || contents.isDestroyed() || !host.tabContext(contents)?.promptsEnabled) return false + const requests = pendingByContents.get(contents) ?? {} + pendingByContents.set(contents, requests) + const slot = kind === 'app-launch-notice' ? 'notice' : 'prompt' + const existing = requests[slot] + if (existing && slot === 'prompt') { + if (!joinable || existing.key !== key) return false + existing.answers.push(answer) + return 'joined' + } + if (existing) settle(existing, null) + const id = generateId() + const entry: PendingRequest = { + id, + contents, + kind, + view: { ...view, requestId: id } as BrowserPagePrompt | BrowserAppLaunchNotice, + key, + answers: [answer], + timer: setTimeout( + () => settle(entry, null), + slot === 'notice' ? APP_LAUNCH_NOTICE_TIMEOUT_MS : PAGE_PROMPT_TIMEOUT_MS + ), + } + pendingById.set(id, entry) + requests[slot] = entry + host.publish(contents) + if (focus) host.focus(contents) + return 'held' +} + +function settle(entry: PendingRequest, response: BrowserPagePromptResponse | null): void { + if (pendingById.get(entry.id) !== entry) return + pendingById.delete(entry.id) + const requests = pendingByContents.get(entry.contents) + if (requests?.prompt === entry) requests.prompt = undefined + if (requests?.notice === entry) requests.notice = undefined + clearTimeout(entry.timer) + if (!entry.contents.isDestroyed()) host?.publish(entry.contents) + for (const answer of entry.answers) { + try { + answer(response) + } catch (error) { + logger.warn('Browser page prompt answer failed', { error: getErrorMessage(error) }) + } + } +} + +/** The decision this tab's page waits on, for its page state. */ +export function pagePromptForContents(contents: WebContents): BrowserPagePrompt | undefined { + return pendingByContents.get(contents)?.prompt?.view as BrowserPagePrompt | undefined +} + +export function appLaunchNoticeForContents( + contents: WebContents +): BrowserAppLaunchNotice | undefined { + return pendingByContents.get(contents)?.notice?.view as BrowserAppLaunchNotice | undefined +} + +/** Refuses what the previous document asked for: the tab moved on, crashed, or closed. */ +export function cancelPagePrompts(contents: WebContents): void { + const requests = pendingByContents.get(contents) + if (requests?.prompt) settle(requests.prompt, null) + if (requests?.notice) settle(requests.notice, null) +} + +/** + * Tracks the URL the tab's top-level navigation is requesting (null once it + * commits or fails), so HTTP authentication can tell its own challenge from a + * frame's. + */ +export function noteMainFrameNavigation(contents: WebContents, url: string | null): void { + if (url) mainFrameNavigations.set(contents, url) + else mainFrameNavigations.delete(contents) +} + +/** The renderer's answer for chat `scopeId`; only an on-screen answer of the request's own kind accepts. */ +export function respondToPagePrompt(scopeId: string, action: BrowserPanelAction): void { + const entry = typeof action.requestId === 'string' ? pendingById.get(action.requestId) : undefined + if (!entry || entry.contents.isDestroyed()) return + const context = host?.tabContext(entry.contents) + if (!context || context.scopeId !== scopeId) return + const { response } = action + if (response?.allowed === false) { + settle(entry, response) + return + } + const accepted = response?.allowed === true && response.kind === entry.kind && context.onScreen + settle(entry, accepted ? response : null) +} + +/** + * A challenge worth asking the user about, or null to cancel it: only the + * tab's own top-level navigation, challenged by the host and port it + * requested, or a proxy. Frames, subresources, and cross-origin challenges + * never get a prompt. + */ +export function httpAuthChallenge( + details: Pick, + authInfo: Pick, + mainFrameUrl: string | undefined +): { origin: string; proxy: boolean; insecure: boolean } | null { + const challengedHost = authInfo.host.toLowerCase().replace(/^\[|\]$/g, '') + if (!challengedHost || !Number.isInteger(authInfo.port) || authInfo.port <= 0) return null + if (authInfo.isProxy) { + const hostLabel = challengedHost.includes(':') ? `[${challengedHost}]` : challengedHost + return { origin: `${hostLabel}:${authInfo.port}`, proxy: true, insecure: false } + } + if (!details.isRequestForNavigation || details.url !== mainFrameUrl) return null + const url = parseHttpUrl(details.url) + if (!url) return null + const requestedPort = Number(url.port || (url.protocol === 'https:' ? 443 : 80)) + const requestedHost = url.hostname.toLowerCase().replace(/^\[|\]$/g, '') + if (requestedHost !== challengedHost || requestedPort !== authInfo.port) return null + return { origin: url.origin, proxy: false, insecure: url.protocol === 'http:' } +} + +/** + * `app.on('login')` for agent browser tabs. Other contents keep Electron's + * default, which cancels: the main window's proxy prompt needs a shell form. + * The prompt never takes focus, since a page can trigger it on its own. + */ +export function handleBrowserLogin( + event: { preventDefault(): void }, + contents: WebContents | null, + details: AuthenticationResponseDetails, + authInfo: AuthInfo, + callback: (username?: string, password?: string) => void +): void { + if (!contents || contents.isDestroyed() || !isAgentWebContents(contents)) return + event.preventDefault() + withPrompt('HTTP sign-in', callback, (answer) => { + const challenge = httpAuthChallenge(details, authInfo, mainFrameNavigations.get(contents)) + if (!challenge) return false + const prompts = recentAttempts( + contents, + `http-auth|${challenge.origin}`, + HTTP_AUTH_PROMPT_WINDOW_MS + ) + if (prompts.length >= MAX_HTTP_AUTH_PROMPTS) return false + const held = hold( + contents, + 'http-auth', + { kind: 'http-auth', ...challenge, retry: !details.firstAuthAttempt }, + `http-auth|${challenge.proxy}|${challenge.origin}|${authInfo.scheme}`, + (response) => { + if ( + response?.allowed === true && + response.kind === 'http-auth' && + response.username.length <= MAX_CREDENTIAL_LENGTH && + response.password.length <= MAX_CREDENTIAL_LENGTH + ) { + answer(response.username, response.password) + } else { + answer() + } + }, + { joinable: true } + ) + if (held === 'held') { + prompts.push(Date.now()) + if (host?.tabContext(contents)?.agentDrives) { + host.noticeForAgent( + contents, + `${challenge.origin} asked for a username and password. The user was asked to sign in in the browser panel; wait for them instead of retrying or entering credentials.` + ) + } + } + return Boolean(held) + }) +} + +/** + * `select-webauthn-account` for every session a browser tab uses, the borrowed + * app session included. Without a listener Electron cancels every discoverable + * passkey sign-in; requests from any non-browser contents still cancel. The + * prompt never takes focus: a page can ask without the user's involvement. + */ +export function handleSelectPasskeyAccount( + _event: unknown, + details: { relyingPartyId: string; accounts: WebAuthnAccount[]; frame: WebFrameMain | null }, + callback: (credentialId?: string | null) => void +): void { + withPrompt('a passkey account choice', callback, (answer) => { + const contents = details.frame ? electronWebContents.fromFrame(details.frame) : undefined + if (!contents || contents.isDestroyed() || !isAgentWebContents(contents)) return false + const relyingPartyId = displayText(details.relyingPartyId, 253) + const accounts = uniqueChoices( + details.accounts ?? [], + (account) => account?.credentialId, + (id, account) => ({ + id, + name: displayText(account.name), + displayName: displayText(account.displayName), + }) + ) + if (accounts.length === 0 || !relyingPartyId) return false + const held = hold( + contents, + 'passkey-account', + { kind: 'passkey-account', relyingPartyId, accounts }, + `passkey|${relyingPartyId}|${accounts.map((account) => account.id).join(',')}`, + (response) => { + const chosen = + response?.allowed === true && response.kind === 'passkey-account' + ? accounts.find((account) => account.id === response.choiceId) + : undefined + if (chosen) answer(chosen.id) + else answer() + }, + { joinable: true } + ) + return Boolean(held) + }) +} + +/** + * `app.on('select-client-certificate')` for agent browser tabs. Electron's + * default sends the first certificate to any site that asks; a browser tab + * sends one only after the user picks it, and the prompt never takes focus, + * since a subresource can ask. The main window keeps the default. + */ +export function handleSelectClientCertificate( + event: { preventDefault(): void }, + contents: WebContents | null, + url: string, + certificates: Certificate[], + callback: (certificate?: Certificate) => void +): void { + if (!contents || contents.isDestroyed() || !isAgentWebContents(contents)) return + event.preventDefault() + withPrompt('a client certificate choice', callback, (answer) => { + const requestUrl = parseHttpUrl(url) + const requestHost = requestUrl?.protocol === 'https:' ? requestUrl.host.toLowerCase() : null + const choices = uniqueChoices( + certificates, + (certificate) => certificate?.fingerprint, + (id, certificate) => ({ + id, + subject: displayText(certificate.subjectName || certificate.subject?.commonName), + issuer: displayText(certificate.issuerName || certificate.issuer?.commonName), + expiresAt: Number.isFinite(certificate.validExpiry) ? certificate.validExpiry * 1000 : 0, + }) + ) + if (!requestHost || choices.length === 0) return false + const decisions = certificateDecisions.get(contents.session) ?? new Map() + certificateDecisions.set(contents.session, decisions) + const remembered = decisions.get(requestHost) + if (remembered === null) return false + const rememberedCertificate = certificates.find( + (certificate) => certificate.fingerprint === remembered + ) + if (rememberedCertificate) { + answer(rememberedCertificate) + return true + } + const held = hold( + contents, + 'client-certificate', + { kind: 'client-certificate', host: requestHost, certificates: choices }, + `client-certificate|${requestHost}`, + (response) => { + if (response?.allowed === false) { + rememberBounded(decisions, requestHost, null) + answer() + return + } + const certificate = + response?.allowed === true && response.kind === 'client-certificate' + ? certificates.find((candidate) => candidate.fingerprint === response.choiceId) + : undefined + if (!certificate) { + answer() + return + } + rememberBounded(decisions, requestHost, certificate.fingerprint) + answer(certificate) + }, + { joinable: true } + ) + return Boolean(held) + }) +} + +/** The tab's own top-level document asked, not a frame inside it. */ +function isTopDocument( + contents: WebContents, + frame: WebFrameMain | null | undefined, + origin: string +): boolean { + try { + return Boolean( + frame && + frame.parent === null && + frame.frameTreeNodeId === contents.mainFrame.frameTreeNodeId && + frame.origin === origin + ) + } catch { + return false + } +} + +function launchAppLink(url: string): void { + void openExternalSafe(url, false, { appLink: true }) +} + +/** + * A browser tab's navigation to a non-web URL, with the frame that initiated + * it. Agent input never launches anything, and neither do frames; `window.open` + * never reaches here, since it names no initiating frame. The top document's + * request right after the user's own click asks them to confirm the app (mail + * links open directly); one without that click only gets a passive notice the + * user can act on. Nothing launches without a renderer that shows prompts, and + * a pending launch prompt refuses further requests. + */ +export function requestExternalLaunch( + contents: WebContents, + rawUrl: string, + initiator: WebFrameMain | null | undefined +): void { + if (!host || contents.isDestroyed() || !isAgentWebContents(contents)) return + const target = appLinkTarget(rawUrl) + if (!target) { + logger.warn('Denied external app link in agent browser', { + scheme: displayText(rawUrl.split(':', 1)[0], 32), + }) + return + } + let appName = '' + try { + appName = displayText(app.getApplicationNameForProtocol(target.url), 80) + } catch {} + const context = host.tabContext(contents) + if (!appName || !context?.promptsEnabled) return + if (context.agentActedLast) { + host.noticeForAgent( + contents, + `The page tried to open ${appName}. Links to other apps never open from agent actions; ask the user to open it themselves if the task needs it.` + ) + return + } + const origin = parseHttpUrl(contents.getURL())?.origin + if (!origin) return + if (!isTopDocument(contents, initiator, origin)) { + logger.warn('Denied an app link from a frame in agent browser', { scheme: target.scheme }) + return + } + const attempts = recentAttempts(contents, 'app-launch', LAUNCH_RATE_WINDOW_MS) + if (attempts.length >= MAX_LAUNCHES_PER_WINDOW) return + const launchKey = `${origin}|${target.scheme}` + if (context.userActivated && !context.agentDrives) { + if (target.scheme === 'mailto' || alwaysAllowedLaunches.has(launchKey)) { + attempts.push(Date.now()) + launchAppLink(target.url) + return + } + const canAlwaysAllow = origin.startsWith('https:') + const held = hold( + contents, + 'external-app', + { kind: 'external-app', origin, appName, canAlwaysAllow }, + `external-app|${launchKey}`, + (response) => { + if (response?.allowed !== true || response.kind !== 'external-app') return + if (response.alwaysAllow && canAlwaysAllow) { + rememberBounded(alwaysAllowedLaunches, launchKey, true) + } + launchAppLink(target.url) + }, + // The prompt answers the user's own click in the page, so it takes the focus that click had. + { focus: true } + ) + if (held) attempts.push(Date.now()) + return + } + const held = hold( + contents, + 'app-launch-notice', + { origin, appName }, + `app-launch-notice|${launchKey}`, + (response) => { + if (response?.allowed === true && response.kind === 'app-launch-notice') { + launchAppLink(target.url) + } + } + ) + if (!held) return + attempts.push(Date.now()) + if (context.agentDrives) { + host.noticeForAgent( + contents, + `The page offered to open ${appName}. Only the user can open it, from the notice in the browser panel.` + ) + } +} diff --git a/apps/desktop/src/main/browser-agent/session.test.ts b/apps/desktop/src/main/browser-agent/session.test.ts index 8f04b61520c..92e4b1ff448 100644 --- a/apps/desktop/src/main/browser-agent/session.test.ts +++ b/apps/desktop/src/main/browser-agent/session.test.ts @@ -12,6 +12,7 @@ vi.mock('node:dns/promises', () => ({ })) import { + app, BrowserWindow, dialog, session as electronSession, @@ -19,6 +20,7 @@ import { shell, WebContentsView, } from 'electron' +import { appLaunchNoticeForContents } from '@/main/browser-agent/page-prompts' import * as panel from '@/main/browser-agent/panel' import { agentAppOrigin, routeAgentNavigation } from '@/main/browser-agent/registry' import * as sessionModule from '@/main/browser-agent/session' @@ -1257,6 +1259,11 @@ describe('browser-agent session', () => { contents.loadURL.mockClear() expect(openHandler({ url: 'file:///etc/passwd' })).toEqual({ action: 'deny' }) expect(contents.loadURL).not.toHaveBeenCalled() + // window.open names no initiating frame, so an app link from it is never offered either. + session.enablePagePrompts() + app.getApplicationNameForProtocol = vi.fn(() => 'Zoom') + expect(openHandler({ url: 'zoommtg://zoom.us/join' })).toEqual({ action: 'deny' }) + expect(appLaunchNoticeForContents(contents as unknown as WebContents)).toBeUndefined() }) it('opens a background-disposition popup by URL and keeps cross-scheme popups denied', () => { @@ -1566,9 +1573,11 @@ describe('browser-agent session', () => { it('leaves nothing of the signed-out user behind in the browser profile', async () => { const clearStorageData = vi.fn(async () => {}) const clearCache = vi.fn(async () => {}) + const clearAuthCache = vi.fn(async () => {}) vi.mocked(electronSession.fromPartition).mockReturnValue({ clearStorageData, clearCache, + clearAuthCache, } as unknown as ReturnType) const { persistence, snapshots } = memoryBrowserPersistence() session = freshSession(win, {}, persistence) @@ -1587,6 +1596,8 @@ describe('browser-agent session', () => { expect(snapshots.get('chat-test')).toMatchObject({ tabs: [], activeIndex: -1 }) expect(clearStorageData).toHaveBeenCalled() expect(clearCache).toHaveBeenCalled() + // Chromium keeps typed HTTP sign-in credentials outside storage and cache. + expect(clearAuthCache).toHaveBeenCalled() }) it('hides the panel when the renderer stops renewing its bounds lease', async () => { diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index f96d0645f04..6131d25a56f 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -56,6 +56,13 @@ import { steppedZoomFactor, } from '@/main/browser-agent/context-menu' import type { BrowserCookieSignal } from '@/main/browser-agent/known-sessions' +import { + cancelPagePrompts, + handleSelectPasskeyAccount, + noteMainFrameNavigation, + resetPagePromptDecisions, + setPagePromptHost, +} from '@/main/browser-agent/page-prompts' import { activatePanelScope, detachIfAttached, @@ -104,6 +111,7 @@ export interface BrowserAppSession { let browserAppSession: BrowserAppSession | undefined const configuredDownloadSessions = new WeakSet() +const configuredPasskeySessions = new WeakSet() const routedNavigations = new WeakMap() class SessionError extends Error {} @@ -121,6 +129,15 @@ export interface AgentTab { pendingMediaPermission?: PendingMediaPermission mediaPermissionGrant?: MediaPermissionGrant lastRealUserGestureAt?: number + /** + * Last real click or keypress in this tab. Unlike the media gesture, navigation keeps it: a + * click on an app link starts a navigation, clearing that one, before the launch request. + */ + lastUserActivationAt?: number + /** When the current document committed; activation before it belonged to another page. */ + documentCommittedAt?: number + /** Last click or keypress the agent dispatched; activation belongs to whoever acted last. */ + lastAgentInputAt?: number /** The tab whose page opened this one; agent work returns there when this tab closes. */ openerTabId?: string /** Closing this tab while active shows its opener, until the user switches tabs (as in Chrome). */ @@ -195,6 +212,8 @@ export interface AgentSessionEvents { onTabThemeChanged: (contents: WebContents, theme: BrowserTheme) => void /** A download started, progressed, or finished inside one chat's browser. */ onDownloadsChanged?: (state: BrowserDownloadsState) => void + /** Something a page tried that the agent should learn from its next result in this chat. */ + onAgentNotice?: (notice: string) => void } /** @@ -226,6 +245,8 @@ const BACKGROUND_TAB_RESTORE_TIMEOUT_MS = 15_000 const FOREGROUND_TAB_RESTORE_TIMEOUT_MS = 20_000 const MEDIA_PERMISSION_GESTURE_WINDOW_MS = 10_000 const MEDIA_PERMISSION_PROMPT_TIMEOUT_MS = 30_000 +/** Chromium's transient user activation lifetime. */ +const USER_ACTIVATION_WINDOW_MS = 5_000 export type BrowserShortcut = | 'focus-omnibox' @@ -337,6 +358,8 @@ interface BrowserScopeState { browserChromeFocused: boolean /** The renderer shows page dialogs, so the user can be asked instead of the shell answering. */ pageDialogsEnabled: boolean + /** The renderer shows sign-in and app-launch prompts; until then the shell refuses them. */ + pagePromptsEnabled: boolean automationActive: boolean automationNeedsAttention: boolean /** @@ -371,6 +394,7 @@ function createBrowserScopeState(): BrowserScopeState { focusedBrowserClearTimer: null, browserChromeFocused: false, pageDialogsEnabled: false, + pagePromptsEnabled: false, automationActive: false, automationNeedsAttention: false, findingTabId: null, @@ -1105,6 +1129,47 @@ export function initSession( getMainWindow = mainWindowProvider browserSessionPersistence = persistence ?? null browserDownloadSettings = downloadSettings ?? null + setPagePromptHost({ + tabContext: (contents) => { + const scoped = scopedTabForContents(contents) + if (!scoped) return null + const { scopeId, tab } = scoped + const onScreen = tabIsOnScreen(scopeId, tab) + const agentActedLast = + isDispatchingAgentInput(contents) || + (tab.lastAgentInputAt !== undefined && + tab.lastAgentInputAt >= (tab.lastUserActivationAt ?? 0) && + withinWindow(tab.lastAgentInputAt, USER_ACTIVATION_WINDOW_MS)) + const userAt = tab.lastUserActivationAt + return { + scopeId, + promptsEnabled: browserScopeStates.get(scopeId)?.pagePromptsEnabled === true, + onScreen, + userActivated: + onScreen && + userAt !== undefined && + userAt > Math.max(tab.documentCommittedAt ?? 0, tab.lastAgentInputAt ?? 0) && + withinWindow(userAt, USER_ACTIVATION_WINDOW_MS), + agentActedLast, + agentDrives: withBrowserScope(scopeId, () => agentOwnsPopupFrom(contents)), + } + }, + publish: (contents) => { + const scopeId = browserScopeIdForContents(contents) + if (scopeId) withBrowserScope(scopeId, () => events?.onPageStateChanged(contents)) + }, + focus: (contents) => { + const scoped = scopedTabForContents(contents) + const win = panelWindow() + if (scoped && tabIsOnScreen(scoped.scopeId, scoped.tab) && win && !win.isDestroyed()) { + win.webContents.focus() + } + }, + noticeForAgent: (contents, notice) => { + const scopeId = browserScopeIdForContents(contents) + if (scopeId) withBrowserScope(scopeId, () => events?.onAgentNotice?.(notice)) + }, + }) initPanel({ onGeometryChanged: () => events?.onPanelGeometryChanged?.(), getMainWindow: () => getMainWindow(), @@ -1496,16 +1561,28 @@ function scopedTabForContents(contents: WebContents): { scopeId: string; tab: Ag return tab ? { scopeId, tab } : null } -function mediaRequestIsUserInitiated(scopeId: string, tab: AgentTab): boolean { - const win = panelWindow() +function tabIsOnScreen(scopeId: string, tab: AgentTab): boolean { return ( resolveBrowserScopeId(scopeId) === getActiveBrowserScopeId() && browserScopeStates.get(scopeId)?.activeTabId === tab.id && - isPanelVisible() && + isPanelVisible() + ) +} + +/** Whether `at` lies in the last `windowMs`; a clock stepping backwards never passes. */ +function withinWindow(at: number, windowMs: number): boolean { + const elapsed = Date.now() - at + return elapsed >= 0 && elapsed < windowMs +} + +function mediaRequestIsUserInitiated(scopeId: string, tab: AgentTab): boolean { + const win = panelWindow() + return ( + tabIsOnScreen(scopeId, tab) && Boolean(win && !win.isDestroyed() && win.isFocused()) && tab.view.webContents.isFocused() && typeof tab.lastRealUserGestureAt === 'number' && - Date.now() - tab.lastRealUserGestureAt <= MEDIA_PERMISSION_GESTURE_WINDOW_MS + withinWindow(tab.lastRealUserGestureAt, MEDIA_PERMISSION_GESTURE_WINDOW_MS) ) } @@ -1694,6 +1771,7 @@ function configureAgentPartition(ses: Session): void { if (configuredPartitions.has(ses)) return configuredPartitions.add(ses) keepSessionCookiesAcrossRestarts(ses) + configureBrowserPasskeys(ses) ses.setPermissionRequestHandler(browserPermissions.request) ses.setPermissionCheckHandler(browserPermissions.check) ses.webRequest.onBeforeRequest((details, callback) => { @@ -1702,6 +1780,16 @@ function configureAgentPartition(ses: Session): void { configureBrowserDownloads(ses) } +/** + * Passkey account choice for browser tabs. The borrowed app session gets the + * listener too, so the handler cancels requests from any non-browser contents. + */ +function configureBrowserPasskeys(ses: Session): void { + if (configuredPasskeySessions.has(ses)) return + configuredPasskeySessions.add(ses) + ses.on('select-webauthn-account', handleSelectPasskeyAccount) +} + /** Borrowing app authentication must not replace its permission or request handlers. */ function configureBrowserDownloads(ses: Session): void { if (configuredDownloadSessions.has(ses)) return @@ -1936,6 +2024,11 @@ export function enablePageDialogs(): void { currentScope.pageDialogsEnabled = true } +/** Lets this scope's renderer show sign-in and app-launch prompts from now on. */ +export function enablePagePrompts(): void { + currentScope.pagePromptsEnabled = true +} + /** * Whether a dialog on this tab is the user's to answer: the renderer can show * it, the page is on screen, and it is the user's page rather than the agent's. @@ -2098,6 +2191,7 @@ export function recordPageLoadFailure( contents: WebContents, issue: Extract ): void { + noteMainFrameNavigation(contents, null) if ((issue.code === -2 || issue.code === -3) && routedNavigations.has(contents)) return const tab = tabForContents(contents) if (!tab) return @@ -2542,8 +2636,10 @@ function initializeTabView( view.setBackgroundColor(browserBackgroundColor()) const contents = view.webContents registerAgentWebContents(contents, appOrigin, browserPermissions) - if (appOrigin) configureBrowserDownloads(contents.session) - else configureAgentPartition(contents.session) + if (appOrigin) { + configureBrowserDownloads(contents.session) + configureBrowserPasskeys(contents.session) + } else configureAgentPartition(contents.session) const routeNavigation = (url: string, method: string): boolean => { if (!/^https?:\/\//i.test(url) || !browserAppSession) return false const wantsAppSession = isAppOrigin(url, browserAppSession.origin) @@ -2637,17 +2733,19 @@ function initializeTabView( contents.on( 'before-mouse-event', bindToBrowserScope(scopeId, (_event, mouse) => { - if ( - isDispatchingAgentInput(contents) || - !['mouseDown', 'contextMenu', 'mouseWheel'].includes(mouse.type) - ) { + if (!['mouseDown', 'contextMenu', 'mouseWheel'].includes(mouse.type)) return + const tab = tabs.find((entry) => entry.view.webContents === contents) + if (isDispatchingAgentInput(contents)) { + if (tab && mouse.type === 'mouseDown') tab.lastAgentInputAt = Date.now() return } - const tab = tabs.find((entry) => entry.view.webContents === contents) if (tab && tab.id === currentScope.automationTabId) noteUserIntervention(tab.id) if (tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true - if (mouse.type === 'mouseDown') tab.lastRealUserGestureAt = Date.now() + if (mouse.type === 'mouseDown') { + tab.lastRealUserGestureAt = Date.now() + tab.lastUserActivationAt = tab.lastRealUserGestureAt + } } }) ) @@ -2732,6 +2830,7 @@ function initializeTabView( if (!tab) return tab.nativePageDialogOpen = false invalidatePageDialog(tab) + cancelPagePrompts(contents) if (tab.recoveringUnresponsive) { tab.recoveringUnresponsive = false contents.reload() @@ -2777,6 +2876,7 @@ function initializeTabView( bindToBrowserScope(scopeId, (event, input) => { const tab = tabs.find((entry) => entry.view === view) const agentInput = isDispatchingAgentInput(contents) + if (agentInput && input.type === 'keyDown' && tab) tab.lastAgentInputAt = Date.now() if ( !agentInput && input.type === 'keyDown' && @@ -2787,7 +2887,10 @@ function initializeTabView( } if (!agentInput && tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true - if (input.type === 'keyDown' && !input.isAutoRepeat) tab.lastRealUserGestureAt = Date.now() + if (input.type === 'keyDown' && !input.isAutoRepeat) { + tab.lastRealUserGestureAt = Date.now() + tab.lastUserActivationAt = tab.lastRealUserGestureAt + } } const shortcut = browserShortcutForInput(input) if (!shortcut) return @@ -2869,14 +2972,27 @@ function initializeTabView( 'did-navigate', bindToBrowserScope(scopeId, () => { const tab = tabs.find((entry) => entry.view.webContents === contents) - if (tab) tab.pendingRestoreUrl = undefined + if (tab) { + tab.pendingRestoreUrl = undefined + tab.documentCommittedAt = Date.now() + } + noteMainFrameNavigation(contents, null) persistBrowserSession() }) ) + contents.on('did-redirect-navigation', (details) => { + if (details.isMainFrame && !details.isSameDocument) + noteMainFrameNavigation(contents, details.url) + }) + contents.on( 'did-start-navigation', bindToBrowserScope(scopeId, (details) => { if (!details.isMainFrame) return + if (!details.isSameDocument) { + cancelPagePrompts(contents) + noteMainFrameNavigation(contents, details.url) + } const tab = tabForContents(contents) if (!tab) return invalidatePageDialog(tab) @@ -2921,6 +3037,7 @@ function initializeTabView( // the view's contents, so remove it without touching them. const tab = tabs.find((entry) => entry.view === view) if (tab) removeTab(tab, null) + cancelPagePrompts(contents) events?.onTabClosed(contents) }) ) @@ -4016,6 +4133,17 @@ export async function clearProfileStorage(): Promise { // allowlist would silently miss whatever Chromium adds next. await ses.clearStorageData() await ses.clearCache() + await clearBrowserSignIns(ses) +} + +/** + * Sign-out forgets every HTTP credential Chromium cached for browser tabs, in the + * borrowed app session too, and the remembered certificate and app-launch choices. + */ +async function clearBrowserSignIns(agentSession: Session): Promise { + resetPagePromptDecisions() + await agentSession.clearAuthCache() + await browserAppSession?.session.clearAuthCache() } /** @@ -4049,6 +4177,10 @@ export async function clearAgentData(kinds: readonly BrowserDataKind[]): Promise if (storages.length > 0) { await ses.clearStorageData({ storages } as Parameters[0]) } + if (kinds.includes('cookies')) { + await ses.clearAuthCache() + resetPagePromptDecisions(ses) + } if (kinds.includes('cache')) { await ses.clearCache() // Resolved-host verdicts are a cache too, and a user clearing the cache diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 71a8d3a3c1a..b88167462ac 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -1320,6 +1320,9 @@ export function registerIpcHandlers(deps: IpcDeps): void { needsUserActivation: ([action]) => { if (!isRecordLike(action)) return false if (action.action === 'navigate') return true + if (action.action === 'respond-prompt') { + return isRecordLike(action.response) && action.response.allowed === true + } return action.action === 'respond-media-permission' && action.allowed === true }, handler: (sender, action, rawScope) => { diff --git a/apps/desktop/src/main/navigation.test.ts b/apps/desktop/src/main/navigation.test.ts index 9503a573056..bb4c69a2d17 100644 --- a/apps/desktop/src/main/navigation.test.ts +++ b/apps/desktop/src/main/navigation.test.ts @@ -4,6 +4,7 @@ vi.mock('electron', () => import('@/test/electron-mock')) import { shell } from 'electron' import { + appLinkTarget, classifyNavigation, classifyWindowOpen, isSafeExternalUrl, @@ -138,3 +139,48 @@ describe('openExternalSafe', () => { expect(shell.openExternal).not.toHaveBeenCalled() }) }) + +describe('appLinkTarget', () => { + it.each([ + 'file:///etc/passwd', + 'javascript:alert(1)', + 'ms-msdt:id', + 'search-ms:query=x', + 'ms-cxh-full://0', + 'local+app://x', + 'jnlp://evil.example/app.jnlp', + 'x-man-page://ls', + 'c:/Windows/System32/calc.exe', + 'https://example.com', + ])('refuses %s', (url) => { + expect(appLinkTarget(url)).toBeNull() + }) + + it('refuses credentials, control characters, oversize URLs, and unsafe mail links', () => { + expect(appLinkTarget('zoommtg://user:pass@zoom.us/join')).toBeNull() + expect(appLinkTarget('zoommtg://zoom.us/join?x=1\n')).toBeNull() + expect(appLinkTarget('zoommtg://zoom.us/join?x= 1')).toBeNull() + expect(appLinkTarget(`zoommtg://zoom.us/join?x=${'a'.repeat(2_048)}`)).toBeNull() + expect(appLinkTarget('mailto:a@example.com?attach=/etc/passwd')).toBeNull() + }) + + it('hands the operating system the parser-rebuilt URL of an app link', () => { + expect(appLinkTarget('ZoomMtg://zoom.us/join?confno=123&pwd=abc')).toEqual({ + url: 'zoommtg://zoom.us/join?confno=123&pwd=abc', + scheme: 'zoommtg', + }) + expect(appLinkTarget('mailto:a@example.com?subject=Hi')?.scheme).toBe('mailto') + }) + + it('opens an app link only on the app-link path, and only in its rebuilt form', async () => { + vi.mocked(shell.openExternal).mockClear() + await expect(openExternalSafe('zoommtg://zoom.us/join')).resolves.toBe(false) + await expect( + openExternalSafe('ZoomMtg://zoom.us/join', false, { appLink: true }) + ).resolves.toBe(false) + await expect( + openExternalSafe('zoommtg://zoom.us/join', false, { appLink: true }) + ).resolves.toBe(true) + expect(shell.openExternal).toHaveBeenCalledExactlyOnceWith('zoommtg://zoom.us/join') + }) +}) diff --git a/apps/desktop/src/main/navigation.ts b/apps/desktop/src/main/navigation.ts index 530eb49d6a2..5d69e18e4d6 100644 --- a/apps/desktop/src/main/navigation.ts +++ b/apps/desktop/src/main/navigation.ts @@ -267,11 +267,111 @@ export function isSafeExternalUrl(raw: string, allowHttpLocalhost = false): bool } /** - * Opens a URL in the system browser, or a `mailto:` in the system mail client, after validation. - * Every openExternal in the app goes through here — menu items, IPC, and navigation policy. + * Schemes that reach local files, network shares, script, browser internals, + * or Windows/macOS handlers with a record of remote-code abuse. A browser page + * never opens them, even with a registered handler and the user's click. */ -export async function openExternalSafe(raw: string, allowHttpLocalhost = false): Promise { - if (!isSafeExternalUrl(raw, allowHttpLocalhost) && !isSafeMailtoUrl(raw)) { +const DENIED_APP_LINK_SCHEMES: ReadonlySet = new Set([ + 'about', + 'afp', + 'applescript', + 'blob', + 'chrome', + 'chrome-extension', + 'chrome-untrusted', + 'data', + 'dav', + 'devtools', + 'disk', + 'disks', + 'file', + 'filesystem', + 'ftp', + 'hcp', + 'help', + 'ie.http', + 'its', + 'jar', + 'javascript', + 'jnlp', + 'localfs', + 'mk', + 'ms-appinstaller', + 'ms-excel', + 'ms-help', + 'ms-its', + 'ms-msdt', + 'ms-officecmd', + 'ms-powerpoint', + 'ms-settings', + 'ms-word', + 'nfs', + 'nntp', + 'res', + 'search', + 'search-ms', + 'shell', + 'sim-shell', + 'smb', + 'vbscript', + 'view-source', + 'vnd.ms.radio', + 'webdav', + 'x-man-page', +]) +const DENIED_APP_LINK_SCHEME_PREFIXES = ['ms-cxh', 'local+'] as const +const MAX_APP_LINK_LENGTH = 2_048 +/** C0 controls, space, DEL, and C1 controls: never valid in a URL handed to the OS. */ +const URL_CONTROL_CHARACTERS = /[\u0000-\u0020\u007f-\u009f]/ + +function isDeniedAppLinkScheme(scheme: string): boolean { + return ( + scheme.length < 2 || + scheme === 'http' || + scheme === 'https' || + DENIED_APP_LINK_SCHEMES.has(scheme) || + DENIED_APP_LINK_SCHEME_PREFIXES.some((prefix) => scheme.startsWith(prefix)) + ) +} + +/** + * The link to another app a browser page may hand the OS, rebuilt by the URL + * parser, or null when the scheme is denied or the URL is malformed, oversized, + * carries credentials or control characters, or is an unsafe mail link. + */ +export function appLinkTarget(raw: unknown): { url: string; scheme: string } | null { + if (typeof raw !== 'string' || raw.length > MAX_APP_LINK_LENGTH) return null + if (URL_CONTROL_CHARACTERS.test(raw)) return null + let url: URL + try { + url = new URL(raw) + } catch { + return null + } + const scheme = url.protocol.slice(0, -1).toLowerCase() + if (!/^[a-z][a-z0-9+.-]*$/.test(scheme) || isDeniedAppLinkScheme(scheme)) return null + if (url.username || url.password) return null + if (scheme === 'mailto' && !isSafeMailtoUrl(raw)) return null + const rebuilt = url.href + if (rebuilt.length > MAX_APP_LINK_LENGTH || URL_CONTROL_CHARACTERS.test(rebuilt)) return null + return { url: rebuilt, scheme } +} + +/** + * Opens a URL in the system browser, a `mailto:` in the system mail client, or, with + * `appLink`, a validated link to another app, after validation. Every openExternal in the + * app goes through here — menu items, IPC, navigation policy, and browser-tab app links. + */ +export async function openExternalSafe( + raw: string, + allowHttpLocalhost = false, + { appLink = false }: { appLink?: boolean } = {} +): Promise { + if ( + !isSafeExternalUrl(raw, allowHttpLocalhost) && + !isSafeMailtoUrl(raw) && + !(appLink && appLinkTarget(raw)?.url === raw) + ) { logger.warn('Blocked unsafe external URL', { url: scrubUrl(raw) }) return false } diff --git a/apps/desktop/src/main/security-guards.ts b/apps/desktop/src/main/security-guards.ts index 93d719e5e9b..e69dbb8c669 100644 --- a/apps/desktop/src/main/security-guards.ts +++ b/apps/desktop/src/main/security-guards.ts @@ -1,6 +1,11 @@ import { createLogger } from '@sim/logger' -import type { WebContents } from 'electron' +import type { WebContents, WebFrameMain } from 'electron' import { app } from 'electron' +import { + handleBrowserLogin, + handleSelectClientCertificate, + requestExternalLaunch, +} from '@/main/browser-agent/page-prompts' import { isAgentWebContents } from '@/main/browser-agent/registry' import { trackInputActivity } from '@/main/input-activity' import { classifyNavigation, openExternalSafe } from '@/main/navigation' @@ -21,14 +26,25 @@ export interface GuardDeps { * Applies the top-level navigation policy to both will-navigate and * will-redirect — OAuth chains can be redirect-injected, so redirects get the * same classifier as direct navigations. + * + * Agent-browser tabs never navigate off http(s). A link to another app (Zoom, + * Teams, mail) becomes a launch request the user decides only when the tab's + * own top-level document initiated it: a navigation a frame initiated, even + * one targeting the top level, is refused, as is the `openExternal` + * permission that backs frame launches. */ export function attachNavigationGuards(contents: WebContents, deps: GuardDeps): void { - const handle = (event: { preventDefault(): void }, url: string) => { + const handle = ( + event: { preventDefault(): void; isMainFrame?: boolean; initiator?: WebFrameMain | null }, + url: string + ) => { if (isAgentWebContents(contents)) { const hasAllowedAgentBrowserScheme = /^https?:/i.test(url) if (!hasAllowedAgentBrowserScheme) { event.preventDefault() - logger.warn('Denied non-http navigation in agent browser', { url: scrubUrl(url) }) + if (event.isMainFrame === true) { + requestExternalLaunch(contents, url, event.initiator) + } else logger.warn('Denied non-http navigation in agent browser', { url: scrubUrl(url) }) } return } @@ -69,7 +85,9 @@ export function attachNavigationGuards(contents: WebContents, deps: GuardDeps): * Agent-browser tabs swap the classifier for a free-http(s) policy (their job * is browsing arbitrary sites, isolated on their own partition). TLS errors * are always fatal when packaged; self-host private CAs must be - * system-trusted rather than bypassed in-app. + * system-trusted rather than bypassed in-app. HTTP authentication and client + * certificate requests from agent-browser tabs wait on the user; every other + * contents keeps Electron's defaults. */ export function installGlobalGuards(deps: GuardDeps): void { app.on('web-contents-created', (_event, contents) => { @@ -87,6 +105,9 @@ export function installGlobalGuards(deps: GuardDeps): void { attachNavigationGuards(contents, deps) }) + app.on('login', handleBrowserLogin) + app.on('select-client-certificate', handleSelectClientCertificate) + app.on('certificate-error', (event, _webContents, url, error, _certificate, callback) => { event.preventDefault() callback(false) diff --git a/apps/desktop/src/test/electron-mock.ts b/apps/desktop/src/test/electron-mock.ts index c0f180897d0..105aa55f117 100644 --- a/apps/desktop/src/test/electron-mock.ts +++ b/apps/desktop/src/test/electron-mock.ts @@ -107,6 +107,11 @@ export const session = { fromPartition: vi.fn(), } +/** Resolves a frame to its WebContents; tests return the contents they registered. */ +export const webContents = { + fromFrame: vi.fn(), +} + export const protocol = { registerSchemesAsPrivileged: vi.fn(), handle: vi.fn(), diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-prompt.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-prompt.tsx new file mode 100644 index 00000000000..fd0d823e864 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-prompt.tsx @@ -0,0 +1,347 @@ +'use client' + +import { type ReactNode, useEffect, useState } from 'react' +import type { + BrowserAppLaunchNotice, + BrowserClientCertificatePrompt, + BrowserExternalAppPrompt, + BrowserHttpAuthPrompt, + BrowserPagePrompt, + BrowserPagePromptResponse, + BrowserPasskeyPrompt, +} from '@sim/browser-protocol' +import { + Banner, + Checkbox, + Chip, + ChipConfirmModal, + ChipModal, + ChipModalBody, + ChipModalDescription, + ChipModalField, + ChipModalFooter, + ChipModalHeader, + ChipSelect, + type ChipSelectOption, + Label, +} from '@sim/emcn' +import { X } from '@sim/emcn/icons' + +/** Keeps a click or keystroke aimed at the page from answering a prompt as it appears. */ +const ACTIVATION_DELAY_MS = 500 + +type RespondToPrompt = (requestId: string, response: BrowserPagePromptResponse) => void + +/** + * False until the prompt has been visible long enough to be read. Hiding it + * disarms it again, so a prompt shown after waiting behind another dialog + * starts its delay only once it appears. + */ +function useActivationDelay(visible: boolean): boolean { + const [armed, setArmed] = useState(false) + useEffect(() => { + if (!visible) { + setArmed(false) + return + } + const timer = setTimeout(() => setArmed(true), ACTIVATION_DELAY_MS) + return () => clearTimeout(timer) + }, [visible]) + return armed +} + +interface PromptModalProps { + prompt: Prompt + open: boolean + onRespond: RespondToPrompt +} + +/** + * A page raised this, so it never takes focus, and signing in takes a click on + * Sign in: Enter in a field never submits credentials typed for the page. + */ +function HttpAuthPromptModal({ prompt, open, onRespond }: PromptModalProps) { + const armed = useActivationDelay(open) + const [username, setUsername] = useState('') + const [password, setPassword] = useState('') + const cancel = () => onRespond(prompt.requestId, { allowed: false }) + const submit = () => + onRespond(prompt.requestId, { allowed: true, kind: 'http-auth', username, password }) + return ( + !next && cancel()} + srTitle='Sign in' + size='sm' + focusOnOpen={false} + > + Sign in + + + {prompt.proxy ? `The proxy ${prompt.origin}` : prompt.origin} requires a username and + password.{prompt.insecure ? ' Your connection to this site is not private.' : ''} + + + + + + + ) +} + +interface ChoicePromptModalProps { + open: boolean + title: string + description: string + fieldTitle: string + placeholder: string + options: ChipSelectOption[] + hintFor?: (choiceId: string) => string | undefined + cancelLabel?: string + onCancel: () => void + onChoose: (choiceId: string) => void +} + +/** + * Picks one of the page's offered identities. It never takes focus, nothing is + * chosen up front, Enter dismisses, and Continue waits out the activation delay, + * so a keystroke meant for the page can never send an identity. + */ +function ChoicePromptModal({ + open, + title, + description, + fieldTitle, + placeholder, + options, + hintFor, + cancelLabel, + onCancel, + onChoose, +}: ChoicePromptModalProps) { + const armed = useActivationDelay(open) + const [choiceId, setChoiceId] = useState('') + return ( + !next && onCancel()} + srTitle={title} + size='sm' + focusOnOpen={false} + > + {title} + + {description} + + + + + onChoose(choiceId), + disabled: !armed || !choiceId, + }} + /> + + ) +} + +function passkeyAccountLabel(account: BrowserPasskeyPrompt['accounts'][number]): string { + const { displayName, name } = account + if (displayName && name && displayName !== name) return `${displayName} (${name})` + return displayName || name || 'Unnamed account' +} + +function PasskeyPromptModal({ prompt, open, onRespond }: PromptModalProps) { + return ( + ({ + label: passkeyAccountLabel(account), + value: account.id, + }))} + onCancel={() => onRespond(prompt.requestId, { allowed: false })} + onChoose={(choiceId) => + onRespond(prompt.requestId, { allowed: true, kind: 'passkey-account', choiceId }) + } + /> + ) +} + +function ClientCertificatePromptModal({ + prompt, + open, + onRespond, +}: PromptModalProps) { + const hintFor = (choiceId: string) => { + const chosen = prompt.certificates.find((certificate) => certificate.id === choiceId) + if (!chosen) return undefined + return `Issued by ${chosen.issuer || 'unknown issuer'} · Expires ${new Date(chosen.expiresAt).toLocaleDateString()}` + } + return ( + ({ + label: certificate.subject || certificate.id, + value: certificate.id, + }))} + hintFor={hintFor} + cancelLabel="Don't send" + onCancel={() => onRespond(prompt.requestId, { allowed: false })} + onChoose={(choiceId) => + onRespond(prompt.requestId, { allowed: true, kind: 'client-certificate', choiceId }) + } + /> + ) +} + +function ExternalAppPromptModal({ + prompt, + open, + onRespond, +}: PromptModalProps) { + const armed = useActivationDelay(open) + const [alwaysAllow, setAlwaysAllow] = useState(false) + const checkboxId = `browser-always-open-${prompt.requestId}` + return ( + !next && onRespond(prompt.requestId, { allowed: false })} + title={`Open ${prompt.appName}?`} + text={[{ text: prompt.origin, bold: true }, ' wants to open this app.']} + defaultAction='dismiss' + confirm={{ + label: 'Open', + variant: 'primary', + disabled: !armed, + onClick: () => + onRespond(prompt.requestId, { + allowed: true, + kind: 'external-app', + alwaysAllow: prompt.canAlwaysAllow && alwaysAllow, + }), + }} + > + {prompt.canAlwaysAllow && ( +
+ setAlwaysAllow(checked === true)} + /> + +
+ )} + + ) +} + +type PromptModal = (props: PromptModalProps) => ReactNode + +const PROMPT_MODALS: { + [Kind in BrowserPagePrompt['kind']]: ( + props: PromptModalProps> + ) => ReactNode +} = { + 'http-auth': HttpAuthPromptModal, + 'passkey-account': PasskeyPromptModal, + 'client-certificate': ClientCertificatePromptModal, + 'external-app': ExternalAppPromptModal, +} + +interface BrowserPagePromptModalProps { + prompt: BrowserPagePrompt | undefined + open: boolean + onRespond: RespondToPrompt +} + +/** The sign-in or app-launch decision the visible page is waiting on. */ +export function BrowserPagePromptModal({ prompt, open, onRespond }: BrowserPagePromptModalProps) { + if (!prompt) return null + const Modal = PROMPT_MODALS[prompt.kind] as PromptModal | undefined + if (!Modal) return null + return +} + +interface BrowserAppLaunchNoticeBarProps { + notice: BrowserAppLaunchNotice + onRespond: RespondToPrompt +} + +/** A page's unprompted request to open another app; nothing opens until the user chooses Open. */ +export function BrowserAppLaunchNoticeBar({ notice, onRespond }: BrowserAppLaunchNoticeBarProps) { + const armed = useActivationDelay(true) + return ( + +
+

+ {notice.origin} wants to open {notice.appName}. +

+
+ + onRespond(notice.requestId, { allowed: true, kind: 'app-launch-notice' }) + } + > + Open + + onRespond(notice.requestId, { allowed: false })} + /> +
+
+ + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx index 511e39d9825..4fae5497216 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx @@ -4,6 +4,7 @@ import { type CSSProperties, useCallback, useEffect, useMemo, useRef, useState } import type { BrowserMediaPermissionRequest, BrowserOmniboxFocusMode, + BrowserPagePromptResponse, BrowserPanelAnchor, BrowserPanelBounds, BrowserPanelSnapshot, @@ -75,6 +76,10 @@ import { BrowserFindBar } from '@/app/workspace/[workspaceId]/home/components/mo import { BrowserLoadingBar } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar' import { BrowserPageDialogModal } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-dialog' import { BrowserPageIssueView } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-issue' +import { + BrowserAppLaunchNoticeBar, + BrowserPagePromptModal, +} from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-prompt' import { type BrowserPanelOverlay, type BrowserPanelOverlayController, @@ -453,6 +458,7 @@ export function BrowserSession({ const mediaPermissionRequest = pageState?.mediaPermissionRequest const sitePermissionRequest = pageState?.sitePermissionRequest const permissionRequest = sitePermissionRequest ?? mediaPermissionRequest + const pagePrompt = pageState?.prompt const panelRef = useRef(null) const hostRef = useRef(null) // Lets the occlusion handshake reject a capture taken before a modal's @@ -538,12 +544,14 @@ export function BrowserSession({ ) }, [permissionRequest, respondToPermission, visible]) - // The shell answers page dialogs itself until this renderer declares it shows them. + // The shell answers page dialogs and refuses page prompts until this renderer declares it shows them. useEffect(() => { - sendBrowserPanelAction('enable-page-dialogs', {}, scopeId) + sendBrowserPanelAction('enable-page-dialogs', { pagePrompts: true }, scopeId) }, [scopeId]) const answerPageDialog = (requestId: string, accept: boolean) => sendBrowserPanelAction('respond-dialog', { requestId, allowed: accept }, scopeId) + const respondToPagePrompt = (requestId: string, response: BrowserPagePromptResponse) => + sendBrowserPanelAction('respond-prompt', { requestId, response }, scopeId) const permissionModalOpen = shouldShowBrowserPermissionRequest( permissionRequest?.requestId, @@ -971,7 +979,7 @@ export function BrowserSession({ // Keep the page's exact captured frame underneath it while it is open so // pointer events reach the Sim popover instead of the WebContentsView. useEffect(() => { - if (mediaPermissionRequest || sitePermissionRequest) { + if (mediaPermissionRequest || sitePermissionRequest || pagePrompt) { void closeOverlay('suggestions') return } @@ -988,6 +996,7 @@ export function BrowserSession({ closeOverlay, hasRendererPage, mediaPermissionRequest, + pagePrompt, requestOverlay, sitePermissionRequest, suggestions.length, @@ -1350,6 +1359,13 @@ export function BrowserSession({ )} + {pageState?.appLaunchNotice && ( + + )} {/* Host area: the real page is overlaid exactly on this rect. */} @@ -1405,6 +1421,11 @@ export function BrowserSession({ open={visible && !permissionModalOpen} onAnswer={answerPageDialog} /> + ) } diff --git a/apps/sim/stores/browser-session/store.ts b/apps/sim/stores/browser-session/store.ts index 4192dc679c2..2730cb1fdf5 100644 --- a/apps/sim/stores/browser-session/store.ts +++ b/apps/sim/stores/browser-session/store.ts @@ -187,6 +187,8 @@ function pageStateEqual(a: BrowserPageState | null, b: BrowserPageState | null): a.canGoBack === b.canGoBack && a.canGoForward === b.canGoForward && a.dialog?.requestId === b.dialog?.requestId && + a.prompt?.requestId === b.prompt?.requestId && + a.appLaunchNotice?.requestId === b.appLaunchNotice?.requestId && pageIssueEqual(a.issue, b.issue) && mediaPermissionRequestEqual(a.mediaPermissionRequest, b.mediaPermissionRequest) && sitePermissionRequestEqual(a.sitePermissionRequest, b.sitePermissionRequest) diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index 874bdca7a26..962192bbb82 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -295,11 +295,19 @@ export interface BrowserPanelAction { /** * Declares that this renderer shows page dialogs. Until it arrives, the * shell keeps answering them itself, so an older renderer never leaves a - * page blocked on a dialog nobody can see. + * page blocked on a dialog nobody can see. With `pagePrompts`, it also + * declares {@link BrowserPageState.prompt} and + * {@link BrowserPageState.appLaunchNotice}; without it the shell cancels + * those requests at once. */ | 'enable-page-dialogs' /** Answers {@link BrowserPageState.dialog}; `allowed` is Leave. */ | 'respond-dialog' + /** + * Answers {@link BrowserPageState.prompt} or {@link BrowserPageState.appLaunchNotice} + * with `response`. Installed shells without page prompts never emit one to answer. + */ + | 'respond-prompt' /** Absolute URL for `navigate` (typed into the panel's URL bar). */ url?: string /** Stable tab id for `switch-tab` and `close-tab`. */ @@ -316,6 +324,100 @@ export interface BrowserPanelAction { requestId?: string /** User decision for a permission or dialog response. */ allowed?: boolean + /** `enable-page-dialogs` only: this renderer also shows page prompts. */ + pagePrompts?: boolean + /** `respond-prompt` only: the user's answer. */ + response?: BrowserPagePromptResponse +} + +/** + * The user's answer to a {@link BrowserPagePrompt} or {@link BrowserAppLaunchNotice}. + * Anything but an accepted answer of the request's own kind is a refusal. + */ +export type BrowserPagePromptResponse = + | { allowed: false } + | { allowed: true; kind: 'http-auth'; username: string; password: string } + | { allowed: true; kind: 'passkey-account' | 'client-certificate'; choiceId: string } + /** `alwaysAllow` stops asking for this site and app until restart. */ + | { allowed: true; kind: 'external-app'; alwaysAllow: boolean } + | { allowed: true; kind: 'app-launch-notice' } + +/** + * Credentials requested by a server for a top-level page, or by a proxy. + * Shown only for the tab whose navigation was challenged. + */ +export interface BrowserHttpAuthPrompt { + requestId: string + kind: 'http-auth' + /** Scheme, host, and port of the server, or host and port of a proxy. */ + origin: string + proxy: boolean + /** The credentials would cross an unencrypted connection. */ + insecure: boolean + /** The previous credentials were rejected. */ + retry: boolean +} + +interface BrowserPasskeyAccount { + /** The credential id the shell resolves the choice by. */ + id: string + name: string + displayName: string +} + +/** A passkey sign-in asking which of the authenticator's accounts to use. */ +export interface BrowserPasskeyPrompt { + requestId: string + kind: 'passkey-account' + relyingPartyId: string + accounts: BrowserPasskeyAccount[] +} + +interface BrowserClientCertificate { + /** Certificate fingerprint; the shell resolves the choice by it. */ + id: string + subject: string + issuer: string + /** Expiry, in milliseconds since the epoch. */ + expiresAt: number +} + +/** A site asking which client certificate, if any, identifies the user. */ +export interface BrowserClientCertificatePrompt { + requestId: string + kind: 'client-certificate' + /** Host and port of the server asking. */ + host: string + certificates: BrowserClientCertificate[] +} + +/** A page the user just interacted with asks to open another app. */ +export interface BrowserExternalAppPrompt { + requestId: string + kind: 'external-app' + /** Origin of the page that asked. */ + origin: string + /** The app the operating system would open. */ + appName: string + /** Only secure origins may be remembered. */ + canAlwaysAllow: boolean +} + +/** A decision the page is waiting on, owned by the user and never by the agent. */ +export type BrowserPagePrompt = + | BrowserHttpAuthPrompt + | BrowserPasskeyPrompt + | BrowserClientCertificatePrompt + | BrowserExternalAppPrompt + +/** + * A page asked to open another app without a recent click in it. Shown + * passively; nothing opens unless the user chooses Open. + */ +export interface BrowserAppLaunchNotice { + requestId: string + origin: string + appName: string } /** A leave-site decision awaiting the user's answer in browser chrome. */ @@ -360,6 +462,10 @@ export interface BrowserPageState { sitePermissionRequest?: BrowserSitePermissionRequest /** Page dialog awaiting the user's answer. Optional for older shells. */ dialog?: BrowserPageDialog + /** Sign-in or app-launch decision awaiting the user. Optional for older shells. */ + prompt?: BrowserPagePrompt + /** Passive offer to open another app. Optional for older shells. */ + appLaunchNotice?: BrowserAppLaunchNotice } /** A recoverable top-level page problem rendered by Sim instead of a blank native view. */ diff --git a/packages/emcn/src/components/chip-modal/chip-modal.tsx b/packages/emcn/src/components/chip-modal/chip-modal.tsx index 4aee794b535..09222492316 100644 --- a/packages/emcn/src/components/chip-modal/chip-modal.tsx +++ b/packages/emcn/src/components/chip-modal/chip-modal.tsx @@ -156,6 +156,10 @@ function focusChipModalDefaultAction(event: Event): void { focusChipModalContent(event.currentTarget as HTMLElement | null) } +function keepFocusOnOpen(event: Event): void { + event.preventDefault() +} + /** * Supplies implicit submission for single-line custom inputs that are not a * {@link ChipModalField}. Canonical fields handle Enter at the control so they @@ -216,6 +220,13 @@ interface ChipModalProps { * @default false */ dismissDisabled?: boolean + /** + * Whether opening moves focus into the modal. Set `false` for a modal the + * user did not ask for, such as a prompt a web page raised: typing meant for + * whatever had focus never lands in it, and the user clicks into it instead. + * @default true + */ + focusOnOpen?: boolean children?: React.ReactNode } @@ -276,6 +287,7 @@ function ChipModal({ size = 'md', className, dismissDisabled = false, + focusOnOpen = true, children, 'aria-describedby': ariaDescribedBy, }: ChipModalProps) { @@ -287,7 +299,7 @@ function ChipModal({ srTitle={srTitle} size={size} dismissDisabled={dismissDisabled} - onOpenAutoFocus={focusChipModalDefaultAction} + onOpenAutoFocus={focusOnOpen ? focusChipModalDefaultAction : keepFocusOnOpen} aria-describedby={ariaDescribedBy} > {children}