Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/desktop/e2e/fixtures/updater.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ void app.whenReady().then(async () => {
computerUseApp: null,
browserToolRunning,
agentWorkRunning: false,
downloadCount: 0,
...(await countWindowsWithUnsavedChanges(
hungPage ? [page.webContents, hungPage.webContents] : [page.webContents]
)),
Expand Down
1 change: 1 addition & 0 deletions apps/desktop/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
"@sim/utils": "workspace:*",
"@xterm/headless": "6.0.0",
"electron-updater": "6.8.9",
"lru-cache": "11.5.3",
"micromatch": "4.0.8",
"safe-regex2": "5.1.2",
"tldts": "7.4.18",
Expand Down
20 changes: 18 additions & 2 deletions apps/desktop/src/main/browser-agent/page-prompts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,16 @@ describe('HTTP sign-in', () => {
answer(contents, { allowed: true, kind: 'http-auth', username: 'ada', password: 'pw' })
expect(callback).toHaveBeenCalledExactlyOnceWith('ada', 'pw')
})

it('cancels instead of leaving the page waiting when an accepted answer is malformed', () => {
const contents = agentContents()
noteMainFrameNavigation(contents, url)
const callback = login(contents)
// double-cast-allowed: an IPC payload is untyped; this is a renderer's malformed answer
answer(contents, { allowed: true, kind: 'http-auth' } as unknown as BrowserPagePromptResponse)
expect(callback).toHaveBeenCalledExactlyOnceWith()
expect(pagePromptForContents(contents)).toBeUndefined()
})
})

describe('client certificates', () => {
Expand Down Expand Up @@ -228,13 +238,19 @@ describe('client certificates', () => {
expect(request(agentContents(otherSession), [first]).callback).not.toHaveBeenCalled()
})

it("remembers Don't send, but not a prompt nobody answered", () => {
it("remembers Don't send, but not a closed picker or a prompt nobody answered", () => {
const declined = agentContents()
const refused = request(declined, [first])
answer(declined, { allowed: false })
answer(declined, { allowed: false, remember: true })
expect(refused.callback).toHaveBeenCalledExactlyOnceWith()
expect(request(agentContents(declined.session), [first]).callback).toHaveBeenCalledWith()

const closed = agentContents()
const dismissed = request(closed, [first])
answer(closed, { allowed: false })
expect(dismissed.callback).toHaveBeenCalledExactlyOnceWith()
expect(request(agentContents(closed.session), [first]).callback).not.toHaveBeenCalled()

const ignored = agentContents()
const unanswered = request(ignored, [first])
vi.advanceTimersByTime(120_000)
Expand Down
67 changes: 35 additions & 32 deletions apps/desktop/src/main/browser-agent/page-prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ import type {
WebFrameMain,
} from 'electron'
import { app, webContents as electronWebContents } from 'electron'
import { LRUCache } from 'lru-cache'
import { isAgentWebContents } from '@/main/browser-agent/registry'
import { withinWindow } from '@/main/input-activity'
import { appLinkTarget, openExternalSafe, parseHttpUrl } from '@/main/navigation'

const logger = createLogger('BrowserPagePrompts')
Expand Down Expand Up @@ -92,10 +94,11 @@ let host: PagePromptHost | null = null
const pendingById = new Map<string, PendingRequest>()
const pendingByContents = new WeakMap<WebContents, TabRequests>()
const mainFrameNavigations = new WeakMap<WebContents, string>()
const recentAttemptsByContents = new WeakMap<WebContents, Map<string, number[]>>()
/** `null` remembers an explicit "no certificate" for that host. */
let certificateDecisions = new WeakMap<Session, Map<string, string | null>>()
const alwaysAllowedLaunches = new Map<string, true>()
const recentAttemptsByContents = new WeakMap<WebContents, LRUCache<string, number[]>>()
/** A host's remembered certificate, or `'declined'` for an explicit "Don't send". */
type CertificateDecision = { fingerprint: string } | 'declined'
let certificateDecisions = new WeakMap<Session, LRUCache<string, CertificateDecision>>()
const alwaysAllowedLaunches = new LRUCache<string, true>({ max: MAX_REMEMBERED_DECISIONS })

export function setPagePromptHost(next: PagePromptHost | null): void {
host = next
Expand All @@ -110,27 +113,14 @@ export function resetPagePromptDecisions(session?: Session): void {
alwaysAllowedLaunches.clear()
}

function withinWindow(at: number, windowMs: number): boolean {
const elapsed = Date.now() - at
return elapsed >= 0 && elapsed < windowMs
}

function rememberBounded<Value>(map: Map<string, Value>, key: string, value: Value): void {
map.delete(key)
map.set(key, value)
while (map.size > MAX_REMEMBERED_DECISIONS) {
const oldest = map.keys().next().value
if (oldest === undefined) break
map.delete(oldest)
}
}

/** The live list of this tab's recent attempts under `key`, pruned to the window. */
function recentAttempts(contents: WebContents, key: string, windowMs: number): number[] {
const byKey = recentAttemptsByContents.get(contents) ?? new Map<string, number[]>()
const byKey =
recentAttemptsByContents.get(contents) ??
new LRUCache<string, number[]>({ max: MAX_REMEMBERED_DECISIONS })
recentAttemptsByContents.set(contents, byKey)
const recent = (byKey.get(key) ?? []).filter((at) => withinWindow(at, windowMs))
rememberBounded(byKey, key, recent)
byKey.set(key, recent)
return recent
}

Expand Down Expand Up @@ -247,6 +237,12 @@ function settle(entry: PendingRequest, response: BrowserPagePromptResponse | nul
answer(response)
} catch (error) {
logger.warn('Browser page prompt answer failed', { error: getErrorMessage(error) })
// A malformed response must still release the page, which otherwise waits forever.
if (response !== null) {
try {
answer(null)
} catch {}
}
}
}
}
Expand All @@ -271,8 +267,8 @@ export function cancelPagePrompts(contents: WebContents): void {

/**
* Tracks the URL the tab's top-level navigation is requesting (null once it
* commits or fails), so HTTP authentication can tell its own challenge from a
* frame's.
* commits, fails, or the tab stops loading), so HTTP authentication can tell its
* own challenge from a frame's.
*/
export function noteMainFrameNavigation(contents: WebContents, url: string | null): void {
if (url) mainFrameNavigations.set(contents, url)
Expand Down Expand Up @@ -352,6 +348,8 @@ export function handleBrowserLogin(
if (
response?.allowed === true &&
response.kind === 'http-auth' &&
typeof response.username === 'string' &&
typeof response.password === 'string' &&
response.username.length <= MAX_CREDENTIAL_LENGTH &&
response.password.length <= MAX_CREDENTIAL_LENGTH
) {
Expand Down Expand Up @@ -448,13 +446,15 @@ export function handleSelectClientCertificate(
})
)
if (!requestHost || choices.length === 0) return false
const decisions = certificateDecisions.get(contents.session) ?? new Map<string, string | null>()
const decisions =
certificateDecisions.get(contents.session) ??
new LRUCache<string, CertificateDecision>({ max: MAX_REMEMBERED_DECISIONS })
certificateDecisions.set(contents.session, decisions)
const remembered = decisions.get(requestHost)
if (remembered === null) return false
const rememberedCertificate = certificates.find(
(certificate) => certificate.fingerprint === remembered
)
if (remembered === 'declined') return false
const rememberedCertificate = remembered
? certificates.find((certificate) => certificate.fingerprint === remembered.fingerprint)
: undefined
if (rememberedCertificate) {
answer(rememberedCertificate)
return true
Expand All @@ -466,19 +466,22 @@ export function handleSelectClientCertificate(
`client-certificate|${requestHost}`,
(response) => {
if (response?.allowed === false) {
rememberBounded(decisions, requestHost, null)
// Only an explicit "Don't send" sticks; closing the picker just cancels this request.
if (response.remember === true) decisions.set(requestHost, 'declined')
answer()
return
}
const certificate =
response?.allowed === true && response.kind === 'client-certificate'
response?.allowed === true &&
response.kind === 'client-certificate' &&
typeof response.choiceId === 'string'
? certificates.find((candidate) => candidate.fingerprint === response.choiceId)
: undefined
if (!certificate) {
answer()
return
}
rememberBounded(decisions, requestHost, certificate.fingerprint)
decisions.set(requestHost, { fingerprint: certificate.fingerprint })
answer(certificate)
},
{ joinable: true }
Expand Down Expand Up @@ -568,7 +571,7 @@ export function requestExternalLaunch(
(response) => {
if (response?.allowed !== true || response.kind !== 'external-app') return
if (response.alwaysAllow && canAlwaysAllow) {
rememberBounded(alwaysAllowedLaunches, launchKey, true)
alwaysAllowedLaunches.set(launchKey, true)
}
launchAppLink(target.url)
},
Expand Down
57 changes: 56 additions & 1 deletion apps/desktop/src/main/browser-agent/session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@ import {
shell,
WebContentsView,
} from 'electron'
import { appLaunchNoticeForContents } from '@/main/browser-agent/page-prompts'
import {
appLaunchNoticeForContents,
handleBrowserLogin,
pagePromptForContents,
} from '@/main/browser-agent/page-prompts'
import * as panel from '@/main/browser-agent/panel'
import { agentAppOrigin, routeAgentNavigation } from '@/main/browser-agent/registry'
import * as sessionModule from '@/main/browser-agent/session'
Expand Down Expand Up @@ -1730,6 +1734,57 @@ describe('browser-agent session', () => {
})
})

it('asks about an HTTP sign-in only while its top-level navigation is still loading', () => {
const url = 'https://intranet.example/'
const contents = (session.ensureTab().view as unknown as MockView).webContents
session.enablePagePrompts()
const emit = (eventName: string, ...args: unknown[]) => {
for (const [name, handler] of contents.on.mock.calls) {
if (name === eventName && typeof handler === 'function') handler(...args)
}
}
const signIn = () => {
const callback = vi.fn()
handleBrowserLogin(
{ preventDefault: () => {} },
contents as unknown as WebContents,
{ url, isRequestForNavigation: true, firstAuthAttempt: true } as never,
{ isProxy: false, scheme: 'basic', host: 'intranet.example', port: 443, realm: '' },
callback
)
return callback
}

emit('did-start-navigation', { isMainFrame: true, isSameDocument: false, url })
const loading = signIn()
expect(loading).not.toHaveBeenCalled()
expect(pagePromptForContents(contents as unknown as WebContents)).toMatchObject({
kind: 'http-auth',
})

// The navigation is aborted: Chromium reports ERR_ABORTED, which records no failure.
emit('did-start-navigation', { isMainFrame: true, isSameDocument: false, url })
emit('did-stop-loading')
expect(signIn()).toHaveBeenCalledExactlyOnceWith()
})

it('counts a download as in flight for a restart only until it finishes', async () => {
const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-'))
session = freshSession(win, {}, undefined, {
getDirectory: () => directory,
getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER,
})
const contents = (session.ensureTab().view as unknown as MockView).webContents
const download = mockDownloadItem({ filename: 'report.csv', totalBytes: 100 })

startMockDownload(contents, download)
await vi.waitFor(() => expect(download.item.resume).toHaveBeenCalledOnce())
expect(session.countProgressingBrowserDownloads()).toBe(1)

download.emitDone('completed')
await vi.waitFor(() => expect(session.countProgressingBrowserDownloads()).toBe(0))
})

it('never overwrites a file that takes the allocated name before the download claims it', async () => {
const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-'))
writeFileSync(join(directory, 'taken.bin'), 'user data')
Expand Down
31 changes: 22 additions & 9 deletions apps/desktop/src/main/browser-agent/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ import {
import { clearHostVerdictCache } from '@/main/browser-agent/url-guard'
import type { BrowserSessionSnapshot } from '@/main/desktop-chat-session-store'
import { suggestedFilename, uniqueDownloadPath } from '@/main/downloads'
import { withinWindow } from '@/main/input-activity'
import { isAppOrigin, isBlankPopupUrl, parseHttpUrl } from '@/main/navigation'
import {
type FocusedResourceShortcut,
Expand Down Expand Up @@ -257,12 +258,18 @@ export type BrowserShortcut =
| 'hard-reload'
| UserOnlyShortcut

type UserOnlyShortcut = 'back' | 'forward' | 'previous-tab' | 'next-tab' | 'stop'
/** Shortcuts that move the user's history, tabs, or load; keystrokes the agent sends never trigger them. */
const USER_ONLY_SHORTCUTS = ['back', 'forward', 'previous-tab', 'next-tab', 'stop'] as const
type UserOnlyShortcut = (typeof USER_ONLY_SHORTCUTS)[number]
const USER_ONLY_SHORTCUTS: ReadonlySet<BrowserShortcut> = new Set<UserOnlyShortcut>([
'back',
'forward',
'previous-tab',
'next-tab',
'stop',
])

function isUserOnlyShortcut(shortcut: BrowserShortcut): shortcut is UserOnlyShortcut {
return USER_ONLY_SHORTCUTS.some((candidate) => candidate === shortcut)
return USER_ONLY_SHORTCUTS.has(shortcut)
}

type BrowserShortcutInput = Pick<
Expand Down Expand Up @@ -635,6 +642,15 @@ function activeBrowserDownloadCount(scopeId?: string): number {
return count
}

/** Agent-browser downloads still receiving bytes, which a restart would cut off. */
export function countProgressingBrowserDownloads(): number {
let count = 0
for (const active of activeBrowserDownloads) {
if (active.download.state === 'progressing') count += 1
}
return count
}

function withBrowserDownloadTimeout<T>(
operation: Promise<T>,
timeoutMs: number,
Expand Down Expand Up @@ -1569,12 +1585,6 @@ function tabIsOnScreen(scopeId: string, tab: AgentTab): boolean {
)
}

/** Whether `at` lies in the last `windowMs`; a clock stepping backwards never passes. */
function withinWindow(at: number, windowMs: number): boolean {
const elapsed = Date.now() - at
return elapsed >= 0 && elapsed < windowMs
}

function mediaRequestIsUserInitiated(scopeId: string, tab: AgentTab): boolean {
const win = panelWindow()
return (
Expand Down Expand Up @@ -2984,6 +2994,9 @@ function initializeTabView(
if (details.isMainFrame && !details.isSameDocument)
noteMainFrameNavigation(contents, details.url)
})
// An aborted navigation (ERR_ABORTED) reports no failure the driver records; once nothing is
// loading, no top-level request is pending, so a later frame cannot pass for it.
contents.on('did-stop-loading', () => noteMainFrameNavigation(contents, null))

contents.on(
'did-start-navigation',
Expand Down
8 changes: 8 additions & 0 deletions apps/desktop/src/main/in-flight-work.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ export interface InFlightWork {
browserToolRunning: boolean
/** Whether a chat has desktop work claimed on this machine. */
agentWorkRunning: boolean
/** Agent-browser downloads still in progress. */
downloadCount: number
unsavedWindowCount: number
unresponsiveWindowCount: number
}
Expand Down Expand Up @@ -101,6 +103,12 @@ export function describeInFlightWork(work: InFlightWork): string[] {
sentences.push('An agent is still working on this computer.')
}

if (work.downloadCount === 1) {
sentences.push('A download is still in progress.')
} else if (work.downloadCount > 1) {
sentences.push(`${work.downloadCount} downloads are still in progress.`)
}

if (work.unsavedWindowCount === 1) {
sentences.push('A window has unsaved changes.')
} else if (work.unsavedWindowCount > 1) {
Expand Down
2 changes: 2 additions & 0 deletions apps/desktop/src/main/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
setPanelOccluded as setBrowserAgentPanelOccluded,
} from '@/main/browser-agent/panel'
import {
countProgressingBrowserDownloads,
handleFocusedShortcut as handleFocusedBrowserShortcut,
isBrowserScopeSuspended,
quiesceBrowserSessions,
Expand Down Expand Up @@ -1180,6 +1181,7 @@ function main(): void {
computerUseApp: computerUseActivity ? (computerUseActivity.appName ?? '') : null,
browserToolRunning: isBrowserToolRunning(),
agentWorkRunning,
downloadCount: countProgressingBrowserDownloads(),
...windows,
})
},
Expand Down
7 changes: 6 additions & 1 deletion apps/desktop/src/main/input-activity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,11 @@ function isWithin(
if (contents.isDestroyed()) return false
const activity = activityByContents.get(contents)
if (!activity) return false
const elapsed = Date.now() - stamp(activity)
return withinWindow(stamp(activity), windowMs)
}

/** Whether `at` lies in the last `windowMs`; a clock stepping backwards never passes. */
export function withinWindow(at: number, windowMs: number): boolean {
const elapsed = Date.now() - at
return elapsed >= 0 && elapsed < windowMs
}
Loading
Loading