From 5f7f0873ebd63b1c1f572fc8c55cb9047ece5efe Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 29 Oct 2026 14:35:51 -0700 Subject: [PATCH 1/9] feat(extension): bring Sim chat and page context to Chrome --- .github/workflows/browser-extension.yml | 122 ++++ apps/extension/DESIGN.md | 155 ++++ apps/extension/README.md | 67 ++ apps/extension/package.json | 37 + apps/extension/scripts/build.ts | 107 +++ apps/extension/scripts/test-e2e.ts | 495 +++++++++++++ apps/extension/src/background.ts | 90 +++ apps/extension/src/capture.ts | 56 ++ apps/extension/src/config.ts | 6 + apps/extension/src/extract.ts | 111 +++ apps/extension/src/intent.ts | 10 + apps/extension/src/shell.css | 14 + apps/extension/src/sidepanel.html | 13 + apps/extension/src/sidepanel.tsx | 304 ++++++++ apps/extension/src/styles.d.ts | 5 + apps/extension/src/utils.ts | 5 + apps/extension/tsconfig.json | 11 + apps/extension/turbo.json | 15 + apps/sim/.env.example | 1 + apps/sim/app/extension/[workspaceId]/chat.tsx | 249 +++++++ .../app/extension/[workspaceId]/header.tsx | 103 +++ .../extension/[workspaceId]/page-context.tsx | 85 +++ apps/sim/app/extension/[workspaceId]/page.tsx | 16 + .../extension/[workspaceId]/search-params.ts | 13 + .../app/extension/[workspaceId]/workspace.tsx | 62 ++ .../app/extension/hooks/use-page-context.ts | 95 +++ apps/sim/app/extension/layout.tsx | 52 ++ apps/sim/app/extension/loading.tsx | 10 + apps/sim/app/extension/page.tsx | 5 + apps/sim/app/extension/privacy.tsx | 16 + apps/sim/app/extension/providers.tsx | 58 ++ apps/sim/app/extension/sign-in.tsx | 23 + apps/sim/app/extension/theme.tsx | 24 + apps/sim/app/extension/workspaces.tsx | 34 + .../chat-context-kind-registry.tsx | 4 + .../mothership-chat/mothership-chat.tsx | 11 +- .../home/components/user-input/user-input.tsx | 22 +- .../[workspaceId]/home/hooks/use-chat.ts | 28 +- .../app/workspace/[workspaceId]/home/types.ts | 2 + .../components/user-input/utils.test.ts | 32 +- .../copilot/components/user-input/utils.ts | 2 + .../settings/settings-navigation-provider.tsx | 22 +- apps/sim/hooks/use-settings-navigation.ts | 8 +- apps/sim/hooks/use-speech-to-text.ts | 8 +- .../api/contracts/mothership-chat-context.ts | 106 +-- apps/sim/lib/core/config/env.ts | 1 + apps/sim/lib/core/security/csp.ts | 6 +- apps/sim/lib/core/security/extension.ts | 10 + apps/sim/lib/mothership/chat/context.ts | 2 + apps/sim/lib/mothership/chat/page-context.ts | 17 + apps/sim/lib/mothership/chat/payload.test.ts | 6 +- apps/sim/lib/mothership/chat/payload.ts | 19 +- .../mothership/chat/persisted-message.test.ts | 34 + .../lib/mothership/chat/persisted-message.ts | 3 + apps/sim/lib/mothership/chat/post.ts | 20 +- .../mothership/chat/process-contents.test.ts | 55 ++ .../lib/mothership/chat/process-contents.ts | 24 + apps/sim/next.config.ts | 4 +- apps/sim/package.json | 2 + apps/sim/proxy.test.ts | 35 + apps/sim/proxy.ts | 14 + .../sim/scripts/test-browser-extension-e2e.ts | 670 ++++++++++++++++++ apps/sim/stores/mothership-drafts/store.ts | 9 +- apps/sim/stores/panel/types.ts | 2 + bun.lock | 46 ++ knip.jsonc | 4 + .../browser-extension-protocol/package.json | 29 + .../browser-extension-protocol/src/index.ts | 64 ++ .../browser-extension-protocol/tsconfig.json | 5 + 69 files changed, 3713 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/browser-extension.yml create mode 100644 apps/extension/DESIGN.md create mode 100644 apps/extension/README.md create mode 100644 apps/extension/package.json create mode 100644 apps/extension/scripts/build.ts create mode 100644 apps/extension/scripts/test-e2e.ts create mode 100644 apps/extension/src/background.ts create mode 100644 apps/extension/src/capture.ts create mode 100644 apps/extension/src/config.ts create mode 100644 apps/extension/src/extract.ts create mode 100644 apps/extension/src/intent.ts create mode 100644 apps/extension/src/shell.css create mode 100644 apps/extension/src/sidepanel.html create mode 100644 apps/extension/src/sidepanel.tsx create mode 100644 apps/extension/src/styles.d.ts create mode 100644 apps/extension/src/utils.ts create mode 100644 apps/extension/tsconfig.json create mode 100644 apps/extension/turbo.json create mode 100644 apps/sim/app/extension/[workspaceId]/chat.tsx create mode 100644 apps/sim/app/extension/[workspaceId]/header.tsx create mode 100644 apps/sim/app/extension/[workspaceId]/page-context.tsx create mode 100644 apps/sim/app/extension/[workspaceId]/page.tsx create mode 100644 apps/sim/app/extension/[workspaceId]/search-params.ts create mode 100644 apps/sim/app/extension/[workspaceId]/workspace.tsx create mode 100644 apps/sim/app/extension/hooks/use-page-context.ts create mode 100644 apps/sim/app/extension/layout.tsx create mode 100644 apps/sim/app/extension/loading.tsx create mode 100644 apps/sim/app/extension/page.tsx create mode 100644 apps/sim/app/extension/privacy.tsx create mode 100644 apps/sim/app/extension/providers.tsx create mode 100644 apps/sim/app/extension/sign-in.tsx create mode 100644 apps/sim/app/extension/theme.tsx create mode 100644 apps/sim/app/extension/workspaces.tsx create mode 100644 apps/sim/lib/core/security/extension.ts create mode 100644 apps/sim/lib/mothership/chat/page-context.ts create mode 100644 apps/sim/scripts/test-browser-extension-e2e.ts create mode 100644 packages/browser-extension-protocol/package.json create mode 100644 packages/browser-extension-protocol/src/index.ts create mode 100644 packages/browser-extension-protocol/tsconfig.json diff --git a/.github/workflows/browser-extension.yml b/.github/workflows/browser-extension.yml new file mode 100644 index 00000000000..562f3ee980d --- /dev/null +++ b/.github/workflows/browser-extension.yml @@ -0,0 +1,122 @@ +name: browser-extension + +on: + pull_request: + paths: + - '.github/workflows/browser-extension.yml' + - 'apps/extension/**' + - 'packages/browser-extension-protocol/**' + - 'apps/sim/app/extension/**' + - 'apps/sim/app/workspace/**/home/**' + - 'apps/sim/stores/mothership-drafts/**' + - 'apps/sim/scripts/test-browser-extension-e2e.ts' + - 'apps/sim/lib/core/security/**' + - 'apps/sim/lib/mothership/chat/**' + - 'apps/sim/lib/api/contracts/mothership-chat-context.ts' + - 'apps/sim/proxy.ts' + - 'apps/sim/next.config.ts' + - 'packages/emcn/**' + - 'bun.lock' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: browser-extension-${{ github.ref }} + cancel-in-progress: true + +jobs: + browser: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: extension_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d extension_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:8.2-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/extension_test + REDIS_URL: redis://127.0.0.1:6379 + NEXT_PUBLIC_APP_URL: http://127.0.0.1:3007 + BETTER_AUTH_URL: http://127.0.0.1:3007 + BETTER_AUTH_SECRET: browser-extension-ci-auth-secret-32-characters + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' + API_ENCRYPTION_KEY: '1111111111111111111111111111111111111111111111111111111111111111' + NEXT_TELEMETRY_DISABLED: '1' + DISABLE_AUTH: 'false' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 + with: + node-version: '24' + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.4 + - run: bun install --frozen-lockfile + - run: bunx playwright install --with-deps chromium + working-directory: apps/extension + - run: bun run type-check && bun run build && bun run test:e2e + working-directory: apps/extension + env: + EXTENSION_REPORT_PATH: test-results/extension-report.json + - name: Provision the disposable app database + run: bun run --cwd packages/db db:migrate + - name: Generate the fixture extension identity + run: | + bun - <<'JS' >> "$GITHUB_ENV" + import { generateKeyPairSync, createHash } from "node:crypto"; + const { publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const der = publicKey.export({ type: "spki", format: "der" }); + const id = [...createHash("sha256").update(der).digest("hex").slice(0, 32)] + .map(c => String.fromCharCode(97 + parseInt(c, 16))).join(""); + process.stdout.write(`SIM_EXTENSION_KEY=${der.toString("base64")}\nBROWSER_EXTENSION_IDS=${id}\n`); + JS + - name: Start the real Sim app + working-directory: apps/sim + run: | + node "$(bun -p 'require.resolve("next/dist/bin/next")')" dev --hostname 127.0.0.1 --port 3007 > "$RUNNER_TEMP/extension-app.log" 2>&1 & + for ((attempt=0; attempt<60; attempt++)); do + if curl --fail --silent http://127.0.0.1:3007/api/auth/get-session > /dev/null; then exit 0; fi + sleep 2 + done + exit 1 + - name: Exercise authenticated app integration + working-directory: apps/sim + run: bun run test:browser-extension:e2e + env: + EXTENSION_APP_E2E_BASE_URL: http://127.0.0.1:3007 + EXTENSION_APP_E2E_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/extension_test + EXTENSION_APP_E2E_REPORT_PATH: ../extension/test-results/real-app-report.json + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: browser-extension-results + path: apps/extension/test-results + retention-days: 7 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: failure() + with: + name: browser-extension-app-log + path: ${{ runner.temp }}/extension-app.log + retention-days: 7 diff --git a/apps/extension/DESIGN.md b/apps/extension/DESIGN.md new file mode 100644 index 00000000000..13f79d813fa --- /dev/null +++ b/apps/extension/DESIGN.md @@ -0,0 +1,155 @@ +# Sim browser extension + +Implementation scope and research, 2026-10-29. + +## Product contract + +The extension is another surface of Sim. A native Chrome side panel hosts the actual +Sim composer and transcript, using existing authenticated Mothership requests, +conversation persistence, streaming recovery, model controls, tool permissions, +and workspace authorization. A conversation can continue in the full app. + +The user signs in in a normal Sim tab, chooses a workspace, opens or creates a +conversation, and explicitly attaches the current page or highlighted text. +Pending snapshots sit beside the composer as compact removable attachments. Opening one +shows the captured text, source, capture time, and conversation-retention disclosure. +Partial captures remain labeled in the attachment row. Site access lives in the +extension options menu; successful delivery adds no persistent status banner. +They are immutable: navigating to another page cannot silently change an attached +source. Removing an attachment, changing workspace, or starting a new conversation +clears pending page context. Several explicitly attached pages can support comparison. +Assistant accepts page snapshots while retaining its existing restriction on +workspace-resource contexts. + +The extension does not promise complete capture of virtualized documents, canvas +editors, restricted browser pages, inaccessible cross-origin frames, or PDFs. +Extraction failure is visible; the model must never infer absent page content. +Browser interaction, autonomous navigation, screenshot OCR, and native-app control +are separate capabilities requiring their own permission and execution contracts. +Existing Sim server tools remain subject to their existing authorization and approval. + +## Ownership and data flow + +1. `apps/extension`: packaged MV3 service worker, side-panel host, bounded content + extraction, per-site permission controls, build artifacts, browser verification. +2. `packages/browser-extension-protocol`: versioned, strictly validated messages and + immutable page snapshots shared by the extension, app, and server. +3. `apps/sim/app/extension`: session gate, workspace picker, minimal app providers, + real chat components, history and a narrow parent-window bridge. +4. Existing chat context pipeline: `browser_page` input validation, durable message + projection, restored context chips, untrusted-content formatting, Assistant admission. +5. Existing Sim API and Mothership: authentication, authorization, billing, tool + execution and streaming; no alternate inference service or extension API key. + +The content script returns text to its extension caller. The extension verifies the +captured document still belongs to the intended tab, then sends a typed snapshot to +the exact hosted frame. The frame validates its parent origin and identity, and puts +the snapshot into pending UI context. The existing send path persists it with the +message and passes quoted content to Mothership. A page snapshot grants no access to +other workspace resources or browser-control tools. + +## Permission and lifetime rules + +- Required browser capabilities: side panel, scripting, active tab, context menus, + and short-lived session storage for selection handoff. Required host access is + limited to the configured Sim origin. +- Other HTTP(S) origins are optional grants requested for the current site from a + user gesture. Each grant can be revoked. Keeping the panel open is not a grant + to read every subsequently selected tab. +- No history, bookmarks, cookie API, debugger, network interception, native + messaging, external messaging, or arbitrary remote code execution. +- No always-running page reader. Extraction is tied to an explicit attachment action. +- Page bytes are not written to extension local/sync storage. Temporary context-menu + handoff uses trusted-context session storage, expiration and navigation cleanup. +- Sent snapshots become chat data and follow Sim's chat retention and deletion + policy. This must be disclosed in the interface and store listing. No claim of + zero retention is made once a message is sent. +- No page content or full page URL in extension telemetry or routine logs. + +## Boundary controls + +- Package all privileged JavaScript. The hosted iframe has no extension API access. +- Only exact configured Sim origins and exact configured extension IDs are trusted. + Both directions validate source window, origin, protocol version and message shape. + There is no arbitrary fetch, JavaScript, navigation or extension-API command bridge. +- `BROWSER_EXTENSION_IDS` is a bounded comma-separated list of Chrome IDs. Missing + or malformed configuration disables embedding. Only `/extension` and its children + receive the corresponding frame policy; ordinary app pages keep their frame denial. +- Extension pages use a restrictive CSP; the hosted app keeps its app CSP with only + the embedding ancestor list changed. Sign-in happens in a top-level tab. +- Allow HTTPS production app origins; HTTP is development-only on loopback. +- Capture uses bounded traversal and output size, skips hidden and editable/form + content, and records truncation. It never serializes the entire DOM or reads + cookies, browser storage, input values, or page-world JavaScript state. +- Validate snapshots again server-side, including headless execution paths. Bound + individual and aggregate text/byte sizes. Only HTTP(S) source URLs without embedded + credentials are accepted; URL secrets should not be carried as source metadata. +- Snapshot content is explicitly untrusted quoted JSON. This is a model cue, not + an authorization mechanism: tool access remains independently enforced by Sim. + +## Verification planned before implementation + +The credible failure modes are cross-origin/frame message forgery; schema-version +confusion; hostile URL metadata; hidden/form data leakage; unbounded DOM traversal; +selection or extraction delivered after navigation; stale context on workspace +switch; persistence mutating the captured text; Assistant silently dropping context; +and an embed exception accidentally allowing unrelated app pages to be framed. + +Use a real Chromium extension against controlled HTTP pages for extraction, +navigation, permission and message-boundary checks. Emit a caller-configured JSON +report and screenshot/trace artifacts. Security-boundary tests may cover header +isolation and durable-context semantics where the running app needs unavailable +services. Do not replace real Chrome APIs with mocks and describe that as E2E. + +The authenticated live-app acceptance gate is: sign in, choose a permitted workspace, +capture a real page, receive a grounded Mothership answer, reopen the conversation, +stop/reconnect streaming, switch workspaces, and verify access denial. An absent +backend or account is an unverified gate, not a pass. + +## Research and decisions + +- [Glean sidebar](https://docs.glean.com/user-guide/apps/extension-sidebar) and + [page permissions](https://docs.glean.com/user-guide/apps/why-does-the-chrome-extension-ask-for-those-permissions): + page context can use address and text without indexing the page. Adopt explicit + page attachment alongside the existing Sim knowledge and tool experience. +- [Chrome activeTab](https://developer.chrome.com/docs/extensions/develop/concepts/activeTab) + and [optional permissions](https://developer.chrome.com/docs/extensions/reference/api/permissions): + temporary access follows user invocation; cross-origin navigation requires fresh + authority. Request persistent access per site only when selected by the user. +- [Chrome extension security](https://developer.chrome.com/docs/extensions/develop/security-privacy/stay-secure) + and [OWASP extension guidance](https://cheatsheetseries.owasp.org/cheatsheets/Browser_Extension_Vulnerabilities_Cheat_Sheet.html): + minimize privileges and treat all incoming page messages as attacker-controlled. + Do not expose a generic privileged bridge. +- [Service worker lifecycle](https://developer.chrome.com/docs/extensions/develop/concepts/service-workers/lifecycle): + workers are disposable. Stream from the app document; bound temporary handoffs + independently of worker globals. +- [Storage and cookies](https://developer.chrome.com/docs/extensions/develop/concepts/storage-and-cookies): + extension frames have special partition and cookie behavior. Reuse Sim sessions, + verify enterprise policy and blocked-cookie cases, and show a top-level sign-in + recovery path instead of copying credentials into extension storage. +- [MV3 remote-code policy](https://developer.chrome.com/docs/webstore/program-policies/mv3-requirements): + isolated remote frames are permitted subject to review and user-data rules. All + privileged capture and message logic remains packaged and inspectable. +- [User-data disclosures](https://developer.chrome.com/docs/webstore/program-policies/user-data-faq): + local processing of website content still requires disclosure. Publish actual + data handling and permission justifications with the store submission. + +## Release scope and remaining expansion + +Planning estimate for one engineer familiar with Sim: 5–8 engineering days for +the core surface, context pipeline and security boundaries; 2–4 more days for +authenticated deployment QA, enterprise cookie-policy checks and release packaging. +These are estimates for production readiness, not a promise that a working local +build has completed those gates. Chrome Web Store review is an external timeline. +Richer document adapters and screenshot capture are roughly another 1–2 weeks; +autonomous browser actions need a separate design and estimate. + +The first implementation targets Chrome MV3 and a configured Sim deployment. It +includes the shared chat UI and engine, explicit page/selection context, Assistant +context support, workspace/history navigation, permission controls and verification. +It does not constitute store publication or proof of every third-party site's DOM. + +Follow-on work: screenshot/image attachment through existing authorized upload paths; +document-specific adapters with measured coverage; Firefox/Safari shells; managed +enterprise distribution; and separately authorized browser-action tools. Each must +retain the same source, identity, consent and lifecycle boundaries. diff --git a/apps/extension/README.md b/apps/extension/README.md new file mode 100644 index 00000000000..b8a340cf943 --- /dev/null +++ b/apps/extension/README.md @@ -0,0 +1,67 @@ +# Sim browser extension + +The Chrome extension attaches an explicitly captured page or selection to the existing Sim chat. The native side panel hosts the authenticated `/extension` app surface. Chat rendering, execution, permissions, billing, and history belong to Sim and Mothership. + +## Build and load + +From the repository root: + +```sh +bun install --frozen-lockfile +bun run --cwd apps/extension build +``` + +In Chrome, open `chrome://extensions`, enable Developer mode, choose **Load unpacked**, and select `apps/extension/dist`. Copy the extension ID into the Sim server's `BROWSER_EXTENSION_IDS` configuration. It accepts comma-separated Chrome extension IDs. Restart the app after changing that configuration. Unconfigured IDs cannot embed the app. + +Click the Sim toolbar icon to open chat and attach the current page. **Attach page** adds a fresh snapshot. Attach up to three pages or selections to summarize, extract facts, or compare sources. To quote a passage, select it and choose **Ask Sim about selection** from the context menu. Review or remove the attachment before sending. The **Extension options** menu keeps site permissions out of the chat: **Allow access to [site]** grants access only to that site; **Remove access to [site]** revokes it. Persistent site permission never enables background capture. + +For local development: + +```sh +SIM_EXTENSION_DEV=1 SIM_EXTENSION_ORIGIN=http://localhost:3000 bun run --cwd apps/extension build +``` + +`SIM_EXTENSION_ORIGIN` is baked into the package. Production builds require an HTTPS origin. Explicit development builds may use HTTP on loopback. `SIM_EXTENSION_KEY` optionally supplies the public manifest key for a stable extension ID; never supply a private signing key. Chrome Web Store signing and publisher credentials stay outside this repository. + +## Data and permissions + +- Required capabilities: `activeTab`, `scripting`, `sidePanel`, `storage`, `contextMenus`, and access to the configured Sim origin. +- Optional HTTP/HTTPS site access is requested only by a menu action naming that site. Chrome's `activeTab` access is temporary and does not follow arbitrary new sites. +- Page capture runs on demand in Chrome's isolated world. It excludes editable fields, inputs, hidden elements, scripts, and embedded documents. It reads only the top frame. PDF viewers and canvas documents may have no accessible text. +- Extraction stops at 50,000 text characters, 150,000 serialized UTF-8 bytes, 15,000 visited nodes, or 150ms. The composer exposes truncation. +- Source URLs omit credentials, query parameters, and fragments. Human-readable text can still contain sensitive information; it is the user-selected material they review and send into their conversation. +- Snapshots stay in memory until handed to Sim and acknowledged or expired after 30 seconds. Extension storage contains only a short-lived intent with a tab/document identity and a SHA-256 URL digest, never raw URLs, captured text, or credentials. Closing a tab or navigating invalidates its pending intent. Sim applies its normal conversation retention once a message is sent. +- No browser history, bookmarks, cookie access, debugger access, remote extension JavaScript, arbitrary network proxy, or background content collection is requested. Incognito use is disabled. +- The iframe bridge checks both the exact configured origin and the exact frame, then validates versioned messages. The hosted page may acknowledge readiness and receipt; it cannot request privileged browser actions through the bridge. +- Captures and queued toolbar intents are scoped to the receiving conversation. Changing conversation, workspace, or account discards an unfinished handoff. Captured attachments remain explicit snapshots when the source tab changes. +- The complete hosted document, including portaled source dialogs and menus, is blocked from session replay. Clipboard writing is delegated only to the configured Sim origin. Clipboard reading and microphone capture are not delegated; voice input remains available in the main app. + +## Verification + +```sh +bun run --cwd apps/extension type-check +EXTENSION_REPORT_PATH=test-results/extension-report.json bun run --cwd apps/extension test:e2e +bun run --cwd apps/extension build +``` + +The E2E suite uses real Chromium, the packaged extension, and a disposable HTTP app fixture. It pregrants only the fixture page origin in a temporary package to exercise extraction without automating the browser's permission dialog. It checks HttpOnly session-cookie transport, origin/frame spoofing, sensitive-field exclusion, source URL minimization, selected excerpts, UTF-8 limits, stale-document handoffs, denied host access, and absence of page bytes in extension storage. The cookie check uses a fixture, not Better Auth or SSO. JSON results and a screenshot are written beside the caller-supplied report path. The E2E build points `dist` at its temporary fixture origin; rebuild normally before loading or distributing it. + +Release verification must additionally exercise the native toolbar and shortcut, Chrome's real grant/revoke prompt, sign-in/SSO and expiry against the running Sim app, side-panel closing/reopening, workspace changes, narrow widths, and the real Mothership streaming response. The fixture suite does not establish those integrations. The design document records launch scope and release criteria. + +The authenticated acceptance suite runs against a real local app, Better Auth, and a disposable Postgres database whose name contains `test`: + +```sh +EXTENSION_APP_E2E_BASE_URL=http://127.0.0.1:3007 \ +EXTENSION_APP_E2E_DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/extension_test \ +EXTENSION_APP_E2E_REPORT_PATH=../extension/test-results/real-app-report.json \ +SIM_EXTENSION_KEY="$PUBLIC_EXTENSION_KEY" \ +bun run --cwd apps/sim test:browser-extension:e2e +``` + +Configure the matching extension ID in the running app. The suite creates its own accounts/workspaces and removes them afterward. It verifies real authentication, workspace denial, page limits, navigation, source links, clipboard writes, actual replay-recorder exclusion, themes, widths, and sign-out. CI provisions this environment automatically. Set `EXTENSION_APP_E2E_LIVE_CHAT=1` when the app is connected to a working local Mothership instance to additionally verify live rendering, saved page context, queue editing, and drafts through chat admission. Provider credentials belong only in that server's environment. Reports and screenshots are written beside the supplied report path. Native toolbar/shortcut/context-menu and permission-prompt checks remain manual release checks. + +## Distribution + +Publish only the production contents of `dist`, after rebuilding against the deployed Sim origin and configuring the issued store extension ID on that server. The store listing and privacy disclosure must describe that page text is sent to Sim when attached and used in a conversation, and link to the applicable Sim privacy policy. Keep the publisher account protected with hardware-backed MFA and limited release access. + +Research: [Chrome extension security](https://developer.chrome.com/docs/extensions/develop/security-privacy/stay-secure), [activeTab](https://developer.chrome.com/docs/extensions/develop/concepts/activeTab), [storage and cookies](https://developer.chrome.com/docs/extensions/develop/concepts/storage-and-cookies), [Manifest V3 policies](https://developer.chrome.com/docs/webstore/program-policies/mv3-requirements), and [OWASP extension vulnerabilities](https://cheatsheetseries.owasp.org/cheatsheets/Browser_Extension_Vulnerabilities_Cheat_Sheet.html). diff --git a/apps/extension/package.json b/apps/extension/package.json new file mode 100644 index 00000000000..796a3a2d843 --- /dev/null +++ b/apps/extension/package.json @@ -0,0 +1,37 @@ +{ + "name": "@sim/extension", + "version": "0.1.0", + "private": true, + "type": "module", + "license": "Apache-2.0", + "scripts": { + "build": "bun scripts/build.ts", + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format .", + "test:e2e": "bun scripts/test-e2e.ts" + }, + "dependencies": { + "@sim/browser-extension-protocol": "workspace:*", + "@sim/emcn": "workspace:*", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", + "react": "19.2.4", + "react-dom": "19.2.4", + "zod": "4.3.6" + }, + "devDependencies": { + "@playwright/test": "1.61.1", + "@sim/tsconfig": "workspace:*", + "@types/bun": "1.4.3", + "@types/chrome": "0.3.6", + "@types/react": "^19", + "@types/react-dom": "^19", + "esbuild": "0.28.2", + "postcss": "^8.5.30", + "postcss-load-config": "6.0.1", + "typescript": "^7.0.2" + } +} diff --git a/apps/extension/scripts/build.ts b/apps/extension/scripts/build.ts new file mode 100644 index 00000000000..058d327f78d --- /dev/null +++ b/apps/extension/scripts/build.ts @@ -0,0 +1,107 @@ +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { build } from 'esbuild' +import postcss from 'postcss' +import loadPostcssConfig from 'postcss-load-config' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const origin = new URL(process.env.SIM_EXTENSION_ORIGIN ?? 'https://www.sim.ai') +const development = process.env.SIM_EXTENSION_DEV === '1' +const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(origin.hostname) +if ( + origin.username || + origin.password || + origin.pathname !== '/' || + origin.search || + origin.hash || + (origin.protocol !== 'https:' && !(development && loopback && origin.protocol === 'http:')) +) { + throw new Error( + 'SIM_EXTENSION_ORIGIN must be an HTTPS origin; explicit development builds may use HTTP loopback.' + ) +} +const key = process.env.SIM_EXTENSION_KEY +if (key && (!/^[A-Za-z0-9+/]+={0,2}$/.test(key) || key.length > 4_096)) + throw new Error('SIM_EXTENSION_KEY must be a base64 public key.') + +const dist = resolve(root, 'dist') +await rm(dist, { recursive: true, force: true }) +await mkdir(dist, { recursive: true }) +await build({ + absWorkingDir: root, + entryPoints: { background: 'src/background.ts', sidepanel: 'src/sidepanel.tsx' }, + outdir: dist, + bundle: true, + platform: 'browser', + format: 'iife', + target: 'chrome120', + minify: !development, + sourcemap: development ? 'external' : false, + tsconfig: resolve(root, 'tsconfig.json'), + loader: { '.module.css': 'local-css', '.woff2': 'file' }, + define: { + __SIM_ORIGIN__: JSON.stringify(origin.origin), + 'process.env.NODE_ENV': '"production"', + 'process.env': '{}', + 'globalThis.process': 'undefined', + }, + plugins: [ + { + name: 'extension-tailwind', + setup(builder) { + builder.onResolve({ filter: /SeasonSansUprightsVF\.woff2$/ }, () => ({ + path: resolve(root, '../sim/app/_styles/fonts/season/SeasonSansUprightsVF.woff2'), + })) + builder.onLoad({ filter: /shell\.css$/ }, async ({ path }) => { + const config = await loadPostcssConfig({}, resolve(root, '../sim')) + const result = await postcss(config.plugins).process(await readFile(path, 'utf8'), { + from: path, + }) + return { contents: result.css, loader: 'css', resolveDir: dirname(path) } + }) + }, + }, + ], +}) +await cp(resolve(root, 'src/sidepanel.html'), resolve(dist, 'sidepanel.html')) +for (const [size, file] of [ + [16, 'favicon-16x16.png'], + [32, 'favicon-32x32.png'], + [128, 'android-chrome-512x512.png'], +] as const) { + await cp(resolve(root, '../sim/public/favicon', file), resolve(dist, `icon-${size}.png`)) +} +await writeFile( + resolve(dist, 'manifest.json'), + JSON.stringify( + { + manifest_version: 3, + name: development ? 'Sim (Development)' : 'Sim', + version: '0.1.0', + description: + 'Ask Sim about the page you are reading, with your existing Sim conversations and tools.', + minimum_chrome_version: '120', + permissions: ['activeTab', 'scripting', 'sidePanel', 'storage', 'contextMenus'], + host_permissions: [`${origin.origin}/*`], + optional_host_permissions: ['https://*/*', 'http://*/*'], + background: { service_worker: 'background.js' }, + side_panel: { default_path: 'sidepanel.html' }, + action: { default_title: 'Open Sim' }, + icons: { 16: 'icon-16.png', 32: 'icon-32.png', 128: 'icon-128.png' }, + commands: { + 'open-sim': { + suggested_key: { default: 'Alt+Shift+S', mac: 'Command+Shift+S' }, + description: 'Open Sim and attach the current page', + }, + }, + content_security_policy: { + extension_pages: `default-src 'none'; script-src 'self'; style-src 'self'; font-src 'self'; img-src 'self'; frame-src ${origin.origin}; connect-src 'none'; object-src 'none'; base-uri 'none'`, + }, + incognito: 'not_allowed', + ...(key ? { key } : {}), + }, + null, + 2 + ) +) diff --git a/apps/extension/scripts/test-e2e.ts b/apps/extension/scripts/test-e2e.ts new file mode 100644 index 00000000000..ba7661432ab --- /dev/null +++ b/apps/extension/scripts/test-e2e.ts @@ -0,0 +1,495 @@ +import { spawnSync } from 'node:child_process' +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { type BrowserContext, chromium, type Frame, type Page } from '@playwright/test' +import { pageSnapshotSchema } from '@sim/browser-extension-protocol' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const logger = createLogger('BrowserExtensionE2E') +const reportPath = process.env.EXTENSION_REPORT_PATH +if (!reportPath) throw new Error('Set EXTENSION_REPORT_PATH to the output JSON report path.') +const output = resolve(root, reportPath) +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] +const fixture = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + fetch(request) { + const url = new URL(request.url) + if (url.pathname === '/sign-in') + return new Response('Fixture session created', { + headers: { 'set-cookie': 'fixture-session=present; Path=/; HttpOnly; SameSite=Lax' }, + }) + if (url.pathname === '/extension') + return new Response( + ` +

Sim bridge fixture

Packaged extension transport verification

[]
+

${request.headers.get('cookie')?.includes('fixture-session=present') ? 'present' : 'missing'}

+ + `, + { headers: { 'content-type': 'text/html' } } + ) + return new Response( + `Private work page +

Customer escalation

The customer needs a replacement by Friday.

Reference: visible-reference-482

+ +
EDITABLE_SENTINEL
+
CSS_HIDDEN_SENTINEL
+ + `, + { headers: { 'content-type': 'text/html' } } + ) + }, +}) +const temporary = await mkdtemp(resolve(tmpdir(), 'sim-extension-e2e-')) +const extensionPath = resolve(temporary, 'extension') +let context: BrowserContext | undefined + +function assert(value: unknown, message: string): asserts value { + if (!value) throw new Error(message) +} +async function check(name: string, run: () => Promise) { + const started = performance.now() + logger.info(`Checking ${name}`) + try { + await run() + checks.push({ name, status: 'passed', durationMs: Math.round(performance.now() - started) }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: Math.round(performance.now() - started), + error: getErrorMessage(error), + }) + throw error + } +} + +try { + const origin = `http://127.0.0.1:${fixture.port}` + const pageOrigin = `http://localhost:${fixture.port}` + const built = spawnSync(process.execPath, ['scripts/build.ts'], { + cwd: root, + env: { ...process.env, SIM_EXTENSION_ORIGIN: origin, SIM_EXTENSION_DEV: '1' }, + encoding: 'utf8', + }) + assert(built.status === 0, `Extension build failed: ${built.stderr}`) + await cp(resolve(root, 'dist'), extensionPath, { recursive: true }) + const manifest = JSON.parse(await readFile(resolve(extensionPath, 'manifest.json'), 'utf8')) + // The test package pregrants its disposable fixture site, equivalent to an accepted site prompt. + manifest.host_permissions.push(`${pageOrigin}/*`) + await writeFile(resolve(extensionPath, 'manifest.json'), JSON.stringify(manifest)) + context = await chromium.launchPersistentContext(resolve(temporary, 'profile'), { + channel: 'chromium', + headless: true, + args: [`--disable-extensions-except=${extensionPath}`, `--load-extension=${extensionPath}`], + }) + const worker = context.serviceWorkers()[0] ?? (await context.waitForEvent('serviceworker')) + context.setDefaultTimeout(15_000) + const extensionId = worker.url().split('/')[2] + const signIn = await context.newPage() + await signIn.goto(`${origin}/sign-in`) + await signIn.close() + const panel: Page = await context.newPage() + await panel.goto(`chrome-extension://${extensionId}/sidepanel.html`) + await panel.getByRole('button', { name: 'Attach page', exact: true }).waitFor() + const frame: Frame = panel.frames()[1] ?? (await panel.waitForEvent('frameattached')) + const app = panel.frames().find((candidate) => candidate.url().startsWith(origin)) ?? frame + assert(app, 'Hosted frame did not load') + await app.waitForSelector('#received') + await check('A normal-tab HttpOnly session cookie reaches the hosted frame', async () => { + assert( + (await app.locator('#session').textContent()) === 'present', + 'Existing session cookie was not sent to the hosted app' + ) + }) + const page = await context.newPage() + await page.goto(`${pageOrigin}/ticket?access_token=URL_SECRET#secret`) + await page.bringToFront() + const capture = (name = 'Attach page') => + panel + .getByRole('button', { name, exact: true }) + .evaluate((button) => (button as HTMLButtonElement).click()) + const snapshots = async () => + pageSnapshotSchema.array().parse(JSON.parse(await app.locator('#received').innerText())) + let receiverSessionId = generateId() + await app.evaluate((sessionId) => { + document.body.dataset.sessionId = sessionId + }, receiverSessionId) + const ready = () => + app.evaluate( + ({ targetOrigin, sessionId }) => { + document.body.dataset.sessionId = sessionId + parent.postMessage( + { channel: 'sim-extension', version: 1, type: 'ready', sessionId }, + targetOrigin + ) + }, + { targetOrigin: `chrome-extension://${extensionId}`, sessionId: receiverSessionId } + ) + const pendingIntent = () => + worker.evaluate(async () => { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) + if (tab.id === undefined || !tab.url) throw new Error('Missing source tab') + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(tab.url)) + const urlDigest = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, '0') + ).join('') + const [result] = await chrome.scripting.executeScript({ + target: { tabId: tab.id, frameIds: [0] }, + func: () => null, + }) + return { + tabId: tab.id, + windowId: tab.windowId, + kind: 'page', + createdAt: Date.now(), + documentId: result.documentId, + urlDigest, + } + }) + + await check('Capture stays local until the trusted hosted frame is ready', async () => { + await capture() + await panel.getByRole('status').filter({ hasText: 'Waiting for Sim' }).waitFor() + assert((await snapshots()).length === 0, 'Capture crossed before readiness') + await panel.evaluate( + (sessionId) => + new Promise((resolve) => { + window.addEventListener('message', function listener(event) { + if (event.source !== window) return + window.removeEventListener('message', listener) + resolve() + }) + window.postMessage( + { channel: 'sim-extension', version: 1, type: 'ready', sessionId }, + '*' + ) + }), + receiverSessionId + ) + assert((await snapshots()).length === 0, 'Wrong-origin readiness was accepted') + const wrongFrameDelivered = panel.evaluate( + () => + new Promise((resolve) => { + window.addEventListener('message', function listener(event) { + if ( + event.source === window || + event.source === document.querySelector('iframe')?.contentWindow + ) + return + window.removeEventListener('message', listener) + resolve() + }) + }) + ) + await app.evaluate((sessionId) => { + const child = document.createElement('iframe') + child.srcdoc = ` + + diff --git a/apps/extension/src/sidepanel.tsx b/apps/extension/src/sidepanel.tsx new file mode 100644 index 00000000000..99b3b46887d --- /dev/null +++ b/apps/extension/src/sidepanel.tsx @@ -0,0 +1,304 @@ +import { useEffect, useRef, useState } from 'react' +import { + appToExtensionMessageSchema, + type ExtensionToAppMessage, + type PageSnapshot, +} from '@sim/browser-extension-protocol' +import { + Button, + Chip, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@sim/emcn' +import { Globe, MoreHorizontal } from '@sim/emcn/icons' +import { getErrorMessage } from '@sim/utils/errors' +import { createRoot } from 'react-dom/client' +import { captureActivePage } from '@/capture' +import { CAPTURE_INTENT_KEY, CAPTURE_INTENT_TTL_MS, SIM_ORIGIN } from '@/config' +import { captureIntentSchema } from '@/intent' +import '@/shell.css' + +interface SidePanelProps { + origin: string +} + +function SidePanel({ origin }: SidePanelProps) { + const frameRef = useRef(null) + const pendingRef = useRef(null) + const sessionRef = useRef(null) + const deliveredRef = useRef(null) + const captureGenerationRef = useRef(0) + const intentCutoffRef = useRef(0) + const busyRef = useRef(false) + const [busy, setBusy] = useState(false) + const [status, setStatus] = useState('') + const [site, setSite] = useState<{ origin: string; allowed: boolean } | null>(null) + + function deliver() { + const snapshot = pendingRef.current + const sessionId = sessionRef.current + if (!snapshot || !sessionId || deliveredRef.current === snapshot.id) return + if (Date.now() - Date.parse(snapshot.capturedAt) > CAPTURE_INTENT_TTL_MS) { + pendingRef.current = null + setStatus('The capture expired. Attach the page again.') + return + } + const message: ExtensionToAppMessage = { + channel: 'sim-extension', + version: 1, + type: 'context', + sessionId, + snapshot, + } + frameRef.current?.contentWindow?.postMessage(message, SIM_ORIGIN) + deliveredRef.current = snapshot.id + } + + function invalidateSession() { + intentCutoffRef.current = Date.now() + if (pendingRef.current || busyRef.current) + setStatus('The conversation changed before delivery. Attach the page again.') + captureGenerationRef.current++ + pendingRef.current = null + deliveredRef.current = null + sessionRef.current = null + busyRef.current = false + setBusy(false) + } + + async function capture( + kind: 'page' | 'selection', + expected?: { tabId: number; documentId: string; urlDigest: string } + ) { + if (busyRef.current) return + if (pendingRef.current) { + setStatus('Waiting for Sim to receive the previous capture. Try again after it appears.') + return + } + busyRef.current = true + setBusy(true) + const generation = ++captureGenerationRef.current + setStatus('Reading the page…') + try { + const snapshot = await captureActivePage(kind, expected) + if (generation !== captureGenerationRef.current) return + pendingRef.current = snapshot + setStatus('Waiting for Sim to receive the attachment…') + deliver() + } catch (error) { + if (generation === captureGenerationRef.current) + setStatus( + getErrorMessage( + error, + 'Could not read this page. Click the Sim toolbar icon and try again.' + ) + ) + } finally { + if (generation === captureGenerationRef.current) { + busyRef.current = false + setBusy(false) + } + } + } + + useEffect(() => { + let disposed = false + let appTheme: 'light' | 'dark' | undefined + const systemTheme = window.matchMedia('(prefers-color-scheme: dark)') + const applyTheme = () => { + const theme = appTheme ?? (systemTheme.matches ? 'dark' : 'light') + document.documentElement.classList.toggle('dark', theme === 'dark') + document.documentElement.classList.toggle('light', theme === 'light') + } + applyTheme() + systemTheme.addEventListener('change', applyTheme) + let metadataGeneration = 0 + const updateSite = async () => { + const generation = ++metadataGeneration + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) + let next: { origin: string; allowed: boolean } | null = null + if (tab?.url && !tab.incognito) { + const url = new URL(tab.url) + if (['http:', 'https:'].includes(url.protocol) && url.origin !== SIM_ORIGIN) { + next = { + origin: url.origin, + allowed: await chrome.permissions.contains({ origins: [`${url.origin}/*`] }), + } + } + } + if (!disposed && generation === metadataGeneration) setSite(next) + } + const update = () => { + void updateSite().catch(() => { + if (!disposed) setSite(null) + }) + } + let consumingIntent = false + const consumeIntent = async () => { + if (disposed || consumingIntent || busyRef.current || pendingRef.current) return + consumingIntent = true + try { + const window = await chrome.windows.getCurrent() + if (window.id === undefined) return + const stored = await chrome.storage.session.get(null) + if (disposed || busyRef.current || pendingRef.current) return + const keys = Object.keys(stored).filter((key) => + key.startsWith(`${CAPTURE_INTENT_KEY}:${window.id}:`) + ) + const intents = keys + .flatMap((key) => { + const intent = captureIntentSchema.safeParse(stored[key]) + return intent.success && intent.data.windowId === window.id ? [intent.data] : [] + }) + .sort((left, right) => right.createdAt - left.createdAt) + if (keys.length) await chrome.storage.session.remove(keys) + const intent = intents[0] + if (intent && intent.createdAt <= intentCutoffRef.current) { + if (!disposed) + setStatus( + 'A capture from the previous conversation was discarded. Attach the page again.' + ) + return + } + if (!disposed && intent && Date.now() - intent.createdAt <= CAPTURE_INTENT_TTL_MS) + await capture(intent.kind, intent) + } catch { + if (!disposed) setStatus('Could not receive the capture request. Attach the page again.') + } finally { + consumingIntent = false + } + } + const storageChanged = ( + changes: Record, + area: string + ) => { + if (area === 'session' && Object.values(changes).some((change) => change.newValue)) + void consumeIntent() + } + const message = (event: MessageEvent) => { + if (event.origin !== SIM_ORIGIN || event.source !== frameRef.current?.contentWindow) return + const parsed = appToExtensionMessageSchema.safeParse(event.data) + if (!parsed.success) return + if (parsed.data.type === 'theme') { + appTheme = parsed.data.theme + applyTheme() + } else if (parsed.data.type === 'ready') { + if (sessionRef.current && sessionRef.current !== parsed.data.sessionId) invalidateSession() + sessionRef.current = parsed.data.sessionId + deliver() + } else if (parsed.data.sessionId !== sessionRef.current) { + return + } else if (parsed.data.type === 'unavailable') { + invalidateSession() + } else if (pendingRef.current?.id === parsed.data.snapshotId) { + pendingRef.current = null + setStatus('') + void consumeIntent() + } + } + const expire = window.setInterval(() => { + if ( + pendingRef.current && + Date.now() - Date.parse(pendingRef.current.capturedAt) > CAPTURE_INTENT_TTL_MS + ) { + pendingRef.current = null + setStatus('The capture expired. Attach the page again.') + } + void consumeIntent() + }, 5_000) + window.addEventListener('message', message) + chrome.storage.onChanged.addListener(storageChanged) + chrome.tabs.onActivated.addListener(update) + chrome.tabs.onUpdated.addListener(update) + chrome.permissions.onAdded.addListener(update) + chrome.permissions.onRemoved.addListener(update) + update() + void consumeIntent() + return () => { + disposed = true + captureGenerationRef.current++ + pendingRef.current = null + sessionRef.current = null + window.clearInterval(expire) + systemTheme.removeEventListener('change', applyTheme) + window.removeEventListener('message', message) + chrome.storage.onChanged.removeListener(storageChanged) + chrome.tabs.onActivated.removeListener(update) + chrome.tabs.onUpdated.removeListener(update) + chrome.permissions.onAdded.removeListener(update) + chrome.permissions.onRemoved.removeListener(update) + } + }, []) + + async function toggleSite() { + if (!site) return + const origins = [`${site.origin}/*`] + try { + const changed = site.allowed + ? await chrome.permissions.remove({ origins }) + : await chrome.permissions.request({ origins }) + if (!changed) + setStatus('Site access was not changed. You can still use the Sim toolbar icon.') + else setStatus('') + } catch { + setStatus('Could not change site access. Use Chrome’s extension settings.') + } + } + + return ( +
+
+
+ void capture('page')}> + Attach page + + void capture('selection')}> + Selection + + + + + + + {site ? ( + void toggleSite()}> + {site.allowed ? 'Remove access to' : 'Allow access to'}{' '} + {new URL(site.origin).hostname} + + ) : ( + No site access to manage + )} + + +
+

+ {status} +

+
+