diff --git a/docs/onchain-governance.md b/docs/onchain-governance.md index 777675399..2fca42cb3 100644 --- a/docs/onchain-governance.md +++ b/docs/onchain-governance.md @@ -51,13 +51,15 @@ export RPC_URL="https://eth-sepolia.g.alchemy.com/v2/" ## Configure KMS for On-Chain Auth -The KMS CVM includes an auth-api service that connects to your DstackKms contract. Configure it via environment variables in the KMS CVM: +The KMS CVM includes an auth-api service that connects to your DstackKms contract. The default [KMS Compose](../dstack/kms/dstack-app/docker-compose.yaml) reads Phala L2 through a proof-verifying light client; see the [light-client guide](../dstack/kms/light-client/README.md). Configure it via environment variables in the KMS CVM: ```bash KMS_CONTRACT_ADDR= -ETH_RPC_URL= +ETH_MAX_BLOCK_AGE_SECONDS=300 ``` +For other chains, run auth-api with `ETH_RPC_URL` pointing to your RPC endpoint. + The auth-api validates boot requests against the smart contract. See [Deployment Guide](./deployment.md#2-deploy-kms-as-cvm) for complete setup instructions. ## Whitelist OS Image diff --git a/dstack/kms/README.md b/dstack/kms/README.md index f837488d9..aa3f13c91 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -103,7 +103,12 @@ certificate. `core.onboard.public_key_handover = true` (the default) keeps ### Deployment The first two components are deployed as an dstack app on dstack in Local-Key-Provider mode. -The docker compose file would look like [this](dstack-app/docker-compose.yaml). +The default [Docker Compose file](dstack-app/docker-compose.yaml) uses the +[Phala L2 light client](light-client/README.md). Set `KMS_CONTRACT_ADDR` to a +Phala contract compatible with the current auth API, and use a published KMS +image digest; deployment does not build images. Reads use proof-verified state +signed by the L1-registered Phala sequencer, not L1-settled state. See the +light-client guide for deployment requirements. The solidity contracts are deployed on an ethereum compatible chain. diff --git a/dstack/kms/auth-eth/jest.config.js b/dstack/kms/auth-eth/jest.config.js index dafec6254..d097ee000 100644 --- a/dstack/kms/auth-eth/jest.config.js +++ b/dstack/kms/auth-eth/jest.config.js @@ -6,7 +6,7 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', roots: ['/src'], - testMatch: ['**/main.test.ts'], + testMatch: ['**/*.test.ts'], moduleFileExtensions: ['ts', 'js', 'json', 'node'], collectCoverageFrom: [ 'src/**/*.ts', diff --git a/dstack/kms/auth-eth/src/ethereum.test.ts b/dstack/kms/auth-eth/src/ethereum.test.ts new file mode 100644 index 000000000..4086c49aa --- /dev/null +++ b/dstack/kms/auth-eth/src/ethereum.test.ts @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// SPDX-License-Identifier: Apache-2.0 + +import { ethers } from 'ethers'; +import { EthereumBackend } from './ethereum'; +import { BootInfo } from './types'; + +jest.mock('ethers', () => ({ + ethers: { ...jest.requireActual('ethers').ethers, Contract: jest.fn() }, +})); + +const bootInfo: BootInfo = { + appId: '00'.repeat(20), instanceId: '00'.repeat(20), + composeHash: '00'.repeat(32), deviceId: '00'.repeat(32), + mrAggregated: '00'.repeat(32), mrSystem: '00'.repeat(32), + osImageHash: '00'.repeat(32), tcbStatus: 'UpToDate', advisoryIds: [], +}; + +describe('verified authorization reads', () => { + const contract = { + isAppAllowed: jest.fn(), isKmsAllowed: jest.fn(), gatewayAppId: jest.fn(), + }; + const provider = { + getNetwork: jest.fn(), getBlockNumber: jest.fn(), getBlock: jest.fn(), + }; + let backend: EthereumBackend; + + beforeEach(() => { + jest.resetAllMocks(); + (ethers.Contract as unknown as jest.Mock).mockReturnValue(contract); + provider.getNetwork.mockResolvedValue({ chainId: 2035n }); + provider.getBlockNumber.mockResolvedValue(100); + provider.getBlock.mockResolvedValue({ number: 98, timestamp: Math.floor(Date.now() / 1000) - 20 }); + contract.isAppAllowed.mockResolvedValue([true, '']); + contract.isKmsAllowed.mockResolvedValue([true, '']); + contract.gatewayAppId.mockResolvedValue('gateway'); + backend = new EthereumBackend(provider as unknown as ethers.JsonRpcProvider, + ethers.ZeroAddress, { chainId: 2035, blockLag: 2, maxAgeSeconds: 60 }); + }); + + it.each([false, true])('pins all boot reads to the same fresh block (kms=%s)', async (isKms) => { + await expect(backend.checkBoot(bootInfo, isKms)).resolves.toEqual({ + isAllowed: true, reason: '', gatewayAppId: 'gateway', + }); + expect(provider.getBlock).toHaveBeenCalledWith(98); + expect(isKms ? contract.isKmsAllowed : contract.isAppAllowed) + .toHaveBeenCalledWith(expect.any(Object), { blockTag: 98 }); + expect(contract.gatewayAppId).toHaveBeenCalledWith({ blockTag: 98 }); + }); + + it('rejects a different chain before contract execution', async () => { + provider.getNetwork.mockResolvedValue({ chainId: 1n }); + await expect(backend.checkBoot(bootInfo, false)).rejects.toThrow('chain ID mismatch'); + expect(contract.isAppAllowed).not.toHaveBeenCalled(); + }); + + it.each([null, { number: 98, timestamp: 0 }, { number: 98, timestamp: 9999999999 }])( + 'rejects unavailable, stale, or future state: %j', async (block) => { + provider.getBlock.mockResolvedValue(block); + await expect(backend.checkBoot(bootInfo, false)).rejects.toThrow('not fresh'); + expect(contract.isAppAllowed).not.toHaveBeenCalled(); + }, + ); + + it('rejects state that expires while contract calls are in flight', async () => { + const started = Date.now(); + const clock = jest.spyOn(Date, 'now').mockReturnValue(started); + contract.gatewayAppId.mockImplementation(async () => { + clock.mockReturnValue(started + 61_000); + return 'gateway'; + }); + try { + await expect(backend.checkBoot(bootInfo, true)).rejects.toThrow('not fresh'); + } finally { + clock.mockRestore(); + } + }); + + it('does not fallback when proof execution fails', async () => { + contract.isAppAllowed.mockRejectedValue(new Error('invalid proof')); + await expect(backend.checkBoot(bootInfo, false)).rejects.toThrow('invalid proof'); + expect(contract.isAppAllowed).toHaveBeenCalledTimes(1); + expect(contract.gatewayAppId).not.toHaveBeenCalled(); + }); +}); diff --git a/dstack/kms/auth-eth/src/ethereum.ts b/dstack/kms/auth-eth/src/ethereum.ts index ebd655140..149010b01 100644 --- a/dstack/kms/auth-eth/src/ethereum.ts +++ b/dstack/kms/auth-eth/src/ethereum.ts @@ -13,11 +13,17 @@ const DSTACK_KMS_ABI = [ "function appImplementation() view returns (address)" ]; +export interface VerifiedReadPolicy { + chainId: number; + blockLag: number; + maxAgeSeconds: number; +} + export class EthereumBackend { private provider: ethers.JsonRpcProvider; private kmsContract: ethers.Contract; - constructor(provider: ethers.JsonRpcProvider, kmsContractAddr: string) { + constructor(provider: ethers.JsonRpcProvider, kmsContractAddr: string, private readPolicy?: VerifiedReadPolicy) { this.provider = provider; this.kmsContract = new ethers.Contract( ethers.getAddress(kmsContractAddr), @@ -26,6 +32,28 @@ export class EthereumBackend { ); } + private async authorizationBlock(): Promise<{ blockTag: ethers.BlockTag; timestamp?: number }> { + if (!this.readPolicy) return { blockTag: await this.provider.getBlockNumber() }; + const { chainId, blockLag } = this.readPolicy; + if (Number((await this.provider.getNetwork()).chainId) !== chainId) { + throw new Error('authorization chain ID mismatch'); + } + const head = await this.provider.getBlockNumber(); + if (head < blockLag) throw new Error('authorization head unavailable'); + const block = await this.provider.getBlock(head - blockLag); + if (!block) throw new Error('authorization block is not fresh'); + this.checkFreshness(block.timestamp); + return { blockTag: block.number, timestamp: block.timestamp }; + } + + private checkFreshness(timestamp?: number): void { + if (!this.readPolicy || timestamp === undefined) return; + const now = Math.floor(Date.now() / 1000); + if (now - timestamp > this.readPolicy.maxAgeSeconds || timestamp > now + 5) { + throw new Error('authorization block is not fresh'); + } + } + private decodeHex(hex: string, sz: number = 32): string { // Remove '0x' prefix if present hex = hex.startsWith('0x') ? hex.slice(2) : hex; @@ -50,8 +78,7 @@ export class EthereumBackend { tcbStatus: bootInfo.tcbStatus, advisoryIds: bootInfo.advisoryIds }; - // Read the whole decision from one block. - const blockTag = await this.provider.getBlockNumber(); + const { blockTag, timestamp } = await this.authorizationBlock(); let response; if (isKms) { response = await this.kmsContract.isKmsAllowed(bootInfoStruct, { blockTag }); @@ -60,6 +87,7 @@ export class EthereumBackend { } const [isAllowed, reason] = response; const gatewayAppId = await this.kmsContract.gatewayAppId({ blockTag }); + this.checkFreshness(timestamp); return { isAllowed, reason, @@ -68,7 +96,7 @@ export class EthereumBackend { } async getGatewayAppId(): Promise { - return await this.kmsContract.gatewayAppId(); + return await this.kmsContract.gatewayAppId({ blockTag: (await this.authorizationBlock()).blockTag }); } async getChainId(): Promise { @@ -77,6 +105,6 @@ export class EthereumBackend { } async getAppImplementation(): Promise { - return await this.kmsContract.appImplementation(); + return await this.kmsContract.appImplementation({ blockTag: (await this.authorizationBlock()).blockTag }); } } diff --git a/dstack/kms/auth-eth/src/main.test.ts b/dstack/kms/auth-eth/src/main.test.ts index 842f1e36b..5c9b63002 100644 --- a/dstack/kms/auth-eth/src/main.test.ts +++ b/dstack/kms/auth-eth/src/main.test.ts @@ -129,3 +129,25 @@ describe('Server', () => { }); }); }); + +describe('verified-read configuration', () => { + const settings = ['ETH_CHAIN_ID', 'ETH_BLOCK_LAG', 'ETH_MAX_BLOCK_AGE_SECONDS']; + const saved = settings.map(name => process.env[name]); + + afterEach(() => { + settings.forEach((name, index) => { + if (saved[index] === undefined) delete process.env[name]; + else process.env[name] = saved[index]; + }); + }); + + it.each(['ETH_BLOCK_LAG', 'ETH_MAX_BLOCK_AGE_SECONDS'])( + 'requires an explicit %s when verified reads are enabled', async missing => { + Object.assign(process.env, { + ETH_CHAIN_ID: '2035', ETH_BLOCK_LAG: '2', ETH_MAX_BLOCK_AGE_SECONDS: '60', + }); + delete process.env[missing]; + await expect(build()).rejects.toThrow(`invalid ${missing}`); + }, + ); +}); diff --git a/dstack/kms/auth-eth/src/server.ts b/dstack/kms/auth-eth/src/server.ts index 54dd14346..290c85e27 100644 --- a/dstack/kms/auth-eth/src/server.ts +++ b/dstack/kms/auth-eth/src/server.ts @@ -57,7 +57,22 @@ export async function build(): Promise { const rpcUrl = process.env.ETH_RPC_URL || 'http://localhost:8545'; const kmsContractAddr = process.env.KMS_CONTRACT_ADDR || '0x0000000000000000000000000000000000000000'; const provider = new ethers.JsonRpcProvider(rpcUrl); - server.decorate('ethereum', new EthereumBackend(provider, kmsContractAddr)); + const integerSetting = (name: string): number => { + const value = process.env[name] ?? ''; + if (!/^(0|[1-9][0-9]*)$/.test(value) || !Number.isSafeInteger(Number(value))) { + throw new Error(`invalid ${name}`); + } + return Number(value); + }; + const readPolicy = process.env.ETH_CHAIN_ID === undefined ? undefined : { + chainId: integerSetting('ETH_CHAIN_ID'), + blockLag: integerSetting('ETH_BLOCK_LAG'), + maxAgeSeconds: integerSetting('ETH_MAX_BLOCK_AGE_SECONDS'), + }; + if (!readPolicy) { + server.log.warn('ETH_CHAIN_ID is unset: authorization reads skip chain ID and freshness checks'); + } + server.decorate('ethereum', new EthereumBackend(provider, kmsContractAddr, readPolicy)); const publicRpcEndpoint = (value: string): string => { try { diff --git a/dstack/kms/dstack-app/builder/Dockerfile b/dstack/kms/dstack-app/builder/Dockerfile index 426dd6ab9..854e2936b 100644 --- a/dstack/kms/dstack-app/builder/Dockerfile +++ b/dstack/kms/dstack-app/builder/Dockerfile @@ -26,6 +26,27 @@ RUN rustup target add x86_64-unknown-linux-musl RUN cd repo/dstack && cargo build --release --locked -p dstack-kms --target x86_64-unknown-linux-musl RUN echo "${DSTACK_REV}" > /build/.GIT_REV +FROM rust:1.94.1-bookworm@sha256:6ae102bdbf528294bc79ad6e1fae682f6f7c2a6e6621506ba959f9685b308a55 AS helios-builder +# Vendored OpenSSL embeds its build date unless SOURCE_DATE_EPOCH is set. +ARG SOURCE_DATE_EPOCH +WORKDIR /build/helios +RUN git init && git remote add origin https://github.com/Dstack-TEE/helios.git \ + && git fetch --depth 1 origin ff3820aba2f218c75fae6b542b6067580e9f4ec4 \ + && git checkout --detach FETCH_HEAD +RUN cargo test --locked --release -p helios-opstack --lib \ + && cargo build --locked --release -p helios-cli -p helios-opstack --bins \ + && install -D target/release/helios /out/helios \ + && install -D target/release/opstack-server /out/opstack-server \ + && rm -rf target + +FROM node:22.23.4-bookworm-slim@sha256:de170b39439eb0e5a3444ad7d60b015614a381e10fecae2e30ec9e735122f38d AS auth-builder +WORKDIR /app +COPY --from=kms-builder /build/repo/dstack/kms/auth-eth/package*.json ./ +RUN npm ci --ignore-scripts +COPY --from=kms-builder /build/repo/dstack/kms/auth-eth/tsconfig.json ./ +COPY --from=kms-builder /build/repo/dstack/kms/auth-eth/src/ src/ +RUN npm run build && npm prune --omit=dev --ignore-scripts + FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe COPY --from=build-shared /pin-packages.sh /build/ COPY ./shared/pinned-packages.txt /build/ @@ -41,6 +62,11 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache WORKDIR / COPY --from=kms-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms /usr/local/bin/dstack-kms +COPY --from=helios-builder /out/helios /usr/local/bin/helios +COPY --from=helios-builder /out/opstack-server /usr/local/bin/opstack-server +COPY --from=auth-builder /usr/local/bin/node /usr/local/bin/node +COPY --from=auth-builder /app/dist/ /opt/auth-eth/dist/ +COPY --from=auth-builder /app/node_modules/ /opt/auth-eth/node_modules/ COPY --from=kms-builder /build/.GIT_REV /etc/ # The OCI metadata is generated once by build-lib.sh and injected here, as # the config labels and as the manifest annotations, so the three views of diff --git a/dstack/kms/dstack-app/compose-dev.yaml b/dstack/kms/dstack-app/compose-dev.yaml deleted file mode 100644 index 02e17c4b5..000000000 --- a/dstack/kms/dstack-app/compose-dev.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: © 2025 Phala Network -# SPDX-FileCopyrightText: © 2025 Test in Prod -# -# SPDX-License-Identifier: Apache-2.0 - -services: - auth-api: - build: - context: . - dockerfile_inline: | - FROM node:22.23.4-bookworm-slim@sha256:de170b39439eb0e5a3444ad7d60b015614a381e10fecae2e30ec9e735122f38d - WORKDIR /app - - RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* - RUN git clone ${GIT_REPOSITORY} && \ - cd dstack && \ - git checkout ${GIT_REV} - WORKDIR /app/dstack/dstack/kms/auth-eth - RUN npm ci - RUN npm run build - CMD node dist/main.js - environment: - - HOST=0.0.0.0 - - PORT=8000 - - ETH_RPC_URL=${ETH_RPC_URL} - - KMS_CONTRACT_ADDR=${KMS_CONTRACT_ADDR} - restart: unless-stopped - ports: - - 8001:8000 - - kms: - image: ${KMS_IMAGE} - volumes: - - kms-volume:/kms - - /var/run/dstack.sock:/var/run/dstack.sock - ports: - - 8000:8000 - depends_on: - - auth-api - restart: unless-stopped - configs: - - source: kms_config - target: /kms/kms.toml - command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' - -volumes: - kms-volume: - -configs: - kms_config: - content: | - [rpc] - address = "0.0.0.0" - port = 8000 - - [rpc.tls] - key = "/kms/certs/rpc.key" - certs = "/kms/certs/rpc.crt" - - [core] - cert_dir = "/kms/certs" - - [core.image] - verify = true - cache_dir = "/kms/images" - download_url = "${IMAGE_DOWNLOAD_URL}" - download_timeout = "2m" - - [core.auth_api] - type = "webhook" - - [core.auth_api.webhook] - url = "http://auth-api:8000" - - [core.onboard] - enabled = true - auto_bootstrap_domain = "" - address = "0.0.0.0" - port = 8000 diff --git a/dstack/kms/dstack-app/deploy-to-vmm.sh b/dstack/kms/dstack-app/deploy-to-vmm.sh index 23f8c2917..31b500e47 100755 --- a/dstack/kms/dstack-app/deploy-to-vmm.sh +++ b/dstack/kms/dstack-app/deploy-to-vmm.sh @@ -26,9 +26,6 @@ else # The address of the KMS contract # KMS_CONTRACT_ADDR=0x59E4a36B01a87fD9D1A4C12377253FE9a7b018Ba -# The address of the auth-api service listening on Host machine -# AUTH_API_RPC_ADDR=0.0.0.0:8001 - # The address of the KMS service listening on Host machine # KMS_RPC_ADDR=0.0.0.0:9201 @@ -41,20 +38,19 @@ else # Image hash verification feature flag VERIFY_IMAGE=true -# The URL of the Ethereum RPC service -ETH_RPC_URL=https://rpc.phala.network - -# The Git repository to deploy -GIT_REPOSITORY=https://github.com/Dstack-TEE/dstack.git - -# The Git revision to deploy -GIT_REV=HEAD +# Maximum age of sequencer-authenticated state accepted for authorization. +# Choose according to the acceptable revocation delay and observed RPC latency. +# ETH_MAX_BLOCK_AGE_SECONDS= # The dstack OS image name to use for the KMS app -OS_IMAGE=dstack-0.5.5 +OS_IMAGE=dstack-0.6.0 + +# Optional user config overriding the light-client data sources +# (see ../light-client/README.md) +# USER_CONFIG=./kms-user-config # The dstack KMS image name to use for the KMS app -KMS_IMAGE=dstacktee/dstack-kms@sha256:11ac59f524a22462ccd2152219b0bec48a28ceb734e32500152d4abefab7a62a +# KMS_IMAGE=ghcr.io/dstack-tee/dstack-kms@sha256: # The admin token for the KMS app ADMIN_TOKEN=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1) @@ -66,13 +62,12 @@ fi required_env_vars=( "VMM_RPC" "KMS_RPC_ADDR" - "AUTH_API_RPC_ADDR" "GUEST_AGENT_ADDR" "KMS_CONTRACT_ADDR" - "ETH_RPC_URL" + "ETH_MAX_BLOCK_AGE_SECONDS" "IMAGE_DOWNLOAD_URL" "VERIFY_IMAGE" - "GIT_REPOSITORY" + "KMS_IMAGE" ) for var in "${required_env_vars[@]}"; do @@ -87,24 +82,20 @@ CLI="../../vmm/src/vmm-cli.py --url $VMM_RPC" COMPOSE_TMP=$(mktemp) -GIT_REV=$(git rev-parse "$GIT_REV") - # shellcheck disable=SC2034 # consumed via `subvar` into compose-*.yaml ADMIN_TOKEN_HASH=$(echo -n "$ADMIN_TOKEN" | sha256sum | cut -d' ' -f1) -cp compose-dev.yaml "$COMPOSE_TMP" +cp docker-compose.yaml "$COMPOSE_TMP" subvar() { - sed -i "s|\${$1}|${!1}|g" "$COMPOSE_TMP" + sed -i "s|\${$1\(:[-?][^}]*\)\?}|${!1}|g" "$COMPOSE_TMP" } -subvar ETH_RPC_URL +subvar ETH_MAX_BLOCK_AGE_SECONDS subvar KMS_CONTRACT_ADDR -subvar GIT_REV subvar IMAGE_DOWNLOAD_URL subvar ADMIN_TOKEN_HASH subvar VERIFY_IMAGE -subvar GIT_REPOSITORY subvar KMS_IMAGE echo "Docker compose file:" @@ -141,8 +132,8 @@ $CLI deploy \ --compose .app-compose.json \ --image "$OS_IMAGE" \ --port tcp:"$KMS_RPC_ADDR":8000 \ - --port tcp:"$AUTH_API_RPC_ADDR":8001 \ --port tcp:"$GUEST_AGENT_ADDR":8090 \ --vcpu 8 \ --memory 8G \ - --disk 50G + --disk 50G \ + --user-config "${USER_CONFIG:-<(echo '# KMS light-client data sources')}" diff --git a/dstack/kms/dstack-app/docker-compose.yaml b/dstack/kms/dstack-app/docker-compose.yaml index 4fb1b603f..5c3a62dcc 100644 --- a/dstack/kms/dstack-app/docker-compose.yaml +++ b/dstack/kms/dstack-app/docker-compose.yaml @@ -2,84 +2,113 @@ # # SPDX-License-Identifier: Apache-2.0 +x-kms-image: &kms-image ${KMS_IMAGE:?Set a published KMS image digest} +# Only this file is mounted: the rest of host-shared holds the app keys. +x-user-config: &user-config /dstack/.host-shared/.user-config:/etc/user-config:ro + services: - # Helios is a Ethereum light client + # Data sources are untrusted, so the host may override them through the + # unmeasured user config without changing the compose hash. + relay: + image: *kms-image + restart: unless-stopped + volumes: [*user-config] + command: + - sh + - -c + - | + cfg() { v=$$(sed -n "s/^$$1=//p" /etc/user-config 2>/dev/null | tail -n 1); echo "$${v:-$$2}"; } + export OP_NODE_P2P_STATIC="$$(cfg OP_NODE_P2P_STATIC /ip4/8.229.99.126/tcp/9222/p2p/16Uiu2HAmA7EmxHixJhunY6dZKi7nriyUfBCWP3aBPvGUsQBbigm1)" + exec opstack-server --network=phala --execution-rpc=https://rpc.phala.network \ + --server-address=0.0.0.0:3000 --gossip-address=0.0.0.0:9876 + environment: + OP_NODE_P2P_STATIC_IDLE_TIMEOUT: "30" + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + helios: - build: - context: . - dockerfile_inline: | - FROM rust:1.92.0@sha256:48851a839d6a67370c9dbe0e709bedc138e3e404b161c5233aedcf2b717366e4 - WORKDIR /app - RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* build-base openssl-dev protobuf protobuf-dev perl - RUN git clone https://github.com/a16z/helios && \ - cd helios && \ - git checkout 5c61864a167c16141a9a12b976c0e9398b332f07 - WORKDIR /app/helios - RUN cargo build --release - command: [ - "./target/release/helios", - "ethereum", - "--network", "mainnet", - "--checkpoint", "0xbee4f32f91e62060d2aa41c652f6c69431829cfb09b02ea3cad92f65bd15dcce", - "--rpc-bind-ip", "0.0.0.0", - "--rpc-port", "8545", - "--consensus-rpc", "https://ethereum.operationsolarstorm.org", - "--execution-rpc", "https://ethereum-rpc.publicnode.com" - ] + image: *kms-image + restart: unless-stopped + volumes: [*user-config] + command: + - sh + - -c + - | + cfg() { v=$$(sed -n "s/^$$1=//p" /etc/user-config 2>/dev/null | tail -n 1); echo "$${v:-$$2}"; } + exec helios opstack --network=phala \ + --execution-rpc="$$(cfg PHALA_RPC_URL https://rpc.phala.network)" \ + --consensus-rpc=http://relay:3000 \ + --ethereum-consensus-rpc="$$(cfg ETHEREUM_CONSENSUS_RPC https://ethereum.operationsolarstorm.org)" \ + --ethereum-execution-rpc="$$(cfg ETHEREUM_EXECUTION_RPC https://eth.drpc.org)" \ + --ethereum-checkpoint=0x3a19ad46aa7391ebfb8fa2690cb8ef56b70d69a2db8e32f4adb63ca7a8697598 \ + --max-head-age=${ETH_MAX_BLOCK_AGE_SECONDS:?Set the maximum accepted block age} \ + --rpc-bind-ip=0.0.0.0 --rpc-port=8545 + depends_on: [relay] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] - # Auth API is a webhook server that authenticates KMS instances and Apps launches auth-api: - build: - context: . - dockerfile_inline: | - FROM node:22.23.4-bookworm-slim@sha256:de170b39439eb0e5a3444ad7d60b015614a381e10fecae2e30ec9e735122f38d - WORKDIR /app - - RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* - RUN git clone https://github.com/Dstack-TEE/dstack.git && \ - cd dstack && \ - git checkout 78057c975fe4b9e21f557fb888d72eeecfb21178 - WORKDIR /app/dstack/kms/auth-eth - RUN npm ci && npm run build - CMD node dist/main.js + image: *kms-image + command: [node, /opt/auth-eth/dist/main.js] + restart: unless-stopped environment: - - HOST=0.0.0.0 - - PORT=8000 - - ETH_RPC_URL=http://helios:8545 - - KMS_CONTRACT_ADDR=0x515cd0dd984Adc75EE8625131FcC3d2D91f7F140 - depends_on: - - helios + HOST: 0.0.0.0 + PORT: "8000" + ETH_RPC_URL: http://helios:8545 + ETH_CHAIN_ID: "2035" + ETH_BLOCK_LAG: "2" + ETH_MAX_BLOCK_AGE_SECONDS: ${ETH_MAX_BLOCK_AGE_SECONDS:?Set the maximum accepted block age} + KMS_CONTRACT_ADDR: ${KMS_CONTRACT_ADDR:?Set the Phala KMS contract address} + depends_on: [helios] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] - # KMS handles the TEE Remote Attestation kms: - build: - context: . - dockerfile_inline: | - FROM rust:1.92.0@sha256:48851a839d6a67370c9dbe0e709bedc138e3e404b161c5233aedcf2b717366e4 - WORKDIR /app - RUN apt-get update && apt-get install -y \ - git \ - build-essential \ - libssl-dev \ - protobuf-compiler \ - libprotobuf-dev \ - clang \ - libclang-dev \ - --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* - RUN git clone https://github.com/Dstack-TEE/dstack.git && \ - cd dstack && \ - git checkout 78057c975fe4b9e21f557fb888d72eeecfb21178 - WORKDIR /app/dstack - RUN cargo build --release --locked -p dstack-kms - CMD ["./target/release/dstack-kms"] + image: *kms-image volumes: - - kms-volume:/etc/kms + - kms-volume:/kms - /var/run/dstack.sock:/var/run/dstack.sock ports: - 8000:8000 depends_on: - auth-api + restart: unless-stopped + configs: + - source: kms_config + target: /kms/kms.toml + command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' volumes: kms-volume: + +configs: + kms_config: + content: | + [rpc] + address = "0.0.0.0" + port = 8000 + + [rpc.tls] + key = "/kms/certs/rpc.key" + certs = "/kms/certs/rpc.crt" + + [core] + cert_dir = "/kms/certs" + + [core.image] + verify = true + cache_dir = "/kms/images" + download_url = "${IMAGE_DOWNLOAD_URL}" + download_timeout = "2m" + + [core.auth_api] + type = "webhook" + + [core.auth_api.webhook] + url = "http://auth-api:8000" + + [core.onboard] + enabled = true + auto_bootstrap_domain = "" + address = "0.0.0.0" + port = 8000 diff --git a/dstack/kms/light-client/README.md b/dstack/kms/light-client/README.md new file mode 100644 index 000000000..0e7f2d72c --- /dev/null +++ b/dstack/kms/light-client/README.md @@ -0,0 +1,95 @@ +# Phala L2 light client + +The default [KMS Compose](../dstack-app/docker-compose.yaml) runs KMS, auth-api, +Helios and its gossip relay inside the same CVM. All services use one published +KMS image; no deployment-time builds or source downloads are required. The KMS +builder pins the [Helios fork](https://github.com/Dstack-TEE/helios) by commit. + +## Deployment + +Set these variables and render Compose before passing it to dstack: + +- `KMS_IMAGE`: a reviewed, published KMS image digest. +- `KMS_CONTRACT_ADDR`: a Phala contract compatible with the current auth API. +- `ETH_MAX_BLOCK_AGE_SECONDS`: the maximum accepted age of authenticated state, + enforced by both Helios and auth-api. It bounds how long revocations can be + missed when the chain feed stalls, e.g. `300` (about 30 Phala blocks). +- `IMAGE_DOWNLOAD_URL`: the verified OS image download URL used by KMS. + +```sh +docker compose -f dstack/kms/dstack-app/docker-compose.yaml config > kms-compose.yaml +``` + +The app must enable `secure_time` (`vmm-cli.py compose --secure-time`, as +`deploy-to-vmm.sh` does). Freshness checks rely on the guest clock; without +NTS-synchronized time, the host can roll the clock back and replay old signed +blocks to hide revocations. `secure_time` is part of `app-compose.json`, so only +approve KMS compose hashes whose `app-compose.json` sets it to `true`. +Secure time needs a guest image whose chrony supports NTS (dstack 0.6.0 or +later); older images hang at boot ([#745](https://github.com/Dstack-TEE/dstack/issues/745)). + +Only the KMS port is published. + +### Data sources + +Helios and the relay read their endpoints from the unmeasured +[user config](../../../docs/vmm-cli-user-guide.md), so they can be changed +without a new compose hash. The file holds `KEY=value` lines; other keys are +ignored and unset keys use the defaults in the Compose file: + +| Key | Default | +|-----|---------| +| `PHALA_RPC_URL` | `https://rpc.phala.network` | +| `ETHEREUM_CONSENSUS_RPC` | `https://ethereum.operationsolarstorm.org` | +| `ETHEREUM_EXECUTION_RPC` | `https://eth.drpc.org` | +| `OP_NODE_P2P_STATIC` | Phala's public op-node | + +Set it with `USER_CONFIG` in `deploy-to-vmm.sh`, or later with +`vmm-cli.py update-user-config ` and a restart. Always deploy +with a non-empty user config (a comment is enough): the VMM skips empty ones, +and Docker then mounts a directory in its place. These sources are +untrusted: every response is checked against signatures or proofs, so a +malicious endpoint can only stall authorization. The Ethereum checkpoint stays +in Compose because it anchors that verification. + +## Trust and verification + +Reads use **sequencer-authenticated state, not L1-settled state**: + +1. Helios follows Ethereum mainnet with its sync-committee light client, starting + from the pinned `--ethereum-checkpoint`. +2. Every minute it proves Phala's unsafe signer from the L1 `SystemConfig` + contract (`0xeBf5859b7646ca9cf8A981613569bF28394F2571`) with an account and + storage proof. No Phala block is accepted before the first proof, or once the + latest proof is older than 15 minutes. A signer rotated on L1 therefore + replaces the old key automatically. +3. Each Phala block must carry that signer's signature and a header that hashes to + the signed block hash. Contract calls run locally against account, storage and + code proofs; RPC servers cannot supply an unchecked `eth_call` result. +4. auth-api checks chain ID 2035, reads two blocks behind the head so proofs are + available, pins one block for the whole decision, and rejects state older than + `ETH_MAX_BLOCK_AGE_SECONDS`, including state that expires during the call. + +The remaining trust is in the Phala sequencer key: a sequencer that signs a false +block can forge authorization results, because L1 settlement is not checked. +The relay, Ethereum RPCs and Phala RPC are data sources only. + +## Operations + +Operational failures fail closed with `authorization backend unavailable`: + +- **Startup.** Authorization is unavailable until Helios proves the signer and + caches three blocks, typically within a minute of boot. +- **Feed stalls.** The relay reconnects its static peer after 30 seconds without + a higher signed block. Longer outages exceed the age limit. +- **L1 unavailable.** Without a fresh signer proof, Helios stops accepting blocks. +- **Signer rotation.** Helios follows L1, but the relay filters gossip by the + signer pinned in the fork; update the fork and image. +- **Phala hard forks.** The fork schedule and header layout are pinned. A fork + that changes headers stops Helios with `payload block hash mismatch`; one that + only changes EVM rules can make local calls diverge. Update the fork and image + before every activation. +- **Checkpoint age.** Refresh `--ethereum-checkpoint` (a finalized beacon block + root) with each release; Helios warns once it is older than 14 days. + +Monitor the auth-api `/` endpoint and Helios `failed to advance` logs.