feat(dev-box): SSH dev box for dstack v0.6 CVMs

.github/workflows/build-dev-box.yml

@@ -0,0 +1,55 @@
+name: build dev-box
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'dev-box/**'
+      - '.github/workflows/build-dev-box.yml'
+  pull_request:
+    paths:
+      - 'dev-box/**'
+      - '.github/workflows/build-dev-box.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      IMAGE: ghcr.io/dstack-tee/dstack-dev-box
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      # On PRs (incl. forks) GITHUB_TOKEN lacks packages:write, so only build.
+      - name: Log in to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=sha,format=short
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: dev-box
+          file: dev-box/Dockerfile
+          platforms: linux/amd64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -164,6 +164,7 @@ Development and debugging tools. **Not for production.**
 | Example | Description |
 |---------|-------------|
 | [webshell](./webshell) | Web-based shell access for debugging |
+| [dev-box](./dev-box) | SSH, debugging tools, and a host shell for dstack v0.6 CVMs (replaces the removed dev OS image) |
 | [ssh-over-gateway](./ssh-over-gateway) | SSH tunneling through dstack gateway |
 | [tcp-port-forwarding](./tcp-port-forwarding) | Arbitrary TCP port forwarding |
 

dev-box/Dockerfile

@@ -0,0 +1,14 @@
+# Debian trixie matches the dstack 0.6 guest OS userland.
+FROM debian:trixie-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        openssh-server openssh-sftp-server rsync \
+        bash-completion ca-certificates curl dnsutils file gdb git htop \
+        iproute2 iputils-ping jq less lsof procps psmisc python3 strace \
+        tcpdump tmux util-linux vim \
+        docker-cli docker-compose \
+    && rm -rf /var/lib/apt/lists/* /etc/ssh/ssh_host_*
+COPY sshd_config /etc/ssh/sshd_config
+COPY motd /etc/motd
+COPY --chmod=755 host-shell /usr/local/bin/host-shell
+COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint
+ENTRYPOINT ["/usr/local/bin/entrypoint"]

dev-box/README.md

@@ -0,0 +1,98 @@
+# dstack Dev Box
+
+dstack v0.6 no longer ships a dev OS image with SSH and debugging tools. Add this one container to your compose file to get them back: root SSH into the CVM, a full Debian toolset, the host filesystem, the host Docker daemon, and a shell in the host OS itself.
+
+> [!WARNING]
+> This gives anyone holding an authorized key full root on the CVM: every container, the decrypted disk, and every secret in memory. It breaks the confidentiality guarantee your users rely on. The dev box is part of the attested `app-compose`, so verifiers can see it's there. Use it for development and debugging only.
+
+## Usage
+
+Add the `dev-box` service from [`docker-compose.yaml`](./docker-compose.yaml) next to your workload:
+
+```yaml
+services:
+  dev-box:
+    image: ghcr.io/dstack-tee/dstack-dev-box:latest
+    restart: unless-stopped
+    privileged: true
+    pid: host
+    network_mode: host
+    uts: host
+    environment:
+      - SSH_AUTHORIZED_KEYS=${SSH_AUTHORIZED_KEYS}
+    volumes:
+      - /:/host:rslave
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /dev/pts:/dev/pts
+      - ssh-host-keys:/etc/ssh/host-keys
+      - home:/root
+
+  your-app:
+    image: ...
+
+volumes:
+  ssh-host-keys:
+  home:
+```
+
+Deploy with your public key or keys in `SSH_AUTHORIZED_KEYS`, in `authorized_keys` format with one key per line. The container exits with an error if the variable is empty.
+
+```bash
+phala deploy -n my-app -c docker-compose.yaml \
+  --image dstack-0.6.1 \
+  -e SSH_AUTHORIZED_KEYS="$(cat ~/.ssh/id_ed25519.pub)"
+```
+
+On a plain dstack deployment, set `SSH_AUTHORIZED_KEYS` as an encrypted environment variable.
+
+## Connect
+
+`sshd` listens on port `1022`. Reach it through the dstack-gateway by adding this to `~/.ssh/config`:
+
+```
+Host my-app
+    User root
+    ProxyCommand openssl s_client -quiet -connect <app-id>-1022.<gateway-domain>:443 -servername <app-id>-1022.<gateway-domain> -verify_hostname <app-id>-1022.<gateway-domain> -verify_return_error
+```
+
+Replace `<app-id>` with your app ID and `<gateway-domain>` with the gateway base domain of the node the CVM runs on, for example `dstack-pha-prod15.phala.network`. Change `443` if your gateway uses a different port.
+
+> [!NOTE]
+> On macOS the system `openssl` is LibreSSL, which can time out with this `ProxyCommand`. Install OpenSSL with `brew install openssl` and make sure it's first on your `PATH`.
+
+## What you get
+
+You land in the dev box: a Debian trixie container, the same base as the dstack 0.6 guest OS, that shares the host's PID, network, and UTS namespaces.
+
+| Need | How |
+|------|-----|
+| Host processes, network, hostname | Visible directly: `ps`, `htop`, `ip`, `ss`, `tcpdump`, `strace -p` |
+| Host files | `/host`, for example `/host/dstack/app-compose.json` |
+| Containers | `docker` and `docker compose` talk to the host daemon |
+| systemd, journald, anything else on the host | `host-shell` opens a shell in the host OS; `host-shell CMD` runs one command there, for example `host-shell journalctl -u docker` |
+| File transfer | `scp`, `sftp`, and `rsync`, for example `rsync my-app:/host/tmp/dump ./` |
+| Port forwarding | `ssh -N -L 8080:127.0.0.1:8080 my-app` reaches any port on the CVM |
+
+Also installed: `vim`, `tmux`, `git`, `curl`, `jq`, `dig`, `gdb`, `lsof`, and `python3`. Run `apt-get install` for anything else. Installed packages last until the container is recreated. `/root` is a volume, so dotfiles and history persist across restarts and updates, as does the SSH host key.
+
+## How it works
+
+The host rootfs is a read-only, dm-verity-protected image, so tools can't be installed on the host itself. The dev box brings its own userland and borrows the host's namespaces instead, the same design as Bottlerocket's admin container and `kubectl debug node`.
+
+| Setting | Why |
+|---------|-----|
+| `privileged: true` | Full device access and capabilities for `tcpdump`, `strace`, `gdb`, and the `setns()` calls in `host-shell`. |
+| `pid: host`, `network_mode: host`, `uts: host` | The box sees the host's processes, network stack, and hostname. |
+| `/:/host:rslave` | The host filesystem, including mounts made after the box starts. |
+| `/var/run/docker.sock` | `docker` drives the host daemon. |
+| `/dev/pts:/dev/pts` | PTYs come from the host's devpts instance, so a terminal still works after `host-shell` switches to the host mount namespace. |
+
+[`host-shell`](./host-shell) runs `nsenter --target 1 --all --root`, joining every namespace of host PID 1 and its root directory. Its shell is the host's own `/bin/bash`, with no dev box tools.
+
+## Build your own image
+
+The compose file uses `ghcr.io/dstack-tee/dstack-dev-box:latest`, built from this directory by [`build-dev-box.yml`](../.github/workflows/build-dev-box.yml). To add packages, edit the [`Dockerfile`](./Dockerfile), push the image to a registry, and change `image:`. For a quick test, [ttl.sh](https://ttl.sh) holds anonymous images for up to 24 hours:
+
+```bash
+docker buildx build --platform linux/amd64 -t ttl.sh/my-dev-box:24h --push .
+```

dev-box/docker-compose.yaml

@@ -0,0 +1,29 @@
+# dstack dev box: SSH access to the CVM with a full debugging toolset.
+#
+# Sessions run in this container but share the host's PID, network, and
+# UTS namespaces, see the host filesystem at /host, and drive the host
+# Docker daemon. `host-shell` drops into the host OS itself.
+#
+# This gives full root on the CVM. Development/debugging only.
+services:
+  dev-box:
+    image: ghcr.io/dstack-tee/dstack-dev-box:latest
+    restart: unless-stopped
+    privileged: true
+    pid: host
+    network_mode: host
+    uts: host
+    environment:
+      # One or more public keys, newline-separated (authorized_keys format).
+      - SSH_AUTHORIZED_KEYS=${SSH_AUTHORIZED_KEYS}
+    volumes:
+      - /:/host:rslave
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Share the host's devpts so `host-shell` keeps a working terminal.
+      - /dev/pts:/dev/pts
+      - ssh-host-keys:/etc/ssh/host-keys
+      - home:/root
+
+volumes:
+  ssh-host-keys:
+  home:

dev-box/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+if [ -z "${SSH_AUTHORIZED_KEYS:-}" ]; then
+  echo "SSH_AUTHORIZED_KEYS is not set" >&2
+  exit 1
+fi
+install -d -m 700 /root/.ssh
+printf '%s\n' "$SSH_AUTHORIZED_KEYS" > /root/.ssh/authorized_keys
+chmod 600 /root/.ssh/authorized_keys
+key=/etc/ssh/host-keys/ssh_host_ed25519_key
+[ -f "$key" ] || ssh-keygen -q -t ed25519 -N '' -f "$key"
+install -d -m 755 /run/sshd
+exec /usr/sbin/sshd -D -e

dev-box/host-shell

@@ -0,0 +1,8 @@
+#!/bin/sh
+# Run a login shell, or the given command, directly on the host: enter
+# every namespace of host PID 1 and chroot to its root. --wdns resolves
+# the directory inside the host mount namespace.
+if [ $# -eq 0 ]; then
+  exec nsenter --target 1 --all --root --wdns=/root -- /bin/bash -l
+fi
+exec nsenter --target 1 --all --root --wdns=/root -- "$@"

dev-box/motd

@@ -0,0 +1,8 @@
+
+dstack dev box: full root on this CVM. Development use only.
+
+  /host          host root filesystem
+  docker         talks to the host Docker daemon
+  host-shell     shell in the host OS (all host namespaces)
+  host-shell CMD run CMD on the host, e.g. host-shell journalctl -u docker
+

dev-box/sshd_config

@@ -0,0 +1,9 @@
+Port 1022
+HostKey /etc/ssh/host-keys/ssh_host_ed25519_key
+AllowUsers root
+PermitRootLogin prohibit-password
+AuthenticationMethods publickey
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+UsePAM no
+Subsystem sftp internal-sftp