From 6331c01c80ef430cad0e61fcba25044d987ee8d5 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Mon, 26 Oct 2026 10:42:59 +0000 Subject: [PATCH] feat(dev-box): SSH dev box for dstack v0.6 CVMs dstack v0.6 drops the dev OS image, so there is no built-in SSH or debugging toolset on the CVM anymore. Add a single container that brings them back: - sshd on port 1022 (public-key root login, keys from the encrypted SSH_AUTHORIZED_KEYS env), reachable through dstack-gateway - Debian trixie userland matching the 0.6 guest OS: rsync, sftp, vim, tmux, gdb, strace, tcpdump, htop, docker CLI + compose, ... - shares the host PID, network and UTS namespaces; host root at /host; host Docker daemon via docker.sock - host-shell: nsenter into every namespace of host PID 1 for a real host shell (systemctl, journalctl, ...) /dev/pts is shared with the host so PTYs stay valid after host-shell switches mount namespaces. The SSH host key and /root persist in volumes. Tested on Phala Cloud prod15 with dstack-0.6.1. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-dev-box.yml | 55 ++++++++++++++++ README.md | 1 + dev-box/Dockerfile | 14 +++++ dev-box/README.md | 98 +++++++++++++++++++++++++++++ dev-box/docker-compose.yaml | 29 +++++++++ dev-box/entrypoint.sh | 13 ++++ dev-box/host-shell | 8 +++ dev-box/motd | 8 +++ dev-box/sshd_config | 9 +++ 9 files changed, 235 insertions(+) create mode 100644 .github/workflows/build-dev-box.yml create mode 100644 dev-box/Dockerfile create mode 100644 dev-box/README.md create mode 100644 dev-box/docker-compose.yaml create mode 100644 dev-box/entrypoint.sh create mode 100644 dev-box/host-shell create mode 100644 dev-box/motd create mode 100644 dev-box/sshd_config diff --git a/.github/workflows/build-dev-box.yml b/.github/workflows/build-dev-box.yml new file mode 100644 index 00000000..c60897cb --- /dev/null +++ b/.github/workflows/build-dev-box.yml @@ -0,0 +1,55 @@ +name: build dev-box + +on: + push: + branches: [main] + paths: + - 'dev-box/**' + - '.github/workflows/build-dev-box.yml' + pull_request: + paths: + - 'dev-box/**' + - '.github/workflows/build-dev-box.yml' + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + build: + runs-on: ubuntu-latest + env: + IMAGE: ghcr.io/dstack-tee/dstack-dev-box + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + # On PRs (incl. forks) GITHUB_TOKEN lacks packages:write, so only build. + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: dev-box + file: dev-box/Dockerfile + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/README.md b/README.md index 5270b9c1..da7300ee 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ Development and debugging tools. **Not for production.** | Example | Description | |---------|-------------| | [webshell](./webshell) | Web-based shell access for debugging | +| [dev-box](./dev-box) | SSH, debugging tools, and a host shell for dstack v0.6 CVMs (replaces the removed dev OS image) | | [ssh-over-gateway](./ssh-over-gateway) | SSH tunneling through dstack gateway | | [tcp-port-forwarding](./tcp-port-forwarding) | Arbitrary TCP port forwarding | diff --git a/dev-box/Dockerfile b/dev-box/Dockerfile new file mode 100644 index 00000000..5da732bb --- /dev/null +++ b/dev-box/Dockerfile @@ -0,0 +1,14 @@ +# Debian trixie matches the dstack 0.6 guest OS userland. +FROM debian:trixie-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssh-server openssh-sftp-server rsync \ + bash-completion ca-certificates curl dnsutils file gdb git htop \ + iproute2 iputils-ping jq less lsof procps psmisc python3 strace \ + tcpdump tmux util-linux vim \ + docker-cli docker-compose \ + && rm -rf /var/lib/apt/lists/* /etc/ssh/ssh_host_* +COPY sshd_config /etc/ssh/sshd_config +COPY motd /etc/motd +COPY --chmod=755 host-shell /usr/local/bin/host-shell +COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint +ENTRYPOINT ["/usr/local/bin/entrypoint"] diff --git a/dev-box/README.md b/dev-box/README.md new file mode 100644 index 00000000..168da428 --- /dev/null +++ b/dev-box/README.md @@ -0,0 +1,98 @@ +# dstack Dev Box + +dstack v0.6 no longer ships a dev OS image with SSH and debugging tools. Add this one container to your compose file to get them back: root SSH into the CVM, a full Debian toolset, the host filesystem, the host Docker daemon, and a shell in the host OS itself. + +> [!WARNING] +> This gives anyone holding an authorized key full root on the CVM: every container, the decrypted disk, and every secret in memory. It breaks the confidentiality guarantee your users rely on. The dev box is part of the attested `app-compose`, so verifiers can see it's there. Use it for development and debugging only. + +## Usage + +Add the `dev-box` service from [`docker-compose.yaml`](./docker-compose.yaml) next to your workload: + +```yaml +services: + dev-box: + image: ghcr.io/dstack-tee/dstack-dev-box:latest + restart: unless-stopped + privileged: true + pid: host + network_mode: host + uts: host + environment: + - SSH_AUTHORIZED_KEYS=${SSH_AUTHORIZED_KEYS} + volumes: + - /:/host:rslave + - /var/run/docker.sock:/var/run/docker.sock + - /dev/pts:/dev/pts + - ssh-host-keys:/etc/ssh/host-keys + - home:/root + + your-app: + image: ... + +volumes: + ssh-host-keys: + home: +``` + +Deploy with your public key or keys in `SSH_AUTHORIZED_KEYS`, in `authorized_keys` format with one key per line. The container exits with an error if the variable is empty. + +```bash +phala deploy -n my-app -c docker-compose.yaml \ + --image dstack-0.6.1 \ + -e SSH_AUTHORIZED_KEYS="$(cat ~/.ssh/id_ed25519.pub)" +``` + +On a plain dstack deployment, set `SSH_AUTHORIZED_KEYS` as an encrypted environment variable. + +## Connect + +`sshd` listens on port `1022`. Reach it through the dstack-gateway by adding this to `~/.ssh/config`: + +``` +Host my-app + User root + ProxyCommand openssl s_client -quiet -connect -1022.:443 -servername -1022. -verify_hostname -1022. -verify_return_error +``` + +Replace `` with your app ID and `` with the gateway base domain of the node the CVM runs on, for example `dstack-pha-prod15.phala.network`. Change `443` if your gateway uses a different port. + +> [!NOTE] +> On macOS the system `openssl` is LibreSSL, which can time out with this `ProxyCommand`. Install OpenSSL with `brew install openssl` and make sure it's first on your `PATH`. + +## What you get + +You land in the dev box: a Debian trixie container, the same base as the dstack 0.6 guest OS, that shares the host's PID, network, and UTS namespaces. + +| Need | How | +|------|-----| +| Host processes, network, hostname | Visible directly: `ps`, `htop`, `ip`, `ss`, `tcpdump`, `strace -p` | +| Host files | `/host`, for example `/host/dstack/app-compose.json` | +| Containers | `docker` and `docker compose` talk to the host daemon | +| systemd, journald, anything else on the host | `host-shell` opens a shell in the host OS; `host-shell CMD` runs one command there, for example `host-shell journalctl -u docker` | +| File transfer | `scp`, `sftp`, and `rsync`, for example `rsync my-app:/host/tmp/dump ./` | +| Port forwarding | `ssh -N -L 8080:127.0.0.1:8080 my-app` reaches any port on the CVM | + +Also installed: `vim`, `tmux`, `git`, `curl`, `jq`, `dig`, `gdb`, `lsof`, and `python3`. Run `apt-get install` for anything else. Installed packages last until the container is recreated. `/root` is a volume, so dotfiles and history persist across restarts and updates, as does the SSH host key. + +## How it works + +The host rootfs is a read-only, dm-verity-protected image, so tools can't be installed on the host itself. The dev box brings its own userland and borrows the host's namespaces instead, the same design as Bottlerocket's admin container and `kubectl debug node`. + +| Setting | Why | +|---------|-----| +| `privileged: true` | Full device access and capabilities for `tcpdump`, `strace`, `gdb`, and the `setns()` calls in `host-shell`. | +| `pid: host`, `network_mode: host`, `uts: host` | The box sees the host's processes, network stack, and hostname. | +| `/:/host:rslave` | The host filesystem, including mounts made after the box starts. | +| `/var/run/docker.sock` | `docker` drives the host daemon. | +| `/dev/pts:/dev/pts` | PTYs come from the host's devpts instance, so a terminal still works after `host-shell` switches to the host mount namespace. | + +[`host-shell`](./host-shell) runs `nsenter --target 1 --all --root`, joining every namespace of host PID 1 and its root directory. Its shell is the host's own `/bin/bash`, with no dev box tools. + +## Build your own image + +The compose file uses `ghcr.io/dstack-tee/dstack-dev-box:latest`, built from this directory by [`build-dev-box.yml`](../.github/workflows/build-dev-box.yml). To add packages, edit the [`Dockerfile`](./Dockerfile), push the image to a registry, and change `image:`. For a quick test, [ttl.sh](https://ttl.sh) holds anonymous images for up to 24 hours: + +```bash +docker buildx build --platform linux/amd64 -t ttl.sh/my-dev-box:24h --push . +``` diff --git a/dev-box/docker-compose.yaml b/dev-box/docker-compose.yaml new file mode 100644 index 00000000..15a3e98b --- /dev/null +++ b/dev-box/docker-compose.yaml @@ -0,0 +1,29 @@ +# dstack dev box: SSH access to the CVM with a full debugging toolset. +# +# Sessions run in this container but share the host's PID, network, and +# UTS namespaces, see the host filesystem at /host, and drive the host +# Docker daemon. `host-shell` drops into the host OS itself. +# +# This gives full root on the CVM. Development/debugging only. +services: + dev-box: + image: ghcr.io/dstack-tee/dstack-dev-box:latest + restart: unless-stopped + privileged: true + pid: host + network_mode: host + uts: host + environment: + # One or more public keys, newline-separated (authorized_keys format). + - SSH_AUTHORIZED_KEYS=${SSH_AUTHORIZED_KEYS} + volumes: + - /:/host:rslave + - /var/run/docker.sock:/var/run/docker.sock + # Share the host's devpts so `host-shell` keeps a working terminal. + - /dev/pts:/dev/pts + - ssh-host-keys:/etc/ssh/host-keys + - home:/root + +volumes: + ssh-host-keys: + home: diff --git a/dev-box/entrypoint.sh b/dev-box/entrypoint.sh new file mode 100644 index 00000000..f33061fa --- /dev/null +++ b/dev-box/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +if [ -z "${SSH_AUTHORIZED_KEYS:-}" ]; then + echo "SSH_AUTHORIZED_KEYS is not set" >&2 + exit 1 +fi +install -d -m 700 /root/.ssh +printf '%s\n' "$SSH_AUTHORIZED_KEYS" > /root/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys +key=/etc/ssh/host-keys/ssh_host_ed25519_key +[ -f "$key" ] || ssh-keygen -q -t ed25519 -N '' -f "$key" +install -d -m 755 /run/sshd +exec /usr/sbin/sshd -D -e diff --git a/dev-box/host-shell b/dev-box/host-shell new file mode 100644 index 00000000..46a077e7 --- /dev/null +++ b/dev-box/host-shell @@ -0,0 +1,8 @@ +#!/bin/sh +# Run a login shell, or the given command, directly on the host: enter +# every namespace of host PID 1 and chroot to its root. --wdns resolves +# the directory inside the host mount namespace. +if [ $# -eq 0 ]; then + exec nsenter --target 1 --all --root --wdns=/root -- /bin/bash -l +fi +exec nsenter --target 1 --all --root --wdns=/root -- "$@" diff --git a/dev-box/motd b/dev-box/motd new file mode 100644 index 00000000..e0b76502 --- /dev/null +++ b/dev-box/motd @@ -0,0 +1,8 @@ + +dstack dev box: full root on this CVM. Development use only. + + /host host root filesystem + docker talks to the host Docker daemon + host-shell shell in the host OS (all host namespaces) + host-shell CMD run CMD on the host, e.g. host-shell journalctl -u docker + diff --git a/dev-box/sshd_config b/dev-box/sshd_config new file mode 100644 index 00000000..c3fcc551 --- /dev/null +++ b/dev-box/sshd_config @@ -0,0 +1,9 @@ +Port 1022 +HostKey /etc/ssh/host-keys/ssh_host_ed25519_key +AllowUsers root +PermitRootLogin prohibit-password +AuthenticationMethods publickey +PasswordAuthentication no +KbdInteractiveAuthentication no +UsePAM no +Subsystem sftp internal-sftp