[WIP] Acknowledg Forge

.github/workflows/acknowledg-deploy.yml

@@ -16,7 +16,8 @@
 # Host, paths, and credentials come from secrets in the `acknowledg`
 # environment (see dev_tools/services/acknowledg/README.md). The SSH setup
 # writes them into a `Host acknowledg` alias so they never appear on a command
-# line.
+# line. The public API URL is the ACKNOWLEDG_API_HOST repository variable,
+# shared with acknowledg-forge-deploy.yml.
 
 name: AcknowlEDG Deploy
 
@@ -27,6 +28,8 @@ on:
     paths:
       - dev_tools/services/acknowledg/**
       - dev_tools/pylibs/edgacknowledg/**
+      # acknowledg-forge is a separate service.
+      - '!dev_tools/pylibs/edgacknowledg/forge/**'
       - .github/workflows/acknowledg-deploy.yml
   workflow_dispatch:
 
@@ -61,7 +64,7 @@ jobs:
       - name: Build client
         working-directory: ${{ env.ACKNOWLEDG_DIR }}/client
         env:
-          VITE_EDG_ACKNOWLEDG_API_HOST: ${{ secrets.ACKNOWLEDG_API_HOST }}
+          VITE_EDG_ACKNOWLEDG_API_HOST: ${{ vars.ACKNOWLEDG_API_HOST }}
         run: |
           # The client opens `${VITE_EDG_ACKNOWLEDG_API_HOST}/review`; without
           # a scheme the browser resolves that relative to the page instead.

.github/workflows/acknowledg-forge-deploy.yml

@@ -0,0 +1,138 @@
+# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
+# Exceptions.
+# See https://edgcpp.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Publish the acknowledg-forge container image and deploy it:
+#
+#   test     the acknowledg-forge typecheck and tests (acknowledg-forge.yml)
+#   publish  build the image (build-edg-env acknowledg-forge) and push it to
+#            ghcr.io/edgcpp/acknowledg-forge:latest (deployments use the
+#            pushed digest)
+#   deploy   over SSH, run the host's deploy helper (an administrator-
+#            installed copy of
+#            dev_tools/services/acknowledg-forge/deploy-remote.sh, the only
+#            thing the deploy user may run as root) to point the service at
+#            the pushed digest, set its AcknowlEDG API and UI URLs, and
+#            restart it (only if any of those changed)
+#
+# The image is public and holds no credentials or configuration: the publish
+# job has no access to the deployment's secrets. Everything else about the
+# service (its unit, tokens, API key) is configured on the host.
+#
+# Host, user, and SSH credentials come from secrets in the
+# `acknowledg-forge` environment; the public AcknowlEDG URLs come from
+# repository variables shared with acknowledg-deploy.yml (see
+# dev_tools/services/acknowledg-forge/README.md).
+
+name: acknowledg-forge Deploy
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      # The image freezes all of dev_tools/bin and dev_tools/pylibs.
+      - dev_tools/bin/**
+      - dev_tools/pylibs/**
+      - dev_tools/services/acknowledg-forge/**
+      - dev_tools/docker/acknowledg-forge/**
+      - .github/workflows/acknowledg-forge.yml
+      - .github/workflows/acknowledg-forge-deploy.yml
+  workflow_dispatch:
+
+concurrency:
+  group: acknowledg-forge-deploy
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test
+    if: github.ref == 'refs/heads/main'
+    uses: ./.github/workflows/acknowledg-forge.yml
+
+  publish:
+    name: Publish image
+    needs: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      image: ${{ steps.push.outputs.image }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v10.3.0
+
+      - name: Build image
+        env:
+          PYTHONPATH: ${{ github.workspace }}/dev_tools/pylibs
+        run: dev_tools/bin/build-edg-env --quiet acknowledg-forge
+
+      - name: Push image
+        id: push
+        env:
+          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          image=ghcr.io/edgcpp/acknowledg-forge
+          printf '%s' "${REGISTRY_TOKEN}" \
+            | docker login ghcr.io --username "${GITHUB_ACTOR}" \
+                --password-stdin
+          docker push --quiet "${image}:latest"
+          docker logout ghcr.io
+          digest="$(docker inspect \
+            --format '{{range .RepoDigests}}{{println .}}{{end}}' \
+            "${image}:latest" | grep "^${image}@sha256:" | head -n 1)"
+          echo "Pushed ${digest}"
+          echo "image=${digest}" >> "${GITHUB_OUTPUT}"
+
+  deploy:
+    name: Deploy
+    needs: publish
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: acknowledg-forge
+    steps:
+      - name: Configure SSH
+        env:
+          SSH_HOST: ${{ secrets.ACKNOWLEDG_FORGE_SSH_HOST }}
+          SSH_USER: ${{ secrets.ACKNOWLEDG_FORGE_SSH_USER }}
+          SSH_PORT: ${{ secrets.ACKNOWLEDG_FORGE_SSH_PORT || '22' }}
+          SSH_PRIVATE_KEY: ${{ secrets.ACKNOWLEDG_FORGE_SSH_PRIVATE_KEY }}
+          SSH_KNOWN_HOSTS: ${{ secrets.ACKNOWLEDG_FORGE_SSH_KNOWN_HOSTS }}
+        run: |
+          install -d -m 700 ~/.ssh
+          install -m 600 /dev/null ~/.ssh/deploy_key
+          printf '%s\n' "${SSH_PRIVATE_KEY}" > ~/.ssh/deploy_key
+          printf '%s\n' "${SSH_KNOWN_HOSTS}" > ~/.ssh/known_hosts
+          cat > ~/.ssh/config <<EOF
+          Host acknowledg-forge
+            HostName ${SSH_HOST}
+            User ${SSH_USER}
+            Port ${SSH_PORT}
+            IdentityFile ~/.ssh/deploy_key
+            IdentitiesOnly yes
+            StrictHostKeyChecking yes
+            BatchMode yes
+          EOF
+          chmod 600 ~/.ssh/config ~/.ssh/known_hosts
+
+      - name: Update and restart the service
+        env:
+          SERVICE: ${{ secrets.ACKNOWLEDG_FORGE_SERVICE || 'acknowledg-forge-github' }}
+          IMAGE: ${{ needs.publish.outputs.image }}
+          API_URL: ${{ vars.ACKNOWLEDG_API_HOST }}
+          UI_URL: ${{ vars.ACKNOWLEDG_UI_HOST }}
+        run: |
+          # The arguments are deliberately expanded (and quoted) here.
+          # shellcheck disable=SC2029
+          ssh acknowledg-forge "sudo -n /usr/local/libexec/acknowledg-forge-deploy \
+            $(printf '%q ' "${SERVICE}" "${IMAGE}" "${API_URL}" \
+              "${UI_URL}")"

.github/workflows/acknowledg-forge.yml

@@ -0,0 +1,61 @@
+# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
+# Exceptions.
+# See https://edgcpp.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Typecheck and test acknowledg-forge: the shared edgacknowledg.forge
+# package, the GitHub and GitLab services, and their integration with the
+# AcknowlEDG client library and server.
+
+name: acknowledg-forge
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - dev_tools/pylibs/edgacknowledg/**
+      - dev_tools/pylibs/edgtest/**
+      - dev_tools/pylibs/edgutil/**
+      - dev_tools/services/acknowledg/server/**
+      - dev_tools/services/acknowledg-forge/**
+      - .github/workflows/acknowledg-forge.yml
+  pull_request:
+    paths:
+      - dev_tools/pylibs/edgacknowledg/**
+      - dev_tools/pylibs/edgtest/**
+      - dev_tools/pylibs/edgutil/**
+      - dev_tools/services/acknowledg/server/**
+      - dev_tools/services/acknowledg-forge/**
+      - .github/workflows/acknowledg-forge.yml
+  # Also run by acknowledg-forge-deploy before publishing an image.
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Typecheck and test
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v10.3.0
+
+      # The sandbox tests need bubblewrap, which needs unprivileged user
+      # namespaces (restricted by AppArmor on Ubuntu 24.04 runners).
+      - name: Install bubblewrap
+        run: |
+          sudo apt-get update -q
+          sudo apt-get install -y -q bubblewrap
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+      - name: Typecheck
+        run: dev_tools/services/acknowledg-forge/typecheck.sh
+
+      - name: Test
+        run: dev_tools/services/acknowledg-forge/tests/run-tests.sh -v

dev_tools/bin/edg-acknowledg-cli

@@ -41,7 +41,7 @@ from edgacknowledg.client.bench import (
   BenchSourceHandler,
   resolve_baseline_dir,
 )
-from edgacknowledg.client.shared import ReviewClientHandler
+from edgacknowledg.client.shared import ReviewClientHandler, SandboxContext
 from edgacknowledg.client.test import TestEditHandler, TestSourceHandler
 from edgutil import eprint
 
@@ -96,10 +96,12 @@ def _make_bench_session(parameters: Dict[str, Any],
   if session is not None:
     return session
 
+  # The run (which may lie elsewhere) and the bench home are all a review
+  # may read.
+  context = SandboxContext(bench_home, root)
   session = BenchReviewSession(
-    bench_home = bench_home,
-    run_dir = root,
-    baseline_dir = resolve_baseline_dir(bench_home, tag),
+    run_dir = context.subpath(root),
+    baseline_dir = resolve_baseline_dir(context.subpath(bench_home), tag),
     tag = tag,
     threshold_slow = float(
       parameters.get('slower_threshold', DEFAULT_THRESHOLD_SLOW)
@@ -131,11 +133,15 @@ def _make_handler(section: Dict[str, Any],
   if section_type == 'test.src':
     if test_home is None:
       raise ValueError('test home required for test.src sections')
+    root = Path(parameters['root'])
+    # The run (which may lie elsewhere) and the test home are all a review
+    # may read.
+    context = SandboxContext(test_home, root)
     return TestSourceHandler(
       section_name,
       slug,
-      test_home,
-      Path(parameters['root'])
+      context.subpath(test_home),
+      context.subpath(root)
     )
   if section_type == 'test.edit':
     return TestEditHandler(section_name, slug, worker_pool)

dev_tools/bin/edg-bench-review

@@ -32,6 +32,7 @@ from edgacknowledg.client.bench import (
   BenchSourceHandler,
   resolve_baseline_dir,
 )
+from edgacknowledg.client.shared import SandboxContext
 from edgacknowledg import IncompatibleProtocolVersion
 from edgutil import eprint
 
@@ -166,14 +167,16 @@ def main() -> None:
     eprint(f"Path does not exist: {run_dir}")
     sys.exit(1)
 
-  baseline_dir = resolve_baseline_dir(bench_home, args.tag)
+  # The run (given with --root, it may lie elsewhere) and the bench home
+  # are all a review may read.
+  context = SandboxContext(bench_home, run_dir)
+  baseline_dir = resolve_baseline_dir(context.subpath(bench_home), args.tag)
   if not baseline_dir.exists():
     eprint(f"Baseline directory does not exist: {baseline_dir}")
     sys.exit(1)
 
   session = BenchReviewSession(
-    bench_home = bench_home,
-    run_dir = run_dir,
+    run_dir = context.subpath(run_dir),
     baseline_dir = baseline_dir,
     tag = args.tag,
     threshold_slow = args.threshold_slow,

dev_tools/bin/edgy-review

@@ -23,6 +23,7 @@ from edgacknowledg.client import (
   add_endpoint_arguments,
   form_section_http_url,
 )
+from edgacknowledg.client.shared import SandboxContext
 from edgacknowledg.client.test import TestEditHandler, TestSourceHandler
 from edgacknowledg import IncompatibleProtocolVersion
 from edgutil import eprint
@@ -163,9 +164,15 @@ def main() -> None:
         review = await session.create_review(
           run_id, review_name, created_at
         )
-        await review.add_section(
-          TestSourceHandler(args.section_name, slug, test_home, run_dir)
-        )
+        # The run (given with --root, it may lie elsewhere) and the test
+        # home are all a review may read.
+        context = SandboxContext(test_home, run_dir)
+        await review.add_section(TestSourceHandler(
+          args.section_name,
+          slug,
+          context.subpath(test_home),
+          context.subpath(run_dir)
+        ))
         await review.add_section(
           TestEditHandler(args.section_name, slug, worker_pool)
         )

dev_tools/docker/acknowledg-forge/Dockerfile

@@ -0,0 +1,56 @@
+# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
+# Exceptions.
+# See https://edgcpp.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# acknowledg-forge services (GitHub and GitLab), with a frozen copy of the EDG
+# tooling they run and their locked Python dependencies. See
+# dev_tools/services/acknowledg-forge/README.md for how to run it (rootless
+# Podman; bubblewrap inside the container needs --security-opt
+# unmask=/proc/*).
+
+FROM fedora:43
+
+# The image is public: it carries no credentials or deployment configuration
+# (those come from the host at run time).
+LABEL org.opencontainers.image.source="https://github.com/edgcpp/compiler"
+LABEL org.opencontainers.image.description="acknowledg-forge: AcknowlEDG reviews for GitHub and GitLab pull requests"
+LABEL org.opencontainers.image.licenses="Apache-2.0 WITH LLVM-exception"
+
+RUN mkdir -p /edg/setup-scripts /edg/tools /edg/acknowledg-forge
+COPY setup-scripts/* /edg/setup-scripts/
+
+# Execute root based setup scripts.
+RUN /edg/setup-scripts/run-all.sh && rm -rf /edg/setup-scripts
+
+# The frozen EDG tooling (dev_tools/bin and dev_tools/pylibs) the service runs
+# (edg-rematerialize-test-run, ifc-dump); see prebuild.py.
+COPY scripts-and-libs.tar.gz /edg/scripts-and-libs.tar.gz
+RUN tar -xzf /edg/scripts-and-libs.tar.gz --directory=/edg/tools && \
+    rm /edg/scripts-and-libs.tar.gz
+
+# One virtual environment per service: the locked (hash-checked)
+# dependencies, then the service wheel itself.
+COPY services/ /edg/acknowledg-forge/dist/
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+RUN for forge in github gitlab; do \
+      venv="/edg/acknowledg-forge/${forge}" && \
+      python3 -m venv "${venv}" && \
+      "${venv}/bin/pip" install --no-cache-dir --require-hashes \
+        -r "/edg/acknowledg-forge/dist/${forge}/requirements.txt" && \
+      "${venv}/bin/pip" install --no-cache-dir --no-deps \
+        /edg/acknowledg-forge/dist/${forge}/*.whl && \
+      ln -s "${venv}/bin/acknowledg-forge-${forge}" /usr/local/bin/ ; \
+    done && \
+    rm -rf /edg/acknowledg-forge/dist
+
+RUN useradd --create-home --uid 1000 acknowledg && \
+    mkdir -p /var/lib/acknowledg-forge && \
+    chown acknowledg:acknowledg /var/lib/acknowledg-forge
+
+ENV ACKNOWLEDG_FORGE_TOOLS_DIR=/edg/tools
+ENV ACKNOWLEDG_FORGE_HOME=/var/lib/acknowledg-forge
+
+USER acknowledg
+WORKDIR /var/lib/acknowledg-forge
+CMD ["acknowledg-forge-github"]