From 888a09cc213c391af1710f63835bdd2925dc62f1 Mon Sep 17 00:00:00 2001 From: jrg Date: Tue, 27 Oct 2026 18:08:26 -0500 Subject: [PATCH] template: build with the yeet CLI's toolchain; cargo-style output build/toolchain.mk now prefers the toolchain managed by the yeet CLI: with `yeet` on PATH every tool runs as `yeet toolchain exec `, sharing the CLI's single install instead of filling a second cache. If none is installed, the `toolchain` target installs the version pinned in build/toolchain.lock as the default; if a different default already exists it warns rather than switching it. `make YEET=` forces the lock-pinned cache path, which CI and yeet-less hosts keep using. Toolchain paths (libbpf headers, veristat, qemu) are resolved lazily and memoized: make expands variables at parse time, and the install happens in a target, so every consumer runs after `| toolchain` while clean and postgen never trigger a lookup or install. A tool may now be a multi-word command: existence checks probe its first word, postgen runs git instead of testing for a file, and gen-vmlinux.sh receives the bpftool command quoted and word-splits it deliberately. kernel-matrix.sh prefers the qemu bundled with the yeet toolchain. New build/ui.mk gives the build cargo-style status lines: a bold green verb right-aligned to 12 columns on stderr, color only on a terminal with NO_COLOR honored, each command echoed under its line, QUIET=1 for status only, and a Finished line with elapsed time. gen-vmlinux.sh drops the partial header on failure and filters bpftool's "loaded as base" note. The toolchain.mk and fetch-toolchain.sh changes are mirrored into toolchain/embed/ and need to land in yeet-src/toolchain before the next `make sync-toolchain`, which overwrites them from the tag. --- template/Makefile | 31 ++++++--- template/README.md | 11 +++ template/build/bpf.mk | 40 ++++++----- template/build/fetch-toolchain.sh | 15 ++-- template/build/gen-vmlinux.sh | 15 +++- template/build/kernel-matrix.sh | 14 ++-- template/build/toolchain.mk | 106 +++++++++++++++++++++++------ template/build/ui.mk | 46 +++++++++++++ toolchain/embed/fetch-toolchain.sh | 15 ++-- toolchain/embed/toolchain.mk | 106 +++++++++++++++++++++++------ 10 files changed, 316 insertions(+), 83 deletions(-) create mode 100644 template/build/ui.mk diff --git a/template/Makefile b/template/Makefile index 70cfe4e..c888292 100644 --- a/template/Makefile +++ b/template/Makefile @@ -7,6 +7,7 @@ # make postgen — finalize a freshly generated project (git init) # make clangd — write a local .clangd pointing at the resolved toolchain # make clean — remove build artifacts +# make QUIET=1 — quiet: status lines only, no echoed commands # # This is the build *frontend*: it orchestrates two independent # compilers — clang for the BPF objects, esbuild for the JS bundle. @@ -16,15 +17,25 @@ # so the default goal must leave the project runnable. # # clang, bpftool and esbuild come from the static toolchain resolved by -# build/toolchain.mk (a shared per-machine cache, or binaries vendored in -# the bootstrap repo) — so the build needs no system C/BPF toolchain. +# build/toolchain.mk (the yeet CLI's installed toolchain when `yeet` is on +# PATH, else a shared per-machine cache pinned by build/toolchain.lock) — so +# the build needs no system C/BPF toolchain. .DEFAULT_GOAL := all +include build/ui.mk include build/toolchain.mk include build/bpf.mk +BUILD_START := $(shell date +%s%N 2>/dev/null) + all: bpf bundle + @end=$$(date +%s%N 2>/dev/null); t=""; \ + case "$(BUILD_START)$$end" in \ + ""|*[!0-9]*) ;; \ + *) t=" in $$(awk "BEGIN { printf \"%.2fs\", ($$end - $(BUILD_START)) / 1e9 }")" ;; \ + esac; \ + printf '$(STATUS_FMT)' 'Finished' "build$$t" >&2 # Bundle the entry with the vendored esbuild. esbuild honors tsconfig `paths` # (so `@/` resolves at bundle time), while `yeet:*` builtins and `*.bpf.o` @@ -45,24 +56,26 @@ ESBUILD_FLAGS := --bundle --format=esm --platform=neutral \ --outfile=dist/index.jsx --jsx=automatic --jsx-import-source=yeet:tui bundle: | toolchain - $(ESBUILD) src/main.jsx $(ESBUILD_FLAGS) '--external:yeet:*' '--external:*.bpf.o' + $(call status,Bundling,src/main.jsx) + $(Q)$(ESBUILD) src/main.jsx $(ESBUILD_FLAGS) --log-level=warning '--external:yeet:*' '--external:*.bpf.o' # Post-generation finalize: initialize a git repository with the vendored git # (fetched via `vendored-git`). Idempotent — skipped if this is already a repo. # The scaffolders (`yeet new`, `scripts/new`) run `make postgen` after creating # the project, so the CLI itself stays a thin caller of make. postgen: | vendored-git - @g="$(GIT)"; [ -x "$$g" ] || g="$$(command -v git 2>/dev/null || true)"; \ + @g="$(GIT)"; $$g --version >/dev/null 2>&1 || g="$$(command -v git 2>/dev/null || true)"; \ if [ -e .git ]; then \ - echo "postgen: already a git repository"; \ + printf '$(STATUS_FMT)' 'Skipping' 'git init (already a repository)' >&2; \ elif [ -n "$$g" ]; then \ - echo "postgen: git init"; \ - "$$g" -c init.templateDir= init -q . || echo "warning: 'git init' failed" >&2; \ + printf '$(STATUS_FMT)' 'Initializing' 'git repository' >&2; \ + $$g -c init.templateDir= init -q . || printf '$(WARN_FMT)' 'git init failed' >&2; \ else \ - echo "warning: no git available (vendored or host); skipping 'git init'" >&2; \ + printf '$(WARN_FMT)' 'no git available (vendored or host); skipping git init' >&2; \ fi clean: clean-bpf - rm -rf node_modules dist + $(call status,Cleaning,node_modules dist) + $(Q)rm -rf node_modules dist .PHONY: all bundle clean postgen diff --git a/template/README.md b/template/README.md index 5f87b87..6938f5d 100644 --- a/template/README.md +++ b/template/README.md @@ -64,6 +64,7 @@ the `@/` alias; `main.jsx` wires them together and owns input. ```sh make # compile BPF (clang + bpftool) + bundle JS (esbuild) +make QUIET=1 # same, quiet: status lines only, no echoed commands yeet run . # runs the bundled dist/index.jsx (needs root for BPF) ``` @@ -144,6 +145,16 @@ feed the components from any source that exposes the same signals. - `clang` and `bpftool` (for the BPF leg; `bpftool` generates `src/bpf/include/vmlinux.h` from the host kernel, which needs `CONFIG_DEBUG_INFO_BTF`) +Both come from the static toolchain, resolved by `build/toolchain.mk`. With +the `yeet` CLI on PATH the build runs them as `yeet toolchain exec ` +from the toolchain `yeet toolchain default` points at — one install shared +with the CLI, kept current with `yeet toolchain update`. If none is installed +yet, the first `make` installs the pinned version. Without `yeet`, the build +fetches the version pinned in `build/toolchain.lock` into a per-machine cache +(`~/.cache/yeet/toolchain/`) once; `make YEET=` forces that path even when +`yeet` is installed. `make clangd` writes a `.clangd` pointing at whichever +one resolved. + No node/npm is required: esbuild is vendored by the toolchain, and the starter has no third-party deps. (You only need a package manager if you add npm/jsr dependencies of your own — see *npm / jsr packages* above.) diff --git a/template/build/bpf.mk b/template/build/bpf.mk index 0650851..f54c804 100644 --- a/template/build/bpf.mk +++ b/template/build/bpf.mk @@ -10,9 +10,11 @@ # To add another independent object, give it its own link target alongside # bin/probe.bpf.o below — there is intentionally no per-object magic here. -# CLANG/BPFTOOL are normally resolved by build/toolchain.mk (vendored static -# toolchain or shared cache), included before this file. These are the last -# resort when neither is available: whatever is on PATH. bpftool frequently +# CLANG/BPFTOOL are normally resolved by build/toolchain.mk (the yeet CLI's +# toolchain via `yeet toolchain exec`, or the lock-pinned shared cache), +# included before this file. These are the last resort when neither is +# available: whatever is on PATH. A resolved tool may be a multi-word command, +# so the existence checks below probe its first word. bpftool frequently # lives in /usr/sbin, which isn't always on a non-root user's PATH; fall back # to it before giving up. CLANG ?= clang @@ -44,25 +46,29 @@ bpf: $(BPF_OUT) # `| toolchain` (order-only) ensures the vendored clang/bpftool are present in # the cache before any rule shells out to them, without forcing rebuilds. $(VMLINUX): | toolchain - @command -v $(BPFTOOL) >/dev/null 2>&1 || { echo "error: bpftool not found — install bpftool / linux-tools"; exit 1; } - sh build/gen-vmlinux.sh $(BPFTOOL) $@ + @command -v $(firstword $(BPFTOOL)) >/dev/null 2>&1 || { $(call fail,bpftool not found — install bpftool / linux-tools); } + $(call status,Generating,$@) + $(Q)sh build/gen-vmlinux.sh "$(BPFTOOL)" $@ # Compile each unit to an intermediate object. .build/bpf/%.bpf.o: src/bpf/%.bpf.c $(VMLINUX) | toolchain - @command -v $(CLANG) >/dev/null 2>&1 || { echo "error: clang not found — install clang"; exit 1; } - @mkdir -p $(dir $@) - $(CLANG) $(BPF_CFLAGS) -c $< -o $@ + @command -v $(firstword $(CLANG)) >/dev/null 2>&1 || { $(call fail,clang not found — install clang); } + $(call status,Compiling,$(notdir $<)) + $(Q)mkdir -p $(dir $@) + $(Q)$(CLANG) $(BPF_CFLAGS) -c $< -o $@ # Statically link every unit into the single loadable object. $(BPF_OUT): $(BPF_OBJS) | bin toolchain - @command -v $(BPFTOOL) >/dev/null 2>&1 || { echo "error: bpftool not found — install bpftool / linux-tools"; exit 1; } - $(BPFTOOL) gen object $@ $(BPF_OBJS) + @command -v $(firstword $(BPFTOOL)) >/dev/null 2>&1 || { $(call fail,bpftool not found — install bpftool / linux-tools); } + $(call status,Linking,$@) + $(Q)$(BPFTOOL) gen object $@ $(BPF_OBJS) bin: mkdir -p bin clean-bpf: - rm -rf $(BPF_OUT) .build $(VMLINUX) + $(call status,Cleaning,$(BPF_OUT) .build $(VMLINUX)) + $(Q)rm -rf $(BPF_OUT) .build $(VMLINUX) # Load the linked object with veristat to confirm THIS kernel's verifier # accepts every program, and to see per-program complexity (insns/states) — a @@ -72,8 +78,9 @@ clean-bpf: # static binary, or `veristat` on PATH). .PHONY: veristat veristat: $(BPF_OUT) | toolchain - @command -v $(VERISTAT) >/dev/null 2>&1 || { echo "error: veristat not found ($(VERISTAT)) — bump build/toolchain.lock to a toolchain that ships veristat, or install veristat on PATH"; exit 1; } - $(VERISTAT) $(BPF_OUT) + @command -v $(firstword $(VERISTAT)) >/dev/null 2>&1 || { $(call fail,veristat not found ($(VERISTAT)) — bump build/toolchain.lock to a toolchain that ships veristat, or install veristat on PATH); } + $(call status,Verifying,$(BPF_OUT)) + $(Q)$(VERISTAT) $(BPF_OUT) # Run the same verifier check across a matrix of kernels locally (Linux + KVM), # the local counterpart to .github/workflows/kernel-matrix.yml. Boots @@ -81,12 +88,13 @@ veristat: $(BPF_OUT) | toolchain # KERNELS="6.6-main bpf-next-main" or rely on the script's default spread. .PHONY: veristat-matrix veristat-matrix: $(BPF_OUT) | toolchain - VERISTAT="$(VERISTAT)" sh build/kernel-matrix.sh $(KERNELS) + $(call status,Verifying,$(BPF_OUT) across kernels) + $(Q)VERISTAT="$(VERISTAT)" TOOLCHAIN_DIR="$(TOOLCHAIN_DIR)" sh build/kernel-matrix.sh $(KERNELS) # Write a local .clangd so the editor resolves vmlinux.h, the libbpf SDK # headers and __u* types using the *resolved* toolchain include path — unlike # the committed .clangd (which only covers in-repo editing), this picks up the -# shared-cache path in a scaffolded project. Run after `make` so vmlinux.h +# yeet toolchain or shared-cache path in a scaffolded project. Run after `make` so vmlinux.h # exists. The result can hold a machine-specific cache path, so leave it # untracked. Falls back to host headers when no vendored toolchain is found. .PHONY: clangd @@ -103,6 +111,6 @@ clangd: ' - -D__TARGET_ARCH_$(ARCH)' \ ' - -D__BPF_TRACING__' \ > .clangd - @echo "wrote .clangd (libbpf headers: $(if $(BPF_SYSINCLUDE),$(BPF_SYSINCLUDE),))" + $(call status,Writing,.clangd (libbpf headers: $(if $(BPF_SYSINCLUDE),$(BPF_SYSINCLUDE),))) .PHONY: bpf clean-bpf diff --git a/template/build/fetch-toolchain.sh b/template/build/fetch-toolchain.sh index eff6a87..9d91222 100755 --- a/template/build/fetch-toolchain.sh +++ b/template/build/fetch-toolchain.sh @@ -12,6 +12,11 @@ set -eu +# Status lines in the build's format (STATUS_FMT comes from the Makefile; the +# default matches it uncolored). curl shows a progress bar only on a terminal. +status() { printf "${STATUS_FMT:-%12s %s\n}" "$1" "$2" >&2; } +if [ -t 2 ]; then CURL_PROGRESS=--progress-bar; else CURL_PROGRESS=-s; fi + DIR="${1:?usage: fetch-toolchain.sh [tool...]}" ARCH="${2:?missing arch}" LOCK="${3:?missing lock}" @@ -66,9 +71,9 @@ fetch_bin() { name="$1" want "$name" || return 0 [ -x "$DIR/$name" ] && return 0 - echo ">> fetch ${name} (${ARCH}, v${TOOLCHAIN_VERSION})" + status Downloading "${name} (${ARCH}, v${TOOLCHAIN_VERSION})" tmp="$DIR/.${name}.$$" - curl -fSL --retry 3 -o "$tmp" "${REL}/${name}-${ARCH}" + curl -fSL $CURL_PROGRESS --retry 3 -o "$tmp" "${REL}/${name}-${ARCH}" verify "$tmp" "$(get_sha "$(echo "$name" | tr a-z A-Z)")" "$name" || { rm -f "$tmp"; exit 1; } chmod +x "$tmp" mv -f "$tmp" "$DIR/$name" @@ -86,13 +91,13 @@ fetch_bin gh # per-arch tool dirs ($key/include/bpf/*.h). INC="$(dirname "$DIR")/include" if want headers && [ ! -e "$INC/bpf/bpf_helpers.h" ]; then - echo ">> fetch libbpf headers (v${TOOLCHAIN_VERSION})" + status Downloading "libbpf headers (v${TOOLCHAIN_VERSION})" td="$(mktemp -d)" - curl -fSL --retry 3 -o "$td/h.tgz" "${REL}/libbpf-headers.tar.gz" + curl -fSL $CURL_PROGRESS --retry 3 -o "$td/h.tgz" "${REL}/libbpf-headers.tar.gz" verify "$td/h.tgz" "${LIBBPF_HEADERS_SHA256:-}" "libbpf-headers" || { rm -rf "$td"; exit 1; } mkdir -p "$INC" tar xzf "$td/h.tgz" -C "$INC" # tarball holds a top-level bpf/ dir rm -rf "$td" fi -echo ">> toolchain ready: $DIR${FILTER:+ ($FILTER)}" +status Toolchain "$DIR${FILTER:+ ($FILTER)}" diff --git a/template/build/gen-vmlinux.sh b/template/build/gen-vmlinux.sh index e217372..4cbbc1d 100644 --- a/template/build/gen-vmlinux.sh +++ b/template/build/gen-vmlinux.sh @@ -3,6 +3,9 @@ # every BPF program in this project includes. # # build/gen-vmlinux.sh +# +# may be a multi-word command (`yeet toolchain exec bpftool`), so it +# is word-split on use, deliberately. set -eu @@ -15,14 +18,20 @@ if [ ! -r /sys/kernel/btf/vmlinux ]; then fi mkdir -p "$(dirname "$OUT")" +err="$(mktemp)" +trap 'rm -f "$err"' EXIT # Mark the generated header as a system header so clang keeps -Wall quiet # about it. The dump is not warning-clean: the kernel builds with tagged # anonymous members (`struct foo;` as an unnamed struct/union member), which # bpftool emits verbatim and clang >= 22 flags with -Wmissing-declarations. # Nothing in it is ours to fix, and the pragma only affects diagnostics — # CO-RE relocations and preserve_access_index are untouched. +# On failure, drop the partial header so make doesn't take it for a good one. { echo "#pragma clang system_header" - "$BPFTOOL" btf dump file /sys/kernel/btf/vmlinux format c -} >"$OUT" -echo "generated $OUT" + # shellcheck disable=SC2086 + $BPFTOOL btf dump file /sys/kernel/btf/vmlinux format c +} >"$OUT" 2>"$err" || { cat "$err" >&2; rm -f "$OUT"; exit 1; } +# bpftool notes that the kernel BTF "will be loaded as base" — expected here, +# so keep it out of the build output; anything else it said goes through. +grep -v 'will be loaded as base' "$err" >&2 || true diff --git a/template/build/kernel-matrix.sh b/template/build/kernel-matrix.sh index 3f128c7..dca6a57 100755 --- a/template/build/kernel-matrix.sh +++ b/template/build/kernel-matrix.sh @@ -42,11 +42,15 @@ need() { command -v "$1" >/dev/null 2>&1 || { echo "error: '$1' not found — $2 sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi; } need ssh "install openssh-client" -# --- resolve qemu: prefer the vendored static qemu from the toolchain release, -# fall back to a system qemu-system on PATH. lvh finds qemu via PATH and the -# static qemu finds its firmware blobs relative to its own binary, so we just -# prepend the extracted bin dir to PATH — no -L plumbing into lvh needed. -if [ -f build/toolchain.lock ]; then +# --- resolve qemu: prefer the static qemu inside the yeet-managed toolchain +# (TOOLCHAIN_DIR, passed by the Makefile), then the vendored one from the +# toolchain release, then a system qemu-system on PATH. lvh finds qemu via +# PATH and the static qemu finds its firmware blobs relative to its own +# binary, so we just prepend the bin dir to PATH — no -L plumbing into lvh. +if [ -n "${TOOLCHAIN_DIR:-}" ] && [ -x "${TOOLCHAIN_DIR%/}/qemu/bin/$QEMU" ]; then + PATH="${TOOLCHAIN_DIR%/}/qemu/bin:$PATH"; export PATH + echo ">> using yeet toolchain qemu: ${TOOLCHAIN_DIR%/}/qemu/bin/$QEMU" +elif [ -f build/toolchain.lock ]; then . ./build/toolchain.lock qsha="$(eval "printf '%s' \"\${QEMU_SHA256_${ARCH}:-}\"")" QDIR="${XDG_CACHE_HOME:-$HOME/.cache}/yeet/toolchain/v${TOOLCHAIN_VERSION}/${ARCH}/qemu" diff --git a/template/build/toolchain.mk b/template/build/toolchain.mk index 6cb5f18..dffc4aa 100644 --- a/template/build/toolchain.mk +++ b/template/build/toolchain.mk @@ -2,33 +2,81 @@ # by the project Makefile before build/bpf.mk, so the tools are set before any # rule uses them. A `make CLANG=…` CLI override beats this. # -# Tools come from a shared, per-machine cache keyed by the project's pinned -# toolchain version (build/toolchain.lock). `make toolchain` fills it, -# downloading each missing tool once from the vendored toolchain release -# (github.com/yeet-src/toolchain). The cache key is the toolchain version, -# never the template version — so updating the template reuses an existing -# cached toolchain, and bumping a tool adds a new entry beside the old one. -# Falls back to host tools on PATH when no lock is present. +# Two sources, tried in this order: # -# The vendored binaries are Linux musl-static, so the cache is only used on -# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH -# unset, falling through to host tools on PATH — macOS is an edit-here, +# 1. The yeet CLI's managed toolchain. When `yeet` is on PATH, each tool runs +# as `yeet toolchain exec ` — from whichever toolchain `yeet toolchain +# default` points at (`yeet toolchain list` shows it). The build then +# shares yeet's single install instead of filling a second cache, and +# `yeet toolchain update` moves the build along with it. If none is +# installed yet, the `toolchain` target installs the version pinned in +# build/toolchain.lock (latest without a lock) and makes it the default; +# it only notes a mismatch when a default already exists. Build with +# `make YEET=` to skip this source. +# +# 2. A shared, per-machine cache keyed by the project's pinned toolchain +# version (build/toolchain.lock). `make toolchain` fills it, downloading +# each missing tool once from the vendored toolchain release +# (github.com/yeet-src/toolchain). The cache key is the toolchain version, +# never the template version — so updating the template reuses an +# existing cached toolchain, and bumping a tool adds a new entry beside +# the old one. This is what CI and hosts without yeet use. +# +# Falls back to host tools on PATH when neither applies (no yeet, no lock). +# +# The vendored binaries are Linux musl-static, so both sources are Linux-only. +# On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH unset, +# falling through to host tools on PATH — macOS is an edit-here, # build-on-Linux host for BPF work. +# Status-line formats, normally set by the project's build/ui.mk (included +# before this file); these fallbacks keep the glue usable on its own. Exported +# so build/fetch-toolchain.sh prints the same way. +STATUS_FMT ?= %12s %s\n +WARN_FMT ?= warning: %s\n +ERROR_FMT ?= error: %s\n +export STATUS_FMT WARN_FMT ERROR_FMT + UNAME_S := $(shell uname -s) UNAME_M := $(shell uname -m) # Apple/BSD report arm64; the release assets are named aarch64. Normalize so # the cache key and asset names line up across hosts. UNAME_M := $(UNAME_M:arm64=aarch64) -# Only engage the vendored cache on Linux; elsewhere TOOLCHAIN_LOCK stays empty -# so the PATH fallbacks below win and the fetch targets become no-ops. +# Only engage the vendored toolchain on Linux; elsewhere YEET and +# TOOLCHAIN_LOCK stay empty so the PATH fallbacks below win and the fetch +# targets become no-ops. ifeq ($(UNAME_S),Linux) +YEET ?= $(shell command -v yeet 2>/dev/null) TOOLCHAIN_LOCK := $(firstword $(wildcard build/toolchain.lock)) endif ifneq ($(TOOLCHAIN_LOCK),) include $(TOOLCHAIN_LOCK) - TOOLCHAIN_KEY := v$(TOOLCHAIN_VERSION) + TOOLCHAIN_KEY := v$(TOOLCHAIN_VERSION) +endif + +ifneq ($(YEET),) + # --- source 1: the yeet-managed toolchain --------------------------------- + YEET_EXEC := $(YEET) toolchain exec + TOOLCHAIN_WANT := $(if $(TOOLCHAIN_KEY),$(TOOLCHAIN_KEY),latest) + # Where the active toolchain lives — ask its clang for its own path. Needed + # for the libbpf headers (-I), `make clangd`, qemu, and veristat (the kernel + # matrix copies the binary into a VM, so it must be a file, not an exec + # wrapper). Resolved lazily, on first use, and memoized: the `toolchain` + # target may install the toolchain first, and every rule that needs the + # path runs after it (order-only `| toolchain`), while targets that don't + # (clean, postgen) never trigger a lookup. Empty if nothing is installed. + TOOLCHAIN_DIR = $(eval TOOLCHAIN_DIR := $(dir $(shell $(YEET_EXEC) clang -print-prog-name=clang 2>/dev/null)))$(TOOLCHAIN_DIR) + TOOLCHAIN_ACTIVE = $(notdir $(patsubst %/,%,$(TOOLCHAIN_DIR))) + CLANG ?= $(YEET_EXEC) clang + BPFTOOL ?= $(YEET_EXEC) bpftool + ESBUILD ?= $(YEET_EXEC) esbuild + GIT ?= $(YEET_EXEC) git + GH ?= $(YEET_EXEC) gh + VERISTAT ?= $(TOOLCHAIN_DIR)veristat + BPF_SYSINCLUDE ?= $(TOOLCHAIN_DIR)libbpf-headers +else ifneq ($(TOOLCHAIN_LOCK),) + # --- source 2: the lock-pinned shared cache ------------------------------- YEET_CACHE_DIR ?= $(if $(XDG_CACHE_HOME),$(XDG_CACHE_HOME),$(HOME)/.cache)/yeet TOOLCHAIN_DIR := $(YEET_CACHE_DIR)/toolchain/$(TOOLCHAIN_KEY)/$(UNAME_M) CLANG ?= $(TOOLCHAIN_DIR)/clang @@ -48,23 +96,41 @@ ESBUILD ?= esbuild GIT ?= git GH ?= gh -# Fill the cache for this arch, downloading any missing tool once. A no-op -# when no lock is present (PATH case). +# Make the toolchain ready for this arch. With yeet: install one if none is +# present (the pinned version, as the default) and note a mismatch against the +# lock's pin otherwise. With the cache: download any missing tool once. A +# no-op when neither applies (PATH case). +# +# The yeet branch works in shell, not via $(TOOLCHAIN_DIR): make expands a +# whole recipe before running it, so expanding the lazy lookup here would +# memoize "not installed" before the install line had run. .PHONY: toolchain toolchain: -ifneq ($(TOOLCHAIN_LOCK),) +ifneq ($(YEET),) + @$(YEET_EXEC) clang -print-prog-name=clang >/dev/null 2>&1 || { \ + printf '$(STATUS_FMT)' 'Installing' 'yeet toolchain $(TOOLCHAIN_WANT) (none installed yet)' >&2; \ + $(YEET) toolchain install $(TOOLCHAIN_WANT) --default; } + @d="$$($(YEET_EXEC) clang -print-prog-name=clang)"; d="$${d%/*}"; v="$${d##*/}"; \ + printf '$(STATUS_FMT)' 'Toolchain' "yeet $$v ($$d)" >&2; \ + if [ -n "$(TOOLCHAIN_KEY)" ] && [ "$$v" != "$(TOOLCHAIN_KEY)" ]; then \ + printf '$(WARN_FMT)' "yeet's default toolchain is $$v but build/toolchain.lock pins $(TOOLCHAIN_KEY) — 'yeet toolchain default $(TOOLCHAIN_KEY)' to match it, or 'make YEET=' to build from the pinned cache" >&2; \ + fi +else ifneq ($(TOOLCHAIN_LOCK),) @sh build/fetch-toolchain.sh "$(TOOLCHAIN_DIR)" "$(UNAME_M)" "$(TOOLCHAIN_LOCK)" else @: endif # Fetch only git into the cache — used by `postgen` so generating a project -# doesn't pull the whole build toolchain just to initialize a repo. +# doesn't pull the whole build toolchain just to initialize a repo. Nothing to +# fetch with yeet: its git is there if any toolchain is, and postgen falls +# back to host git otherwise. .PHONY: vendored-git vendored-git: +ifeq ($(YEET),) ifneq ($(TOOLCHAIN_LOCK),) @sh build/fetch-toolchain.sh "$(TOOLCHAIN_DIR)" "$(UNAME_M)" "$(TOOLCHAIN_LOCK)" git \ - || echo "note: vendored git unavailable; postgen will fall back to host git" >&2 -else - @: + || printf '$(WARN_FMT)' 'vendored git unavailable; postgen will fall back to host git' >&2 +endif endif + @: diff --git a/template/build/ui.mk b/template/build/ui.mk new file mode 100644 index 0000000..5b65d62 --- /dev/null +++ b/template/build/ui.mk @@ -0,0 +1,46 @@ +# Build output styling — cargo-like status lines: +# +# Toolchain yeet v0.10.1 (~/.local/share/yeet/toolchain/linux/x86_64/v0.10.1) +# Generating src/bpf/include/vmlinux.h +# Compiling cpusched.bpf.c +# Compiling runqlat.bpf.c +# Linking bin/probe.bpf.o +# Bundling src/main.jsx +# Finished build in 0.84s +# +# A bold green verb, right-aligned to 12 columns, then the message — on +# stderr, like cargo, so the build's stdout stays clean for piping. Each +# command is echoed under its status line, so the exact invocation is always +# on screen; `make QUIET=1` hides the commands and keeps only the status lines. +# Color is used only when stderr is a terminal, TERM isn't dumb and +# NO_COLOR (https://no-color.org) is unset. Compiler diagnostics pass through +# untouched, so a warning or error is never hidden. +# +# Included first by the project Makefile, so build/toolchain.mk and +# build/bpf.mk can use `$(Q)` and `$(call status,Verb,message)`. + +ifeq ($(QUIET),1) + Q := @ +else + Q := +endif + +COLOR := $(shell [ -t 2 ] && [ -z "$$NO_COLOR" ] && [ "$$TERM" != dumb ] && echo 1) +ifeq ($(COLOR),1) + STATUS_FMT := \033[1;32m%12s\033[0m %s\n + WARN_FMT := \033[1;33mwarning\033[0m: %s\n + ERROR_FMT := \033[1;31merror\033[0m: %s\n +else + STATUS_FMT := %12s %s\n + WARN_FMT := warning: %s\n + ERROR_FMT := error: %s\n +endif +# The shell scripts the build calls print the same way (fetch-toolchain.sh). +export STATUS_FMT WARN_FMT ERROR_FMT + +# $(call status,Verb,message) and $(call warn,message) are whole recipe lines; +# $(call fail,message) is a shell fragment for `|| { $(call fail,…); }`. +# Messages must not contain single quotes. +status = @printf '$(STATUS_FMT)' '$(1)' '$(2)' >&2 +warn = @printf '$(WARN_FMT)' '$(1)' >&2 +fail = printf '$(ERROR_FMT)' '$(1)' >&2; exit 1 diff --git a/toolchain/embed/fetch-toolchain.sh b/toolchain/embed/fetch-toolchain.sh index eff6a87..9d91222 100755 --- a/toolchain/embed/fetch-toolchain.sh +++ b/toolchain/embed/fetch-toolchain.sh @@ -12,6 +12,11 @@ set -eu +# Status lines in the build's format (STATUS_FMT comes from the Makefile; the +# default matches it uncolored). curl shows a progress bar only on a terminal. +status() { printf "${STATUS_FMT:-%12s %s\n}" "$1" "$2" >&2; } +if [ -t 2 ]; then CURL_PROGRESS=--progress-bar; else CURL_PROGRESS=-s; fi + DIR="${1:?usage: fetch-toolchain.sh [tool...]}" ARCH="${2:?missing arch}" LOCK="${3:?missing lock}" @@ -66,9 +71,9 @@ fetch_bin() { name="$1" want "$name" || return 0 [ -x "$DIR/$name" ] && return 0 - echo ">> fetch ${name} (${ARCH}, v${TOOLCHAIN_VERSION})" + status Downloading "${name} (${ARCH}, v${TOOLCHAIN_VERSION})" tmp="$DIR/.${name}.$$" - curl -fSL --retry 3 -o "$tmp" "${REL}/${name}-${ARCH}" + curl -fSL $CURL_PROGRESS --retry 3 -o "$tmp" "${REL}/${name}-${ARCH}" verify "$tmp" "$(get_sha "$(echo "$name" | tr a-z A-Z)")" "$name" || { rm -f "$tmp"; exit 1; } chmod +x "$tmp" mv -f "$tmp" "$DIR/$name" @@ -86,13 +91,13 @@ fetch_bin gh # per-arch tool dirs ($key/include/bpf/*.h). INC="$(dirname "$DIR")/include" if want headers && [ ! -e "$INC/bpf/bpf_helpers.h" ]; then - echo ">> fetch libbpf headers (v${TOOLCHAIN_VERSION})" + status Downloading "libbpf headers (v${TOOLCHAIN_VERSION})" td="$(mktemp -d)" - curl -fSL --retry 3 -o "$td/h.tgz" "${REL}/libbpf-headers.tar.gz" + curl -fSL $CURL_PROGRESS --retry 3 -o "$td/h.tgz" "${REL}/libbpf-headers.tar.gz" verify "$td/h.tgz" "${LIBBPF_HEADERS_SHA256:-}" "libbpf-headers" || { rm -rf "$td"; exit 1; } mkdir -p "$INC" tar xzf "$td/h.tgz" -C "$INC" # tarball holds a top-level bpf/ dir rm -rf "$td" fi -echo ">> toolchain ready: $DIR${FILTER:+ ($FILTER)}" +status Toolchain "$DIR${FILTER:+ ($FILTER)}" diff --git a/toolchain/embed/toolchain.mk b/toolchain/embed/toolchain.mk index 6cb5f18..dffc4aa 100644 --- a/toolchain/embed/toolchain.mk +++ b/toolchain/embed/toolchain.mk @@ -2,33 +2,81 @@ # by the project Makefile before build/bpf.mk, so the tools are set before any # rule uses them. A `make CLANG=…` CLI override beats this. # -# Tools come from a shared, per-machine cache keyed by the project's pinned -# toolchain version (build/toolchain.lock). `make toolchain` fills it, -# downloading each missing tool once from the vendored toolchain release -# (github.com/yeet-src/toolchain). The cache key is the toolchain version, -# never the template version — so updating the template reuses an existing -# cached toolchain, and bumping a tool adds a new entry beside the old one. -# Falls back to host tools on PATH when no lock is present. +# Two sources, tried in this order: # -# The vendored binaries are Linux musl-static, so the cache is only used on -# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH -# unset, falling through to host tools on PATH — macOS is an edit-here, +# 1. The yeet CLI's managed toolchain. When `yeet` is on PATH, each tool runs +# as `yeet toolchain exec ` — from whichever toolchain `yeet toolchain +# default` points at (`yeet toolchain list` shows it). The build then +# shares yeet's single install instead of filling a second cache, and +# `yeet toolchain update` moves the build along with it. If none is +# installed yet, the `toolchain` target installs the version pinned in +# build/toolchain.lock (latest without a lock) and makes it the default; +# it only notes a mismatch when a default already exists. Build with +# `make YEET=` to skip this source. +# +# 2. A shared, per-machine cache keyed by the project's pinned toolchain +# version (build/toolchain.lock). `make toolchain` fills it, downloading +# each missing tool once from the vendored toolchain release +# (github.com/yeet-src/toolchain). The cache key is the toolchain version, +# never the template version — so updating the template reuses an +# existing cached toolchain, and bumping a tool adds a new entry beside +# the old one. This is what CI and hosts without yeet use. +# +# Falls back to host tools on PATH when neither applies (no yeet, no lock). +# +# The vendored binaries are Linux musl-static, so both sources are Linux-only. +# On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH unset, +# falling through to host tools on PATH — macOS is an edit-here, # build-on-Linux host for BPF work. +# Status-line formats, normally set by the project's build/ui.mk (included +# before this file); these fallbacks keep the glue usable on its own. Exported +# so build/fetch-toolchain.sh prints the same way. +STATUS_FMT ?= %12s %s\n +WARN_FMT ?= warning: %s\n +ERROR_FMT ?= error: %s\n +export STATUS_FMT WARN_FMT ERROR_FMT + UNAME_S := $(shell uname -s) UNAME_M := $(shell uname -m) # Apple/BSD report arm64; the release assets are named aarch64. Normalize so # the cache key and asset names line up across hosts. UNAME_M := $(UNAME_M:arm64=aarch64) -# Only engage the vendored cache on Linux; elsewhere TOOLCHAIN_LOCK stays empty -# so the PATH fallbacks below win and the fetch targets become no-ops. +# Only engage the vendored toolchain on Linux; elsewhere YEET and +# TOOLCHAIN_LOCK stay empty so the PATH fallbacks below win and the fetch +# targets become no-ops. ifeq ($(UNAME_S),Linux) +YEET ?= $(shell command -v yeet 2>/dev/null) TOOLCHAIN_LOCK := $(firstword $(wildcard build/toolchain.lock)) endif ifneq ($(TOOLCHAIN_LOCK),) include $(TOOLCHAIN_LOCK) - TOOLCHAIN_KEY := v$(TOOLCHAIN_VERSION) + TOOLCHAIN_KEY := v$(TOOLCHAIN_VERSION) +endif + +ifneq ($(YEET),) + # --- source 1: the yeet-managed toolchain --------------------------------- + YEET_EXEC := $(YEET) toolchain exec + TOOLCHAIN_WANT := $(if $(TOOLCHAIN_KEY),$(TOOLCHAIN_KEY),latest) + # Where the active toolchain lives — ask its clang for its own path. Needed + # for the libbpf headers (-I), `make clangd`, qemu, and veristat (the kernel + # matrix copies the binary into a VM, so it must be a file, not an exec + # wrapper). Resolved lazily, on first use, and memoized: the `toolchain` + # target may install the toolchain first, and every rule that needs the + # path runs after it (order-only `| toolchain`), while targets that don't + # (clean, postgen) never trigger a lookup. Empty if nothing is installed. + TOOLCHAIN_DIR = $(eval TOOLCHAIN_DIR := $(dir $(shell $(YEET_EXEC) clang -print-prog-name=clang 2>/dev/null)))$(TOOLCHAIN_DIR) + TOOLCHAIN_ACTIVE = $(notdir $(patsubst %/,%,$(TOOLCHAIN_DIR))) + CLANG ?= $(YEET_EXEC) clang + BPFTOOL ?= $(YEET_EXEC) bpftool + ESBUILD ?= $(YEET_EXEC) esbuild + GIT ?= $(YEET_EXEC) git + GH ?= $(YEET_EXEC) gh + VERISTAT ?= $(TOOLCHAIN_DIR)veristat + BPF_SYSINCLUDE ?= $(TOOLCHAIN_DIR)libbpf-headers +else ifneq ($(TOOLCHAIN_LOCK),) + # --- source 2: the lock-pinned shared cache ------------------------------- YEET_CACHE_DIR ?= $(if $(XDG_CACHE_HOME),$(XDG_CACHE_HOME),$(HOME)/.cache)/yeet TOOLCHAIN_DIR := $(YEET_CACHE_DIR)/toolchain/$(TOOLCHAIN_KEY)/$(UNAME_M) CLANG ?= $(TOOLCHAIN_DIR)/clang @@ -48,23 +96,41 @@ ESBUILD ?= esbuild GIT ?= git GH ?= gh -# Fill the cache for this arch, downloading any missing tool once. A no-op -# when no lock is present (PATH case). +# Make the toolchain ready for this arch. With yeet: install one if none is +# present (the pinned version, as the default) and note a mismatch against the +# lock's pin otherwise. With the cache: download any missing tool once. A +# no-op when neither applies (PATH case). +# +# The yeet branch works in shell, not via $(TOOLCHAIN_DIR): make expands a +# whole recipe before running it, so expanding the lazy lookup here would +# memoize "not installed" before the install line had run. .PHONY: toolchain toolchain: -ifneq ($(TOOLCHAIN_LOCK),) +ifneq ($(YEET),) + @$(YEET_EXEC) clang -print-prog-name=clang >/dev/null 2>&1 || { \ + printf '$(STATUS_FMT)' 'Installing' 'yeet toolchain $(TOOLCHAIN_WANT) (none installed yet)' >&2; \ + $(YEET) toolchain install $(TOOLCHAIN_WANT) --default; } + @d="$$($(YEET_EXEC) clang -print-prog-name=clang)"; d="$${d%/*}"; v="$${d##*/}"; \ + printf '$(STATUS_FMT)' 'Toolchain' "yeet $$v ($$d)" >&2; \ + if [ -n "$(TOOLCHAIN_KEY)" ] && [ "$$v" != "$(TOOLCHAIN_KEY)" ]; then \ + printf '$(WARN_FMT)' "yeet's default toolchain is $$v but build/toolchain.lock pins $(TOOLCHAIN_KEY) — 'yeet toolchain default $(TOOLCHAIN_KEY)' to match it, or 'make YEET=' to build from the pinned cache" >&2; \ + fi +else ifneq ($(TOOLCHAIN_LOCK),) @sh build/fetch-toolchain.sh "$(TOOLCHAIN_DIR)" "$(UNAME_M)" "$(TOOLCHAIN_LOCK)" else @: endif # Fetch only git into the cache — used by `postgen` so generating a project -# doesn't pull the whole build toolchain just to initialize a repo. +# doesn't pull the whole build toolchain just to initialize a repo. Nothing to +# fetch with yeet: its git is there if any toolchain is, and postgen falls +# back to host git otherwise. .PHONY: vendored-git vendored-git: +ifeq ($(YEET),) ifneq ($(TOOLCHAIN_LOCK),) @sh build/fetch-toolchain.sh "$(TOOLCHAIN_DIR)" "$(UNAME_M)" "$(TOOLCHAIN_LOCK)" git \ - || echo "note: vendored git unavailable; postgen will fall back to host git" >&2 -else - @: + || printf '$(WARN_FMT)' 'vendored git unavailable; postgen will fall back to host git' >&2 +endif endif + @: