diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 5d02685..d8cde24 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -4,6 +4,7 @@ import copy import hashlib +import json import re from collections.abc import Mapping, Sequence from decimal import ROUND_FLOOR, Decimal, InvalidOperation @@ -131,6 +132,11 @@ _HOOK_FILE_MODE = 0o444 # Container modes whose job steps run in a separate pod (the chart's container hooks), which sees neither the runner container's tool cache nor its PATH. _SEPARATE_POD_CONTAINER_MODES = frozenset({"kubernetes", "kubernetes-novolume"}) +# The runner's work directory volume: the name the gha-runner-scale-set chart gives it (templates/_helpers.tpl, dind-work-volume, which keeps a values volume of that name in place of its own) and where the runner keeps every job's checkout, dependencies and caches. +_WORK_VOLUME = "work" +# The media the work volume can sit on: the node's disk (the empty string, the Kubernetes default) or RAM (Memory, a tmpfs). A RAM-backed volume writes nothing to the node's disk, and its pages count against the pod's memory and cannot be reclaimed while the files exist, so a profile that uses it raises its runner pods' memory request and limit by the volume's size and needs a size_limit to bound it. +_WORK_VOLUME_MEDIA = ("", "Memory") +_MEMORY_MEDIUM = "Memory" def _decimal(value: Any) -> Decimal | None: @@ -473,6 +479,30 @@ def _capability_names(profile: Mapping[str, Any], where: str, errors: list[str]) return list(dict.fromkeys(names)) +def _work_volume(profile: Mapping[str, Any], where: str, errors: list[str]) -> tuple[str, str]: + """Return the medium a profile's work volume sits on and its size limit (empty for the node's disk), recording an error when the medium is not one of _WORK_VOLUME_MEDIA, when Memory has no size limit that is a positive Kubernetes quantity, or when a size limit is given for the disk.""" + medium = profile.get("work_volume_medium", "") + size_limit = profile.get("work_volume_size_limit") + if medium not in _WORK_VOLUME_MEDIA: + errors.append(f"{where}.work_volume_medium must be one of {', '.join(repr(value) for value in _WORK_VOLUME_MEDIA)}") + return "", "" + if medium != _MEMORY_MEDIUM: + if size_limit is not None: + errors.append(f"{where}.work_volume_size_limit applies only with work_volume_medium: {_MEMORY_MEDIUM}") + return "", "" + if isinstance(size_limit, bool) or not isinstance(size_limit, (str, int)): + errors.append(f"{where}.work_volume_size_limit is required with work_volume_medium: {_MEMORY_MEDIUM}, as a Kubernetes quantity such as 6Gi, since a tmpfs without one may grow to the pod's memory limit") + return "", "" + try: + positive = arc_quantity(size_limit) > 0 + except ValueError: + positive = False + if not positive: + errors.append(f"{where}.work_volume_size_limit must be a positive Kubernetes quantity such as 6Gi (got {size_limit!r})") + return "", "" + return _MEMORY_MEDIUM, str(size_limit) + + def _explicit_max_runners(profile: Mapping[str, Any], where: str, errors: list[str]) -> int | None: """Return the profile's own max_runners as a whole number, None when unset, recording an error when it is not a whole number of at least 0.""" value = profile.get("max_runners") @@ -615,6 +645,7 @@ def _expand_profile(org_name: str, repository: str, runner_image: str, app_secre max_runners = None namespace = _optional_name(profile, "namespace", where, errors) or f"arc-runners-{slug}{suffix}" release = _optional_name(profile, "release_name", where, errors) or f"{slug}-runners{suffix}" + work_volume_medium, work_volume_size_limit = _work_volume(profile, where, errors) expanded = { "org_name": org_name, "repository": repository, @@ -638,6 +669,8 @@ def _expand_profile(org_name: str, repository: str, runner_image: str, app_secre "max_runners": max_runners, "capabilities": _capability_names(profile, where, errors), "capability_store": capability_store, + "work_volume_medium": work_volume_medium, + "work_volume_size_limit": work_volume_size_limit, "label": f"{scope}{suffix}", "settings": dict(profile), } @@ -658,10 +691,10 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, Args: require_app_id: whether each org must set app_id. The role needs it only when it writes the App Secret; otherwise the App's id is in the existing App Secret, and an app_id that is set is only checked against it. - orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name and repository (a repository name in name's account, without the owner). A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile may set capability_store (a boolean, default false) to copy its tool-cache and system-library payloads, and the dind sidecar's externals, into the capability store, which the pods of a node share, instead of into every pod (see arc_capability_pod). A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. + orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name and repository (a repository name in name's account, without the owner). A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile may set work_volume_medium ('' or Memory) and, with Memory, work_volume_size_limit (a Kubernetes quantity) to keep its runner's work directory in RAM (see arc_work_volume). A profile may set capability_store (a boolean, default false) to copy its tool-cache and system-library payloads, and the dind sidecar's externals, into the capability store, which the pods of a node share, instead of into every pod (see arc_capability_pod). A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. Returns: - A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name (the entry's name), repository (empty for an organisation scope), scope (see arc_scope), config_url (the githubConfigUrl it registers with), suffix, namespace, release, target (namespace/release, joined), app_secret, runner_image (its org's image), scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), capability_store (whether it uses the capability store), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). + A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name (the entry's name), repository (empty for an organisation scope), scope (see arc_scope), config_url (the githubConfigUrl it registers with), suffix, namespace, release, target (namespace/release, joined), app_secret, runner_image (its org's image), scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), capability_store (whether it uses the capability store), work_volume_medium and work_volume_size_limit (the runner's work volume's medium, empty for the node's disk, and its size limit with Memory; see arc_work_volume), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). """ errors: list[str] = [] profiles: list[dict[str, Any]] = [] @@ -722,9 +755,60 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, sized_autoscaled = [profile for profile in autoscaled if profile["sizing"] is not None] if len(sized_autoscaled) > 1: errors.append(f"At most one autoscaled scale-set profile may set sizing, because it derives the shared pool's ceiling; found {', '.join(profile['label'] for profile in sized_autoscaled)}") + # The autoscaler's ceiling (scripts/autoscaler-ceiling.jq) counts only the nodes every pooled scale set's runner pods can be placed on, and the sized profile's measurement counts the nodes its own selector and tolerations allow, so pooled profiles that may use different nodes would leave the ceiling counting a set of nodes some of the pool's pods never use, or never counting nodes they do. + if len({_placement_key(profile) for profile in autoscaled}) > 1: + errors.append(f"Autoscaled scale-set profiles share one pool whose ceiling counts only the nodes every one of them can use, so they must set the same node_selector and tolerations; {', '.join(profile['label'] for profile in autoscaled)} do not") return {"errors": errors, "profiles": profiles, "autoscaled": autoscaled} +def _placement_key(profile: Mapping[str, Any]) -> tuple[tuple[tuple[str, str], ...], tuple[str, ...]]: + """Return what decides which nodes a profile's runner pods may use, apart from the eligibility label every profile shares, in a form two profiles can be compared by: its node_selector and its tolerations, each in a canonical order.""" + selector = tuple(sorted((str(key), str(value)) for key, value in profile["node_selector"].items())) + tolerations = tuple(sorted(json.dumps(toleration, sort_keys=True, default=str) for toleration in profile["tolerations"])) + return selector, tolerations + + +def _measured_by_role(profile: Mapping[str, Any]) -> bool: + """Whether the role measures a ceiling for the profile from the nodes: its sizing is measured and valid, and it is the autoscaled pool's sized profile (install_platform.yml) or an ordinary profile with no max_runners of its own (install_scale_set_profile.yml).""" + sizing = profile["sizing"] + if sizing is None or sizing["errors"] or not sizing["measured"]: + return False + return bool(profile["autoscale"]) or profile["max_runners"] is None + + +def arc_measured_overlap_errors(profiles: Sequence[Mapping[str, Any]], nodes: Sequence[Mapping[str, Any]], eligibility: Mapping[str, Any]) -> list[str]: + """Refuse two profiles whose measured ceilings count the same node. + + A measured ceiling counts the whole of each node its profile's runner pods may use, less the requests of the pods there that are not runner pods (arc_measured_max_runners), so two such profiles that may both use a node each count all of its room, and their ceilings together let more runner pods start than it holds. How to divide one node's room between two scale sets whose pods differ in size is a choice, not something the cluster's figures decide, so the role refuses the overlap rather than guess: the profiles' node_selector and tolerations must keep their pods on disjoint nodes, or all but one of them sets max_runners, which is not measured. A node counts for a profile as it does in arc_measured_max_runners: it carries the eligibility label and the profile's node_selector, and has no repelling taint other than a passing condition's that the profile's tolerations do not match. + + Args: + profiles: the cluster's expanded profiles (arc_cluster_profiles), of which only those the role measures a ceiling for are compared (see _measured_by_role). nodes: Node objects, as kubernetes.core.k8s_info returns them. eligibility: the eligibility label as {key: value}, empty when the role sets none. + + Returns: + One message per pair of measured profiles that may both use a node, naming the nodes; empty when no two overlap. + """ + selector = {key: str(value) for key, value in eligibility.items()} + measured = [profile for profile in profiles if _measured_by_role(profile)] + usable: list[tuple[str, set[str]]] = [] + for profile in measured: + names = { + (node.get("metadata") or {}).get("name", "") + for node in nodes + if not _node_eligibility(node, {**selector, **profile["node_selector"]}, profile["tolerations"])[0] + } + usable.append((profile["label"], names)) + errors: list[str] = [] + for index, (label, names) in enumerate(usable): + for other_label, other_names in usable[index + 1:]: + shared = sorted(names & other_names) + if shared: + errors.append( + f"{label} and {other_label} both measure their runner ceiling from node(s) {', '.join(shared)}, so each counts the whole of their room and together they allow more runner pods than fit there. " + "Keep their runner pods on disjoint nodes with node_selector and tolerations, or give all but one of them max_runners." + ) + return errors + + def _is_list(value: Any) -> bool: """Return whether value is a list in the YAML sense: a sequence that is not a string.""" return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) @@ -1883,6 +1967,52 @@ def arc_registry_cache_configmap(cache: Mapping[str, Any]) -> dict[str, str]: return {**cache["hosts"], _BUILDKITD_DEFAULT_CONFIG: cache["buildkitd"]} +def arc_work_volume(values: Mapping[str, Any], medium: str, size_limit: str) -> dict[str, Any]: + """Put a scale set's work volume, the runner's work directory, on the medium its profile names. + + With the node's disk (an empty medium) the values are returned as given. With Memory the work volume becomes a tmpfs of size_limit (emptyDir medium Memory and sizeLimit), so a job's checkout, dependency installs and cache archives write nothing to the node's disk. A values volume named work, which the chart's Docker-in-Docker mode keeps in place of its own, is changed in place and must be an emptyDir that sets neither field already. Without one, the chart's Docker-in-Docker mode is given one, and otherwise, where the runner keeps its work directory in its own container's filesystem, the volume is added and mounted on the runner container at the work directory. The kubernetes container modes run each job's steps in a separate pod whose work volume the chart claims itself, so they are refused, as are values with no runner container to mount it on. + + Args: + values: the release's rendered Helm values, last of all the role's changes, so the registry cache's dind wiring, which adds a work volume, is already in them. medium: the profile's work_volume_medium, one of _WORK_VOLUME_MEDIA. size_limit: the profile's work_volume_size_limit, a positive Kubernetes quantity, when medium is Memory. + + Returns: + A dict with ``errors`` (empty when the values can carry the volume) and ``values`` (with the volume on the medium, or as given). + """ + if medium != _MEMORY_MEDIUM: + return {"errors": [], "values": values} + result = copy.deepcopy(dict(values)) + spec = result.setdefault("template", {}).setdefault("spec", {}) + mode = (result.get("containerMode") or {}).get("type") or "" + if mode in _SEPARATE_POD_CONTAINER_MODES: + return {"errors": [f"a work volume in memory needs the job's steps to run in the runner pod; containerMode {mode} runs them in a separate pod whose work volume the chart claims itself"], "values": values} + volumes: list[dict[str, Any]] = list(spec.get("volumes") or []) + tmpfs = {"medium": _MEMORY_MEDIUM, "sizeLimit": size_limit} + work = _named(volumes, _WORK_VOLUME) + if work is not None: + source = sorted(key for key in work if key != "name") + if source != ["emptyDir"]: + return {"errors": [f"the values' {_WORK_VOLUME} volume is not an emptyDir ({', '.join(source) or 'no source'}), so it cannot be put in memory"], "values": values} + set_already = sorted(key for key in ("medium", "sizeLimit") if key in (work["emptyDir"] or {})) + if set_already: + return {"errors": [f"the values' {_WORK_VOLUME} volume already sets {' and '.join(set_already)}; set the work volume's medium and size in the profile alone"], "values": values} + work["emptyDir"] = {**(work["emptyDir"] or {}), **tmpfs} + spec["volumes"] = volumes + return {"errors": [], "values": result} + if mode == "dind": + spec["volumes"] = volumes + [{"name": _WORK_VOLUME, "emptyDir": tmpfs}] + return {"errors": [], "values": result} + runner = _runner_container(spec) + if runner is None: + return {"errors": ["a work volume in memory needs the values to define the runner container (template.spec.containers[] named runner), as the role's own values template does"], "values": values} + mounts: list[dict[str, Any]] = list(runner.get("volumeMounts") or []) + taken = [mount.get("name") for mount in mounts if str(mount.get("mountPath", "")).rstrip("/") == _RUNNER_WORK_DIR] + if taken: + return {"errors": [f"the runner container already mounts {taken[0]} at {_RUNNER_WORK_DIR}, the work directory, so a {_WORK_VOLUME} volume cannot be added there"], "values": values} + runner["volumeMounts"] = mounts + [{"name": _WORK_VOLUME, "mountPath": _RUNNER_WORK_DIR}] + spec["volumes"] = volumes + [{"name": _WORK_VOLUME, "emptyDir": tmpfs}] + return {"errors": [], "values": result} + + def arc_image_ref(image: str) -> dict[str, str]: """Split a container image reference into registry, repository and tag or digest. @@ -1971,6 +2101,7 @@ def filters(self) -> dict[str, Any]: "arc_cluster_profiles": arc_cluster_profiles, "arc_max_runners": arc_max_runners, "arc_measured_max_runners": arc_measured_max_runners, + "arc_measured_overlap_errors": arc_measured_overlap_errors, "arc_upgrade_blocker": arc_upgrade_blocker, "arc_helm_server_side": arc_helm_server_side, "arc_image_ref": arc_image_ref, @@ -1998,4 +2129,5 @@ def filters(self) -> dict[str, Any]: "arc_registry_cache_pod": arc_registry_cache_pod, "arc_registry_cache_configmap": arc_registry_cache_configmap, "arc_registry_cache_secret_problems": arc_registry_cache_secret_problems, + "arc_work_volume": arc_work_volume, } diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 950ee9f..700a358 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -18,7 +18,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `suffix`: appended to the namespace (`arc-runners-`) and release (`-runners`) names. Default empty. - `namespace`, `release_name`: the profile's namespace and Helm release name, in place of the names derived from the suffix. The release name is also the scale set's name on GitHub, so two profiles of one scope (an organisation, or one repository) cannot share it, and Helm accepts at most 53 characters. - `values_file`: Helm values for the release, a Jinja template read from `github_runner_arc_values_dir` on the control node (or an absolute path). Without it the role's `templates/runner-scale-set-values.yaml.j2` is used, driven by the profile's own `max_runners`, `min_runners`, `container_mode` (`dind` for the chart's Docker-in-Docker mode) and `resources` (the runner container's requests and limits). - - `node_selector`: a nodeSelector for the runner pods, merged over the eligibility label below. + - `node_selector`: a nodeSelector for the runner pods, merged over the eligibility label below. Measured sizing counts only the nodes carrying it, so a profile pinned to some of the nodes is sized from those alone; two profiles whose ceilings are measured may not count the same node (see [Profiles on different nodes](#profiles-on-different-nodes)). - `max_runners`: the release's `maxRunners`. It takes precedence over the values file and over `sizing`; on the autoscaled profile it is the floor the autoscaler starts from. - `runs_on_label`: the extra `scaleSetLabels` entry the runners carry. Default `-runners`, which pools every profile of the org under one `runs-on` label; set a different one to keep a profile out of that pool. - `scale_set_labels`: the whole `scaleSetLabels` list, in place of `runs_on_label`. An empty list sets no `scaleSetLabels`, so jobs target the scale set by its release name. @@ -26,6 +26,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `autoscale`: `true` makes the profile's scale set one of those the autoscaler manages, pooled against one memory budget; at most one autoscaled profile in a cluster may set `sizing`. - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it bounds the ceiling the autoscaler works out from the nodes, leaving `maxRunners` as the floor. - `capabilities`: names of entries in `github_runner_arc_capabilities` that the profile's runner pods carry, in order (see [Capabilities](#capabilities)). + - `work_volume_medium`, `work_volume_size_limit`: what the runner's work volume sits on: unset or `""` for the node's disk, or `Memory` for a tmpfs of `work_volume_size_limit` (a Kubernetes quantity such as `6Gi`, required with `Memory`), which counts against the pod's memory (see [The work volume in memory](#the-work-volume-in-memory)). - `capability_store`: `true` copies the capabilities' tool-cache and system-library payloads into the capability store, which the pods of a node share, instead of into every pod, and does the same for the dind sidecar's copy of the runner's externals when the profile's values write that sidecar out with an `init-dind-externals` init container (a values file, or the registry cache's wiring), but not when the chart builds the sidecar itself from `containerMode: dind` (see [The capability store](#the-capability-store)). Default `false`. - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`. - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. @@ -95,6 +96,29 @@ A profile's own `max_runners` still takes precedence, and on the autoscaled prof The filter behind it is `exadev.github_runner.arc_max_runners`. +### Profiles on different nodes + +A measured ceiling counts the whole of the room on every node its profile's runner pods may use, leaving every scale set's runner pods out of what it subtracts. Two measured profiles that may both use a node would each count all of it, and together let more runner pods start than it holds, so the role refuses that before it installs anything (the `exadev.github_runner.arc_measured_overlap_errors` filter), naming the profiles and the nodes they share. How one node's room should be divided between scale sets whose pods differ in size is a choice the cluster's figures do not make, so the role does not guess one. Give each measured profile its own nodes: a `node_selector` for the label of the nodes it is pinned to, and, for the others, nodes it cannot use, for example by a `NoSchedule` taint on the pinned nodes that only the pinned profile's `tolerations` match. A profile with its own `max_runners` is not measured and not compared, which is how a fleet divides one set of nodes between scale sets on purpose. Every profile in the cluster is compared, including those another host installs. + +The autoscaled profiles share one pool, whose ceiling the autoscaler works out from the nodes every pooled scale set's runner pods can be placed on, so they must set the same `node_selector` and `tolerations`; validation refuses them otherwise. + +## The work volume in memory + +The runner keeps every job's checkout, dependency installs and cache archives (`actions/cache` writes them under the runner's temporary directory, inside the work directory) in its work directory, `/home/runner/_work`. On a node whose disk sustains far less write throughput than its memory could absorb, those writes can stall every task on the node on I/O while its CPU and memory sit idle. `work_volume_medium: Memory` puts the work directory on a tmpfs of `work_volume_size_limit` instead (an `emptyDir` with `medium: Memory` and `sizeLimit`), so a job writes nothing there to the disk. + +```yaml +work_volume_medium: Memory +work_volume_size_limit: 6Gi +``` + +The values' own volume named `work` is changed in place, and must then be an `emptyDir` that sets neither `medium` nor `sizeLimit`. Without one, the chart's Docker-in-Docker mode is given one (the chart keeps a values volume of that name in place of its own), and otherwise the volume is added and mounted on the runner container at the work directory. The `kubernetes` container modes run a job's steps in a separate pod whose work volume the chart claims, so the role refuses the option there. Unset, the values are left exactly as they were. + +The tmpfs's pages are memory. They are charged to the container that writes them (the runner, and a Docker-in-Docker sidecar that mounts the work directory for the containers it runs), and they cannot be reclaimed while the files exist, so a profile that sets the option raises its runner container's memory request and limit by `work_volume_size_limit` in its values, and its sizing's `pod_memory_request_gib` by the same, or the kubelet evicts its pods when the node runs short and measured sizing places more pods than the memory holds. With the higher request, measured sizing derives fewer pods per node by memory, which is the point: the option suits nodes whose disk, not their memory, bounds how many jobs they serve, pinned with `node_selector` (see [Profiles on different nodes](#profiles-on-different-nodes)) so that the memory is reserved only where the disk is the bound. A write past the limit fails with no space left on the device, as on a full disk. + +Size it from running pods, sampled while the profile's real jobs run: the largest work directory seen (`du -sm /home/runner/_work` in the runner container, or `du -sm` of the `work` volume under `/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/` on the node), plus the largest cache archive a job restores or saves beside it, rounded up. + +Only the work volume is offered. The tool cache and the sysroot hold the capabilities' copies, which the capability store already writes once per node rather than once per pod (see [The capability store](#the-capability-store)), and in memory they would be reserved in every pod whether a job used them or not. The Docker daemon's data root is large only in jobs that build images, so reserving it in every pod's memory would charge the size of a build to every job; a values file whose profile only builds images can mount its own `emptyDir` with `medium: Memory` at the dind container's `/var/lib/docker`. + ## Burst nodes A fixed pool can overflow onto nodes that exist only while there is work for them, such as cloud instances a cluster autoscaler (Kubernetes' Cluster Autoscaler, for one) launches when pods are Pending and removes once they are idle. Such nodes usually carry a label and a matching `NoSchedule` taint so nothing else lands on them, and no eligibility label, so measured sizing never counts them. A profile's `burst` lets its runner pods onto them: diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index 2d4fbde..dfed1b4 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -1,7 +1,7 @@ --- # ARC (Actions Runner Controller) in an existing Kubernetes cluster: the controller, each org's runner scale sets, and the fleet-health platform (heartbeat and autoscaler). Each part is gated on this host's own github_runner_arc_orgs or github_runner_arc_heartbeat_gist_id, so any host that can reach the cluster's API can be the one Ansible installs from: a k3s server host brought up by github_runner_cluster, or localhost with a kubeconfig (playbooks/arc.yml). On a host github_runner_cluster made an agent, this role installs nothing. -# Orgs whose runner scale sets this host installs. Each entry: name, app_id (needed only when the role writes the App Secret; otherwise read from the existing Secret, and checked against it when set), image, private_key (the App's PEM private key, from any source Ansible reads) or private_key_op_reference (an op:// reference the 1Password adapter resolves into private_key), optionally installation_id (resolved from the App when empty), optionally app_secret_name (the App Secret's name in each of the org's namespaces, default -github-app), and scale_set_profiles, a list of {suffix, namespace?, release_name?, values_file?, max_runners?, node_selector?, runs_on_label?, scale_set_labels?, capabilities?, autoscale?, sizing?, burst?}. capabilities names entries of github_runner_arc_capabilities (below) that the profile's runner pods carry. sizing takes uniform node figures, or measured: true to read each eligible node's free capacity from the cluster at install time. burst ({node_label_key, node_label_values?, tolerations?, max_runners, exclusive?}) lets a sized profile's runner pods overflow onto nodes carrying node_label_key, which a cluster autoscaler adds when pods are Pending, and adds max_runners to the sized maxRunners; with exclusive: true the pods run on burst nodes only and max_runners is the whole maxRunners. Each profile becomes one namespace (namespace, default arc-runners-), two Secrets in it, and one Helm release (release_name, default -runners); setting both overrides lets the role take over scale sets an existing install created under its own names. See the README for every profile key. +# Orgs whose runner scale sets this host installs. Each entry: name, app_id (needed only when the role writes the App Secret; otherwise read from the existing Secret, and checked against it when set), image, private_key (the App's PEM private key, from any source Ansible reads) or private_key_op_reference (an op:// reference the 1Password adapter resolves into private_key), optionally installation_id (resolved from the App when empty), optionally app_secret_name (the App Secret's name in each of the org's namespaces, default -github-app), and scale_set_profiles, a list of {suffix, namespace?, release_name?, values_file?, max_runners?, node_selector?, runs_on_label?, scale_set_labels?, capabilities?, work_volume_medium?, work_volume_size_limit?, autoscale?, sizing?, burst?}. work_volume_medium: Memory puts the runner's work volume on a tmpfs of work_volume_size_limit, which counts against the pod's memory. capabilities names entries of github_runner_arc_capabilities (below) that the profile's runner pods carry. sizing takes uniform node figures, or measured: true to read each eligible node's free capacity from the cluster at install time. burst ({node_label_key, node_label_values?, tolerations?, max_runners, exclusive?}) lets a sized profile's runner pods overflow onto nodes carrying node_label_key, which a cluster autoscaler adds when pods are Pending, and adds max_runners to the sized maxRunners; with exclusive: true the pods run on burst nodes only and max_runners is the whole maxRunners. Each profile becomes one namespace (namespace, default arc-runners-), two Secrets in it, and one Helm release (release_name, default -runners); setting both overrides lets the role take over scale sets an existing install created under its own names. See the README for every profile key. github_runner_arc_orgs: [] # Directory on the control node that a relative scale_set_profiles[].values_file is read from. Values files are Jinja templates rendered on the control node, so the target host needs no checkout. A profile with no values_file uses the role's own templates/runner-scale-set-values.yaml.j2. diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index e11ea35..060aafa 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -13,13 +13,15 @@ argument_specs: description: - Scopes whose runner scale sets this host installs. Each entry has name, app_id (needed only when the role writes the App Secret; otherwise the existing Secret supplies it), image, private_key or private_key_op_reference, and optionally repository, installation_id and app_secret_name (the App Secret's name, default -github-app). - Without repository an entry registers organisation runners in the organisation name names (githubConfigUrl https://github.com/). With repository it registers repository runners in that one repository (https://github.com//), where name may be a user or an organisation; its derived names use -- (lower-cased, with any character a DNS label does not allow as '-') in place of , and its App needs the repository-scope permissions. The scope (name, or name/repository) identifies an entry, so one login may have an organisation entry and any number of repository entries. - - Each entry's scale_set_profiles is a non-empty list of profiles, each with suffix and optionally namespace (default arc-runners-), release_name (default -runners, at most 53 characters, Helm's limit), values_file, max_runners, min_runners, container_mode, resources, node_selector, runs_on_label, scale_set_labels, capabilities, capability_store, tolerations, autoscale, sizing and burst. + - Each entry's scale_set_profiles is a non-empty list of profiles, each with suffix and optionally namespace (default arc-runners-), release_name (default -runners, at most 53 characters, Helm's limit), values_file, max_runners, min_runners, container_mode, resources, node_selector, runs_on_label, scale_set_labels, capabilities, capability_store, work_volume_medium, work_volume_size_limit, tolerations, autoscale, sizing and burst. - capability_store, a boolean (default false), makes the profile's runner pods copy each tool-cache and system-library capability's payload, and the runner's externals for a Docker-in-Docker sidecar the values write out, into the capability store (github_runner_arc_capability_store_host_path), which the pods of a node share, instead of into every pod's own volumes, and mount the result read-only. Every image those copies come from must be pinned by digest, and the store needs the node cache pruner, so a profile that sets it is refused while github_runner_arc_node_cache_size_limit_gib is 0 or github_runner_arc_node_label_key is empty, since the pruner runs only on the nodes carrying that label. A job that can reach a privileged Docker daemon can write to the store, so give it only to a profile whose workflows are all of one trust level. - max_runners, when set, is the release's maxRunners and takes precedence over its values file and its sizing. - scale_set_labels replaces runs_on_label with the whole list of scaleSetLabels; an empty list sets none, so jobs target the release name. - tolerations is a list of Kubernetes tolerations for the profile's runner pods, for nodes the eligibility label selects that also carry a taint to keep other workloads off. Measured sizing counts a node whose only repelling taints these match, as the pods can be placed there; without them it leaves such a node out. With burst they come before burst.tolerations. - sizing derives a runner ceiling from uniform node figures, or with measured true from each eligible node's allocatable CPU and memory (and pod capacity, the kubelet's max-pods) less its current requests and reserve_cpu and reserve_memory_gib. - capabilities names entries of github_runner_arc_capabilities that the profile's runner pods carry, in order. + - work_volume_medium, '' (the default, the node's disk) or Memory, is what the runner's work volume sits on. Memory makes it a tmpfs of work_volume_size_limit (a Kubernetes quantity, required with Memory and refused without it), whose pages count against the pod's memory, so the profile's runner memory request and limit, and its sizing's pod_memory_request_gib, must be raised by that size. The kubernetes container modes are refused with it. + - node_selector restricts the profile's runner pods, and the nodes its measured sizing counts, to nodes carrying those labels as well as the eligibility label. Two profiles whose ceilings the role measures may not count the same node (each would count all of its room), and autoscaled profiles must share one node_selector and one set of tolerations, since their pool's ceiling counts only the nodes all of them can use. - burst, on a profile with sizing and without max_runners or autoscale, lets its runner pods overflow onto nodes carrying burst.node_label_key (any value, or one of burst.node_label_values), with burst.tolerations for their taint, and adds burst.max_runners to the sized maxRunners. With burst.exclusive true (and no sizing, max_runners or autoscale) the pods run on burst nodes only and burst.max_runners is the whole maxRunners. github_runner_arc_capabilities: type: dict diff --git a/roles/github_runner_arc/tasks/check_measured_overlap.yml b/roles/github_runner_arc/tasks/check_measured_overlap.yml new file mode 100644 index 0000000..a0067b5 --- /dev/null +++ b/roles/github_runner_arc/tasks/check_measured_overlap.yml @@ -0,0 +1,19 @@ +--- +# Refuses, before any release is installed or upgraded, two profiles in the cluster whose measured ceilings would count the same node (plugins/filter/arc.py's arc_measured_overlap_errors): each measured ceiling counts the whole of every node its profile's runner pods may use, so two that share a node would together let more runner pods start than it holds. Every profile of the cluster is compared, not only this host's, since another host's measured profile counts the same nodes. Reads the Nodes only; read-only, so it also runs under --check. + +- name: Read the nodes the measured profiles may use + kubernetes.core.k8s_info: + kind: Node + register: github_runner_arc_overlap_nodes + +- name: Fail when two measured profiles count the same node + ansible.builtin.fail: + msg: "{{ github_runner_arc_overlap_problems }}" + vars: + github_runner_arc_overlap_problems: >- + {{ + github_runner_arc_fleet.profiles | exadev.github_runner.arc_measured_overlap_errors( + github_runner_arc_overlap_nodes.resources, + {github_runner_arc_node_label_key: github_runner_arc_node_label_value | string} if github_runner_arc_node_label_key | length > 0 else {}) + }} + when: github_runner_arc_overlap_problems | length > 0 diff --git a/roles/github_runner_arc/tasks/main.yml b/roles/github_runner_arc/tasks/main.yml index 0cb6713..95c240c 100644 --- a/roles/github_runner_arc/tasks/main.yml +++ b/roles/github_runner_arc/tasks/main.yml @@ -53,6 +53,11 @@ ansible.builtin.include_tasks: label_nodes.yml when: github_runner_arc_node_label_key | length > 0 and github_runner_arc_labelled_nodes | length > 0 + # After the nodes are labelled, since eligibility reads the label, and before anything is installed (the platform measures the autoscaled pool's ceiling, and each scale set its own), so that a refused overlap installs nothing. Only with two or more measured profiles in the cluster, since one cannot overlap anything. + - name: Refuse two measured profiles that count the same node + ansible.builtin.include_tasks: check_measured_overlap.yml + when: github_runner_arc_fleet.profiles | selectattr('sizing') | selectattr('sizing.measured') | list | length > 1 + - name: Create the priority class the platform's own pods run under ansible.builtin.include_tasks: install_priority_class.yml diff --git a/roles/github_runner_arc/tasks/render_scale_set_values.yml b/roles/github_runner_arc/tasks/render_scale_set_values.yml index 23754f0..17a733c 100644 --- a/roles/github_runner_arc/tasks/render_scale_set_values.yml +++ b/roles/github_runner_arc/tasks/render_scale_set_values.yml @@ -1,5 +1,5 @@ --- -# Renders one scale-set profile's Helm values on the control node, with `profile` (the expanded profile from plugins/filter/arc.py) and `org` (its org entry) set by the includer: the profile's values file (or the role's default template), the role's own overlay merged over it, then the profile's capabilities added (plugins/filter/arc.py's arc_capability_pod), then, with the registry cache on, its Docker daemon pointed at the cache (arc_registry_cache_pod), and last, with the profile's capability store on, the dind sidecar's copy of the runner's externals moved into the store (arc_capability_store_externals). Sets github_runner_arc_scale_set_values, the release's values, github_runner_arc_job_started_data, the job-started ConfigMap's data (empty when the profile needs none), github_runner_arc_registry_cache_wired, whether the profile's pods use the registry cache, and github_runner_arc_registry_cache_data, the data of the registry cache ConfigMap they then mount. Included by install_scale_set_profile.yml, and by tests/capabilities/render.yml so the tests render exactly what the role installs. +# Renders one scale-set profile's Helm values on the control node, with `profile` (the expanded profile from plugins/filter/arc.py) and `org` (its org entry) set by the includer: the profile's values file (or the role's default template), the role's own overlay merged over it, then the profile's capabilities added (plugins/filter/arc.py's arc_capability_pod), then, with the registry cache on, its Docker daemon pointed at the cache (arc_registry_cache_pod), then, with the profile's capability store on, the dind sidecar's copy of the runner's externals moved into the store (arc_capability_store_externals), and last, with the profile's work_volume_medium set, the runner's work volume put on that medium (arc_work_volume). Sets github_runner_arc_scale_set_values, the release's values, github_runner_arc_job_started_data, the job-started ConfigMap's data (empty when the profile needs none), github_runner_arc_registry_cache_wired, whether the profile's pods use the registry cache, and github_runner_arc_registry_cache_data, the data of the registry cache ConfigMap they then mount. Included by install_scale_set_profile.yml, and by tests/capabilities/render.yml so the tests render exactly what the role installs. - name: Set the capability store settings for {{ profile.label }} ansible.builtin.set_fact: @@ -85,3 +85,17 @@ - name: Keep the values with the externals copy in the capability store for {{ profile.label }} ansible.builtin.set_fact: github_runner_arc_scale_set_values: "{{ github_runner_arc_capability_store_render['values'] }}" + +# Last of all, after the registry cache, whose dind wiring adds a work volume of its own, so that whichever work volume the pod ends up with is the one put on the profile's medium. A profile without work_volume_medium keeps the values exactly as rendered above. +- name: Put the work volume on its medium for {{ profile.label }} + ansible.builtin.set_fact: + github_runner_arc_work_volume_render: "{{ github_runner_arc_scale_set_values | exadev.github_runner.arc_work_volume(profile.work_volume_medium, profile.work_volume_size_limit) }}" + +- name: Fail when the values cannot carry the work volume's medium for {{ profile.label }} + ansible.builtin.fail: + msg: "{{ github_runner_arc_work_volume_render.errors | map('regex_replace', '^', profile.label ~ ': ') | list }}" + when: github_runner_arc_work_volume_render.errors | length > 0 + +- name: Keep the values with the work volume on its medium for {{ profile.label }} + ansible.builtin.set_fact: + github_runner_arc_scale_set_values: "{{ github_runner_arc_work_volume_render['values'] }}" diff --git a/tests/unit/test_arc_filters.py b/tests/unit/test_arc_filters.py index f1ca324..d69a3ee 100644 --- a/tests/unit/test_arc_filters.py +++ b/tests/unit/test_arc_filters.py @@ -699,5 +699,178 @@ def test_a_first_install_proceeds(self) -> None: self.assertEqual(arc.arc_upgrade_blocker("", "0.15.0", 5), "") + +ELIGIBLE_LABEL = {"example.com/ci-eligible": "true"} +SLOW_DISK_LABEL = {"example.com/slow-disk": "true"} +SLOW_DISK_TOLERATION = {"key": "example.com/slow-disk", "operator": "Exists", "effect": "NoSchedule"} + + +def slow_disk_node(name: str) -> dict[str, Any]: + """Return an eligible node that carries the slow-disk label and the taint only a profile pinned to it tolerates.""" + return node(name, "8", "32Gi", labels={**ELIGIBLE_LABEL, **SLOW_DISK_LABEL}, taints=[{"key": "example.com/slow-disk", "value": "true", "effect": "NoSchedule"}]) + + +class ProfilesOnDifferentNodesTest(unittest.TestCase): + NODES = [node("shared-1", "4", "8Gi", labels=ELIGIBLE_LABEL), node("shared-2", "4", "8Gi", labels=ELIGIBLE_LABEL), slow_disk_node("slow-1"), slow_disk_node("slow-2")] + GENERAL = {"suffix": "", "sizing": MEASURED} + PINNED = {"suffix": "-slow", "sizing": MEASURED, "node_selector": SLOW_DISK_LABEL, "tolerations": [SLOW_DISK_TOLERATION]} + + def profiles(self, *profiles: dict[str, Any]) -> list[dict[str, Any]]: + result = arc.arc_profiles([org(profiles=list(profiles))]) + self.assertEqual(result["errors"], []) + expanded: list[dict[str, Any]] = result["profiles"] + return expanded + + def test_a_pinned_profile_is_sized_from_its_own_nodes_and_the_other_from_the_rest(self) -> None: + general, pinned = self.profiles(self.GENERAL, self.PINNED) + measured_general = arc.arc_measured_max_runners(MEASURED, self.NODES, [], ELIGIBLE_LABEL | general["node_selector"], (), general["tolerations"]) + measured_pinned = arc.arc_measured_max_runners(MEASURED, self.NODES, [], ELIGIBLE_LABEL | pinned["node_selector"], (), pinned["tolerations"]) + self.assertEqual([entry["name"] for entry in measured_general["nodes"]], ["shared-1", "shared-2"]) + self.assertEqual([entry["name"] for entry in measured_pinned["nodes"]], ["slow-1", "slow-2"]) + # shared: 4 cores and 8 GiB hold four pods of 1 core and 2 GiB each; slow: 8 cores and 32 GiB hold eight by CPU. + self.assertEqual((measured_general["max_runners"], measured_pinned["max_runners"]), (8, 16)) + self.assertEqual(arc.arc_measured_overlap_errors([general, pinned], self.NODES, ELIGIBLE_LABEL), []) + + def test_without_the_selector_a_profile_that_tolerates_the_taint_counts_every_node(self) -> None: + [unpinned] = self.profiles({**self.PINNED, "node_selector": {}}) + measured = arc.arc_measured_max_runners(MEASURED, self.NODES, [], ELIGIBLE_LABEL | unpinned["node_selector"], (), unpinned["tolerations"]) + self.assertEqual([entry["name"] for entry in measured["nodes"]], ["shared-1", "shared-2", "slow-1", "slow-2"]) + self.assertEqual(measured["max_runners"], 24) + + def test_two_measured_profiles_that_may_use_one_node_are_refused_with_the_nodes_named(self) -> None: + general, pinned = self.profiles({**self.GENERAL, "tolerations": [SLOW_DISK_TOLERATION]}, self.PINNED) + self.assertEqual(arc.arc_measured_overlap_errors([general, pinned], self.NODES, ELIGIBLE_LABEL), [ + "Example and Example-slow both measure their runner ceiling from node(s) slow-1, slow-2, so each counts the whole of their room and together they allow more runner pods than fit there. " + "Keep their runner pods on disjoint nodes with node_selector and tolerations, or give all but one of them max_runners." + ]) + + def test_a_profile_with_its_own_max_runners_or_uniform_sizing_is_not_compared(self) -> None: + tolerating = {**self.GENERAL, "tolerations": [SLOW_DISK_TOLERATION]} + for other in ({**self.PINNED, "max_runners": 2}, {**self.PINNED, "sizing": SIZING}, {**self.PINNED, "sizing": None, "max_runners": 1}): + with self.subTest(other=other): + self.assertEqual(arc.arc_measured_overlap_errors(self.profiles(tolerating, other), self.NODES, ELIGIBLE_LABEL), []) + + def test_the_autoscaled_pools_sized_profile_is_compared_even_with_max_runners(self) -> None: + general, pinned = self.profiles({**self.GENERAL, "tolerations": [SLOW_DISK_TOLERATION]}, {**self.PINNED, "autoscale": True, "max_runners": 2}) + self.assertEqual(len(arc.arc_measured_overlap_errors([general, pinned], self.NODES, ELIGIBLE_LABEL)), 1) + + def test_the_eligibility_label_decides_too(self) -> None: + general, pinned = self.profiles({**self.GENERAL, "tolerations": [SLOW_DISK_TOLERATION]}, self.PINNED) + self.assertEqual(arc.arc_measured_overlap_errors([general, pinned], self.NODES, {"example.com/other": "true"}), []) + + def test_autoscaled_profiles_must_share_their_node_selector_and_tolerations(self) -> None: + pooled = {"values_file": "v.yaml", "autoscale": True} + same = arc.arc_profiles([org(profiles=[{**pooled, "suffix": "-a", "tolerations": [SLOW_DISK_TOLERATION]}, {**pooled, "suffix": "-b", "tolerations": [dict(reversed(list(SLOW_DISK_TOLERATION.items())))]}])]) + self.assertEqual(same["errors"], []) + for differing in ({"node_selector": SLOW_DISK_LABEL}, {"tolerations": [SLOW_DISK_TOLERATION]}): + with self.subTest(differing=differing): + result = arc.arc_profiles([org(profiles=[{**pooled, "suffix": "-a"}, {**pooled, "suffix": "-b", **differing}])]) + self.assertEqual(result["errors"], ["Autoscaled scale-set profiles share one pool whose ceiling counts only the nodes every one of them can use, so they must set the same node_selector and tolerations; Example-a, Example-b do not"]) + + def test_the_role_refuses_an_overlap_before_it_installs_anything(self) -> None: + import yaml + + tasks = Path(PLUGIN).parents[2] / "roles" / "github_runner_arc" / "tasks" + [install] = [task for task in yaml.safe_load((tasks / "main.yml").read_text()) if task.get("name") == "Install into the cluster"] + included = [task.get("ansible.builtin.include_tasks") for task in install["block"]] + check = included.index("check_measured_overlap.yml") + self.assertLess(check, included.index("install_platform.yml")) + self.assertLess(check, included.index("install_org.yml")) + self.assertEqual(included.index("label_nodes.yml"), check - 1) + [read, refuse] = yaml.safe_load((tasks / "check_measured_overlap.yml").read_text()) + self.assertEqual(read["kubernetes.core.k8s_info"], {"kind": "Node"}) + self.assertIn("exadev.github_runner.arc_measured_overlap_errors", refuse["vars"]["github_runner_arc_overlap_problems"]) + +class WorkVolumeProfileTest(unittest.TestCase): + def expanded(self, profile: dict[str, Any]) -> dict[str, Any]: + result = arc.arc_profiles([org(profiles=[{"max_runners": 1, **profile}])]) + self.assertEqual(result["errors"], []) + expanded: dict[str, Any] = result["profiles"][0] + return expanded + + def errors(self, profile: dict[str, Any]) -> list[str]: + errors: list[str] = arc.arc_profiles([org(profiles=[{"max_runners": 1, **profile}])])["errors"] + return errors + + def test_the_work_volume_stays_on_the_nodes_disk_unless_the_profile_names_memory(self) -> None: + self.assertEqual((self.expanded({})["work_volume_medium"], self.expanded({})["work_volume_size_limit"]), ("", "")) + self.assertEqual((self.expanded({"work_volume_medium": ""})["work_volume_medium"]), "") + memory = self.expanded({"work_volume_medium": "Memory", "work_volume_size_limit": "6Gi"}) + self.assertEqual((memory["work_volume_medium"], memory["work_volume_size_limit"]), ("Memory", "6Gi")) + self.assertEqual(self.expanded({"work_volume_medium": "Memory", "work_volume_size_limit": 1073741824})["work_volume_size_limit"], "1073741824") + + def test_a_medium_the_work_volume_cannot_use_is_an_error(self) -> None: + for medium in ("memory", "HugePages", 1, None): + with self.subTest(medium=medium): + self.assertEqual(self.errors({"work_volume_medium": medium}), ["Example scale_set_profiles[0].work_volume_medium must be one of '', 'Memory'"]) + + def test_memory_needs_a_positive_size_limit(self) -> None: + self.assertEqual(self.errors({"work_volume_medium": "Memory"}), ["Example scale_set_profiles[0].work_volume_size_limit is required with work_volume_medium: Memory, as a Kubernetes quantity such as 6Gi, since a tmpfs without one may grow to the pod's memory limit"]) + for size_limit in ("0", "six", "-1Gi", "1 Gi"): + with self.subTest(size_limit=size_limit): + self.assertEqual(self.errors({"work_volume_medium": "Memory", "work_volume_size_limit": size_limit}), [f"Example scale_set_profiles[0].work_volume_size_limit must be a positive Kubernetes quantity such as 6Gi (got {size_limit!r})"]) + for size_limit in (True, 1.5, ["6Gi"]): + with self.subTest(size_limit=size_limit): + self.assertTrue(self.errors({"work_volume_medium": "Memory", "work_volume_size_limit": size_limit})[0].startswith("Example scale_set_profiles[0].work_volume_size_limit is required")) + + def test_a_size_limit_without_memory_is_an_error(self) -> None: + self.assertEqual(self.errors({"work_volume_size_limit": "6Gi"}), ["Example scale_set_profiles[0].work_volume_size_limit applies only with work_volume_medium: Memory"]) + + +class WorkVolumeTest(unittest.TestCase): + RUNNER = {"name": "runner", "command": ["/home/runner/run.sh"]} + TMPFS = {"medium": "Memory", "sizeLimit": "6Gi"} + + def test_the_nodes_disk_leaves_the_values_as_given(self) -> None: + values = {"template": {"spec": {"containers": [self.RUNNER]}}} + self.assertIs(arc.arc_work_volume(values, "", "")["values"], values) + + def test_a_values_work_volume_is_put_in_memory_in_place(self) -> None: + values = {"template": {"spec": {"containers": [{**self.RUNNER, "volumeMounts": [{"name": "work", "mountPath": "/home/runner/_work"}]}], "volumes": [{"name": "dind-sock", "emptyDir": {}}, {"name": "work", "emptyDir": {}}]}}} + result = arc.arc_work_volume(values, "Memory", "6Gi") + self.assertEqual(result["errors"], []) + self.assertEqual(result["values"]["template"]["spec"]["volumes"], [{"name": "dind-sock", "emptyDir": {}}, {"name": "work", "emptyDir": self.TMPFS}]) + self.assertEqual(result["values"]["template"]["spec"]["containers"], values["template"]["spec"]["containers"]) + self.assertEqual(values["template"]["spec"]["volumes"][1], {"name": "work", "emptyDir": {}}) + + def test_a_values_work_volume_that_cannot_be_put_in_memory_is_an_error(self) -> None: + cases = { + "the values' work volume is not an emptyDir (persistentVolumeClaim), so it cannot be put in memory": {"name": "work", "persistentVolumeClaim": {"claimName": "c"}}, + "the values' work volume already sets medium; set the work volume's medium and size in the profile alone": {"name": "work", "emptyDir": {"medium": "Memory"}}, + "the values' work volume already sets sizeLimit; set the work volume's medium and size in the profile alone": {"name": "work", "emptyDir": {"sizeLimit": "1Gi"}}, + } + for message, volume in cases.items(): + with self.subTest(message=message): + values = {"template": {"spec": {"containers": [self.RUNNER], "volumes": [volume]}}} + result = arc.arc_work_volume(values, "Memory", "6Gi") + self.assertEqual(result["errors"], [message]) + self.assertIs(result["values"], values) + + def test_dind_mode_gets_a_work_volume_the_chart_keeps_in_place_of_its_own(self) -> None: + values = {"containerMode": {"type": "dind"}, "template": {"spec": {"containers": [self.RUNNER]}}} + result = arc.arc_work_volume(values, "Memory", "6Gi") + self.assertEqual(result["errors"], []) + self.assertEqual(result["values"]["template"]["spec"]["volumes"], [{"name": "work", "emptyDir": self.TMPFS}]) + self.assertNotIn("volumeMounts", result["values"]["template"]["spec"]["containers"][0]) + + def test_without_a_work_volume_the_runner_gets_one_at_its_work_directory(self) -> None: + values = {"template": {"spec": {"containers": [{**self.RUNNER, "volumeMounts": [{"name": "cache", "mountPath": "/cache"}]}], "volumes": [{"name": "cache", "emptyDir": {}}]}}} + spec = arc.arc_work_volume(values, "Memory", "6Gi")["values"]["template"]["spec"] + self.assertEqual(spec["volumes"], [{"name": "cache", "emptyDir": {}}, {"name": "work", "emptyDir": self.TMPFS}]) + self.assertEqual(spec["containers"][0]["volumeMounts"], [{"name": "cache", "mountPath": "/cache"}, {"name": "work", "mountPath": "/home/runner/_work"}]) + + def test_values_that_cannot_take_a_work_volume_are_an_error(self) -> None: + cases = { + "a work volume in memory needs the job's steps to run in the runner pod; containerMode kubernetes runs them in a separate pod whose work volume the chart claims itself": {"containerMode": {"type": "kubernetes"}, "template": {"spec": {"containers": [self.RUNNER]}}}, + "a work volume in memory needs the values to define the runner container (template.spec.containers[] named runner), as the role's own values template does": {"template": {"spec": {}}}, + "the runner container already mounts scratch at /home/runner/_work, the work directory, so a work volume cannot be added there": {"template": {"spec": {"containers": [{**self.RUNNER, "volumeMounts": [{"name": "scratch", "mountPath": "/home/runner/_work/"}]}], "volumes": [{"name": "scratch", "emptyDir": {}}]}}}, + } + for message, values in cases.items(): + with self.subTest(message=message): + result = arc.arc_work_volume(values, "Memory", "6Gi") + self.assertEqual(result["errors"], [message]) + self.assertIs(result["values"], values) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 4887047..4061e90 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -492,6 +492,15 @@ def test_the_role_renders_a_node_cache_capability_into_a_per_profile_node_direct self.assertEqual(runner(rendered["values"])["volumeMounts"], [{"name": "node-cache", "mountPath": "/node-cache"}]) self.assertEqual(rendered["job_started"], {}) + def test_the_role_renders_the_work_volume_in_memory_last_of_all(self) -> None: + tmpfs = {"name": "work", "emptyDir": {"medium": "Memory", "sizeLimit": "6Gi"}} + rendered = self.render({"max_runners": 1, "capabilities": ["node"], "work_volume_medium": "Memory", "work_volume_size_limit": "6Gi"}) + self.assertEqual(rendered["values"]["template"]["spec"]["volumes"], [{"name": "tool-cache", "emptyDir": {}}, tmpfs]) + self.assertEqual(runner(rendered["values"])["volumeMounts"], [{"name": "tool-cache", "mountPath": TOOL_CACHE}, {"name": "work", "mountPath": "/home/runner/_work"}]) + dind = self.render({"max_runners": 1, "container_mode": "dind", "work_volume_medium": "Memory", "work_volume_size_limit": "6Gi"}) + self.assertEqual(dind["values"]["template"]["spec"]["volumes"], [tmpfs]) + self.assertNotIn("volumes", self.render({"max_runners": 1, "container_mode": "dind"})["values"]["template"]["spec"]) + def test_a_profile_without_capabilities_renders_as_before(self) -> None: rendered = self.render({"max_runners": 3, "container_mode": "dind"}) self.assertEqual(rendered["job_started"], {})