From 341d5780c6e4113c45b5dbf224fc24637de53bde Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Sun, 11 Oct 2026 20:10:39 +0200 Subject: [PATCH 001/181] Orchestrator M0: database, event chain, actions, worker, API --- CHANGELOG.md | 2 + console-backend/README.md | 28 + .../bits_console_backend.egg-info/PKG-INFO | 10 + .../bits_console_backend.egg-info/SOURCES.txt | 50 ++ .../dependency_links.txt | 1 + .../requires.txt | 5 + .../top_level.txt | 1 + console-backend/console_backend/config.py | 6 + console-backend/console_backend/main.py | 3 + .../console_backend/orch/__init__.py | 4 + .../console_backend/orch/actions.py | 357 +++++++++++++ console-backend/console_backend/orch/api.py | 213 ++++++++ .../console_backend/orch/authority.py | 48 ++ console-backend/console_backend/orch/db.py | 172 ++++++ .../console_backend/orch/events.py | 99 ++++ console-backend/console_backend/orch/kinds.py | 78 +++ .../console_backend/orch/worker.py | 112 ++++ console-backend/pyproject.toml | 2 +- console-backend/tests/test_orch.py | 503 ++++++++++++++++++ 19 files changed, 1693 insertions(+), 1 deletion(-) create mode 100644 console-backend/bits_console_backend.egg-info/PKG-INFO create mode 100644 console-backend/bits_console_backend.egg-info/SOURCES.txt create mode 100644 console-backend/bits_console_backend.egg-info/dependency_links.txt create mode 100644 console-backend/bits_console_backend.egg-info/requires.txt create mode 100644 console-backend/bits_console_backend.egg-info/top_level.txt create mode 100644 console-backend/console_backend/orch/__init__.py create mode 100644 console-backend/console_backend/orch/actions.py create mode 100644 console-backend/console_backend/orch/api.py create mode 100644 console-backend/console_backend/orch/authority.py create mode 100644 console-backend/console_backend/orch/db.py create mode 100644 console-backend/console_backend/orch/events.py create mode 100644 console-backend/console_backend/orch/kinds.py create mode 100644 console-backend/console_backend/orch/worker.py create mode 100644 console-backend/tests/test_orch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a7249f0b..554408be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ --- # Unreleased +- **[Feature]** console-backend orchestrator, milestone M0 (`console_backend/orch/`): a worker process (`python -m console_backend.orch.worker`) and a shared SQLite database; a hash-chained event log; typed actions with JSON schemas, approvals (an advisor can only propose), authority re-checked at run time, retries with backoff, one action at a time per subject; `pipeline.retry` and `pipeline.cancel` as the first kinds; `/ops/actions` and `/ops/events`. + - **[Feature]** `bits cvmfs release --pipeline `: makes a release (its links and merged view) from one build, at any time, from the S3 store alone; the BOMs now carry the build's `cvmfs_templates`, requested packages, `view:` rules and `requires`, and each pipeline's builds are indexed under `BUILDS/by-pipeline/`. - **[Improvement]** A merged view is one build's packages again, replaced when it differs (no more merging across builds), and is independent of the release links: a group with `cvmfs_views_template` but no `cvmfs_releases_template` gets views. It records its top-level packages; one made from others is replaced only with `--force-view`. `bits cvmfs publish --wait` waits for every job, the packages before the release. - **[Feature]** `BUILD_EPOCH` (`bits_helpers/build_epoch.py`): bumping it rebuilds every package once, for changes to how bits builds that the package hashes do not see. At 0 (now) it is not hashed, so no hash changed. A test fails when the build template, `init.sh` generation or relocation scripts change, until their checksum is updated and the epoch considered. diff --git a/console-backend/README.md b/console-backend/README.md index b0256cde..a16dec7c 100644 --- a/console-backend/README.md +++ b/console-backend/README.md @@ -25,3 +25,31 @@ authenticated GitHub read proxy for package browsing — see `BITS_GITHUB_TOKEN` # from bits/console-backend, with the bits repo on the path for bits_helpers: PYTHONPATH=.:.. python3 -m unittest discover -s tests PYTHONPATH=.:.. uvicorn console_backend.main:app --port 8080 # /healthz + +## Orchestrator + +`console_backend/orch/` is the bits orchestrator: a worker that records builds, +runs actions, and (in later milestones) signs per architecture, starts +schedules and collects failures. It runs as its own process from the same +image, beside this API, and shares one SQLite database with it: + + BITS_ORCH_DB=/orch/orch.db python3 -m console_backend.orch.worker + +The worker owns the database: it takes a lock beside it (a second worker +exits), applies the numbered migrations, and runs one pass every +`BITS_ORCH_INTERVAL` seconds (default 30), touching `.heartbeat` after each. +The API only writes requests and decisions, and answers 503 until the worker +has created and migrated the database. The worker has no user token, so when +the admin policy names GitLab groups (`&group` lines or `BITS_ADMINS_GROUP`) it +needs `BITS_ADMIN_RESOLVE_TOKEN`, and refuses to start without it. + +Every change of state is an **action**, of a kind with a JSON schema +(`GET /ops/actions/catalogue`). A person who administers the action's community +requests it (`POST /ops/actions`); kinds marked `requester` are approved by +that request, `approver` kinds need `POST /ops/actions/{id}/approve`. An +`advisor` actor can only propose. The worker re-checks the approver's authority +when it runs an action, retries transient failures with backoff (at most 5 +attempts), and runs one action at a time per subject. + +Everything that happens is an **event** (`GET /ops/events?after=&wait=`, overall +admins): an append-only log whose hash chain is the audit trail. diff --git a/console-backend/bits_console_backend.egg-info/PKG-INFO b/console-backend/bits_console_backend.egg-info/PKG-INFO new file mode 100644 index 00000000..b8c35dc3 --- /dev/null +++ b/console-backend/bits_console_backend.egg-info/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 2.4 +Name: bits-console-backend +Version: 0.1.0 +Summary: bits-console backend — relying party and the security-proxy's only client +Requires-Python: >=3.10 +Requires-Dist: fastapi==0.138.0 +Requires-Dist: uvicorn==0.49.0 +Requires-Dist: httpx==0.28.1 +Requires-Dist: pyjwt[crypto]==2.15.1 +Requires-Dist: webauthn==2.5.2 diff --git a/console-backend/bits_console_backend.egg-info/SOURCES.txt b/console-backend/bits_console_backend.egg-info/SOURCES.txt new file mode 100644 index 00000000..ed9c254b --- /dev/null +++ b/console-backend/bits_console_backend.egg-info/SOURCES.txt @@ -0,0 +1,50 @@ +README.md +pyproject.toml +bits_console_backend.egg-info/PKG-INFO +bits_console_backend.egg-info/SOURCES.txt +bits_console_backend.egg-info/dependency_links.txt +bits_console_backend.egg-info/requires.txt +bits_console_backend.egg-info/top_level.txt +console_backend/__init__.py +console_backend/audit.py +console_backend/auth_oidc.py +console_backend/authz.py +console_backend/catalog.py +console_backend/ci_auth.py +console_backend/config.py +console_backend/credentials.py +console_backend/forge_ops.py +console_backend/identity.py +console_backend/main.py +console_backend/session.py +console_backend/signproxy.py +console_backend/webauthn_rp.py +console_backend/orch/__init__.py +console_backend/orch/actions.py +console_backend/orch/api.py +console_backend/orch/authority.py +console_backend/orch/db.py +console_backend/orch/events.py +console_backend/orch/kinds.py +console_backend/orch/worker.py +tests/test_auth.py +tests/test_auth_oidc.py +tests/test_authz.py +tests/test_authz_tree.py +tests/test_catalog.py +tests/test_ci.py +tests/test_cli_preapprove.py +tests/test_cli_sign.py +tests/test_enrollment_authority.py +tests/test_health.py +tests/test_ops_admin.py +tests/test_ops_build.py +tests/test_ops_file.py +tests/test_ops_pipeline.py +tests/test_orch.py +tests/test_preapprove.py +tests/test_sign.py +tests/test_sign_approve.py +tests/test_sign_preapproved.py +tests/test_signproxy.py +tests/test_webauthn.py \ No newline at end of file diff --git a/console-backend/bits_console_backend.egg-info/dependency_links.txt b/console-backend/bits_console_backend.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/console-backend/bits_console_backend.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/console-backend/bits_console_backend.egg-info/requires.txt b/console-backend/bits_console_backend.egg-info/requires.txt new file mode 100644 index 00000000..705232af --- /dev/null +++ b/console-backend/bits_console_backend.egg-info/requires.txt @@ -0,0 +1,5 @@ +fastapi==0.138.0 +uvicorn==0.49.0 +httpx==0.28.1 +pyjwt[crypto]==2.15.1 +webauthn==2.5.2 diff --git a/console-backend/bits_console_backend.egg-info/top_level.txt b/console-backend/bits_console_backend.egg-info/top_level.txt new file mode 100644 index 00000000..d819de81 --- /dev/null +++ b/console-backend/bits_console_backend.egg-info/top_level.txt @@ -0,0 +1 @@ +console_backend diff --git a/console-backend/console_backend/config.py b/console-backend/console_backend/config.py index 58a18a62..20d112f6 100644 --- a/console-backend/console_backend/config.py +++ b/console-backend/console_backend/config.py @@ -116,6 +116,12 @@ def __init__(self, env=None): # then set 1 to enforce — otherwise a never-enrolled admin signs single-shot. self.webauthn_required = e.get("BITS_WEBAUTHN_REQUIRED", "0") == "1" + # The orchestrator (orch/): its SQLite database, shared by the worker (which + # migrates and owns it) and this API (requests, approvals). Empty = the + # /ops/actions and /ops/events endpoints answer 503. + self.orch_db = e.get("BITS_ORCH_DB", "") + self.orch_interval = float(e.get("BITS_ORCH_INTERVAL", "30") or "30") + def oidc_login_configured(self) -> bool: # Only issuer + client id + redirect are required. The authorize/token/jwks # endpoints are DISCOVERED from the issuer's well-known doc (the explicit diff --git a/console-backend/console_backend/main.py b/console-backend/console_backend/main.py index 8f4557aa..dd9f8e81 100644 --- a/console-backend/console_backend/main.py +++ b/console-backend/console_backend/main.py @@ -48,6 +48,9 @@ app = FastAPI(title="bits-console backend", version="0.1.0") +from .orch import api as orch_api # noqa: E402 (the orchestrator's routes) +app.include_router(orch_api.router) + _STATIC = os.path.join(os.path.dirname(__file__), "static") diff --git a/console-backend/console_backend/orch/__init__.py b/console-backend/console_backend/orch/__init__.py new file mode 100644 index 00000000..0f030ec5 --- /dev/null +++ b/console-backend/console_backend/orch/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The bits orchestrator: records builds, runs actions, signs per architecture, +starts schedules and collects failures. See the README, "Orchestrator".""" diff --git a/console-backend/console_backend/orch/actions.py b/console-backend/console_backend/orch/actions.py new file mode 100644 index 00000000..e405d704 --- /dev/null +++ b/console-backend/console_backend/orch/actions.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Actions: every change of state the orchestrator makes is one. + +An action is requested (by a person, CI, a schedule, the worker, or later the +advisor), approved, then run by the worker. Each kind has a JSON schema for its +parameters (the catalogue the console reads, and later the advisor's tool list), +the community it acts for, and the approval it needs: + +* "requester": an authorized requester is enough; the action is approved when + requested. +* "approver": someone authorized must approve it separately. + +An advisor can only propose: its actions start in "proposed" whatever the kind, +and only a person can approve them. The worker checks the approver's authority +again when it runs the action, not only when it was approved. + +States: proposed -> approved -> running -> done | failed; proposed -> rejected. +""" + +import json +import re +import uuid +from dataclasses import dataclass, field +from typing import Callable, Optional + +from . import db, events + +STATES = ("proposed", "approved", "running", "done", "failed", "rejected") +APPROVALS = ("requester", "approver") +MAX_ATTEMPTS = 5 +MAX_PARAMS = 16 * 1024 + + +class ActionError(Exception): + """A request that cannot be accepted (unknown kind, bad params, wrong state).""" + + +class Retry(Exception): + """Raised by a runner for a transient failure: the action is tried again + later, up to MAX_ATTEMPTS.""" + + +@dataclass(frozen=True) +class Kind: + name: str + summary: str + params: dict # JSON schema (subset, see validate) + approval: str # "requester" | "approver" + subject: Callable[[dict], str] # what it acts on, e.g. "pipeline:123" + scope: Callable[[dict, object], str] # its community; "" = overall admins only + run: Callable[[dict, object], Optional[dict]] + safe_to_rerun: bool = False # may run again after a crash mid-run + requesters: frozenset = field(default=frozenset(events.ACTOR_KINDS)) + + def __post_init__(self): + if self.approval not in APPROVALS: + raise ValueError("approval must be one of %s" % (APPROVALS,)) + + +REGISTRY = {} + + +def register(kind): + if kind.name in REGISTRY: + raise ValueError("action kind %r registered twice" % kind.name) + REGISTRY[kind.name] = kind + return kind + + +def kind(name): + k = REGISTRY.get(name) + if k is None: + raise ActionError("unknown action kind %r" % name) + return k + + +def catalogue(): + """Every kind with its parameter schema and the approval it needs.""" + return [{"kind": k.name, "summary": k.summary, "approval": k.approval, + "params": k.params} for k in sorted(REGISTRY.values(), key=lambda k: k.name)] + + +# --- parameters ------------------------------------------------------------- + +_TYPES = {"string": str, "integer": int, "number": (int, float), "boolean": bool, + "array": list, "object": dict, "null": type(None)} + + +def validate(schema, value, path="params"): + """Validate against the JSON-schema subset the catalogue uses: type, properties, + required, additionalProperties: false, enum, pattern, maxLength, minimum, + maximum, items, maxItems. Raises ActionError naming the first problem.""" + t = schema.get("type") + if t: + py = _TYPES[t] + if not isinstance(value, py) or (t in ("integer", "number") and isinstance(value, bool)): + raise ActionError("%s: expected %s" % (path, t)) + if "enum" in schema and not any(value == e and type(value) is type(e) + for e in schema["enum"]): + raise ActionError("%s: must be one of %s" % (path, schema["enum"])) + if isinstance(value, str): + if len(value) > schema.get("maxLength", 1024): + raise ActionError("%s: too long" % path) + if "pattern" in schema and not re.fullmatch(schema["pattern"], value): + raise ActionError("%s: does not match %s" % (path, schema["pattern"])) + if isinstance(value, (int, float)) and not isinstance(value, bool): + if "minimum" in schema and value < schema["minimum"]: + raise ActionError("%s: below %s" % (path, schema["minimum"])) + if "maximum" in schema and value > schema["maximum"]: + raise ActionError("%s: above %s" % (path, schema["maximum"])) + if isinstance(value, list): + if len(value) > schema.get("maxItems", 256): + raise ActionError("%s: too many items" % path) + for i, item in enumerate(value): + validate(schema.get("items", {}), item, "%s[%d]" % (path, i)) + if isinstance(value, dict): + props = schema.get("properties", {}) + for name in schema.get("required", []): + if name not in value: + raise ActionError("%s: %s is required" % (path, name)) + for name, v in value.items(): + if name not in props: + if schema.get("additionalProperties", True) is False: + raise ActionError("%s: unexpected %s" % (path, name)) + continue + validate(props[name], v, "%s.%s" % (path, name)) + + +def check_params(k, params): + if not isinstance(params, dict): + raise ActionError("params must be an object") + if len(json.dumps(params)) > MAX_PARAMS: + raise ActionError("params over %d bytes" % MAX_PARAMS) + validate(k.params, params) + + +# --- rows ------------------------------------------------------------------- + +def _row(r): + if r is None: + return None + d = dict(r) + d["params"] = json.loads(d["params"]) + d["result"] = json.loads(d["result"]) if d["result"] else None + return d + + +def get(conn, action_id): + return _row(conn.execute("SELECT * FROM actions WHERE id = ?", (int(action_id),)).fetchone()) + + +def listing(conn, state=None, communities=None, limit=100): + """Newest first. *communities*: only those (lowercase names); None = all.""" + q, args = "SELECT * FROM actions WHERE 1=1", [] + if state: + q += " AND state = ?" + args.append(state) + if communities is not None: + names = sorted(communities) + q += " AND lower(community) IN (%s)" % ",".join("?" * len(names)) if names else " AND 0" + args.extend(names) + q += " ORDER BY id DESC LIMIT ?" + args.append(max(1, min(int(limit), 500))) + return [_row(r) for r in conn.execute(q, args)] + + +def _event(conn, what, row, actor, extra=None, cause=None): + payload = {"action": row["id"], "kind": row["kind"], "community": row["community"]} + payload.update(extra or {}) + events.append(conn, "action." + what, row["subject"], actor, payload, + cause=cause or "action:%d" % row["id"]) + + +# --- lifecycle -------------------------------------------------------------- + +def request(conn, kind_name, params, actor, community, authorized, + idempotency_key=None, cause=None): + """Record a requested action. *authorized* is the caller's verdict that + *actor* may act for *community* (the API and the worker decide it; this + module does no I/O). Returns (row, created). A request repeating an existing + (kind, requester, idempotency_key) returns that action unchanged; the same + key with other parameters is an error.""" + k = kind(kind_name) + check_params(k, params) + if actor.kind not in k.requesters: + raise ActionError("%s may not request %s" % (actor.kind, k.name)) + if actor.kind != "advisor" and not authorized: + raise ActionError("%s %s is not authorized for %s" + % (actor.kind, actor.id, community or "overall admin")) + key = idempotency_key or uuid.uuid4().hex + if len(key) > 200: + raise ActionError("idempotency key too long") + auto = actor.kind != "advisor" and k.approval == "requester" + now = db.now_ms() + with db.transaction(conn): + old = _row(conn.execute( + "SELECT * FROM actions WHERE kind = ? AND requested_by_kind = ? AND" + " requested_by_id = ? AND idempotency_key = ?", + (k.name, actor.kind, actor.id, key)).fetchone()) + if old: + if old["params"] != params or old["community"] != community: + raise ActionError("idempotency key %r was used for another %s" % (key, k.name)) + return old, False + cur = conn.execute( + "INSERT INTO actions (kind, params, idempotency_key, subject, community, state," + " requested_by_kind, requested_by_id, approved_by_kind, approved_by_id, cause," + " created_ms, updated_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (k.name, json.dumps(params, sort_keys=True), key, k.subject(params), community, + "approved" if auto else "proposed", actor.kind, actor.id, + actor.kind if auto else None, actor.id if auto else None, cause, now, now)) + row = _row(conn.execute("SELECT * FROM actions WHERE id = ?", + (cur.lastrowid,)).fetchone()) + _event(conn, "requested", row, actor, {"params": params}, cause=cause) + if auto: + _event(conn, "approved", row, actor, {"by": "requester"}) + return row, True + + +def _decide(conn, action_id, actor, authorized, to, reason=None): + if actor.kind != "person": + raise ActionError("only a person can approve or reject an action") + with db.transaction(conn): + row = get(conn, action_id) + if row is None: + raise ActionError("no action %s" % action_id) + if row["state"] != "proposed": + raise ActionError("action %s is %s, not proposed" % (action_id, row["state"])) + if not authorized: + raise ActionError("%s is not authorized for %s" + % (actor.id, row["community"] or "overall admin")) + k = REGISTRY.get(row["kind"]) + if (to == "approved" and k is not None and k.approval == "approver" + and (row["requested_by_kind"], row["requested_by_id"]) == (actor.kind, actor.id)): + raise ActionError("%s needs an approver other than its requester" % row["kind"]) + if to == "approved": + conn.execute("UPDATE actions SET state = 'approved', approved_by_kind = ?," + " approved_by_id = ?, updated_ms = ? WHERE id = ?", + (actor.kind, actor.id, db.now_ms(), row["id"])) + _event(conn, "approved", row, actor) + else: + conn.execute("UPDATE actions SET state = 'rejected', error = ?, updated_ms = ?" + " WHERE id = ?", (reason, db.now_ms(), row["id"])) + _event(conn, "rejected", row, actor, {"reason": reason}) + return get(conn, action_id) + + +def approve(conn, action_id, actor, authorized): + return _decide(conn, action_id, actor, authorized, "approved") + + +def reject(conn, action_id, actor, authorized, reason=""): + return _decide(conn, action_id, actor, authorized, "rejected", (reason or "")[:500]) + + +def claim_next(conn): + """Take the oldest approved action that is due, whose subject has nothing + running and no older approved action waiting (one backing off, say), and mark + it running: actions on one subject run in the order they were approved for. + Returns it, or None.""" + now = db.now_ms() + with db.transaction(conn): + r = conn.execute( + "SELECT * FROM actions a WHERE state = 'approved' AND not_before_ms <= ?" + " AND NOT EXISTS (SELECT 1 FROM actions b WHERE b.subject = a.subject" + " AND b.state = 'running') AND NOT EXISTS (SELECT 1 FROM actions c" + " WHERE c.subject = a.subject AND c.state = 'approved' AND c.id < a.id)" + " ORDER BY id LIMIT 1", (now,)).fetchone() + if r is None: + return None + conn.execute("UPDATE actions SET state = 'running', attempts = attempts + 1," + " updated_ms = ? WHERE id = ?", (now, r["id"])) + row = get(conn, r["id"]) + _event(conn, "started", row, events.WORKER, {"attempt": row["attempts"]}) + return row + + +def finish(conn, row, result=None, error=None, retry=False): + """Record how a run ended: done, failed, or (retry) approved again after a + backoff, until MAX_ATTEMPTS.""" + now = db.now_ms() + with db.transaction(conn): + if error is None: + conn.execute("UPDATE actions SET state = 'done', result = ?, error = NULL," + " updated_ms = ? WHERE id = ?", + (json.dumps(result) if result is not None else None, now, row["id"])) + _event(conn, "done", row, events.WORKER, {"result": result}) + elif retry and row["attempts"] < MAX_ATTEMPTS: + delay = 30000 * 2 ** (row["attempts"] - 1) + conn.execute("UPDATE actions SET state = 'approved', error = ?, not_before_ms = ?," + " updated_ms = ? WHERE id = ?", + (error[:1000], now + delay, now, row["id"])) + _event(conn, "retrying", row, events.WORKER, + {"error": error[:1000], "attempt": row["attempts"], "in_ms": delay}) + else: + conn.execute("UPDATE actions SET state = 'failed', error = ?, updated_ms = ?" + " WHERE id = ?", (error[:1000], now, row["id"])) + _event(conn, "failed", row, events.WORKER, + {"error": error[:1000], "attempt": row["attempts"]}) + + +def recover(conn): + """At worker start: an action left running was interrupted. Run it again if + its kind allows and it has attempts left, else fail it so a person decides + (an action that crashes the worker must not run on every restart).""" + rows = [_row(r) for r in conn.execute("SELECT * FROM actions WHERE state = 'running'")] + for row in rows: + k = REGISTRY.get(row["kind"]) + with db.transaction(conn): + if k is not None and k.safe_to_rerun and row["attempts"] < MAX_ATTEMPTS: + conn.execute("UPDATE actions SET state = 'approved', updated_ms = ?" + " WHERE id = ?", (db.now_ms(), row["id"])) + _event(conn, "interrupted", row, events.WORKER, {"then": "rerun"}) + else: + conn.execute("UPDATE actions SET state = 'failed', error = ?, updated_ms = ?" + " WHERE id = ?", ("interrupted while running", db.now_ms(), + row["id"])) + _event(conn, "interrupted", row, events.WORKER, {"then": "failed"}) + return len(rows) + + +def run_due(conn, deps, authorize, budget=20): + """Run up to *budget* due actions. *authorize(actor, community)* re-checks the + approver's authority now; a lapsed authority fails the action.""" + n = 0 + while n < budget: + row = claim_next(conn) + if row is None: + break + n += 1 + k = REGISTRY.get(row["kind"]) + if k is None: + finish(conn, row, error="unknown action kind %s" % row["kind"]) + continue + try: + approver = events.Actor(row["approved_by_kind"], row["approved_by_id"]) + except (TypeError, ValueError): + finish(conn, row, error="action has no valid approver") + continue + try: + ok = authorize(approver, row["community"]) + except Exception as exc: # policy unreadable: try again later + finish(conn, row, error="authority check failed: %s" % exc, retry=True) + continue + if not ok: + finish(conn, row, error="%s is no longer authorized for %s" + % (approver.id, row["community"] or "overall admin")) + continue + try: + result = k.run(row["params"], deps) + except Retry as exc: + finish(conn, row, error=str(exc) or "transient failure", retry=True) + except Exception as exc: + finish(conn, row, error="%s: %s" % (type(exc).__name__, exc)) + else: + finish(conn, row, result=result) + return n diff --git a/console-backend/console_backend/orch/api.py b/console-backend/console_backend/orch/api.py new file mode 100644 index 00000000..0ce4b119 --- /dev/null +++ b/console-backend/console_backend/orch/api.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's API, mounted on the console backend. + + GET /ops/actions/catalogue every action kind, its schema, its approval + POST /ops/actions request an action {kind, params, idempotency_key?} + GET /ops/actions actions of the communities the caller administers + GET /ops/actions/{id} + POST /ops/actions/{id}/approve + POST /ops/actions/{id}/reject {reason?} + GET /ops/events?after=&limit=&wait= the event log (overall admins), long-polled + +The API writes only requests and decisions; the worker runs actions. Identity +and the admin policy are the backend's own (`main._current`, `_resolved_policy`). +""" + +import asyncio +import contextlib +import json + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse +from starlette.concurrency import run_in_threadpool + +from .. import audit, authz +from . import actions, db, events, kinds + +router = APIRouter() +_MAX_BODY = 64 * 1024 + + +def _m(): + from .. import main # imported late: main mounts this router + return main + + +@contextlib.contextmanager +def _open(): + path = _m().settings.orch_db + if not path: + raise HTTPException(503, "orchestrator not configured (BITS_ORCH_DB)") + try: + conn = db.open_checked(path) + except db.SchemaError as e: + raise HTTPException(503, str(e)) + try: + yield conn + finally: + conn.close() + + +async def _caller(request): + data = await _m()._current_async(request) + if not data: + raise HTTPException(401, "not authenticated") + return data + + +def _policy(data): + return _m()._resolved_policy(data["token"]) + + +def _may_act(data, community): + resolved = _policy(data) + if community: + return authz.is_admin_for(data["user"], community, resolved) + return authz.admin_groups(data["user"], resolved)[0] + + +async def _body(request): + raw = await _m()._read_capped(request, _MAX_BODY) + try: + payload = json.loads(raw or b"{}") + except (ValueError, RecursionError): + raise HTTPException(400, "expected a JSON object") + if not isinstance(payload, dict): + raise HTTPException(400, "expected a JSON object") + return payload + + +def _public(row): + keep = ("id", "kind", "params", "subject", "community", "state", "requested_by_kind", + "requested_by_id", "approved_by_kind", "approved_by_id", "attempts", + "result", "error", "created_ms", "updated_ms") + return {k: row[k] for k in keep} + + +@router.get("/ops/actions/catalogue") +async def catalogue(request: Request): + await _caller(request) + return {"actions": actions.catalogue()} + + +@router.post("/ops/actions") +async def request_action(request: Request): + data = await _caller(request) + payload = await _body(request) + name, params = payload.get("kind"), payload.get("params") + key = payload.get("idempotency_key") + if not isinstance(name, str) or (key is not None and not isinstance(key, str)): + raise HTTPException(400, "expected {kind, params, idempotency_key?}") + try: + k = actions.kind(name) + actions.check_params(k, params) + community = await run_in_threadpool(k.scope, params, kinds.Deps(_m().settings)) + except actions.ActionError as e: + raise HTTPException(400, str(e)) + except kinds.NotConfigured as e: + raise HTTPException(503, str(e)) + except Exception as e: # e.g. the forge could not be read + raise HTTPException(502, "cannot resolve the action's community: %s" % e) + allowed = await run_in_threadpool(_may_act, data, community) + if not allowed: + audit.record("ops_denied", op="action", kind=name, user=data["user"], + community=community or "*", principal="human") + raise HTTPException(403, "%s is not an admin for %s" + % (data["user"], community or "every community")) + + def write(): + with _open() as conn: + return actions.request(conn, name, params, events.Actor("person", data["user"]), + community, True, idempotency_key=key) + try: + row, created = await run_in_threadpool(write) + except actions.ActionError as e: + raise HTTPException(400, str(e)) + audit.record("ops_action", kind=name, action=row["id"], user=data["user"], + community=community or "*", principal="human") + return JSONResponse(_public(row), status_code=201 if created else 200) + + +@router.get("/ops/actions") +async def list_actions(request: Request, state: str = "", limit: int = 100): + data = await _caller(request) + if state and state not in actions.STATES: + raise HTTPException(400, "unknown state") + overall, groups = await run_in_threadpool(lambda: authz.admin_groups(data["user"], _policy(data))) + + mine = None if overall else {g.lower() for g in groups} + + def read(): + with _open() as conn: + return [_public(r) for r in actions.listing( + conn, state=state or None, communities=mine, limit=limit)] + return {"actions": await run_in_threadpool(read)} + + +async def _visible(data, action_id): + def read(): + with _open() as conn: + return actions.get(conn, action_id) + row = await run_in_threadpool(read) + if row is None or not await run_in_threadpool(_may_act, data, row["community"]): + raise HTTPException(404, "no such action") + return row + + +@router.get("/ops/actions/{action_id}") +async def get_action(action_id: int, request: Request): + data = await _caller(request) + return _public(await _visible(data, action_id)) + + +async def _decide(action_id, request, approve): + data = await _caller(request) + payload = await _body(request) + row = await _visible(data, action_id) # also the authority check + actor = events.Actor("person", data["user"]) + + def write(): + with _open() as conn: + if approve: + return actions.approve(conn, row["id"], actor, True) + return actions.reject(conn, row["id"], actor, True, + str(payload.get("reason") or "")) + try: + out = await run_in_threadpool(write) + except actions.ActionError as e: + raise HTTPException(409, str(e)) + audit.record("ops_action_" + ("approve" if approve else "reject"), action=row["id"], + kind=row["kind"], user=data["user"], community=row["community"] or "*", + principal="human") + return _public(out) + + +@router.post("/ops/actions/{action_id}/approve") +async def approve_action(action_id: int, request: Request): + return await _decide(action_id, request, True) + + +@router.post("/ops/actions/{action_id}/reject") +async def reject_action(action_id: int, request: Request): + return await _decide(action_id, request, False) + + +@router.get("/ops/events") +async def list_events(request: Request, after: int = 0, limit: int = 200, wait: int = 0): + """Events after *after*. With wait (seconds, at most 25) and nothing new yet, + answer as soon as an event arrives or the wait ends. Overall admins only until + events carry their community (M1).""" + data = await _caller(request) + if not await run_in_threadpool(lambda: authz.admin_groups(data["user"], _policy(data))[0]): + raise HTTPException(403, "%s is not an overall admin" % data["user"]) + + def read(): + with _open() as conn: + return events.since(conn, after, limit) + deadline = asyncio.get_running_loop().time() + max(0, min(int(wait), 25)) + while True: + out = await run_in_threadpool(read) + if out or asyncio.get_running_loop().time() >= deadline: + return JSONResponse({"events": out}, headers={"Cache-Control": "no-store"}) + await asyncio.sleep(1) diff --git a/console-backend/console_backend/orch/authority.py b/console-backend/console_backend/orch/authority.py new file mode 100644 index 00000000..ed78666e --- /dev/null +++ b/console-backend/console_backend/orch/authority.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Who may act for a community, for the API (at request and approval time) and +the worker (again, when it runs an action). Same admin policy as the rest of the +backend: an admin of the community, or an overall admin; an action with no +community is for overall admins only.""" + +from .. import authz + +try: + from bits_helpers import forge as _forge +except Exception: # pragma: no cover - bits_helpers is on the path in every image + _forge = None + + +def person_allowed(settings, user, community, token=None): + resolved = authz.resolved_admin_policy(settings, token) + if community: + return authz.is_admin_for(user, community, resolved) + overall, _ = authz.admin_groups(user, resolved) + return overall + + +def unresolvable(settings): + """Why the worker could not check authority, or "". It has no user token, so + an admin policy naming GitLab groups (&group lines, or BITS_ADMINS_GROUP) + needs BITS_ADMIN_RESOLVE_TOKEN; without it every action would fail at run + time as "no longer authorized".""" + if settings.admin_resolve_token: + return "" + if settings.admins_group: + return "BITS_ADMINS_GROUP is set but BITS_ADMIN_RESOLVE_TOKEN is not" + if _forge is not None and _forge.admin_policy_grouprefs(authz.load_policy(settings)): + return "the admin policy names GitLab groups but BITS_ADMIN_RESOLVE_TOKEN is not set" + return "" + + +def checker(settings): + """authorize(actor, community) for the worker, which holds no user token: the + policy is resolved with BITS_ADMIN_RESOLVE_TOKEN. Only a person's authority is + checked against the policy. The worker's own actions are decided by its code. + Schedules and CI get their approval rules (standing approvals, CI identity) in + later milestones; until then they are not authorized.""" + def authorize(actor, community): + if actor.kind == "person": + return person_allowed(settings, actor.id, community) + return actor.kind == "worker" + return authorize diff --git a/console-backend/console_backend/orch/db.py b/console-backend/console_backend/orch/db.py new file mode 100644 index 00000000..04659bc4 --- /dev/null +++ b/console-backend/console_backend/orch/db.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's database: one SQLite file in WAL mode, shared by the worker +(which owns it and applies migrations) and the API (which writes only requests, +approvals and corrections). + +Schema changes are numbered migrations (each a list of statements), applied in order by the worker at start +and recorded in PRAGMA user_version. The API refuses a schema newer than the one +it knows, so an old API never writes rows a newer schema reads differently. +Each later milestone adds its tables as a new migration. +""" + +import contextlib +import fcntl +import os +import sqlite3 +import time +from urllib.parse import quote + +MIGRATIONS = [ + # 1 — the event log (also the audit chain) and the actions every change of + # state goes through. + [ + """CREATE TABLE events ( + seq INTEGER PRIMARY KEY, + ts_ms INTEGER NOT NULL, + kind TEXT NOT NULL, + subject TEXT NOT NULL, + actor_kind TEXT NOT NULL, + actor_id TEXT NOT NULL, + cause TEXT, + payload TEXT NOT NULL, + prev_hash TEXT NOT NULL, + hash TEXT NOT NULL + )""", + "CREATE INDEX events_subject ON events(subject, seq)", + """CREATE TABLE actions ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + params TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + subject TEXT NOT NULL, + community TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN + ('proposed','approved','running','done','failed','rejected')), + requested_by_kind TEXT NOT NULL, + requested_by_id TEXT NOT NULL, + approved_by_kind TEXT, + approved_by_id TEXT, + approval_id INTEGER, + cause TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + not_before_ms INTEGER NOT NULL DEFAULT 0, + result TEXT, + error TEXT, + created_ms INTEGER NOT NULL, + updated_ms INTEGER NOT NULL, + UNIQUE (kind, requested_by_kind, requested_by_id, idempotency_key) + )""", + "CREATE INDEX actions_state ON actions(state, id)", + "CREATE INDEX actions_subject ON actions(subject, state, id)", + ], +] + +SCHEMA_VERSION = len(MIGRATIONS) + + +class SchemaError(Exception): + pass + + +class LockedError(Exception): + """Another worker holds the database lock.""" + + +def now_ms(): + return int(time.time() * 1000) + + +def connect(path, create=True): + """A connection with the settings both processes rely on. Autocommit mode: + writes happen only inside `transaction()`. With create=False a missing file + is an error instead of a new empty database.""" + if create: + conn = sqlite3.connect(path, timeout=10, isolation_level=None) + else: + conn = sqlite3.connect("file:%s?mode=rw" % quote(path), uri=True, + timeout=10, isolation_level=None) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA busy_timeout=10000") + return conn + + +@contextlib.contextmanager +def transaction(conn): + """BEGIN IMMEDIATE: take the write lock up front, so two writers serialise + instead of one failing mid-transaction, and the event chain reads a stable + last hash.""" + conn.execute("BEGIN IMMEDIATE") + try: + yield conn + conn.execute("COMMIT") + except BaseException: + # SQLite may already have rolled back (disk full, I/O error); never let + # a second error hide the first, nor leave the write lock held. + if conn.in_transaction: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise + + +def version(conn): + return conn.execute("PRAGMA user_version").fetchone()[0] + + +def migrate(conn): + """Apply every migration newer than the database. Worker only.""" + have = version(conn) + if have > SCHEMA_VERSION: + raise SchemaError("database schema %d is newer than this code (%d)" + % (have, SCHEMA_VERSION)) + for n in range(have, SCHEMA_VERSION): + with transaction(conn): + for stmt in MIGRATIONS[n]: + conn.execute(stmt) + conn.execute("PRAGMA user_version=%d" % (n + 1)) + return SCHEMA_VERSION + + +def check(conn): + """For the API: the schema must be one this code knows, and present.""" + have = version(conn) + if have > SCHEMA_VERSION: + raise SchemaError("database schema %d is newer than this code (%d)" + % (have, SCHEMA_VERSION)) + if have < SCHEMA_VERSION: + raise SchemaError("database schema %d is older than this code (%d): the " + "worker has not migrated it yet" % (have, SCHEMA_VERSION)) + + +def open_checked(path): + """For the API: an existing database the worker has migrated.""" + try: + conn = connect(path, create=False) + except sqlite3.OperationalError as e: + raise SchemaError("cannot open %s (%s): has the worker started?" % (path, e)) + try: + check(conn) + except Exception: + conn.close() + raise + return conn + + +@contextlib.contextmanager +def worker_lock(path): + """Exclusive, non-blocking lock beside the database: a second worker exits + instead of running the same steps twice. Released when the process dies.""" + fd = os.open(path + ".lock", os.O_RDWR | os.O_CREAT, 0o600) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise LockedError("another worker holds %s.lock" % path) + yield + finally: + os.close(fd) diff --git a/console-backend/console_backend/orch/events.py b/console-backend/console_backend/orch/events.py new file mode 100644 index 00000000..84107f88 --- /dev/null +++ b/console-backend/console_backend/orch/events.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The append-only event log: everything that happened, who did it and why. + +It is also the audit chain. Each event's hash covers its own fields and the +previous event's hash, so editing or removing any event breaks every hash after +it; `verify` finds the first break. The outside copy (M4) is a second place the +chain is kept, so a rewrite on this host is detectable. + +Events are appended inside the caller's transaction, so a change of state and +its event are written together or not at all. +""" + +import hashlib +import json +from dataclasses import dataclass + +from . import db + +ACTOR_KINDS = ("person", "ci", "schedule", "worker", "advisor") +GENESIS = "0" * 64 +MAX_PAYLOAD = 64 * 1024 + + +@dataclass(frozen=True) +class Actor: + kind: str + id: str + + def __post_init__(self): + if self.kind not in ACTOR_KINDS: + raise ValueError("unknown actor kind %r" % self.kind) + if not isinstance(self.id, str) or not self.id or len(self.id) > 128: + raise ValueError("actor id must be 1-128 characters") + + +WORKER = Actor("worker", "orchestrator") + + +def _canonical(obj): + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _digest(prev_hash, row): + body = _canonical([row["seq"], row["ts_ms"], row["kind"], row["subject"], + row["actor_kind"], row["actor_id"], row["cause"], + row["payload"]]) + return hashlib.sha256((prev_hash + body).encode()).hexdigest() + + +def append(conn, kind, subject, actor, payload=None, cause=None): + """Append one event; call inside `db.transaction`. Returns its seq.""" + if not conn.in_transaction: + raise RuntimeError("events.append must run inside a transaction") + text = _canonical(payload or {}) + if len(text) > MAX_PAYLOAD: + raise ValueError("event payload over %d bytes" % MAX_PAYLOAD) + last = conn.execute("SELECT seq, hash FROM events ORDER BY seq DESC LIMIT 1").fetchone() + row = {"seq": (last["seq"] + 1) if last else 1, "ts_ms": db.now_ms(), + "kind": kind, "subject": subject, "actor_kind": actor.kind, + "actor_id": actor.id, "cause": cause, "payload": text} + prev = last["hash"] if last else GENESIS + row["prev_hash"] = prev + row["hash"] = _digest(prev, row) + conn.execute("INSERT INTO events (seq, ts_ms, kind, subject, actor_kind, actor_id," + " cause, payload, prev_hash, hash) VALUES (:seq, :ts_ms, :kind," + " :subject, :actor_kind, :actor_id, :cause, :payload, :prev_hash, :hash)", + row) + return row["seq"] + + +def _public(r): + return {"seq": r["seq"], "ts_ms": r["ts_ms"], "kind": r["kind"], + "subject": r["subject"], "actor": {"kind": r["actor_kind"], "id": r["actor_id"]}, + "cause": r["cause"], "payload": json.loads(r["payload"]), "hash": r["hash"]} + + +def since(conn, after=0, limit=200, subject=None): + """Events after *after*, oldest first.""" + limit = max(1, min(int(limit), 1000)) + if subject: + rows = conn.execute("SELECT * FROM events WHERE seq > ? AND subject = ?" + " ORDER BY seq LIMIT ?", (int(after), subject, limit)) + else: + rows = conn.execute("SELECT * FROM events WHERE seq > ? ORDER BY seq LIMIT ?", + (int(after), limit)) + return [_public(r) for r in rows] + + +def verify(conn): + """Walk the chain. Returns (count, None) when intact, or (count, seq) with the + first event whose link or hash does not hold.""" + prev, expect, n = GENESIS, 1, 0 + for r in conn.execute("SELECT * FROM events ORDER BY seq"): + n += 1 + if r["seq"] != expect or r["prev_hash"] != prev or _digest(prev, r) != r["hash"]: + return n, r["seq"] + prev, expect = r["hash"], expect + 1 + return n, None diff --git a/console-backend/console_backend/orch/kinds.py b/console-backend/console_backend/orch/kinds.py new file mode 100644 index 00000000..ca25ebfa --- /dev/null +++ b/console-backend/console_backend/orch/kinds.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The action kinds. M0 has the two pipeline operations the console already +offers, so the whole path (request, approval, run, events) is exercised before +certification depends on it; later milestones add certify, release, renew, +revoke, start-build, schedule changes and signature proposals here.""" + +from .. import forge_ops +from . import actions + +_PIPELINE = { + "type": "object", + "properties": {"pipeline": {"type": "string", "pattern": "^(0|[1-9][0-9]{0,19})$", + "description": "GitLab pipeline id in the build project"}}, + "required": ["pipeline"], + "additionalProperties": False, +} + + +class NotConfigured(actions.Retry): + """A service an action needs is not configured here: 503 to a caller, and a + transient failure to the worker (the deployment is fixed, the action runs).""" + + +class Deps: + """What runners and scope functions may use. Built per call from the + settings, so a test or a settings change takes effect immediately.""" + + def __init__(self, settings): + self.settings = settings + + def forge(self): + f = forge_ops.GitLabForge.from_settings(self.settings) + if f is None: + raise NotConfigured("forge not configured (BITS_FORGE_OPS_TOKEN/PROJECT)") + return f + + +def _transient(e): + return e.status == 0 or e.status == 429 or e.status >= 500 + + +def _pipeline_scope(params, deps): + """A pipeline's community is its own COMMUNITY variable; none means + overall admins only (as the console's pipeline operations decide today).""" + return deps.forge().pipeline_community(params["pipeline"]) or "" + + +def _forge_call(fn, pid): + try: + fn(pid) + except forge_ops.ForgeError as e: + if _transient(e): + raise actions.Retry("forge %s: %s" % (e.status, e.message)) + raise RuntimeError("forge %s: %s" % (e.status, e.message)) + + +def _retry(params, deps): + _forge_call(deps.forge().retry_pipeline, params["pipeline"]) + return {"status": "retried"} + + +def _cancel(params, deps): + _forge_call(deps.forge().cancel_pipeline, params["pipeline"]) + return {"status": "canceled"} + + +actions.register(actions.Kind( + name="pipeline.retry", summary="Retry the failed jobs of a pipeline", + params=_PIPELINE, approval="requester", + subject=lambda p: "pipeline:%s" % p["pipeline"], scope=_pipeline_scope, run=_retry, + safe_to_rerun=True)) + +actions.register(actions.Kind( + name="pipeline.cancel", summary="Cancel a running pipeline", + params=_PIPELINE, approval="requester", + subject=lambda p: "pipeline:%s" % p["pipeline"], scope=_pipeline_scope, run=_cancel, + safe_to_rerun=True)) diff --git a/console-backend/console_backend/orch/worker.py b/console-backend/console_backend/orch/worker.py new file mode 100644 index 00000000..c6477180 --- /dev/null +++ b/console-backend/console_backend/orch/worker.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator worker: one process, one loop. + +Each pass runs the steps in order. A step reads the database and the outside +world, then writes what it decided; a step that fails is logged, recorded as an +event when its error changes, and tried again on the next pass. M0 has one step, +running due actions; reconcile, builds, signing and schedules are added as steps +by later milestones. + + python -m console_backend.orch.worker # BITS_ORCH_DB=/orch/orch.db +""" + +import logging +import os +import signal +import threading +import time + +from .. import config +from . import actions, authority, db, events, kinds + +log = logging.getLogger("bits_orchestrator") + + +class Worker: + def __init__(self, conn, settings, interval=30.0, heartbeat=None): + self.conn = conn + self.settings = settings + self.interval = interval + self.heartbeat = heartbeat + self.deps = kinds.Deps(settings) + self.authorize = authority.checker(settings) + self.steps = [("actions", self._run_actions)] + self._errors = {} # step -> last error text + self._stop = threading.Event() + + def _run_actions(self): + actions.run_due(self.conn, self.deps, self.authorize) + + def _note(self, step, error): + """Record a step's failure or recovery as an event, but only when it + changes, so a step failing every pass does not flood the log.""" + if self._errors.get(step) == error: + return + self._errors[step] = error + try: + with db.transaction(self.conn): + events.append(self.conn, "worker.step_failed" if error else "worker.step_recovered", + "worker", events.WORKER, {"step": step, "error": error}) + except Exception: + log.exception("cannot record step state for %s", step) + + def run_once(self): + for name, step in self.steps: + try: + step() + except Exception as exc: + log.exception("step %s failed", name) + self._note(name, "%s: %s" % (type(exc).__name__, exc)) + else: + if self._errors.get(name): + self._note(name, None) + if self.heartbeat: + with open(self.heartbeat, "w") as fh: + fh.write(str(int(time.time()))) + + def start(self): + with db.transaction(self.conn): + events.append(self.conn, "worker.started", "worker", events.WORKER, + {"schema": db.SCHEMA_VERSION}) + n = actions.recover(self.conn) + if n: + log.warning("recovered %d interrupted action(s)", n) + + def loop(self): + self.start() + while not self._stop.is_set(): + began = time.monotonic() + self.run_once() + self._stop.wait(max(1.0, self.interval - (time.monotonic() - began))) + with db.transaction(self.conn): + events.append(self.conn, "worker.stopped", "worker", events.WORKER, {}) + + def stop(self, *_): + self._stop.set() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + settings = config.Settings() + if not settings.orch_db: + raise SystemExit("BITS_ORCH_DB is not set") + problem = authority.unresolvable(settings) + if problem: + raise SystemExit(problem) + path = settings.orch_db + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with db.worker_lock(path): + conn = db.connect(path) + db.migrate(conn) + w = Worker(conn, settings, interval=settings.orch_interval, + heartbeat=path + ".heartbeat") + signal.signal(signal.SIGTERM, w.stop) + signal.signal(signal.SIGINT, w.stop) + log.info("orchestrator worker on %s (schema %d, every %ss)", + path, db.SCHEMA_VERSION, settings.orch_interval) + w.loop() + + +if __name__ == "__main__": + main() diff --git a/console-backend/pyproject.toml b/console-backend/pyproject.toml index ef5ed089..fa109d49 100644 --- a/console-backend/pyproject.toml +++ b/console-backend/pyproject.toml @@ -19,4 +19,4 @@ dependencies = [ # alongside in the image), not a PyPI dependency. [tool.setuptools] -packages = ["console_backend"] +packages = ["console_backend", "console_backend.orch"] diff --git a/console-backend/tests/test_orch.py b/console-backend/tests/test_orch.py new file mode 100644 index 00000000..54fba9d3 --- /dev/null +++ b/console-backend/tests/test_orch.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M0: database, event chain, actions, worker, API.""" + +import multiprocessing +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, forge_ops, main +from console_backend.orch import actions, db, events, kinds, worker + +ALICE = events.Actor("person", "alice") +ADVISOR = events.Actor("advisor", "triage") + + +class FakeForge: + """Pipelines 1-9 belong to 'testbed', 10+ to no community.""" + + def __init__(self, fail=None): + self.calls = [] + self.fail = fail or {} + + def pipeline_community(self, pid): + return "testbed" if int(pid) < 10 else "" + + def _do(self, what, pid): + self.calls.append((what, pid)) + if what in self.fail: + raise self.fail[what] + + def retry_pipeline(self, pid): + self._do("retry", pid) + + def cancel_pipeline(self, pid): + self._do("cancel", pid) + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + +class DatabaseTest(_DB): + def test_migrate_is_idempotent_and_versioned(self): + self.assertEqual(db.migrate(self.conn), db.SCHEMA_VERSION) + self.assertEqual(db.version(self.conn), db.SCHEMA_VERSION) + db.check(self.conn) + self.assertEqual(self.conn.execute("PRAGMA journal_mode").fetchone()[0], "wal") + + def test_newer_or_unmigrated_schema_is_refused(self): + self.conn.execute("PRAGMA user_version=%d" % (db.SCHEMA_VERSION + 1)) + with self.assertRaises(db.SchemaError): + db.check(self.conn) + with self.assertRaises(db.SchemaError): + db.migrate(self.conn) + fresh = db.connect(os.path.join(self.dir, "new.db")) + self.addCleanup(fresh.close) + with self.assertRaises(db.SchemaError): + db.check(fresh) + + def test_a_failed_transaction_leaves_nothing(self): + with self.assertRaises(ZeroDivisionError): + with db.transaction(self.conn): + events.append(self.conn, "x", "s", ALICE) + 1 / 0 + self.assertEqual(events.since(self.conn), []) + + +def _hold_lock(path, ready, release): + with db.worker_lock(path): + ready.set() + release.wait(10) + + +class LockTest(_DB): + def test_a_second_worker_is_refused(self): + ready, release = multiprocessing.Event(), multiprocessing.Event() + p = multiprocessing.Process(target=_hold_lock, args=(self.path, ready, release)) + p.start() + try: + self.assertTrue(ready.wait(10)) + with self.assertRaises(db.LockedError): + with db.worker_lock(self.path): + pass + finally: + release.set() + p.join(10) + with db.worker_lock(self.path): # free again once the holder exits + pass + + +class EventsTest(_DB): + def _add(self, n): + for i in range(n): + with db.transaction(self.conn): + events.append(self.conn, "test.e", "s%d" % (i % 2), ALICE, {"i": i}) + + def test_append_needs_a_transaction(self): + with self.assertRaises(RuntimeError): + events.append(self.conn, "x", "s", ALICE) + + def test_chain_verifies_and_since_pages(self): + self._add(5) + self.assertEqual(events.verify(self.conn), (5, None)) + got = events.since(self.conn, after=2, limit=2) + self.assertEqual([e["seq"] for e in got], [3, 4]) + self.assertEqual(got[0]["actor"], {"kind": "person", "id": "alice"}) + self.assertEqual([e["seq"] for e in events.since(self.conn, subject="s1")], [2, 4]) + + def test_an_edited_or_removed_event_breaks_the_chain(self): + self._add(5) + self.conn.execute("UPDATE events SET payload = '{\"i\":99}' WHERE seq = 3") + self.assertEqual(events.verify(self.conn), (3, 3)) + self.conn.execute("DELETE FROM events") + self._add(5) + self.conn.execute("DELETE FROM events WHERE seq = 2") + self.assertEqual(events.verify(self.conn)[1], 3) + + def test_actor_and_payload_limits(self): + with self.assertRaises(ValueError): + events.Actor("robot", "x") + with self.assertRaises(ValueError): + events.Actor("person", "") + with self.assertRaises(ValueError): + with db.transaction(self.conn): + events.append(self.conn, "x", "s", ALICE, {"big": "x" * events.MAX_PAYLOAD}) + + +class ValidateTest(unittest.TestCase): + def test_schema_subset(self): + s = {"type": "object", "required": ["a"], "additionalProperties": False, + "properties": {"a": {"type": "string", "pattern": "[0-9]+"}, + "b": {"type": "integer", "minimum": 1}, + "c": {"type": "array", "items": {"enum": ["x", "y"]}}}} + actions.validate(s, {"a": "12", "b": 3, "c": ["x"]}) + for bad in ({}, {"a": "1x"}, {"a": "1", "b": 0}, {"a": "1", "b": True}, + {"a": "1", "c": ["z"]}, {"a": "1", "d": 1}, {"a": "1" * 2000}): + with self.assertRaises(actions.ActionError, msg=bad): + actions.validate(s, bad) + + +class ActionsTest(_DB): + def setUp(self): + super().setUp() + self.forge = FakeForge() + self.deps = kinds.Deps(None) + self.deps.forge = lambda: self.forge + self.allow = lambda actor, community: actor.kind == "person" and actor.id == "alice" + + def _req(self, pid="5", actor=ALICE, kind="pipeline.retry", **kw): + return actions.request(self.conn, kind, {"pipeline": pid}, actor, "testbed", + kw.pop("authorized", True), **kw) + + def _kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + def test_catalogue_lists_kinds_with_schemas(self): + cat = {c["kind"]: c for c in actions.catalogue()} + self.assertIn("pipeline.retry", cat) + self.assertEqual(cat["pipeline.cancel"]["params"]["required"], ["pipeline"]) + + def test_an_authorized_person_is_approved_at_request(self): + row, created = self._req() + self.assertTrue(created) + self.assertEqual((row["state"], row["approved_by_id"], row["subject"]), + ("approved", "alice", "pipeline:5")) + self.assertEqual(self._kinds(), ["action.requested", "action.approved"]) + + def test_the_same_idempotency_key_returns_the_first_action(self): + a, _ = self._req(idempotency_key="k1") + b, created = self._req(idempotency_key="k1") + self.assertEqual((a["id"], created), (b["id"], False)) + c, _ = self._req() + self.assertNotEqual(c["id"], a["id"]) + with self.assertRaises(actions.ActionError): # same key, other params + self._req(pid="6", idempotency_key="k1") + # Keys belong to their requester: another person's "k1" is their own action. + d, created = self._req(actor=events.Actor("person", "carol"), idempotency_key="k1") + self.assertTrue(created) + self.assertNotEqual(d["id"], a["id"]) + + def test_bad_requests(self): + with self.assertRaises(actions.ActionError): + self._req(pid="12a") + with self.assertRaises(actions.ActionError): + self._req(pid="05") # one spelling per pipeline + with self.assertRaises(actions.ActionError): + self._req(kind="pipeline.explode") + with self.assertRaises(actions.ActionError): + self._req(authorized=False) + + def test_an_advisor_only_proposes_and_only_a_person_decides(self): + row, _ = self._req(actor=ADVISOR, authorized=False) + self.assertEqual(row["state"], "proposed") + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 0) + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, row["id"], ADVISOR, True) + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, row["id"], events.Actor("person", "mallory"), False) + ok = actions.approve(self.conn, row["id"], ALICE, True) + self.assertEqual((ok["state"], ok["approved_by_id"]), ("approved", "alice")) + with self.assertRaises(actions.ActionError): + actions.reject(self.conn, row["id"], ALICE, True) # no longer proposed + other, _ = self._req(actor=ADVISOR, authorized=False, pid="6") + rej = actions.reject(self.conn, other["id"], ALICE, True, "not now") + self.assertEqual((rej["state"], rej["error"]), ("rejected", "not now")) + + def test_run_due_runs_and_records(self): + self._req() + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 1) + self.assertEqual(self.forge.calls, [("retry", "5")]) + row = actions.listing(self.conn)[0] + self.assertEqual((row["state"], row["result"], row["attempts"]), + ("done", {"status": "retried"}, 1)) + self.assertEqual(self._kinds()[-2:], ["action.started", "action.done"]) + self.assertEqual(events.verify(self.conn)[1], None) + + def test_transient_errors_back_off_then_fail(self): + self.forge.fail["retry"] = forge_ops.ForgeError(503, "busy") + row, _ = self._req() + for attempt in range(1, actions.MAX_ATTEMPTS + 1): + self.conn.execute("UPDATE actions SET not_before_ms = 0") + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 1) + state = actions.get(self.conn, row["id"]) + self.assertEqual(state["attempts"], attempt) + self.assertEqual(state["state"], "failed") + self.assertIn("busy", state["error"]) + self.assertEqual(self._kinds().count("action.retrying"), actions.MAX_ATTEMPTS - 1) + + def test_backoff_delays_the_next_attempt(self): + self.forge.fail["retry"] = forge_ops.ForgeError(0, "timeout") + self._req() + actions.run_due(self.conn, self.deps, self.allow) + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 0) + + def test_permanent_errors_fail_at_once(self): + self.forge.fail["cancel"] = forge_ops.ForgeError(404, "no pipeline") + row, _ = self._req(kind="pipeline.cancel") + actions.run_due(self.conn, self.deps, self.allow) + got = actions.get(self.conn, row["id"]) + self.assertEqual((got["state"], got["attempts"]), ("failed", 1)) + + def test_a_lapsed_authority_fails_the_action_unrun(self): + self._req() + actions.run_due(self.conn, self.deps, lambda a, c: False) + self.assertEqual(self.forge.calls, []) + self.assertIn("no longer authorized", actions.listing(self.conn)[0]["error"]) + + def test_an_unreadable_policy_retries(self): + self._req() + + def broken(actor, community): + raise OSError("gitlab down") + actions.run_due(self.conn, self.deps, broken) + row = actions.listing(self.conn)[0] + self.assertEqual(row["state"], "approved") + self.assertIn("gitlab down", row["error"]) + + def test_a_later_action_waits_for_an_earlier_one_backing_off(self): + self.forge.fail["retry"] = forge_ops.ForgeError(503, "busy") + self._req() + self._req(kind="pipeline.cancel") + actions.run_due(self.conn, self.deps, self.allow) + self.assertEqual(self.forge.calls, [("retry", "5")]) # the cancel waits + del self.forge.fail["retry"] + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, self.allow) + self.assertEqual(self.forge.calls, [("retry", "5"), ("retry", "5"), ("cancel", "5")]) + + def test_an_approver_kind_needs_someone_else(self): + k = actions.Kind(name="test.second", summary="", params={"type": "object"}, + approval="approver", subject=lambda p: "t", scope=lambda p, d: "", + run=lambda p, d: None) + actions.register(k) + self.addCleanup(actions.REGISTRY.pop, "test.second") + row, _ = actions.request(self.conn, "test.second", {}, ALICE, "testbed", True) + self.assertEqual(row["state"], "proposed") + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, row["id"], ALICE, True) + ok = actions.approve(self.conn, row["id"], events.Actor("person", "carol"), True) + self.assertEqual(ok["state"], "approved") + + def test_listing_filters_communities_before_the_limit(self): + self._req() + for pid in range(10, 15): + actions.request(self.conn, "pipeline.cancel", {"pipeline": str(pid)}, ALICE, "", + True) + got = actions.listing(self.conn, communities={"testbed"}, limit=2) + self.assertEqual([r["community"] for r in got], ["testbed"]) + self.assertEqual(actions.listing(self.conn, communities=set()), []) + + def test_one_running_action_per_subject(self): + a, _ = self._req() + b, _ = self._req() + first = actions.claim_next(self.conn) + self.assertEqual(first["id"], a["id"]) + self.assertIsNone(actions.claim_next(self.conn)) # b waits for a + actions.finish(self.conn, first, result={}) + self.assertEqual(actions.claim_next(self.conn)["id"], b["id"]) + + def test_recover_reruns_or_fails_interrupted_actions(self): + a, _ = self._req() + actions.claim_next(self.conn) + unsafe = actions.Kind(name="test.unsafe", summary="", params={"type": "object"}, + approval="requester", subject=lambda p: "u", + scope=lambda p, d: "", run=lambda p, d: None, + safe_to_rerun=False) + actions.register(unsafe) + self.addCleanup(actions.REGISTRY.pop, "test.unsafe") + b, _ = actions.request(self.conn, "test.unsafe", {}, ALICE, "", True) + actions.claim_next(self.conn) + self.assertEqual(actions.recover(self.conn), 2) + self.assertEqual(actions.get(self.conn, a["id"])["state"], "approved") + self.assertEqual(actions.get(self.conn, b["id"])["state"], "failed") + + def test_an_action_that_keeps_crashing_the_worker_stops(self): + a, _ = self._req() + for _ in range(actions.MAX_ATTEMPTS): + self.assertIsNotNone(actions.claim_next(self.conn)) + actions.recover(self.conn) + self.assertEqual(actions.get(self.conn, a["id"])["state"], "failed") + + +class WorkerTest(_DB): + def setUp(self): + super().setUp() + self.settings = config.Settings(env={}) + self.w = worker.Worker(self.conn, self.settings, + heartbeat=os.path.join(self.dir, "hb")) + self.forge = FakeForge() + self.w.deps.forge = lambda: self.forge + self.w.authorize = lambda actor, community: True + + def test_a_pass_runs_actions_and_beats(self): + self.w.start() + actions.request(self.conn, "pipeline.retry", {"pipeline": "3"}, ALICE, "testbed", True) + self.w.run_once() + self.assertEqual(self.forge.calls, [("retry", "3")]) + self.assertTrue(os.path.exists(os.path.join(self.dir, "hb"))) + + def test_a_failing_step_is_recorded_once_and_its_recovery(self): + calls = [] + + def flaky(): + calls.append(1) + if len(calls) < 3: + raise OSError("disk") + self.w.steps = [("flaky", flaky)] + with self.assertLogs("bits_orchestrator", "ERROR"): + for _ in range(4): + self.w.run_once() + kinds_ = [e["kind"] for e in events.since(self.conn)] + self.assertEqual(kinds_, ["worker.step_failed", "worker.step_recovered"]) + + def test_the_worker_needs_a_resolve_token_for_group_policies(self): + from console_backend.orch import authority + self.assertEqual(authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_POLICY": "testbed @alice"})), "") + self.assertIn("RESOLVE_TOKEN", authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_POLICY": "testbed &cern/bits-admins"}))) + self.assertIn("RESOLVE_TOKEN", authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_GROUP": "bits-admins"}))) + self.assertEqual(authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_GROUP": "bits-admins", "BITS_ADMIN_RESOLVE_TOKEN": "t"})), "") + + def test_the_real_checker_needs_a_person_admin(self): + from console_backend.orch import authority + s = config.Settings(env={"BITS_ADMINS_POLICY": "testbed @alice\n* @carol"}) + ok = authority.checker(s) + self.assertTrue(ok(ALICE, "testbed")) + self.assertFalse(ok(ALICE, "other")) + self.assertFalse(ok(ALICE, "")) + self.assertTrue(ok(events.Actor("person", "carol"), "")) + self.assertFalse(ok(events.Actor("schedule", "nightly"), "testbed")) + self.assertFalse(ok(ADVISOR, "testbed")) + self.assertTrue(ok(events.WORKER, "testbed")) + + +class ApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "testbed @alice\nother @bob\n* @carol", + "BITS_FORGE_OPS_TOKEN": "optok", "BITS_FORGE_PROJECT": "42", + "BITS_ORCH_DB": self.path}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.forge = FakeForge() + p = patch.object(forge_ops.GitLabForge, "from_settings", lambda s: self.forge) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + + def _h(self, user): + return {"Authorization": "Bearer %s" % user} if user else {} + + def _post(self, user, body, path="/ops/actions"): + return self.client.post(path, headers=self._h(user), json=body) + + def test_catalogue_needs_a_login(self): + self.assertEqual(self.client.get("/ops/actions/catalogue").status_code, 401) + r = self.client.get("/ops/actions/catalogue", headers=self._h("dave")) + self.assertIn("pipeline.retry", [a["kind"] for a in r.json()["actions"]]) + + def test_a_community_admin_requests_and_the_worker_runs(self): + r = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}, + "idempotency_key": "x"}) + self.assertEqual(r.status_code, 201, r.text) + self.assertEqual((r.json()["state"], r.json()["community"]), ("approved", "testbed")) + again = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}, + "idempotency_key": "x"}) + self.assertEqual((again.status_code, again.json()["id"]), (200, r.json()["id"])) + actions.run_due(self.conn, kinds.Deps(main.settings), lambda a, c: True) + self.assertEqual(self.forge.calls, [("retry", "5")]) + got = self.client.get("/ops/actions/%d" % r.json()["id"], headers=self._h("alice")) + self.assertEqual(got.json()["state"], "done") + + def test_authority_is_the_target_community(self): + body = {"kind": "pipeline.retry", "params": {"pipeline": "5"}} + self.assertEqual(self._post(None, body).status_code, 401) + self.assertEqual(self._post("bob", body).status_code, 403) + unscoped = {"kind": "pipeline.cancel", "params": {"pipeline": "12"}} + self.assertEqual(self._post("alice", unscoped).status_code, 403) + self.assertEqual(self._post("carol", unscoped).status_code, 201) + self.assertEqual(self._post("alice", {"kind": "pipeline.retry", + "params": {"pipeline": "x"}}).status_code, 400) + self.assertEqual(self._post("alice", {"kind": "nope", "params": {}}).status_code, 400) + + def test_lists_and_reads_are_limited_to_ones_communities(self): + mine = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}}).json() + self._post("carol", {"kind": "pipeline.cancel", "params": {"pipeline": "12"}}) + seen = lambda u: [a["id"] for a in self.client.get( + "/ops/actions", headers=self._h(u)).json()["actions"]] + self.assertEqual(seen("alice"), [mine["id"]]) + self.assertEqual(seen("bob"), []) + self.assertEqual(len(seen("carol")), 2) + self.assertEqual(self.client.get("/ops/actions/%d" % mine["id"], + headers=self._h("bob")).status_code, 404) + + def test_a_proposal_is_approved_by_an_admin_of_its_community(self): + row, _ = actions.request(self.conn, "pipeline.retry", {"pipeline": "5"}, ADVISOR, + "testbed", False) + url = "/ops/actions/%d/approve" % row["id"] + self.assertEqual(self._post("bob", {}, url).status_code, 404) + r = self._post("alice", {}, url) + self.assertEqual((r.status_code, r.json()["state"]), (200, "approved")) + self.assertEqual(self._post("alice", {}, url).status_code, 409) + other, _ = actions.request(self.conn, "pipeline.retry", {"pipeline": "6"}, ADVISOR, + "testbed", False) + r = self._post("alice", {"reason": "flaky runner"}, + "/ops/actions/%d/reject" % other["id"]) + self.assertEqual((r.json()["state"], r.json()["error"]), ("rejected", "flaky runner")) + + def test_events_are_for_overall_admins(self): + self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}}) + self.assertEqual(self.client.get("/ops/events", headers=self._h("alice")).status_code, 403) + r = self.client.get("/ops/events?after=1", headers=self._h("carol")) + self.assertEqual([e["kind"] for e in r.json()["events"]], ["action.approved"]) + r = self.client.get("/ops/events?after=99&wait=1", headers=self._h("carol")) + self.assertEqual(r.json()["events"], []) + + def test_an_unconfigured_forge_is_503_to_the_caller(self): + with patch.object(forge_ops.GitLabForge, "from_settings", lambda s: None): + r = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}}) + self.assertEqual(r.status_code, 503) + + def test_another_users_key_is_not_theirs(self): + body = {"kind": "pipeline.cancel", "params": {"pipeline": "12"}, "idempotency_key": "k"} + first = self._post("carol", body).json() + mine = self._post("alice", dict(body, params={"pipeline": "5"})) + self.assertEqual(mine.status_code, 201) + self.assertNotEqual(mine.json()["id"], first["id"]) + self.assertEqual(self._post("carol", dict(body, params={"pipeline": "13"})).status_code, + 400) + + def test_unconfigured_or_unmigrated_is_503(self): + main.settings.orch_db = "" + r = self.client.get("/ops/actions", headers=self._h("alice")) + self.assertEqual(r.status_code, 503) + main.settings.orch_db = os.path.join(self.dir, "empty.db") + r = self.client.get("/ops/actions", headers=self._h("alice")) + self.assertEqual(r.status_code, 503) + self.assertIn("has the worker started", r.json()["detail"]) + self.assertFalse(os.path.exists(main.settings.orch_db)) # not created + db.connect(main.settings.orch_db).close() + r = self.client.get("/ops/actions", headers=self._h("alice")) + self.assertIn("not migrated", r.json()["detail"]) + + +if __name__ == "__main__": + unittest.main() From 87b6f3d81255a3d627a654c72ea0270fbd4f8213 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Sun, 11 Oct 2026 22:53:13 +0200 Subject: [PATCH 002/181] Orchestrator M1: builds reconciled from GitLab and S3; /ops/builds; CI hints --- CHANGELOG.md | 1 + console-backend/README.md | 14 + console-backend/console_backend/ci_auth.py | 12 +- console-backend/console_backend/config.py | 13 + console-backend/console_backend/forge_ops.py | 25 + console-backend/console_backend/orch/api.py | 120 +++- console-backend/console_backend/orch/db.py | 52 ++ console-backend/console_backend/orch/kinds.py | 27 + .../console_backend/orch/reconcile.py | 467 +++++++++++++++ .../console_backend/orch/worker.py | 44 +- console-backend/tests/test_orch_reconcile.py | 560 ++++++++++++++++++ 11 files changed, 1314 insertions(+), 21 deletions(-) create mode 100644 console-backend/console_backend/orch/reconcile.py create mode 100644 console-backend/tests/test_orch_reconcile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 554408be..4c8d0da7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ # Unreleased - **[Feature]** console-backend orchestrator, milestone M0 (`console_backend/orch/`): a worker process (`python -m console_backend.orch.worker`) and a shared SQLite database; a hash-chained event log; typed actions with JSON schemas, approvals (an advisor can only propose), authority re-checked at run time, retries with backoff, one action at a time per subject; `pipeline.retry` and `pipeline.cancel` as the first kinds; `/ops/actions` and `/ops/events`. +- **[Feature]** Orchestrator M1: the worker records every build pipeline from GitLab (outcome decided from its jobs) and the builds it published from the S3 build index; `GET /ops/builds`, `GET /ops/builds/{pipeline}`; CI jobs' hints `POST /ops/ci/published|failed` with an ID token of its own audience. `ci_auth.verify_ci_token` takes the audience to check. - **[Feature]** `bits cvmfs release --pipeline `: makes a release (its links and merged view) from one build, at any time, from the S3 store alone; the BOMs now carry the build's `cvmfs_templates`, requested packages, `view:` rules and `requires`, and each pipeline's builds are indexed under `BUILDS/by-pipeline/`. - **[Improvement]** A merged view is one build's packages again, replaced when it differs (no more merging across builds), and is independent of the release links: a group with `cvmfs_views_template` but no `cvmfs_releases_template` gets views. It records its top-level packages; one made from others is replaced only with `--force-view`. `bits cvmfs publish --wait` waits for every job, the packages before the release. diff --git a/console-backend/README.md b/console-backend/README.md index a16dec7c..e6053d48 100644 --- a/console-backend/README.md +++ b/console-backend/README.md @@ -53,3 +53,17 @@ attempts), and runs one action at a time per subject. Everything that happens is an **event** (`GET /ops/events?after=&wait=`, overall admins): an append-only log whose hash chain is the audit trail. + +Each pass first **reconciles** builds (`orch/reconcile.py`), trusting GitLab and +S3 rather than what jobs report: it lists the build project's pipelines updated +since the last pass (`BITS_ORCH_LOOKBACK_DAYS`, default 7, on an empty +database), reads a new pipeline's variables once, decides each build pipeline's +outcome from its CI jobs (bits' progress status can hold a pipeline "running" or +fail it late), and reads a finished pipeline's builds from the S3 build index in +`BITS_ORCH_STORE` (`b3://`). GitLab is read with `BITS_FORGE_READ_TOKEN` +(read_api) when set. `GET /ops/builds` and `GET /ops/builds/{pipeline}` serve +what it recorded, for the communities the caller administers. A build job may +send a hint (`POST /ops/ci/published|failed {pipeline}`) with a CI ID token for +the audience `BITS_ORCH_CI_AUDIENCE` (default `bits-orchestrator`), from the +build project on a protected ref; the worker then looks at that pipeline on +its next pass. diff --git a/console-backend/console_backend/ci_auth.py b/console-backend/console_backend/ci_auth.py index bae3de20..ca075528 100644 --- a/console-backend/console_backend/ci_auth.py +++ b/console-backend/console_backend/ci_auth.py @@ -47,10 +47,14 @@ def is_ci_authorized(project: str, group: str, policy: dict) -> bool: return "*" in allowed or group in allowed -def verify_ci_token(token: str, settings, signing_key=None) -> dict: +def verify_ci_token(token: str, settings, signing_key=None, audience=None) -> dict: """Verify a GitLab CI ID token and return its claims, or raise. Enforces - RS256, issuer, audience and the presence of exp/iat/aud/iss/sub.""" - if not (settings.oidc_issuer and settings.oidc_ci_audience + RS256, issuer, audience and the presence of exp/iat/aud/iss/sub. *audience* + overrides the signing audience, so a token minted for one service (the + orchestrator's hints) is never accepted by another (signing).""" + if audience is None: + audience = settings.oidc_ci_audience + if not (settings.oidc_issuer and audience and (signing_key or settings.jwks_url)): raise ValueError("CI token verification is not configured") key = signing_key @@ -60,5 +64,5 @@ def verify_ci_token(token: str, settings, signing_key=None) -> dict: key = _jwks_client(settings.jwks_url).get_signing_key_from_jwt(token).key return jwt.decode( token, key=key, algorithms=["RS256"], - audience=settings.oidc_ci_audience, issuer=settings.oidc_issuer, + audience=audience, issuer=settings.oidc_issuer, options={"require": ["exp", "iat", "aud", "iss", "sub"]}) diff --git a/console-backend/console_backend/config.py b/console-backend/console_backend/config.py index 20d112f6..f8569e25 100644 --- a/console-backend/console_backend/config.py +++ b/console-backend/console_backend/config.py @@ -121,6 +121,19 @@ def __init__(self, env=None): # /ops/actions and /ops/events endpoints answer 503. self.orch_db = e.get("BITS_ORCH_DB", "") self.orch_interval = float(e.get("BITS_ORCH_INTERVAL", "30") or "30") + # Read-only (read_api) GitLab token the worker reads pipelines, jobs and + # job logs with; falls back to the ops token when unset. + self.forge_read_token = e.get("BITS_FORGE_READ_TOKEN", "") + # How far back the worker looks for pipelines when it starts with an empty + # database (days); older history is the failure backfill's job. + self.orch_lookback_days = float(e.get("BITS_ORCH_LOOKBACK_DAYS", "7") or "7") + # The store the builds' BOMs and build index are in (b3://; a + # trailing ::rw is ignored: the worker only reads). Empty = builds are + # recorded from GitLab only, without their BOMs. + self.orch_store = e.get("BITS_ORCH_STORE", "") + # The audience of the CI ID tokens build jobs send with their hints + # (/ops/ci/*): its own, so a hint token can never sign anything. + self.orch_ci_audience = e.get("BITS_ORCH_CI_AUDIENCE", "bits-orchestrator") def oidc_login_configured(self) -> bool: # Only issuer + client id + redirect are required. The authorize/token/jwks diff --git a/console-backend/console_backend/forge_ops.py b/console-backend/console_backend/forge_ops.py index a6ee62a1..219e34a3 100644 --- a/console-backend/console_backend/forge_ops.py +++ b/console-backend/console_backend/forge_ops.py @@ -147,6 +147,31 @@ def set_ci_variable(self, key, value): "protected": False, "masked": False}) raise + def _get_all(self, path, per_page=100, max_pages=50): + """Every page of a list endpoint (GitLab pages by `page`/`per_page`).""" + sep = "&" if "?" in path else "?" + out = [] + for page in range(1, max_pages + 1): + got = self._get("%s%spage=%d&per_page=%d" % (path, sep, page, per_page)) or [] + out.extend(got) + if len(got) < per_page: + break + return out + + def list_pipelines(self, updated_after, page=1, per_page=100): + """One page of the project's pipelines updated after *updated_after* + (ISO 8601), oldest update first.""" + return self._get("/pipelines?updated_after=%s&order_by=updated_at&sort=asc" + "&page=%d&per_page=%d" % (quote(updated_after, safe=""), + page, per_page)) or [] + + def get_pipeline(self, pid): + return self._get("/pipelines/%s" % pid) + + def pipeline_jobs(self, pid): + """Every job of the pipeline, retried attempts included.""" + return self._get_all("/pipelines/%s/jobs?include_retried=true" % pid) + def pipeline_variables(self, pid): """The pipeline's variables as GitLab lists them ([{key, value, ...}]).""" return self._get("/pipelines/%s/variables" % pid) or [] diff --git a/console-backend/console_backend/orch/api.py b/console-backend/console_backend/orch/api.py index 0ce4b119..04da8774 100644 --- a/console-backend/console_backend/orch/api.py +++ b/console-backend/console_backend/orch/api.py @@ -9,6 +9,11 @@ POST /ops/actions/{id}/approve POST /ops/actions/{id}/reject {reason?} GET /ops/events?after=&limit=&wait= the event log (overall admins), long-polled + GET /ops/builds?community=&outcome=&pipelines=&limit= + build pipelines and their builds + GET /ops/builds/{pipeline} one, with its jobs, variables and events + POST /ops/ci/published, /ops/ci/failed {pipeline}: a CI job's hint to look + at its pipeline now (GitLab CI ID token) The API writes only requests and decisions; the worker runs actions. Identity and the admin policy are the backend's own (`main._current`, `_resolved_policy`). @@ -22,8 +27,8 @@ from fastapi.responses import JSONResponse from starlette.concurrency import run_in_threadpool -from .. import audit, authz -from . import actions, db, events, kinds +from .. import audit, authz, ci_auth +from . import actions, db, events, kinds, reconcile router = APIRouter() _MAX_BODY = 64 * 1024 @@ -211,3 +216,114 @@ def read(): if out or asyncio.get_running_loop().time() >= deadline: return JSONResponse({"events": out}, headers={"Cache-Control": "no-store"}) await asyncio.sleep(1) + + +# --- builds ------------------------------------------------------------------- + +async def _scope(data): + """None for an overall admin (every community), else the communities the + caller administers (lowercase).""" + overall, groups = await run_in_threadpool(lambda: authz.admin_groups(data["user"], _policy(data))) + return None if overall else {g.lower() for g in groups} + + +@router.get("/ops/builds") +async def list_builds(request: Request, community: str = "", outcome: str = "", + pipelines: str = "", limit: int = 50): + """Build pipelines newest first, with the builds they published: those of the + communities the caller administers (all, for an overall admin). *pipelines* + is a comma-separated list of ids (the console asks for the rows it shows).""" + data = await _caller(request) + mine = await _scope(data) + if community: + c = community.strip().lower() + if mine is not None and c not in mine: + return {"builds": []} + mine = {c} + if outcome and outcome not in ("running", "success", "failed", "canceled"): + raise HTTPException(400, "unknown outcome") + ids = None + if pipelines: + try: + ids = [int(x) for x in pipelines.split(",") if x.strip()] + except ValueError: + raise HTTPException(400, "pipelines must be comma-separated ids") + + def read(): + with _open() as conn: + return reconcile.list_builds(conn, communities=mine, outcome=outcome or None, + pipelines=ids, limit=limit) + return JSONResponse({"builds": await run_in_threadpool(read)}, + headers={"Cache-Control": "no-store"}) + + +@router.get("/ops/builds/{pipeline}") +async def get_build(pipeline: int, request: Request): + data = await _caller(request) + mine = await _scope(data) + + def read(): + with _open() as conn: + return reconcile.get_build(conn, pipeline) + b = await run_in_threadpool(read) + if b is None or (mine is not None and b["community"] not in mine): + raise HTTPException(404, "no such build") + return JSONResponse(b, headers={"Cache-Control": "no-store"}) + + +# --- CI hints ------------------------------------------------------------------- + +async def _ci_claims(request, settings): + """The verified claims of the caller's CI ID token, from the build project on + a protected ref; else HTTPException. Verified off the event loop: fetching + GitLab's keys is network I/O.""" + auth = request.headers.get("authorization", "") + if not auth.startswith("Bearer "): + raise HTTPException(401, "CI ID token required") + aud = settings.orch_ci_audience + if not aud or aud == settings.oidc_ci_audience: + raise HTTPException(503, "BITS_ORCH_CI_AUDIENCE must be set, and differ from the" + " signing audience") + try: + claims = await run_in_threadpool(ci_auth.verify_ci_token, + auth[len("Bearer "):].strip(), settings, None, aud) + except ValueError as e: + raise HTTPException(503, str(e)) + except Exception: + raise HTTPException(401, "invalid CI ID token") + project = str(settings.forge_project) + if project not in (str(claims.get("project_id", "")), str(claims.get("project_path", ""))): + raise HTTPException(403, "not the build project") + if str(claims.get("ref_protected", "")).lower() != "true": + raise HTTPException(403, "not a protected ref") + return claims + + +async def _hint(request, kind): + settings = _m().settings + claims = await _ci_claims(request, settings) + payload = await _body(request) + try: + pid = int(str(payload.get("pipeline", ""))) + except ValueError: + raise HTTPException(400, "expected {pipeline}") + if str(claims.get("pipeline_id", "")) != str(pid): + raise HTTPException(403, "the token is not from pipeline %d" % pid) + + def write(): + with _open() as conn: + reconcile.add_hint(conn, pid, kind) + await run_in_threadpool(write) + return {"status": "noted"} + + +@router.post("/ops/ci/published") +async def ci_published(request: Request): + """A hint only: the worker looks at the pipeline on its next pass and reads + what was published from S3, never from this request.""" + return await _hint(request, "published") + + +@router.post("/ops/ci/failed") +async def ci_failed(request: Request): + return await _hint(request, "failed") diff --git a/console-backend/console_backend/orch/db.py b/console-backend/console_backend/orch/db.py index 04659bc4..7b5a957d 100644 --- a/console-backend/console_backend/orch/db.py +++ b/console-backend/console_backend/orch/db.py @@ -60,6 +60,58 @@ "CREATE INDEX actions_state ON actions(state, id)", "CREATE INDEX actions_subject ON actions(subject, state, id)", ], + # 2 — what the worker knows of GitLab pipelines and the builds they published + # (M1). Both can be rebuilt from GitLab and S3. + [ + """CREATE TABLE kv ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )""", + """CREATE TABLE pipelines ( + id INTEGER PRIMARY KEY, + is_build INTEGER NOT NULL, + community TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + ref TEXT NOT NULL DEFAULT '', + sha TEXT NOT NULL DEFAULT '', + web_url TEXT NOT NULL DEFAULT '', + package TEXT NOT NULL DEFAULT '', + version TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT '', + variables TEXT NOT NULL DEFAULT '{}', + gl_status TEXT NOT NULL DEFAULT '', + outcome TEXT NOT NULL DEFAULT 'running' CHECK (outcome IN + ('running','success','failed','canceled')), + jobs TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT '', + finished_at TEXT NOT NULL DEFAULT '', + first_seen_ms INTEGER NOT NULL, + checked_ms INTEGER NOT NULL DEFAULT 0, + indexed_ms INTEGER NOT NULL DEFAULT 0 + )""", + "CREATE INDEX pipelines_outcome ON pipelines(is_build, outcome, id)", + "CREATE INDEX pipelines_community ON pipelines(community, id)", + """CREATE TABLE builds ( + pipeline_id INTEGER NOT NULL REFERENCES pipelines(id), + build_id TEXT NOT NULL, + community TEXT NOT NULL DEFAULT '', + state TEXT NOT NULL DEFAULT 'published', + arches TEXT NOT NULL DEFAULT '[]', + bom_keys TEXT NOT NULL DEFAULT '[]', + top_level TEXT NOT NULL DEFAULT '[]', + packages INTEGER NOT NULL DEFAULT 0, + published_at TEXT NOT NULL DEFAULT '', + recorded_ms INTEGER NOT NULL, + PRIMARY KEY (pipeline_id, build_id) + )""", + """CREATE TABLE hints ( + pipeline_id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + ts_ms INTEGER NOT NULL + )""", + ], ] SCHEMA_VERSION = len(MIGRATIONS) diff --git a/console-backend/console_backend/orch/kinds.py b/console-backend/console_backend/orch/kinds.py index ca25ebfa..d536ac7a 100644 --- a/console-backend/console_backend/orch/kinds.py +++ b/console-backend/console_backend/orch/kinds.py @@ -35,6 +35,33 @@ def forge(self): raise NotConfigured("forge not configured (BITS_FORGE_OPS_TOKEN/PROJECT)") return f + def reader(self): + """GitLab for reading (pipelines, jobs, logs): the read_api token when + set, else the ops token.""" + s = self.settings + token = s.forge_read_token or s.forge_ops_token + if not (s.gitlab_api_url and token and s.forge_project): + raise NotConfigured("GitLab not configured (GITLAB_API_URL, BITS_FORGE_PROJECT," + " BITS_FORGE_READ_TOKEN)") + return forge_ops.GitLabForge(s.gitlab_api_url, token, s.forge_project) + + _store = None + + def store(self): + """(s3 client, bucket) of BITS_ORCH_STORE, or None when it is not set. + Built once: the client keeps its connection.""" + url = (self.settings.orch_store or "").split("::", 1)[0] + if not url: + return None + if not url.startswith("b3://"): + raise NotConfigured("BITS_ORCH_STORE must be b3://") + if self._store is None or self._store[0] != url: + import tempfile + from bits_helpers.sync import remote_from_url + r = remote_from_url(url, None, "", tempfile.gettempdir()) + self._store = (url, r.s3, r.remoteStore) + return self._store[1], self._store[2] + def _transient(e): return e.status == 0 or e.status == 429 or e.status >= 500 diff --git a/console-backend/console_backend/orch/reconcile.py b/console-backend/console_backend/orch/reconcile.py new file mode 100644 index 00000000..7ae26803 --- /dev/null +++ b/console-backend/console_backend/orch/reconcile.py @@ -0,0 +1,467 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Reconcile: what the worker knows of builds, from GitLab and S3, never from +what a job reports about itself. + +Each pass: + +1. hints - pipelines a CI job pointed at (POST /ops/ci/*): looked at now, + even when older than the discovery window. +2. discover - pipelines of the build project updated since the last pass + (with a few minutes' overlap); a new one's variables are read once. +3. settle - for every build pipeline still running, or changed on GitLab, the + outcome is decided from its CI jobs, as the console does: bits' + own progress status can hold a pipeline "running" after every job + finished, or fail it long after its jobs succeeded. +4. index - a finished pipeline's builds are read from the S3 build index + (BUILDS/by-pipeline//.json) and their BOMs. + +Each change is an event: pipeline.seen, pipeline.success|failed|canceled|running +(running again: a retry), build.published. +""" + +import datetime +import json +import logging + +from .. import forge_ops +from . import db, events + +log = logging.getLogger("bits_orchestrator") + +BUILD_SOURCES = {"api", "trigger", "web"} +ACTIVE = {"created", "waiting_for_resource", "preparing", "pending", "running", + "scheduled", "manual"} +# A manual job nobody started is finished as far as the build is concerned. +TERMINAL_JOB = {"success", "failed", "canceled", "skipped", "manual"} +_VARS = ("COMMUNITY", "PACKAGE", "VERSION", "PLATFORM") +MAX_VARIABLES = 16 * 1024 +OVERLAP = datetime.timedelta(minutes=5) +RECHECK_MS = 120 * 1000 # a running pipeline's jobs, at most this often +INDEX_RETRY_MS = 10 * 60 * 1000 # a finished pipeline with no index, looked at again +INDEX_GIVE_UP_MS = 6 * 3600 * 1000 +STALE_AFTER = datetime.timedelta(days=3) # a build job times out after 24 h +HINT_MAX_AGE_MS = 24 * 3600 * 1000 +MAX_PER_STEP = 50 # pipelines settled or indexed per pass + + +def _iso(dt): + return dt.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _when(text): + """GitLab's time (any offset, fractional seconds) as an aware datetime, or None.""" + try: + dt = datetime.datetime.fromisoformat(str(text).replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + return dt if dt.tzinfo else dt.replace(tzinfo=datetime.timezone.utc) + + +def _gone(e): + return isinstance(e, forge_ops.ForgeError) and e.status == 404 + + +def _kv_get(conn, key, default=None): + r = conn.execute("SELECT value FROM kv WHERE key = ?", (key,)).fetchone() + return r["value"] if r else default + + +def _kv_set(conn, key, value): + conn.execute("INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", (key, value)) + + +def _subject(pid): + return "pipeline:%d" % int(pid) + + +# --- outcome from jobs ---------------------------------------------------- + +def outcome_from_jobs(gl_status, jobs): + """The pipeline's outcome: 'running', 'success', 'failed' or 'canceled'. + + From its latest job attempts, not from GitLab's pipeline status: decided once + every job is finished, or as soon as a build-stage job has failed (a later + status job may be wedged and cannot rescue it). With no jobs to go by, + GitLab's own status decides.""" + latest = {} + for j in jobs: + key = (j.get("stage"), j.get("name")) + if key not in latest or j.get("id", 0) > latest[key].get("id", 0): + latest[key] = j + jl = list(latest.values()) + if not jl: + if gl_status in ACTIVE: + return "running" + return {"success": "success", "canceled": "canceled", "skipped": "canceled"}.get( + gl_status, "failed") + hard_fail = [j for j in jl if j.get("status") == "failed" and not j.get("allow_failure")] + build = [j for j in jl if j.get("stage") == "build"] + build_failed = (build and all(j.get("status") in TERMINAL_JOB for j in build) + and any(j in hard_fail for j in build)) + if not build_failed and not all(j.get("status") in TERMINAL_JOB for j in jl): + return "running" + if hard_fail: + return "failed" + if any(j.get("status") == "canceled" for j in jl): + return "canceled" + if any(j.get("status") == "success" for j in jl): + return "success" + return "canceled" + + +def job_summary(jobs): + """What the console and the failure collection (M2) need of each job.""" + keep = ("id", "name", "stage", "status", "allow_failure", "failure_reason", + "started_at", "finished_at", "web_url") + out = [] + for j in sorted(jobs, key=lambda j: j.get("id", 0)): + d = {k: j.get(k) for k in keep if j.get(k) is not None} + runner = j.get("runner") or {} + if runner.get("description"): + d["runner"] = runner["description"] + out.append(d) + return out + + +# --- recording pipelines -------------------------------------------------- + +def _vars_dict(variables): + out = {} + for v in variables or []: + if isinstance(v, dict) and v.get("key"): + out[str(v["key"])] = str(v.get("value") or "") + return out + + +def _var(variables, name): + """A variable by name, case-insensitively (as the backend's pipeline + community check reads COMMUNITY).""" + for k, v in variables.items(): + if k.upper() == name: + return v + return "" + + +def record_pipeline(conn, reader, p): + """Insert or update one pipeline from GitLab's listing *p*. A new one's + variables are read once (they never change). Returns its row.""" + pid = int(p["id"]) + old = conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + # Only what this response carries: the listing has no finished_at, the + # single-pipeline read does; a field it lacks is kept, not blanked. + names = {"status": "gl_status", "name": "name", "ref": "ref", "sha": "sha", + "web_url": "web_url", "created_at": "created_at", "updated_at": "updated_at", + "finished_at": "finished_at"} + fields = {col: str(p.get(key) or "") for key, col in names.items() if key in p} + if old is not None: + if all(old[k] == v for k, v in fields.items()): + return old + # Changed on GitLab: its jobs are looked at again on this pass (a retry + # makes it running again; a wedged status job after a failed build does not). + sets = ", ".join("%s = :%s" % (c, c) for c in sorted(fields)) + with db.transaction(conn): + conn.execute("UPDATE pipelines SET %s, checked_ms = 0 WHERE id = :id" % sets, + dict(fields, id=pid)) + return conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + is_build = (p.get("source") in BUILD_SOURCES) + variables = _vars_dict(reader.pipeline_variables(pid)) if is_build else {} + is_build = is_build and bool(_var(variables, "PACKAGE")) + text = json.dumps(variables, sort_keys=True) + if len(text) > MAX_VARIABLES: + text = json.dumps({k: _var(variables, k) for k in _VARS}, sort_keys=True) + row = dict({col: "" for col in names.values()}, **fields) + row.update(id=pid, is_build=int(is_build), source=p.get("source") or "", + community=_var(variables, "COMMUNITY").strip().lower(), + package=_var(variables, "PACKAGE"), version=_var(variables, "VERSION"), + platform=_var(variables, "PLATFORM"), variables=text, + first_seen_ms=db.now_ms()) + with db.transaction(conn): + conn.execute("INSERT INTO pipelines (id, is_build, community, name, source, ref, sha," + " web_url, package, version, platform, variables, gl_status, created_at," + " updated_at, finished_at, first_seen_ms) VALUES (:id, :is_build," + " :community, :name, :source, :ref, :sha, :web_url, :package, :version," + " :platform, :variables, :gl_status, :created_at, :updated_at," + " :finished_at, :first_seen_ms)", row) + if is_build: + events.append(conn, "pipeline.seen", _subject(pid), events.WORKER, + {"pipeline": pid, "community": row["community"], + "package": row["package"], "version": row["version"], + "platform": row["platform"], "status": row["gl_status"]}) + return conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + + +def discover(conn, reader, lookback_days=7.0, max_rounds=20): + """Record the pipelines updated since the last pass. Returns how many. + + Always the first page after a moving start: a listing sorted by update time + shifts while it is read page by page (a pipeline that updates moves to the + end), which can skip one for good. The start moves to the newest update read + (minus a second; already-recorded pipelines cost nothing); when a full page + shares one second, the next page of that second is read instead. The next + pass starts a few minutes back to catch updates recorded out of order, unless + this pass stopped at max_rounds: then it continues exactly from here.""" + cursor = _when(_kv_get(conn, "gitlab.updated_after")) + page = 1 + if cursor is None: + since = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=lookback_days) + elif _kv_get(conn, "gitlab.continue") == "1": + since, page = cursor, int(_kv_get(conn, "gitlab.page", "1") or 1) + else: + since = cursor - OVERLAP + newest, n, truncated = cursor, 0, True + for _ in range(max_rounds): + got = reader.list_pipelines(_iso(since), page=page) + for p in got: + record_pipeline(conn, reader, p) + n += 1 + t = _when(p.get("updated_at")) + if t and (newest is None or t > newest): + newest = t + if len(got) < 100: + truncated = False + break + step = max((_when(p.get("updated_at")) for p in got if _when(p.get("updated_at"))), + default=since) - datetime.timedelta(seconds=1) + if step > since: + since, page = step, 1 + else: + page += 1 + with db.transaction(conn): + if truncated: # the next pass reads exactly this next + _kv_set(conn, "gitlab.updated_after", _iso(since)) + _kv_set(conn, "gitlab.page", str(page)) + elif newest is not None: + _kv_set(conn, "gitlab.updated_after", _iso(newest)) + _kv_set(conn, "gitlab.continue", "1" if truncated else "0") + return n + + +def apply_hints(conn, reader): + """Look now at the pipelines CI jobs pointed at. Unknown ones are fetched + directly; known ones are re-checked and re-indexed on this pass. A hint for + a pipeline GitLab does not have, or older than a day, is dropped.""" + hints = conn.execute("SELECT * FROM hints ORDER BY ts_ms LIMIT ?", + (MAX_PER_STEP,)).fetchall() + now = db.now_ms() + for h in hints: + pid, done = h["pipeline_id"], True + try: + if conn.execute("SELECT 1 FROM pipelines WHERE id = ?", (pid,)).fetchone() is None: + record_pipeline(conn, reader, reader.get_pipeline(pid)) + except Exception as exc: + done = _gone(exc) or now - h["ts_ms"] > HINT_MAX_AGE_MS + log.warning("hint for pipeline %s: %s%s", pid, exc, " (dropped)" if done else "") + with db.transaction(conn): + if done: + conn.execute("UPDATE pipelines SET checked_ms = 0, indexed_ms = 0" + " WHERE id = ?", (pid,)) + conn.execute("DELETE FROM hints WHERE pipeline_id = ? AND ts_ms = ?", + (pid, h["ts_ms"])) + return len(hints) + + +def _set_outcome(conn, r, outcome, jobs_json, extra=None): + with db.transaction(conn): + conn.execute("UPDATE pipelines SET outcome = ?, jobs = ?, checked_ms = ?" + " WHERE id = ?", (outcome, jobs_json, db.now_ms(), r["id"])) + if outcome == r["outcome"]: + return False + # Finished (or running again): its builds are indexed when it ends. + conn.execute("UPDATE pipelines SET indexed_ms = 0 WHERE id = ?", (r["id"],)) + payload = {"pipeline": r["id"], "community": r["community"], + "gl_status": r["gl_status"]} + payload.update(extra or {}) + events.append(conn, "pipeline." + outcome, _subject(r["id"]), events.WORKER, payload) + return True + + +def settle(conn, reader): + """Decide the outcome of build pipelines not settled yet. Returns how many + changed. One pipeline GitLab cannot answer for does not hold up the others: + a deleted one (404) is settled from its last known status, any other error + is tried again on a later pass.""" + now = db.now_ms() + # Running ones at most every RECHECK_MS; any one GitLab changed (or a hint + # pointed at) now. + rows = conn.execute( + "SELECT * FROM pipelines WHERE is_build = 1 AND (checked_ms = 0 OR" + " (outcome = 'running' AND checked_ms <= ?)) ORDER BY checked_ms, id LIMIT ?", + (now - RECHECK_MS, MAX_PER_STEP)).fetchall() + stale_before = datetime.datetime.now(datetime.timezone.utc) - STALE_AFTER + changed = 0 + for r in rows: + try: + jobs = reader.pipeline_jobs(r["id"]) + except Exception as exc: + if _gone(exc): + out = outcome_from_jobs(r["gl_status"], []) + changed += _set_outcome(conn, r, "canceled" if out == "running" else out, + r["jobs"], {"gone": True}) + else: + log.warning("jobs of pipeline %s: %s", r["id"], exc) + with db.transaction(conn): + conn.execute("UPDATE pipelines SET checked_ms = ? WHERE id = ?", + (db.now_ms(), r["id"])) + continue + outcome = outcome_from_jobs(r["gl_status"], jobs) + summary = job_summary(jobs) + extra = {"failed_jobs": [j["name"] for j in summary + if j.get("status") == "failed" and not j.get("allow_failure")]} + created = _when(r["created_at"]) + if outcome == "running" and created and created < stale_before: + # Nothing runs this long: GitLab's word, or failed, and no more polling. + outcome = outcome_from_jobs(r["gl_status"], []) + outcome = "failed" if outcome == "running" else outcome + extra["stale"] = True + changed += _set_outcome(conn, r, outcome, json.dumps(summary), extra) + return changed + + +# --- builds from the S3 index --------------------------------------------- + +def _bom_facts(s3, bucket, keys): + """arches, package count, top-level packages and publish time of a build's + BOMs; an unreadable or non-BOM key is skipped (and logged).""" + from bits_helpers.cvmfs_release import _get_json, _is_bom + arches, packages, top, published = set(), 0, [], "" + for key in keys: + try: + bom = _get_json(s3, bucket, key) + except Exception as exc: + log.warning("unreadable BOM %s: %s", key, exc) + continue + if not _is_bom(bom): + continue + arches.add(str(bom["effective_architecture"])) + packages += len(bom["packages"]) + if not top and isinstance(bom.get("requested_packages"), list): + top = [str(x) for x in bom["requested_packages"]] + published = max(published, str(bom.get("published_at") or "")) + return sorted(arches), packages, top, published + + +def _index_one(conn, s3, bucket, r): + """Read one pipeline's build index and record its builds, all in one + transaction with indexed_ms, so a failure part way records nothing.""" + from bits_helpers.cvmfs_release import _get_json, _list_keys + found = {} + for key in _list_keys(s3, bucket, "BUILDS/by-pipeline/%d/" % r["id"]): + try: + idx = _get_json(s3, bucket, key) + except Exception as exc: + log.warning("unreadable build index %s: %s", key, exc) + continue + if isinstance(idx, dict) and idx.get("build_id"): + found[str(idx["build_id"])] = [k for k in idx.get("keys") or [] if isinstance(k, str)] + known = {b["build_id"] for b in conn.execute( + "SELECT build_id FROM builds WHERE pipeline_id = ?", (r["id"],))} + new = [(b, keys, _bom_facts(s3, bucket, keys)) for b, keys in sorted(found.items()) + if b not in known] + with db.transaction(conn): + conn.execute("UPDATE pipelines SET indexed_ms = ? WHERE id = ?", (db.now_ms(), r["id"])) + for build_id, keys, (arches, n, top, published) in new: + conn.execute("INSERT INTO builds (pipeline_id, build_id, community, arches," + " bom_keys, top_level, packages, published_at, recorded_ms)" + " VALUES (?,?,?,?,?,?,?,?,?)", + (r["id"], build_id, r["community"], json.dumps(arches), + json.dumps(keys), json.dumps(top), n, published, db.now_ms())) + events.append(conn, "build.published", _subject(r["id"]), events.WORKER, + {"pipeline": r["id"], "build_id": build_id, + "community": r["community"], "arches": arches, "packages": n}) + return len(new) + + +def index(conn, store): + """Record the builds of finished pipelines from the S3 build index. Returns + how many builds were recorded. *store* is (s3, bucket) or None.""" + if store is None: + return 0 + s3, bucket = store + now = db.now_ms() + # Indexed when it finishes (indexed_ms reset to 0), when a hint points at it, + # and, while it has no builds yet, every INDEX_RETRY_MS until INDEX_GIVE_UP_MS + # after it finished: the build index is written before the job ends, so a + # miss is a slow store or a build that published nothing. + rows = conn.execute( + "SELECT * FROM pipelines p WHERE is_build = 1 AND outcome != 'running' AND" + " (indexed_ms = 0 OR (indexed_ms <= ? AND checked_ms >= ? AND NOT EXISTS" + " (SELECT 1 FROM builds b WHERE b.pipeline_id = p.id))) ORDER BY indexed_ms, id" + " LIMIT ?", (now - INDEX_RETRY_MS, now - INDEX_GIVE_UP_MS, MAX_PER_STEP)).fetchall() + recorded = 0 + for r in rows: + try: + recorded += _index_one(conn, s3, bucket, r) + except Exception as exc: + log.warning("build index of pipeline %s: %s", r["id"], exc) + with db.transaction(conn): # tried again after INDEX_RETRY_MS + conn.execute("UPDATE pipelines SET indexed_ms = ? WHERE id = ?", + (db.now_ms(), r["id"])) + return recorded + + +# --- reads for the API ---------------------------------------------------- + +def _pipeline_public(r, builds): + return {"pipeline": r["id"], "community": r["community"], "name": r["name"], + "package": r["package"], "version": r["version"], "platform": r["platform"], + "source": r["source"], "ref": r["ref"], "web_url": r["web_url"], + "gl_status": r["gl_status"], "outcome": r["outcome"], + "created_at": r["created_at"], "updated_at": r["updated_at"], + "finished_at": r["finished_at"], + "builds": [{"build_id": b["build_id"], "state": b["state"], + "arches": json.loads(b["arches"]), "packages": b["packages"], + "top_level": json.loads(b["top_level"]), + "published_at": b["published_at"]} for b in builds]} + + +def list_builds(conn, communities=None, outcome=None, pipelines=None, limit=50): + """Build pipelines newest first, with their builds. *communities* (lowercase) + None = all; *pipelines* restricts to those ids.""" + q, args = "SELECT * FROM pipelines WHERE is_build = 1", [] + if communities is not None: + names = sorted(communities) + if not names: + return [] + q += " AND community IN (%s)" % ",".join("?" * len(names)) + args.extend(names) + if outcome: + q += " AND outcome = ?" + args.append(outcome) + if pipelines is not None: + ids = sorted({int(p) for p in pipelines})[:200] + if not ids: + return [] + q += " AND id IN (%s)" % ",".join("?" * len(ids)) + args.extend(ids) + q += " ORDER BY id DESC LIMIT ?" + args.append(max(1, min(int(limit), 200))) + rows = conn.execute(q, args).fetchall() + out = [] + for r in rows: + b = conn.execute("SELECT * FROM builds WHERE pipeline_id = ? ORDER BY build_id", + (r["id"],)).fetchall() + out.append(_pipeline_public(r, b)) + return out + + +def get_build(conn, pid): + r = conn.execute("SELECT * FROM pipelines WHERE id = ? AND is_build = 1", + (int(pid),)).fetchone() + if r is None: + return None + b = conn.execute("SELECT * FROM builds WHERE pipeline_id = ? ORDER BY build_id", + (r["id"],)).fetchall() + out = _pipeline_public(r, b) + out["jobs"] = json.loads(r["jobs"]) + out["variables"] = json.loads(r["variables"]) + out["events"] = events.since(conn, 0, 200, subject=_subject(r["id"])) + return out + + +def add_hint(conn, pid, kind): + with db.transaction(conn): + conn.execute("INSERT INTO hints (pipeline_id, kind, ts_ms) VALUES (?, ?, ?)" + " ON CONFLICT(pipeline_id) DO UPDATE SET kind = excluded.kind," + " ts_ms = excluded.ts_ms", (int(pid), kind, db.now_ms())) diff --git a/console-backend/console_backend/orch/worker.py b/console-backend/console_backend/orch/worker.py index c6477180..78a3c33a 100644 --- a/console-backend/console_backend/orch/worker.py +++ b/console-backend/console_backend/orch/worker.py @@ -4,9 +4,10 @@ Each pass runs the steps in order. A step reads the database and the outside world, then writes what it decided; a step that fails is logged, recorded as an -event when its error changes, and tried again on the next pass. M0 has one step, -running due actions; reconcile, builds, signing and schedules are added as steps -by later milestones. +event when its error changes, and tried again on the next pass. The steps: +running due actions, then hints, discover, settle and index (reconcile.py: what +is true of builds, from GitLab and S3). Signing and schedules are added as +steps by later milestones. python -m console_backend.orch.worker # BITS_ORCH_DB=/orch/orch.db """ @@ -18,7 +19,7 @@ import time from .. import config -from . import actions, authority, db, events, kinds +from . import actions, authority, db, events, kinds, reconcile log = logging.getLogger("bits_orchestrator") @@ -31,7 +32,15 @@ def __init__(self, conn, settings, interval=30.0, heartbeat=None): self.heartbeat = heartbeat self.deps = kinds.Deps(settings) self.authorize = authority.checker(settings) - self.steps = [("actions", self._run_actions)] + # Actions first, so an approved cancel never waits behind a long catch-up. + self.steps = [ + ("actions", self._run_actions), + ("hints", lambda: reconcile.apply_hints(self.conn, self.deps.reader())), + ("discover", lambda: reconcile.discover(self.conn, self.deps.reader(), + settings.orch_lookback_days)), + ("settle", lambda: reconcile.settle(self.conn, self.deps.reader())), + ("index", lambda: reconcile.index(self.conn, self.deps.store())), + ] self._errors = {} # step -> last error text self._stop = threading.Event() @@ -53,6 +62,8 @@ def _note(self, step, error): def run_once(self): for name, step in self.steps: + if self._stop.is_set(): + break try: step() except Exception as exc: @@ -96,16 +107,19 @@ def main(): raise SystemExit(problem) path = settings.orch_db os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with db.worker_lock(path): - conn = db.connect(path) - db.migrate(conn) - w = Worker(conn, settings, interval=settings.orch_interval, - heartbeat=path + ".heartbeat") - signal.signal(signal.SIGTERM, w.stop) - signal.signal(signal.SIGINT, w.stop) - log.info("orchestrator worker on %s (schema %d, every %ss)", - path, db.SCHEMA_VERSION, settings.orch_interval) - w.loop() + try: + with db.worker_lock(path): + conn = db.connect(path) + db.migrate(conn) + w = Worker(conn, settings, interval=settings.orch_interval, + heartbeat=path + ".heartbeat") + signal.signal(signal.SIGTERM, w.stop) + signal.signal(signal.SIGINT, w.stop) + log.info("orchestrator worker on %s (schema %d, every %ss)", + path, db.SCHEMA_VERSION, settings.orch_interval) + w.loop() + except db.LockedError as e: + raise SystemExit("bits-orchestrator: %s; not starting a second worker" % e) if __name__ == "__main__": diff --git a/console-backend/tests/test_orch_reconcile.py b/console-backend/tests/test_orch_reconcile.py new file mode 100644 index 00000000..da3ff9a1 --- /dev/null +++ b/console-backend/tests/test_orch_reconcile.py @@ -0,0 +1,560 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M1: reconciling builds from GitLab and S3, the read API, +and the CI hints.""" + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import ci_auth, config, forge_ops, main +from console_backend.orch import db, events, kinds, reconcile, worker + + +def _job(i, name, status, stage="build", **kw): + return dict({"id": i, "name": name, "stage": stage, "status": status, + "allow_failure": False}, **kw) + + +class FakeReader: + """GitLab as the worker reads it: pipelines (by id), their variables, jobs.""" + + def __init__(self): + self.pipelines, self.variables, self.jobs = {}, {}, {} + self.calls = [] + + def add(self, pid, status="running", updated="2026-10-11T10:00:00Z", source="api", + community="testbed", package="ROOT", jobs=None): + self.pipelines[pid] = {"id": pid, "status": status, "source": source, + "name": "[%s] %s 6.32 (el9)" % (community, package), + "ref": "main", "sha": "abc", "web_url": "https://gl/p/%d" % pid, + "created_at": "2026-10-11T09:00:00Z", "updated_at": updated} + self.variables[pid] = ([{"key": "COMMUNITY", "value": community.upper()}, + {"key": "PACKAGE", "value": package}, + {"key": "VERSION", "value": "6.32"}, + {"key": "PLATFORM", "value": "el9"}] if package else []) + self.jobs[pid] = jobs if jobs is not None else [_job(pid * 10, "build", "running")] + + def list_pipelines(self, updated_after, page=1, per_page=100): + self.calls.append(("list", updated_after, page)) + w = reconcile._when + got = sorted((p for p in self.pipelines.values() if w(p["updated_at"]) > w(updated_after)), + key=lambda p: (w(p["updated_at"]), p["id"])) + return [dict(p) for p in got[(page - 1) * per_page: page * per_page]] # a snapshot + + def get_pipeline(self, pid): + self.calls.append(("get", pid)) + if pid not in self.pipelines: + raise forge_ops.ForgeError(404, "404 Not found") + return dict(self.pipelines[pid]) + + def pipeline_variables(self, pid): + self.calls.append(("vars", pid)) + return self.variables[pid] + + def pipeline_jobs(self, pid): + self.calls.append(("jobs", pid)) + if isinstance(self.jobs.get(pid), Exception): + raise self.jobs[pid] + if pid not in self.pipelines: + raise forge_ops.ForgeError(404, "404 Not found") + return self.jobs[pid] + + +class FakeS3: + def __init__(self, objects=None, page=2): + self.objects = {k: json.dumps(v).encode() for k, v in (objects or {}).items()} + self.page = page + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise KeyError(Key) + + class B: + def __init__(s, d): + s.d = d + + def read(s): + return s.d + return {"Body": B(self.objects[Key])} + + def list_objects_v2(self, Bucket, Prefix, ContinuationToken=None): + keys = sorted(k for k in self.objects if k.startswith(Prefix)) + start = int(ContinuationToken or 0) + chunk = keys[start:start + self.page] + more = start + self.page < len(keys) + return {"Contents": [{"Key": k} for k in chunk], "IsTruncated": more, + "NextContinuationToken": str(start + self.page) if more else None} + + +def _bom(arch, n=2, build_id="b-1"): + return {"build_id": build_id, "effective_architecture": arch, "published_at": "2026-10-11T11:00:00Z", + "requested_packages": ["ROOT"], + "packages": [{"package": "p%d" % i, "hash": "h%d" % i} for i in range(n)]} + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.gl = FakeReader() + + def _kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + def _row(self, pid): + return self.conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + + def _pass(self, store=None): + reconcile.apply_hints(self.conn, self.gl) + reconcile.discover(self.conn, self.gl) + reconcile.settle(self.conn, self.gl) + reconcile.index(self.conn, store) + + +class OutcomeTest(unittest.TestCase): + def test_outcome_comes_from_the_latest_job_attempts(self): + o = reconcile.outcome_from_jobs + self.assertEqual(o("running", [_job(1, "build", "running")]), "running") + self.assertEqual(o("running", [_job(1, "build", "success"), + _job(2, "status", "success", stage="status")]), "success") + # bits' progress status holds the pipeline running after every job ended. + self.assertEqual(o("running", [_job(1, "build", "success")]), "success") + # GitLab failed it on a timed-out status, but every job succeeded. + self.assertEqual(o("failed", [_job(1, "build", "success")]), "success") + # A failed build decides, even with a later job wedged. + self.assertEqual(o("running", [_job(1, "build", "failed"), + _job(2, "status", "waiting_for_resource", stage="status")]), + "failed") + # A retried job: the latest attempt counts. + self.assertEqual(o("running", [_job(1, "build", "failed"), _job(3, "build", "running")]), + "running") + self.assertEqual(o("success", [_job(1, "build", "failed"), _job(3, "build", "success")]), + "success") + self.assertEqual(o("failed", [_job(1, "build", "success"), + _job(2, "lint", "failed", stage="test", allow_failure=True)]), + "success") + self.assertEqual(o("canceled", [_job(1, "build", "canceled")]), "canceled") + self.assertEqual(o("pending", []), "running") + self.assertEqual(o("failed", []), "failed") + + +class ReconcileTest(_DB): + def test_a_build_is_seen_settled_and_its_builds_recorded(self): + self.gl.add(1) + self._pass() + r = self._row(1) + self.assertEqual((r["is_build"], r["community"], r["package"], r["outcome"]), + (1, "testbed", "ROOT", "running")) + self.assertEqual(self._kinds(), ["pipeline.seen"]) + # It finishes; the build index is in S3. + self.gl.add(1, status="success", updated="2026-10-11T12:00:00Z", + jobs=[_job(10, "build", "success", runner={"description": "bits-build-01"})]) + s3 = FakeS3({"BUILDS/by-pipeline/1/b-1.json": {"build_id": "b-1", + "keys": ["MANIFESTS/b-1/x.el9.json", + "MANIFESTS/b-1/x.share.json"]}, + "MANIFESTS/b-1/x.el9.json": _bom("el9", 3), + "MANIFESTS/b-1/x.share.json": _bom("share", 1)}, page=1) + self._pass((s3, "bkt")) + r = self._row(1) + self.assertEqual(r["outcome"], "success") + self.assertEqual(json.loads(r["jobs"])[0]["runner"], "bits-build-01") + got = reconcile.list_builds(self.conn)[0]["builds"] + self.assertEqual(got, [{"build_id": "b-1", "state": "published", "arches": ["el9", "share"], + "packages": 4, "top_level": ["ROOT"], + "published_at": "2026-10-11T11:00:00Z"}]) + self.assertEqual(self._kinds(), ["pipeline.seen", "pipeline.success", "build.published"]) + # Nothing new: nothing is written again. + self._pass((s3, "bkt")) + self.assertEqual(len(self._kinds()), 3) + self.assertEqual(events.verify(self.conn)[1], None) + + def test_variables_are_read_once_and_non_builds_are_kept_aside(self): + self.gl.add(1) + self.gl.add(2, source="push", package="") # Pages / push pipeline + self.gl.add(3, package="") # an api pipeline that is no build + self._pass() + self._pass() + self.assertEqual([c for c in self.gl.calls if c[0] == "vars"], [("vars", 1), ("vars", 3)]) + self.assertEqual([b["pipeline"] for b in reconcile.list_builds(self.conn)], [1]) + + def test_the_cursor_moves_and_overlaps(self): + self.gl.add(1, updated="2026-10-11T10:00:00Z") + reconcile.discover(self.conn, self.gl) + self.assertEqual(reconcile._kv_get(self.conn, "gitlab.updated_after"), + "2026-10-11T10:00:00Z") + self.gl.calls.clear() + reconcile.discover(self.conn, self.gl) + self.assertEqual(self.gl.calls[0][1], "2026-10-11T09:55:00Z") # 5 minutes back + + def _count(self): + return self.conn.execute("SELECT count(*) FROM pipelines").fetchone()[0] + + def test_many_pages_continue_on_the_next_pass(self): + for i in range(1, 251): + self.gl.add(i, updated="2026-10-11T10:%02d:%02dZ" % (i // 60, i % 60)) + reconcile.discover(self.conn, self.gl, max_rounds=1) + self.assertEqual(self._count(), 100) + reconcile.discover(self.conn, self.gl, max_rounds=1) + reconcile.discover(self.conn, self.gl, max_rounds=1) + self.assertEqual(self._count(), 250) + + def test_more_than_a_page_in_one_second(self): + for i in range(1, 251): + self.gl.add(i, updated="2026-10-11T10:00:00Z") + for _ in range(4): + reconcile.discover(self.conn, self.gl, max_rounds=1) + self.assertEqual(self._count(), 250) + reconcile.discover(self.conn, self.gl) # and then it settles down + self.assertEqual(reconcile._kv_get(self.conn, "gitlab.continue"), "0") + + def test_a_pipeline_updating_while_pages_are_read_is_not_skipped(self): + for i in range(1, 151): + self.gl.add(i, status="success", updated="2026-10-11T10:%02d:%02dZ" % (i // 60, i % 60), + jobs=[_job(i * 10, "build", "success")]) + real = self.gl.list_pipelines + + def shifting(updated_after, page=1, per_page=100): + got = real(updated_after, page, per_page) + if 1 in self.gl.pipelines and self.gl.pipelines[1]["updated_at"] < "2026-10-11T11": + self.gl.pipelines[1]["updated_at"] = "2026-10-11T11:00:00Z" # moves to the end + return got + self.gl.list_pipelines = shifting + reconcile.discover(self.conn, self.gl) + self.assertEqual(self._count(), 150) + + def test_times_with_offsets_and_fractions(self): + # After the clocks go back: the later update has the smaller local time. + self.gl.add(1, updated="2026-10-25T02:50:00.250+02:00") # 00:50 UTC + self.gl.add(2, updated="2026-10-25T02:10:00+01:00", community="x") # 01:10 UTC + reconcile.discover(self.conn, self.gl) + self.assertEqual(reconcile._kv_get(self.conn, "gitlab.updated_after"), + "2026-10-25T01:10:00Z") + + def test_a_running_pipeline_is_rechecked_at_most_every_two_minutes(self): + self.gl.add(1) + self._pass() + self.gl.calls.clear() + self._pass() + self.assertNotIn(("jobs", 1), self.gl.calls) + self.conn.execute("UPDATE pipelines SET checked_ms = 1") + self._pass() + self.assertIn(("jobs", 1), self.gl.calls) + + def test_a_failed_build_with_a_wedged_status_job(self): + self.gl.add(1, status="running", jobs=[ + _job(10, "build", "failed", failure_reason="script_failure"), + _job(11, "update-status", "waiting_for_resource", stage="status")]) + self._pass() + self.assertEqual(self._row(1)["outcome"], "failed") + e = events.since(self.conn)[-1] + self.assertEqual((e["kind"], e["payload"]["failed_jobs"]), ("pipeline.failed", ["build"])) + # Later updates to the wedged job do not flap the outcome. + self.gl.add(1, status="running", updated="2026-10-11T13:00:00Z", + jobs=self.gl.jobs[1]) + self._pass() + self.assertEqual(self._kinds(), ["pipeline.seen", "pipeline.failed"]) + + def test_a_retry_runs_again_and_publishes(self): + self.gl.add(1, status="failed", jobs=[_job(10, "build", "failed")]) + self._pass() + self.gl.add(1, status="running", updated="2026-10-11T13:00:00Z", + jobs=[_job(10, "build", "failed"), _job(12, "build", "running")]) + self._pass() + self.assertEqual(self._row(1)["outcome"], "running") + self.gl.add(1, status="success", updated="2026-10-11T14:00:00Z", + jobs=[_job(10, "build", "failed"), _job(12, "build", "success")]) + s3 = FakeS3({"BUILDS/by-pipeline/1/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "M/b.json": _bom("el9", build_id="b")}) + self._pass((s3, "bkt")) + self.assertEqual(self._kinds(), ["pipeline.seen", "pipeline.failed", "pipeline.running", + "pipeline.success", "build.published"]) + + def test_several_builds_in_one_pipeline_and_bad_index_entries(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + s3 = FakeS3({"BUILDS/by-pipeline/1/opt.json": {"build_id": "opt", "keys": ["M/o.json"]}, + "BUILDS/by-pipeline/1/dbg.json": {"build_id": "dbg", + "keys": ["M/d.json", "M/missing.json"]}, + "BUILDS/by-pipeline/1/junk.json": ["not", "an", "index"], + "BUILDS/by-pipeline/10/x.json": {"build_id": "other", "keys": []}, + "M/o.json": _bom("el9-opt", build_id="opt"), + "M/d.json": _bom("el9-dbg", build_id="dbg")}) + with self.assertLogs("bits_orchestrator", "WARNING"): + self._pass((s3, "bkt")) + got = {b["build_id"]: b["arches"] for b in reconcile.list_builds(self.conn)[0]["builds"]} + self.assertEqual(got, {"opt": ["el9-opt"], "dbg": ["el9-dbg"]}) + + def test_no_index_yet_is_retried_then_given_up(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + s3 = FakeS3({}) + self._pass((s3, "bkt")) + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 0) + s3.objects.update({k: json.dumps(v).encode() for k, v in { + "BUILDS/by-pipeline/1/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "M/b.json": _bom("el9", build_id="b")}.items()}) + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 0) # not yet due + self.conn.execute("UPDATE pipelines SET indexed_ms = 1") + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 1) + # Long after it finished, a miss is not looked at again (without a hint). + self.gl.add(2, status="success", jobs=[_job(20, "build", "success")]) + self._pass((s3, "bkt")) + self.conn.execute("UPDATE pipelines SET indexed_ms = 1, checked_ms = 1 WHERE id = 2") + s3.objects["BUILDS/by-pipeline/2/c.json"] = json.dumps( + {"build_id": "c", "keys": ["M/b.json"]}).encode() + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 0) + reconcile.add_hint(self.conn, 2, "published") + self._pass((s3, "bkt")) + self.assertEqual(len(reconcile.list_builds(self.conn, pipelines=[2])[0]["builds"]), 1) + + def test_a_hint_fetches_a_pipeline_older_than_the_window(self): + self.gl.add(7, status="success", updated="2020-01-01T00:00:00Z", + jobs=[_job(70, "build", "success")]) + self._pass() + self.assertIsNone(self._row(7)) + reconcile.add_hint(self.conn, 7, "published") + self._pass() + self.assertEqual(self._row(7)["outcome"], "success") + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 0) + + def test_hints_gitlab_does_not_know_are_dropped_others_kept(self): + reconcile.add_hint(self.conn, 99, "failed") # GitLab: 404 + with self.assertLogs("bits_orchestrator", "WARNING"): + reconcile.apply_hints(self.conn, self.gl) + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 0) + self.gl.get_pipeline = lambda pid: (_ for _ in ()).throw(forge_ops.ForgeError(502, "x")) + reconcile.add_hint(self.conn, 98, "failed") # GitLab unwell: kept + with self.assertLogs("bits_orchestrator", "WARNING"): + reconcile.apply_hints(self.conn, self.gl) + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 1) + self.conn.execute("UPDATE hints SET ts_ms = 1") # a day later: dropped + with self.assertLogs("bits_orchestrator", "WARNING"): + reconcile.apply_hints(self.conn, self.gl) + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 0) + + def test_one_pipeline_gitlab_cannot_answer_for_does_not_hold_up_the_others(self): + self.gl.add(1) + self.gl.add(2, updated="2026-10-11T10:01:00Z") + self._pass() + del self.gl.pipelines[1] # deleted on GitLab + self.gl.jobs[2] = [_job(20, "build", "success")] + self.conn.execute("UPDATE pipelines SET checked_ms = 1") + self._pass() + self.assertEqual((self._row(1)["outcome"], self._row(2)["outcome"]), ("canceled", "success")) + gone = [e for e in events.since(self.conn) if e["subject"] == "pipeline:1"][-1] + self.assertTrue(gone["payload"]["gone"]) + self.gl.add(3, updated="2026-10-11T10:02:00Z") + self.gl.jobs[3] = forge_ops.ForgeError(502, "bad gateway") + with self.assertLogs("bits_orchestrator", "WARNING"): + self._pass() + self.assertEqual(self._row(3)["outcome"], "running") + self.assertGreater(self._row(3)["checked_ms"], 0) # backs off, tried later + + def test_a_manual_job_does_not_hold_a_pipeline_running(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success"), + _job(11, "deploy", "manual", stage="deploy", + allow_failure=True)]) + self._pass() + self.assertEqual(self._row(1)["outcome"], "success") + + def test_a_pipeline_running_for_days_is_settled_from_gitlab(self): + self.gl.add(1) + self.gl.pipelines[1]["created_at"] = "2026-01-01T00:00:00Z" + self._pass() + self.assertEqual(self._row(1)["outcome"], "failed") + self.assertTrue(events.since(self.conn)[-1]["payload"]["stale"]) + + def test_a_failing_index_records_nothing_and_does_not_block_others(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + self.gl.add(2, status="success", updated="2026-10-11T10:01:00Z", + jobs=[_job(20, "build", "success")]) + s3 = FakeS3({"BUILDS/by-pipeline/1/a.json": {"build_id": "a", "keys": ["M/a.json"]}, + "BUILDS/by-pipeline/1/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "BUILDS/by-pipeline/2/c.json": {"build_id": "c", "keys": ["M/c.json"]}, + "M/a.json": _bom("el9", build_id="a"), "M/c.json": _bom("el9", build_id="c")}) + real = reconcile._bom_facts + + def boom(s3_, bucket, keys): + if keys == ["M/b.json"]: + raise RuntimeError("store hiccup") + return real(s3_, bucket, keys) + with patch.object(reconcile, "_bom_facts", boom), \ + self.assertLogs("bits_orchestrator", "WARNING"): + self._pass((s3, "bkt")) + got = {b["pipeline"]: [x["build_id"] for x in b["builds"]] + for b in reconcile.list_builds(self.conn)} + self.assertEqual(got, {1: [], 2: ["c"]}) # all of 1 or none of it + reconcile.add_hint(self.conn, 1, "published") + self._pass((s3, "bkt")) + got = [x["build_id"] for x in reconcile.list_builds(self.conn, pipelines=[1])[0]["builds"]] + self.assertEqual(got, ["a", "b"]) + + def test_variable_names_are_case_insensitive(self): + self.gl.add(1) + self.gl.variables[1] = [{"key": "community", "value": "TestBed"}, + {"key": "Package", "value": "ROOT"}] + self._pass() + self.assertEqual((self._row(1)["community"], self._row(1)["package"]), ("testbed", "ROOT")) + + def test_the_listing_without_finished_at_does_not_count_as_a_change(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + self._pass() + self.gl.pipelines[1]["finished_at"] = "2026-10-11T10:30:00Z" + reconcile.record_pipeline(self.conn, self.gl, self.gl.get_pipeline(1)) + self.assertEqual(self._row(1)["finished_at"], "2026-10-11T10:30:00Z") + listed = {k: v for k, v in self.gl.pipelines[1].items() if k != "finished_at"} + before = self._row(1)["checked_ms"] + reconcile.record_pipeline(self.conn, self.gl, listed) + self.assertEqual((self._row(1)["checked_ms"], self._row(1)["finished_at"]), + (before, "2026-10-11T10:30:00Z")) + + def test_list_filters(self): + self.gl.add(1, community="testbed") + self.gl.add(2, community="other", updated="2026-10-11T10:01:00Z") + self.gl.add(3, community="testbed", status="failed", updated="2026-10-11T10:02:00Z", + jobs=[_job(30, "build", "failed")]) + self._pass() + ids = lambda **kw: [b["pipeline"] for b in reconcile.list_builds(self.conn, **kw)] + self.assertEqual(ids(), [3, 2, 1]) + self.assertEqual(ids(communities={"testbed"}), [3, 1]) + self.assertEqual(ids(communities=set()), []) + self.assertEqual(ids(outcome="failed"), [3]) + self.assertEqual(ids(pipelines=[1, 2]), [2, 1]) + detail = reconcile.get_build(self.conn, 3) + self.assertEqual((detail["jobs"][0]["status"], detail["variables"]["PACKAGE"]), + ("failed", "ROOT")) + self.assertEqual([e["kind"] for e in detail["events"]], ["pipeline.seen", "pipeline.failed"]) + + +class WorkerStepsTest(_DB): + def test_the_worker_runs_reconcile_before_actions(self): + w = worker.Worker(self.conn, config.Settings(env={})) + self.assertEqual([n for n, _ in w.steps], ["actions", "hints", "discover", "settle", "index"]) + w.deps.reader = lambda: self.gl + w.deps.store = lambda: None + self.gl.add(1) + w.run_once() + self.assertEqual(self._row(1)["outcome"], "running") + + def test_unconfigured_gitlab_is_one_event(self): + w = worker.Worker(self.conn, config.Settings(env={})) + w.steps = [s for s in w.steps if s[0] == "hints"] + with self.assertLogs("bits_orchestrator", "ERROR"): + w.run_once() + w.run_once() + got = [e for e in events.since(self.conn) if e["kind"] == "worker.step_failed"] + self.assertEqual(len(got), 1) + self.assertIn("GitLab not configured", got[0]["payload"]["error"]) + + def test_store_settings(self): + d = kinds.Deps(config.Settings(env={})) + self.assertIsNone(d.store()) + d = kinds.Deps(config.Settings(env={"BITS_ORCH_STORE": "https://x/y"})) + with self.assertRaises(kinds.NotConfigured): + d.store() + + +class ApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "testbed @alice\nother @bob\n* @carol", + "BITS_FORGE_PROJECT": "42", "BITS_ORCH_DB": self.path, + "BITS_OIDC_ISSUER": "https://gl", "BITS_OIDC_CI_AUDIENCE": "bits", + "BITS_OIDC_JWKS_URL": "https://gl/jwks"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + self.gl.add(1, community="testbed") + self.gl.add(2, community="other", updated="2026-10-11T10:01:00Z") + self._pass() + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer %s" % user}) + + def test_builds_are_those_of_ones_communities(self): + ids = lambda u, q="": [b["pipeline"] for b in self._get(u, "/ops/builds" + q).json()["builds"]] + self.assertEqual(ids("alice"), [1]) + self.assertEqual(ids("bob"), [2]) + self.assertEqual(ids("carol"), [2, 1]) + self.assertEqual(ids("dave"), []) + self.assertEqual(ids("alice", "?community=other"), []) + self.assertEqual(ids("carol", "?community=OTHER"), [2]) + self.assertEqual(ids("carol", "?pipelines=1,99"), [1]) + self.assertEqual(self._get("alice", "/ops/builds?pipelines=x").status_code, 400) + self.assertEqual(self._get("alice", "/ops/builds?outcome=weird").status_code, 400) + self.assertEqual(self.client.get("/ops/builds").status_code, 401) + + def test_one_build_with_its_jobs_and_events(self): + r = self._get("alice", "/ops/builds/1") + self.assertEqual((r.status_code, r.json()["package"]), (200, "ROOT")) + self.assertEqual([e["kind"] for e in r.json()["events"]], ["pipeline.seen"]) + self.assertEqual(self._get("bob", "/ops/builds/1").status_code, 404) + self.assertEqual(self._get("carol", "/ops/builds/77").status_code, 404) + + def _ci(self, claims, body, kind="published"): + seen = {} + + def verify(t, s, key=None, audience=None): + seen["aud"] = audience + return claims + with patch.object(ci_auth, "verify_ci_token", verify): + r = self.client.post("/ops/ci/" + kind, json=body, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(seen.get("aud"), "bits-orchestrator") # never the signing audience + return r + + def test_ci_hints_need_an_audience_of_their_own(self): + good = {"project_id": "42", "ref_protected": "true", "pipeline_id": "5"} + main.settings.orch_ci_audience = "" + with patch.object(ci_auth, "verify_ci_token", lambda *a, **k: good): + r = self.client.post("/ops/ci/published", json={"pipeline": 5}, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(r.status_code, 503) + main.settings.orch_ci_audience = main.settings.oidc_ci_audience + with patch.object(ci_auth, "verify_ci_token", lambda *a, **k: good): + r = self.client.post("/ops/ci/published", json={"pipeline": 5}, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(r.status_code, 503) + + def test_ci_hints_need_the_build_projects_token_for_that_pipeline(self): + good = {"project_id": "42", "project_path": "bits/console", "ref_protected": "true", + "pipeline_id": "5"} + self.assertEqual(self._ci(good, {"pipeline": 5}).json(), {"status": "noted"}) + with patch.object(ci_auth, "_jwks_client") as jw: # the real check: its own audience + import jwt as _jwt + with patch.object(_jwt, "decode", lambda *a, **k: dict(good, aud=k["audience"])): + jw.return_value.get_signing_key_from_jwt.return_value.key = "k" + c = ci_auth.verify_ci_token("t", main.settings, audience="bits-orchestrator") + self.assertEqual(c["aud"], "bits-orchestrator") + self.assertEqual(self._ci(good, {"pipeline": 5}, "failed").status_code, 200) + self.assertEqual(self.conn.execute("SELECT kind FROM hints").fetchall()[0][0], "failed") + self.assertEqual(self._ci(dict(good, project_id="7", project_path="x/y"), + {"pipeline": 5}).status_code, 403) + self.assertEqual(self._ci(dict(good, ref_protected="false"), {"pipeline": 5}).status_code, + 403) + self.assertEqual(self._ci(good, {"pipeline": 6}).status_code, 403) + self.assertEqual(self._ci(good, {"pipeline": "x"}).status_code, 400) + self.assertEqual(self.client.post("/ops/ci/published", json={"pipeline": 5}).status_code, + 401) + + def bad(t, s, key=None, audience=None): + raise Exception("bad signature") + with patch.object(ci_auth, "verify_ci_token", bad): + r = self.client.post("/ops/ci/published", json={"pipeline": 5}, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(r.status_code, 401) + + +if __name__ == "__main__": + unittest.main() From d3bf607fd3c2107804dd6d095c527dfb128b9ff7 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Sun, 11 Oct 2026 23:15:26 +0200 Subject: [PATCH 003/181] Orchestrator: open the store read-only without a None write store --- console-backend/console_backend/orch/kinds.py | 3 ++- console-backend/tests/test_orch_reconcile.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/console-backend/console_backend/orch/kinds.py b/console-backend/console_backend/orch/kinds.py index d536ac7a..d15d7b72 100644 --- a/console-backend/console_backend/orch/kinds.py +++ b/console-backend/console_backend/orch/kinds.py @@ -58,7 +58,8 @@ def store(self): if self._store is None or self._store[0] != url: import tempfile from bits_helpers.sync import remote_from_url - r = remote_from_url(url, None, "", tempfile.gettempdir()) + # Read only: no write store ("", not None, which the S3 backend cannot take). + r = remote_from_url(url, "", "", tempfile.gettempdir()) self._store = (url, r.s3, r.remoteStore) return self._store[1], self._store[2] diff --git a/console-backend/tests/test_orch_reconcile.py b/console-backend/tests/test_orch_reconcile.py index da3ff9a1..aa963b11 100644 --- a/console-backend/tests/test_orch_reconcile.py +++ b/console-backend/tests/test_orch_reconcile.py @@ -454,6 +454,13 @@ def test_unconfigured_gitlab_is_one_event(self): self.assertEqual(len(got), 1) self.assertIn("GitLab not configured", got[0]["payload"]["error"]) + def test_the_store_is_built_from_the_real_s3_backend(self): + from bits_helpers import sync + d = kinds.Deps(config.Settings(env={"BITS_ORCH_STORE": "b3://bits-store::rw"})) + with patch.object(sync.Boto3RemoteSync, "_s3_init", + lambda self: setattr(self, "s3", "client")): + self.assertEqual(d.store(), ("client", "bits-store")) + def test_store_settings(self): d = kinds.Deps(config.Settings(env={})) self.assertIsNone(d.store()) From 6dae74148193a400f9caaad3afa0befbc449a0fe Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Sun, 11 Oct 2026 23:19:59 +0200 Subject: [PATCH 004/181] Orchestrator: read pipeline variables with the ops token; fall back to the pipeline name --- console-backend/console_backend/orch/kinds.py | 13 +++++++--- .../console_backend/orch/reconcile.py | 25 +++++++++++++++++- console-backend/tests/test_orch_reconcile.py | 26 +++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/console-backend/console_backend/orch/kinds.py b/console-backend/console_backend/orch/kinds.py index d15d7b72..b8b90643 100644 --- a/console-backend/console_backend/orch/kinds.py +++ b/console-backend/console_backend/orch/kinds.py @@ -36,14 +36,21 @@ def forge(self): return f def reader(self): - """GitLab for reading (pipelines, jobs, logs): the read_api token when - set, else the ops token.""" + """GitLab for reading: pipelines, jobs and logs with the read_api token + when set (else the ops token); a pipeline's variables with the ops token, + as the backend's own /ops/pipeline/{id}/variables does, because GitLab + shows them only to a Maintainer and the read token must not be one (a + Maintainer's read_api token can read the project's CI/CD secrets).""" s = self.settings token = s.forge_read_token or s.forge_ops_token if not (s.gitlab_api_url and token and s.forge_project): raise NotConfigured("GitLab not configured (GITLAB_API_URL, BITS_FORGE_PROJECT," " BITS_FORGE_READ_TOKEN)") - return forge_ops.GitLabForge(s.gitlab_api_url, token, s.forge_project) + read = forge_ops.GitLabForge(s.gitlab_api_url, token, s.forge_project) + ops = forge_ops.GitLabForge.from_settings(s) + if ops is not None: + read.pipeline_variables = ops.pipeline_variables + return read _store = None diff --git a/console-backend/console_backend/orch/reconcile.py b/console-backend/console_backend/orch/reconcile.py index 7ae26803..bbee391c 100644 --- a/console-backend/console_backend/orch/reconcile.py +++ b/console-backend/console_backend/orch/reconcile.py @@ -23,6 +23,7 @@ import datetime import json import logging +import re from .. import forge_ops from . import db, events @@ -135,6 +136,20 @@ def _vars_dict(variables): return out +_NAME = re.compile(r"^\s*\[([^\]]+)\]\s*(\S+)\s*(.*?)\s*\(([^)]*)\)") + + +def _from_name(name): + """COMMUNITY, PACKAGE, VERSION, PLATFORM from the build pipeline's name, which + its workflow sets to '[COMMUNITY] PACKAGE VERSION (PLATFORM)': what + the console shows when the variables cannot be read.""" + m = _NAME.match(name or "") + if not m: + return {} + return dict(zip(("COMMUNITY", "PACKAGE", "VERSION", "PLATFORM"), + (g.strip() for g in m.groups()))) + + def _var(variables, name): """A variable by name, case-insensitively (as the backend's pipeline community check reads COMMUNITY).""" @@ -166,7 +181,15 @@ def record_pipeline(conn, reader, p): dict(fields, id=pid)) return conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() is_build = (p.get("source") in BUILD_SOURCES) - variables = _vars_dict(reader.pipeline_variables(pid)) if is_build else {} + variables = {} + if is_build: + # Unreadable variables (no ops token, GitLab refusing) never hold up + # discovery: the pipeline's name carries what is needed to record it. + try: + variables = _vars_dict(reader.pipeline_variables(pid)) + except Exception as exc: + log.warning("variables of pipeline %s: %s; using its name", pid, exc) + variables = _from_name(p.get("name")) is_build = is_build and bool(_var(variables, "PACKAGE")) text = json.dumps(variables, sort_keys=True) if len(text) > MAX_VARIABLES: diff --git a/console-backend/tests/test_orch_reconcile.py b/console-backend/tests/test_orch_reconcile.py index aa963b11..ef7f8ae4 100644 --- a/console-backend/tests/test_orch_reconcile.py +++ b/console-backend/tests/test_orch_reconcile.py @@ -397,6 +397,32 @@ def boom(s3_, bucket, keys): got = [x["build_id"] for x in reconcile.list_builds(self.conn, pipelines=[1])[0]["builds"]] self.assertEqual(got, ["a", "b"]) + def test_unreadable_variables_fall_back_to_the_name(self): + self.gl.add(1, community="lcg", package="ROOT") + self.gl.pipelines[1]["name"] = "[LCG] ROOT 6.32.08 (x86_64-el9) [nightly]" + self.gl.add(2, community="lcg", package="", updated="2026-10-11T10:01:00Z") + self.gl.pipelines[2]["name"] = "Pages" # no build + def refuse(pid): + raise forge_ops.ForgeError(403, "GET /pipelines/%s/variables -> 403" % pid) + self.gl.pipeline_variables = refuse + with self.assertLogs("bits_orchestrator", "WARNING"): + self._pass() + r = self._row(1) + self.assertEqual((r["is_build"], r["community"], r["package"], r["version"], r["platform"]), + (1, "lcg", "ROOT", "6.32.08", "x86_64-el9")) + self.assertEqual(self._row(2)["is_build"], 0) + self.assertEqual(reconcile._from_name("[ALICE] O2 (el9)")["VERSION"], "") + + def test_variables_are_read_with_the_ops_token(self): + s = config.Settings(env={"GITLAB_API_URL": "https://gl/api/v4", "BITS_FORGE_PROJECT": "42", + "BITS_FORGE_READ_TOKEN": "ro", "BITS_FORGE_OPS_TOKEN": "ops"}) + r = kinds.Deps(s).reader() + self.assertEqual(r._token, "ro") + self.assertEqual(r.pipeline_variables.__self__._token, "ops") + s.forge_ops_token = "" + self.assertFalse(hasattr(kinds.Deps(s).reader().pipeline_variables, "__self__") + and kinds.Deps(s).reader().pipeline_variables.__self__._token == "ops") + def test_variable_names_are_case_insensitive(self): self.gl.add(1) self.gl.variables[1] = [{"key": "community", "value": "TestBed"}, From faf6159d988d4b16b112392c09ff81bc267428f2 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 12 Oct 2026 00:10:52 +0200 Subject: [PATCH 005/181] Orchestrator M2: failure records, deduplicated issues, signatures, backfill --- CHANGELOG.md | 1 + console-backend/README.md | 17 + console-backend/console_backend/config.py | 3 + console-backend/console_backend/forge_ops.py | 48 ++ console-backend/console_backend/orch/api.py | 62 +- console-backend/console_backend/orch/db.py | 59 ++ .../console_backend/orch/failures.py | 552 ++++++++++++++++++ .../console_backend/orch/reconcile.py | 10 +- .../console_backend/orch/worker.py | 10 +- console-backend/tests/test_orch_failures.py | 441 ++++++++++++++ console-backend/tests/test_orch_reconcile.py | 3 +- 11 files changed, 1200 insertions(+), 6 deletions(-) create mode 100644 console-backend/console_backend/orch/failures.py create mode 100644 console-backend/tests/test_orch_failures.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c8d0da7..8172eda8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ - **[Feature]** console-backend orchestrator, milestone M0 (`console_backend/orch/`): a worker process (`python -m console_backend.orch.worker`) and a shared SQLite database; a hash-chained event log; typed actions with JSON schemas, approvals (an advisor can only propose), authority re-checked at run time, retries with backoff, one action at a time per subject; `pipeline.retry` and `pipeline.cancel` as the first kinds; `/ops/actions` and `/ops/events`. - **[Feature]** Orchestrator M1: the worker records every build pipeline from GitLab (outcome decided from its jobs) and the builds it published from the S3 build index; `GET /ops/builds`, `GET /ops/builds/{pipeline}`; CI jobs' hints `POST /ops/ci/published|failed` with an ID token of its own audience. `ci_auth.verify_ci_token` takes the audience to check. +- **[Feature]** Orchestrator M2: failed jobs are recorded from their logs (or the `bits-build.log` artifact when GitLab cut the log), redacted, and deduplicated into issues; seeded signatures name known causes; an optional backfill (`BITS_ORCH_BACKFILL_DAYS`) reads older failures; `GET /ops/issues`, `/ops/issues/{id}`, `/ops/failures`, `/ops/signatures`. - **[Feature]** `bits cvmfs release --pipeline `: makes a release (its links and merged view) from one build, at any time, from the S3 store alone; the BOMs now carry the build's `cvmfs_templates`, requested packages, `view:` rules and `requires`, and each pipeline's builds are indexed under `BUILDS/by-pipeline/`. - **[Improvement]** A merged view is one build's packages again, replaced when it differs (no more merging across builds), and is independent of the release links: a group with `cvmfs_views_template` but no `cvmfs_releases_template` gets views. It records its top-level packages; one made from others is replaced only with `--force-view`. `bits cvmfs publish --wait` waits for every job, the packages before the release. diff --git a/console-backend/README.md b/console-backend/README.md index e6053d48..53f00e39 100644 --- a/console-backend/README.md +++ b/console-backend/README.md @@ -67,3 +67,20 @@ send a hint (`POST /ops/ci/published|failed {pipeline}`) with a CI ID token for the audience `BITS_ORCH_CI_AUDIENCE` (default `bits-orchestrator`), from the build project on a protected ref; the worker then looks at that pipeline on its next pass. + +Every failed job of a failed build pipeline is then **recorded** +(`orch/failures.py`): the end of its log, or of its `bits-build.log` artifact +when GitLab cut the log at its size limit, is parsed for bits' `BUILD FAILED: +@` blocks and their error excerpt (else the job's last lines and +GitLab's failure reason), and the excerpt is **redacted** (tokens, auth headers, +URL passwords, key=value secrets, private keys) before it is stored. Failures +with the same package and error lines, once paths, hashes, numbers and times +are taken out, are one **issue**; an active **signature** (a regular expression +over the excerpt and `reason:`) names its cause and suggested +action. Four signatures are seeded once. `BITS_ORCH_BACKFILL_DAYS` (default 0) +reads that many days of older failed pipelines from GitLab, a page per pass, +once. `GET /ops/issues`, `/ops/issues/{id}`, `/ops/failures` and +`/ops/signatures` serve the record, limited to the caller's communities (an +issue seen in several shows a community admin only their own occurrences, +counts and excerpt). A pipeline whose logs cannot be read is tried again every +10 minutes for a day. diff --git a/console-backend/console_backend/config.py b/console-backend/console_backend/config.py index f8569e25..23689753 100644 --- a/console-backend/console_backend/config.py +++ b/console-backend/console_backend/config.py @@ -134,6 +134,9 @@ def __init__(self, env=None): # The audience of the CI ID tokens build jobs send with their hints # (/ops/ci/*): its own, so a hint token can never sign anything. self.orch_ci_audience = e.get("BITS_ORCH_CI_AUDIENCE", "bits-orchestrator") + # Failure backfill (M2): how many days of older failed pipelines to read + # from GitLab, a page per pass, once. 0 = off. + self.orch_backfill_days = float(e.get("BITS_ORCH_BACKFILL_DAYS", "0") or "0") def oidc_login_configured(self) -> bool: # Only issuer + client id + redirect are required. The authorize/token/jwks diff --git a/console-backend/console_backend/forge_ops.py b/console-backend/console_backend/forge_ops.py index 219e34a3..e4a17910 100644 --- a/console-backend/console_backend/forge_ops.py +++ b/console-backend/console_backend/forge_ops.py @@ -165,6 +165,54 @@ def list_pipelines(self, updated_after, page=1, per_page=100): "&page=%d&per_page=%d" % (quote(updated_after, safe=""), page, per_page)) or [] + def list_pipelines_before(self, updated_before, status="failed", page=1, per_page=100): + """One page of the project's pipelines in *status* updated before + *updated_before* (ISO 8601), newest update first (the failure backfill).""" + return self._get("/pipelines?updated_before=%s&status=%s&order_by=updated_at&sort=desc" + "&page=%d&per_page=%d" % (quote(updated_before, safe=""), + quote(status, safe=""), page, per_page)) or [] + + def _tail(self, path, max_bytes, max_total=64 << 20, max_seconds=60): + """The last *max_bytes* of a (possibly huge) raw response; None on 404. + Asks for just the end (Range); a server that sends the whole body is read + streamed, at most *max_total* bytes or *max_seconds*, past which the end + is out of reach and the answer is None.""" + import time as _time + try: + r = requests.get("%s/projects/%s%s" % (self._api, self._project, path), + headers={"PRIVATE-TOKEN": self._token, + "Range": "bytes=-%d" % max_bytes}, + timeout=self._timeout, stream=True) + except requests.RequestException as e: + raise ForgeError(0, str(e)[:200]) + with r: + if r.status_code in (404, 416): + return None + if r.status_code // 100 != 2: + raise ForgeError(r.status_code, "GET %s -> %s" % (path, r.status_code)) + buf, total, began = bytearray(), 0, _time.monotonic() + try: + for chunk in r.iter_content(64 * 1024): + buf += chunk + total += len(chunk) + if len(buf) > 2 * max_bytes: + del buf[:len(buf) - max_bytes] + if total > max_total or _time.monotonic() - began > max_seconds: + return None + except requests.RequestException as e: + raise ForgeError(0, str(e)[:200]) + return bytes(buf[-max_bytes:]) + + def job_log_tail(self, job_id, max_bytes=1 << 20): + """The end of a job's log (GitLab keeps at most its log limit, 4 MB by + default, and cuts the rest).""" + return self._tail("/jobs/%s/trace" % job_id, max_bytes) + + def job_artifact_tail(self, job_id, name, max_bytes=1 << 20): + """The end of one artifact file of a job, or None when it has none (or + it expired).""" + return self._tail("/jobs/%s/artifacts/%s" % (job_id, quote(name, safe="/")), max_bytes) + def get_pipeline(self, pid): return self._get("/pipelines/%s" % pid) diff --git a/console-backend/console_backend/orch/api.py b/console-backend/console_backend/orch/api.py index 04da8774..58414614 100644 --- a/console-backend/console_backend/orch/api.py +++ b/console-backend/console_backend/orch/api.py @@ -14,6 +14,10 @@ GET /ops/builds/{pipeline} one, with its jobs, variables and events POST /ops/ci/published, /ops/ci/failed {pipeline}: a CI job's hint to look at its pipeline now (GitLab CI ID token) + GET /ops/issues?community=&package=&limit= deduplicated failures, last seen first + GET /ops/issues/{id} one, with its occurrences + GET /ops/failures?pipeline=&community=&limit= failed jobs' records, with excerpts + GET /ops/signatures the reviewed patterns that name a cause The API writes only requests and decisions; the worker runs actions. Identity and the admin policy are the backend's own (`main._current`, `_resolved_policy`). @@ -28,7 +32,7 @@ from starlette.concurrency import run_in_threadpool from .. import audit, authz, ci_auth -from . import actions, db, events, kinds, reconcile +from . import actions, db, events, failures, kinds, reconcile router = APIRouter() _MAX_BODY = 64 * 1024 @@ -327,3 +331,59 @@ async def ci_published(request: Request): @router.post("/ops/ci/failed") async def ci_failed(request: Request): return await _hint(request, "failed") + + +# --- failures ------------------------------------------------------------------- + +async def _communities(data, community): + """The communities a read covers: the caller's (None = all, for an overall + admin), narrowed to *community* when given.""" + mine = await _scope(data) + if community: + c = community.strip().lower() + if mine is not None and c not in mine: + return set() + return {c} + return mine + + +def _read(fn, *args, **kw): + with _open() as conn: + return fn(conn, *args, **kw) + + +@router.get("/ops/issues") +async def list_issues(request: Request, community: str = "", package: str = "", limit: int = 100): + """Issues seen in the caller's communities. An issue seen in several shows + its full counts; its occurrences (GET /ops/issues/{id}) are only theirs.""" + data = await _caller(request) + mine = await _communities(data, community) + out = await run_in_threadpool(_read, failures.list_issues, communities=mine, + package=package or None, limit=limit) + return JSONResponse({"issues": out}, headers={"Cache-Control": "no-store"}) + + +@router.get("/ops/issues/{issue_id}") +async def get_issue(issue_id: int, request: Request): + data = await _caller(request) + mine = await _scope(data) + out = await run_in_threadpool(_read, failures.get_issue, issue_id, communities=mine) + if out is None: + raise HTTPException(404, "no such issue") + return JSONResponse(out, headers={"Cache-Control": "no-store"}) + + +@router.get("/ops/failures") +async def list_failures(request: Request, pipeline: int = 0, community: str = "", + limit: int = 100): + data = await _caller(request) + mine = await _communities(data, community) + out = await run_in_threadpool(_read, failures.list_failures, communities=mine, + pipeline=pipeline or None, limit=limit) + return JSONResponse({"failures": out}, headers={"Cache-Control": "no-store"}) + + +@router.get("/ops/signatures") +async def list_signatures(request: Request): + await _caller(request) + return {"signatures": await run_in_threadpool(_read, failures.list_signatures)} diff --git a/console-backend/console_backend/orch/db.py b/console-backend/console_backend/orch/db.py index 7b5a957d..47c1146f 100644 --- a/console-backend/console_backend/orch/db.py +++ b/console-backend/console_backend/orch/db.py @@ -112,6 +112,65 @@ ts_ms INTEGER NOT NULL )""", ], + # 3 — failures (M2): every failed job's record, the deduplicated issues they + # are occurrences of, and the reviewed signatures that name a cause. These + # cannot be rebuilt once GitLab expires the logs: they are backed up. + [ + "ALTER TABLE pipelines ADD COLUMN collected_ms INTEGER NOT NULL DEFAULT 0", + """CREATE TABLE signatures ( + id INTEGER PRIMARY KEY, + pattern TEXT NOT NULL, + cause TEXT NOT NULL, + action TEXT NOT NULL, + safe_to_retry INTEGER NOT NULL DEFAULT 0, + state TEXT NOT NULL DEFAULT 'active' CHECK (state IN + ('proposed','active','retired')), + proposed_by TEXT NOT NULL, + accepted_by TEXT, + version INTEGER NOT NULL DEFAULT 1, + created_ms INTEGER NOT NULL + )""", + """CREATE TABLE issues ( + id INTEGER PRIMARY KEY, + fingerprint TEXT NOT NULL UNIQUE, + package TEXT NOT NULL, + title TEXT NOT NULL, + excerpt TEXT NOT NULL, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + count INTEGER NOT NULL DEFAULT 0, + communities TEXT NOT NULL DEFAULT '[]', + platforms TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'open', + signature_id INTEGER REFERENCES signatures(id), + created_ms INTEGER NOT NULL + )""", + "CREATE INDEX issues_last ON issues(last_seen)", + """CREATE TABLE failures ( + id INTEGER PRIMARY KEY, + job_id INTEGER NOT NULL, + pipeline_id INTEGER NOT NULL REFERENCES pipelines(id), + package TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '', + community TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT '', + job_name TEXT NOT NULL DEFAULT '', + failure_reason TEXT NOT NULL DEFAULT '', + runner TEXT NOT NULL DEFAULT '', + bits_version TEXT NOT NULL DEFAULT '', + excerpt TEXT NOT NULL, + source TEXT NOT NULL, + log_url TEXT NOT NULL DEFAULT '', + failed_at TEXT NOT NULL DEFAULT '', + fingerprint TEXT NOT NULL, + issue_id INTEGER NOT NULL REFERENCES issues(id), + recorded_ms INTEGER NOT NULL, + UNIQUE (job_id, package) + )""", + "CREATE INDEX failures_issue ON failures(issue_id, id)", + "CREATE INDEX failures_pipeline ON failures(pipeline_id)", + "CREATE INDEX failures_community ON failures(community, id)", + ], ] SCHEMA_VERSION = len(MIGRATIONS) diff --git a/console-backend/console_backend/orch/failures.py b/console-backend/console_backend/orch/failures.py new file mode 100644 index 00000000..2d0ba9ff --- /dev/null +++ b/console-backend/console_backend/orch/failures.py @@ -0,0 +1,552 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Failures: every failed build job leaves a record, repeats are one issue, and a +reviewed signature can name an issue's cause. + +collect - for each failed build pipeline not collected yet, each of its failed + jobs (latest attempt): the end of its log (or of its bits-build.log + artifact, when GitLab cut the log at its size limit) is parsed for + bits' "BUILD FAILED: @" blocks and their "Error excerpt" + (one failure per failed package); without one, the job's last lines + and GitLab's failure reason are the excerpt. The excerpt is redacted + before anything is stored. +issue - the fingerprint of a failure is its package and its error lines with + paths, hashes, numbers and times taken out; the same fingerprint is the + same issue, with a count, first and last seen, communities, platforms. +match - active signatures (regular expressions over the excerpt and + "reason:") name an issue's cause and suggested action. +backfill- with BITS_ORCH_BACKFILL_DAYS, a few of the older failed pipelines + GitLab still has are recorded on every pass, newest first, until done. + +Excerpts are untrusted text: stored capped, shown as text, never interpreted. +Whole logs stay in GitLab and are linked. +""" + +import datetime +import hashlib +import json +import logging +import re + +from . import db, events, reconcile + +log = logging.getLogger("bits_orchestrator") + +MAX_EXCERPT = 8 * 1024 +LOG_TAIL = 1 << 20 +COLLECT_PER_PASS = 20 +BACKFILL_PER_PASS = 20 +RETRY_MS = 10 * 60 * 1000 +GIVE_UP_MS = 24 * 3600 * 1000 +ARTIFACT = "bits-build.log" + +# --- redaction ---------------------------------------------------------------- + +_KEY = (r"[\w.-]*(?:password|passwd|passphrase|secret|token|api[_-]?key|access[_-]?key(?:[_-]?id)?" + r"|private[_-]?key|credentials?|auth)[\w.-]*") +_VALUE = r"(?:\"[^\"\n]+\"|'[^'\n]+'|[A-Za-z0-9/+=_.~:@%-]{4,})" +_SECRETS = [ + (re.compile(r"\b(glpat|gldt|glrt|glptt|glcbt|gloas|glsoat|glffct|glimt|glagent)-[A-Za-z0-9_-]{8,}"), + r"\1-"), + (re.compile(r"\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}"), r"\1_"), + (re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}"), "github_pat_"), + (re.compile(r"\b(AKIA|ASIA)[A-Z0-9]{16}\b"), r"\1"), + (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), ""), + (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.S), + ""), + # Authorization: Bearer x, PRIVATE-TOKEN: x, Cookie: x + (re.compile(r"(?i)\b(authorization|private-token|job-token|x-api-key|cookie|set-cookie)" + r"(\s*[:=]\s*)(bearer\s+|basic\s+|token\s+)?\S+"), r"\1\2\3"), + # anything://user:pass@ and anything://token@ + (re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]+@"), r"\1@"), + # ?private_token=x&access_token=x + (re.compile(r"(?i)([?&][\w.-]*(?:token|key|secret|password|sig|signature)=)[^&\s\"']+"), + r"\1"), + # --token x, --password=x, -p x / -px (mysql style) after a password-ish option + (re.compile(r"(?i)(--[\w-]*(?:token|password|passwd|secret|api-key)[\w-]*)([= ]\s*)\S+"), + r"\1\2"), + # KEY=value, KEY: value, "key": "value" (environment, YAML, JSON) + (re.compile(r"(?i)(\b" + _KEY + r"[\"']?)(\s*[=:]\s*)" + _VALUE), r"\1\2"), +] + + +def redact(text): + """The text with credentials replaced: tokens of the known forms, auth + headers, URL user:password, key=value secrets, private keys.""" + for rx, repl in _SECRETS: + text = rx.sub(repl, text) + return text + + +# --- parsing logs --------------------------------------------------------------- + +_ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07]*\x07") +_SECTION = re.compile(r"section_(start|end):\d+:[A-Za-z0-9_.-]+(\[[^\]]*\])?\r?") +_FAILED = re.compile(r"BUILD FAILED:\s*(\S+?)@(\S+)") +_BITS_VERSION = re.compile(r"^\s*bits:\s*(.+?)\s*$", re.M) +_TRUNCATED = "Job's log exceeded limit" +# bits' logger prefixes each line of an error with its level ("ERROR: ...", or in +# debug "2026-10-11@09:00:01:ERROR::: ..."). +_LEVEL = re.compile(r"^(?:\d{4}-\d\d-\d\d@\d\d:\d\d:\d\d:)?(?:ERROR|CRITICAL)(?::[^\s:]+){0,3}: ?") +# GitLab runner's own lines after the script: not the failure. +_RUNNER = re.compile(r"^(Uploading artifacts|Downloading artifacts|Cleaning up|Saving cache|" + r"Restoring cache|Running after_script|Job succeeded|(ERROR: )?Job failed: exit code|" + r"WARNING: .*(artifact|cache)|[a-z_./-]+: found \d+ matching|" + r"Uploading artifacts as|Executing \"step_|Not uploading)", re.I) +# Where bits' failure message ends (its next headings). +_BLOCK_END = re.compile(r"^\s*(Environment:|Development Packages:|Next Steps|BUILD FAILED:)") + + +def clean(raw): + """Log bytes as text lines: ANSI colours, GitLab section markers and progress + redraws (carriage returns) taken out.""" + text = raw.decode("utf-8", "replace") if isinstance(raw, (bytes, bytearray)) else raw + text = _SECTION.sub("", _ANSI.sub("", text)) + return [_LEVEL.sub("", line.rstrip("\r").rsplit("\r", 1)[-1]) for line in text.split("\n")] + + +def parse(lines, before=40): + """[(package, version, excerpt, bits_version)] for each BUILD FAILED block in + the log lines; [] when there is none. The excerpt is bits' own "Error + excerpt" (to its next heading), else the lines just before the block.""" + found = {} + for i, line in enumerate(lines): + m = _FAILED.search(line) + if not m: + continue + block, in_excerpt, bits_version = [], False, "" + for nxt in lines[i + 1:i + 400]: + if _FAILED.search(nxt): + break + v = _BITS_VERSION.match(nxt) + if v: + bits_version = v.group(1) + if nxt.strip().startswith("Error excerpt:"): + in_excerpt = True + continue + if in_excerpt and _BLOCK_END.match(nxt): + in_excerpt = False + if in_excerpt: + block.append(nxt) + excerpt = "\n".join(block).strip("\n") \ + or "\n".join(lines[max(0, i - before):i]).strip("\n") + # A package reported twice (the run summary repeats it): the last one wins. + found[m.group(1)] = (m.group(1), m.group(2), excerpt, bits_version) + return list(found.values()) + + +def last_lines(lines, n=40): + """The job's last lines, without GitLab runner's own trailer.""" + keep = [ln for ln in lines if ln.strip() and not _RUNNER.match(ln.strip())] + return "\n".join(keep[-n:]) + + +# --- fingerprints --------------------------------------------------------------- + +_NORMALIZE = [ + (re.compile(r"(?i)\b(correlation_id|request_id|job_id|pipeline_id)=\S+"), r"\1="), + (re.compile(r"\bcc[A-Za-z0-9]{6}\.(?:s|o|res|ltrans\d*\.\w+)\b"), ""), + (re.compile(r"\b(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{16,}\b"), ""), + (re.compile(r"\b\d{4}-\d\d-\d\d[T ]\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:?\d\d)?\b"), "