From 3ad6c9b369574af3a299cdf8ac1ca21e06c136cc Mon Sep 17 00:00:00 2001 From: Calvin Secrest Date: Tue, 13 Oct 2026 22:46:11 -0400 Subject: [PATCH] Add files via upload Signed-off-by: Calvin Secrest --- CONTRIBUTING.md | 2 + README.md | 11 +++ jfa-conformance-suite.py | 171 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 175 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e37016..fdb6e92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,8 @@ git push --force-with-lease ``` python jfa-conformance-suite.py # check the document python jfa-conformance-suite.py --list # print the invariant registry +python jfa-conformance-suite.py --bindings ../Cloudy # report which delegated + # invariants a repo binds ``` Exit code 0 when every executed check passes, 1 otherwise. It runs on every diff --git a/README.md b/README.md index 84f5fce..82e08c5 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,19 @@ python jfa-conformance-suite.py # check the document python jfa-conformance-suite.py --list # print the invariant registry python jfa-conformance-suite.py --doc PATH # check another copy python jfa-conformance-suite.py --project PATH # check a repo's open-questions deliverable +python jfa-conformance-suite.py --bindings ../Cloudy [../other-repo ...] + # also report which delegated + # invariants those repos bind ``` +`--bindings` scans implementation repositories you have cloned beside this one +for test functions named `TestInvariant__` (with any `-` in the ID +written `_`) and marks each implementation invariant they cite as **BOUND** — +the binding the bylaws define in P1-001 §9.14(b). It reports that a citing test +*exists*; whether it passes is that repository's own CI, so the suite never +reports a delegated invariant as passing (§9.15). A test citing an unregistered +ID fails the run. + Exit code 0 when every executed check passes, 1 otherwise. Run it after any edit to the official document. diff --git a/jfa-conformance-suite.py b/jfa-conformance-suite.py index a38f180..9c585ec 100644 --- a/jfa-conformance-suite.py +++ b/jfa-conformance-suite.py @@ -47,6 +47,27 @@ python jfa-conformance-suite.py --list # print the registry python jfa-conformance-suite.py --project PATH # check a project repo's # open-questions deliverable + python jfa-conformance-suite.py --bindings REPO [REPO ...] + # also report which delegated + # invariants those repos bind + +What --bindings does, and what it does not: + Given one or more implementation repositories (local paths), it scans + their test files for test functions named TestInvariant__, + with any "-" in the ID written "_", and marks each registered + implementation invariant those names cite as BOUND. That is the binding + P1-001 §9.14(b) defines: "complete when a repository ships tests citing + the identifier." It does NOT run those tests: whether a bound test + passes is reported by that repository's own CI, never here, so the + label says "test present" and nothing more (§9.15). A citation of an + unregistered ID fails the run, since a misspelled ID would otherwise + bind nothing silently. A test citing an instrument- or document-bound + ID is listed but binds nothing: instruments bind through their venue + tooling (§9.14(c)). Recognized test files: Go *_test.go + ("func TestInvariant_..."), and JavaScript/TypeScript *.test.* and + *.spec.* ("test(" / "it(" / "describe(" with a "TestInvariant_..." + title). Standard library only; offline; no toolchain for the scanned + language is needed. Exit code 0 when every executed check passes; 1 otherwise. @@ -395,6 +416,91 @@ def check_project(path): return results +# -------------------------------------------------------------------------- +# Bindings: which delegated invariants implementation repos cite in tests +# -------------------------------------------------------------------------- + +Citation = namedtuple("Citation", "repo path test inv_id") + +SKIP_DIRS = {".git", "vendor", "node_modules", "testdata", "dist", "build"} + +GO_TEST = re.compile(r"^func\s+(TestInvariant_\w+)\s*\(\s*\w+\s+\*testing\.T\s*\)", re.M) +JS_TEST = re.compile(r"\b(?:test|it|describe)\s*\(\s*[\'\"`](TestInvariant_\w+)") + + +def _is_test_file(name): + if name.endswith("_test.go"): + return GO_TEST + if re.search(r"\.(test|spec)\.(js|jsx|ts|tsx|mjs|cjs)$", name): + return JS_TEST + return None + + +def resolve_test_id(test_name): + """Map TestInvariant__ to a registered ID by the longest + registered ID (with "-" written "_") that the name starts with and that + is followed by "_" or the end. Returns None when nothing matches.""" + rest = test_name[len("TestInvariant_"):] + best = None + for inv in REGISTRY: + key = inv.id.replace("-", "_") + if rest == key or rest.startswith(key + "_"): + if best is None or len(key) > len(best.id.replace("-", "_")): + best = inv + return best.id if best else None + + +def scan_bindings(repo): + """Return (citations, unknown) for one repository path: every + TestInvariant_ test function found in its test files, resolved to a + registry ID, and the test names that cite no registered ID.""" + citations, unknown = [], [] + label = os.path.basename(os.path.abspath(repo)) + for root, dirs, files in os.walk(repo): + dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) + for name in sorted(files): + pattern = _is_test_file(name) + if pattern is None: + continue + path = os.path.join(root, name) + rel = os.path.relpath(path, repo).replace(os.sep, "/") + text = io.open(path, encoding="utf-8", errors="replace").read() + for test in pattern.findall(text): + inv_id = resolve_test_id(test) + if inv_id is None: + unknown.append(Citation(label, rel, test, None)) + else: + citations.append(Citation(label, rel, test, inv_id)) + return citations, unknown + + +def check_bindings(repos): + """Scan every repo; return (results, citations_by_id, informational).""" + results, by_id, info = [], {}, [] + binding_of = dict((inv.id, inv.binding) for inv in REGISTRY) + for repo in repos: + if not os.path.isdir(repo): + results.append(Result("B1 bindings repo readable: %s" % repo, False, + "not a directory")) + continue + citations, unknown = scan_bindings(repo) + results.append(Result("B1 bindings repo readable: %s" % repo, True, + "%d TestInvariant_ tests found" % (len(citations) + len(unknown)))) + if unknown: + results.append(Result( + "B2 every TestInvariant_ test cites a registered ID: %s" % repo, False, + "; ".join("%s (%s)" % (c.test, c.path) for c in unknown))) + else: + results.append(Result( + "B2 every TestInvariant_ test cites a registered ID: %s" % repo, True)) + for c in citations: + if binding_of[c.inv_id] == "implementation": + by_id.setdefault(c.inv_id, []).append(c) + else: + info.append(c) + return results, by_id, info + + # -------------------------------------------------------------------------- # Reporting # -------------------------------------------------------------------------- @@ -411,18 +517,52 @@ def print_results(title, results): print() -def print_delegation(): +def print_delegation(bound=None, info=None): + """List delegated invariants. With bound (from --bindings), mark those a + scanned repo cites in a test as BOUND — test present, not executed here.""" delegated = [inv for inv in REGISTRY if inv.binding != "document"] - print("Delegated invariants (labeled stand-ins; unbound until a repo or") - print("instrument ships checks citing these IDs)") + if bound is None: + print("Delegated invariants (labeled stand-ins; unbound until a repo or") + print("instrument ships checks citing these IDs)") + print("-" * 60) + for inv in delegated: + print("[DELEGATED:%s] %-22s %s" % (inv.binding, inv.id, inv.note)) + print() + print("%d of %d registered invariants are delegated; this suite binds the" + % (len(delegated), len(REGISTRY))) + print("document layer only. A conformant repo cites the IDs above in its") + print("own test suite; until then, its conformance is self-attested.") + print() + return + + print("Delegated invariants and their bindings (BOUND = a scanned repo ships") + print("a test citing the ID; this suite does not run it -- pass/fail is that") + print("repo's own CI. Everything else stays self-attested.)") print("-" * 60) for inv in delegated: - print("[DELEGATED:%s] %-22s %s" % (inv.binding, inv.id, inv.note)) + cites = bound.get(inv.id, []) + if cites: + where = ", ".join("%s/%s %s" % (c.repo, c.path, c.test) for c in cites) + print("[BOUND:%s] %-22s %s" % (inv.binding, inv.id, where)) + else: + print("[DELEGATED:%s] %-22s %s" % (inv.binding, inv.id, inv.note)) print() - print("%d of %d registered invariants are delegated; this suite binds the" - % (len(delegated), len(REGISTRY))) - print("document layer only. A conformant repo cites the IDs above in its") - print("own test suite; until then, its conformance is self-attested.") + for c in info or []: + print("[NOTE] %s/%s %s cites %s, which binds through an instrument or the" + % (c.repo, c.path, c.test, c.inv_id)) + print(" document, not a test; listed, not counted") + if info: + print() + impl = [inv for inv in delegated if inv.binding == "implementation"] + inst = [inv for inv in delegated if inv.binding == "instrument"] + n_impl = sum(1 for inv in impl if inv.id in bound) + print("%d of %d implementation-bound invariants are bound by tests (test" + % (n_impl, len(impl))) + print("present; results come from each repo's CI). %d of %d instrument-bound" + % (0, len(inst))) + print("invariants are bound here: instruments bind through their venue tooling.") + print("%d of %d delegated invariants remain self-attested." + % (len(delegated) - n_impl, len(delegated))) print() @@ -433,6 +573,9 @@ def main(): ap.add_argument("--project", help="path to a project repo to check for the " "open-questions deliverable") ap.add_argument("--list", action="store_true", help="print the invariant registry") + ap.add_argument("--bindings", nargs="+", metavar="REPO", + help="implementation repo path(s) to scan for TestInvariant_ " + "tests; reports which delegated invariants they bind") args = ap.parse_args() if args.list: @@ -444,6 +587,10 @@ def main(): all_ok = True + if args.project and args.bindings: + print("--bindings reports against the document; run it without --project") + return 1 + if args.project: results = check_project(args.project) print_results("PROJECT CHECKS: %s" % args.project, results) @@ -468,8 +615,14 @@ def main(): check_own_open_questions(doc, args.doc), ] print_results("DOCUMENT CHECKS: %s" % os.path.basename(args.doc), results) - print_delegation() all_ok = all(r.ok for r in results) + if args.bindings: + b_results, bound, info = check_bindings(args.bindings) + print_results("BINDINGS CHECKS", b_results) + print_delegation(bound, info) + all_ok = all_ok and all(r.ok for r in b_results) + else: + print_delegation() print("RESULT: %s" % ("all executed checks PASS" if all_ok else "FAILURES present")) return 0 if all_ok else 1