diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index f903e654b..b0ce60a1f 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -88,6 +88,8 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" +# shellcheck source=stage-rows.sh +. "$SRC/pipeline/stage-rows.sh" # shellcheck source=engine.conf . "$SRC/pipeline/engine.conf" # Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). @@ -817,7 +819,7 @@ while IFS=$'\t' read -r rel csv; do # deterministically on some inputs — takes down a run of tens of thousands of sites. The # header's field count is the contract; a row that does not meet it is dropped and # COUNTED, so the loss is visible rather than fatal and never silent. - if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" + if [ -f "$CLIENT/$csv.csv" ]; then stage_rows "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" else : > "$FACTS/$rel.facts"; fi CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n' done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi) @@ -879,7 +881,7 @@ if [ ! -d "$LIBDIR" ]; then : > "$TMPDIR_L/$rel.facts" for root in "${LIB_ROOTS[@]}"; do while IFS= read -r mod; do - [ -f "$mod/$csv.csv" ] && awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" + [ -f "$mod/$csv.csv" ] && stage_rows "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" done < <(lib_modules "$root") done done < <(printf '%s' "$LIB_SIG_RELS") diff --git a/graph/pipeline/stage-rows.sh b/graph/pipeline/stage-rows.sh new file mode 100644 index 000000000..200ff7670 --- /dev/null +++ b/graph/pipeline/stage-rows.sh @@ -0,0 +1,50 @@ +# stage-rows.sh: copy one IR CSV into a Soufflé fact file, dropping torn rows. Sourced by run-souffle.sh. +# +# stage_rows FILE print FILE's rows without its header, keeping only the rows whose tab-separated +# field count equals the header's; report how many were dropped on stderr. +# +# The filter is line based: a row is one newline-terminated line, a field count is the number of tabs +# plus one (an empty line has none), and a last line without a newline is printed with one. Those are +# exactly awk -F'\t' semantics, which is what this step used to run. +# +# PERL FIRST, AWK AS THE FALLBACK. The two print byte-identical output, but BSD awk (macOS) spends most +# of a build's staging time splitting fields it never reads: 13.6 CPU-s on a 352 MB expressions file, +# against 0.65 s for perl, which only counts tabs. perl ships with macOS, every mainstream Linux +# distribution and Git for Windows' bash (the shell this runs under on win32); a minimal image without +# it (Alpine, distroless) gets the same rows from awk, just slower. graph/test/tools/stage-rows-test.sh +# holds both to the same bytes on the edge cases (trailing tabs, empty lines, CRLF, no final newline). +# +# AXIOM_STAGE_WITH=awk forces the fallback (the test uses it; so can anyone bisecting a staging change). + +if [ "${AXIOM_STAGE_WITH:-}" != awk ] && command -v perl >/dev/null 2>&1 && perl -e 1 >/dev/null 2>&1; then + STAGE_ROWS_TOOL=perl +else + STAGE_ROWS_TOOL=awk +fi + +stage_rows(){ + if [ "$STAGE_ROWS_TOOL" = perl ]; then + # Bytes in, bytes out (no PerlIO layers, whatever PERL_UNICODE says), and $/ pinned to "\n". + perl -e ' + my $f = $ARGV[0]; + open(my $h, "<:raw", $f) or die "cannot read $f: $!\n"; + binmode STDOUT, ":raw"; + local $/ = "\n"; + my $hdr = <$h>; + exit 0 unless defined $hdr; + my $hl = length($hdr) - (substr($hdr, -1) eq "\n" ? 1 : 0); + my $n = $hl ? ($hdr =~ tr/\t//) + 1 : 0; + my $bad = 0; + while (my $l = <$h>) { + my $nl = substr($l, -1) eq "\n"; + my $nf = (length($l) - ($nl ? 1 : 0)) ? ($l =~ tr/\t//) + 1 : 0; + if ($nf == $n) { print($nl ? $l : "$l\n") } else { $bad++ } + } + close($h); + close(STDOUT) or die "cannot write the staged rows of $f: $!\n"; + printf STDERR " ! dropped %d malformed row(s) from %s\n", $bad, $f if $bad > 0; + ' "$1" + else + awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$1" + fi +} diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 18db81e1d..e2a48d386 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -99,6 +99,13 @@ fi if ! bash "$ROOT/graph/test/tools/lib-cache-key-test.sh"; then echo "FAIL: the library-facts cache key is not a function of the staged modules' content (#588)"; exit 1 fi +# ── Staging must not depend on which row filter the machine has ───────────── +# graph/pipeline/stage-rows.sh runs perl where it exists and awk where it does not; both must stage +# the same bytes, or the facts every language solves from would differ by machine. +if ! bash "$ROOT/graph/test/tools/stage-rows-test.sh"; then + echo "aborting: perl and awk stage different rows from the same IR" + exit 1 +fi # ── The -I for soufflé's headers must be the one that actually compiles ────── # Also a preflight, and for the same reason: the old resolution returned a path that only built on # the machine it was written on, and every run here compiles the engine through it. diff --git a/graph/test/tools/stage-rows-test.sh b/graph/test/tools/stage-rows-test.sh new file mode 100644 index 000000000..67abcca69 --- /dev/null +++ b/graph/test/tools/stage-rows-test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Staging copies every IR CSV into a Soufflé fact file and drops the rows whose field count differs +# from the header's (graph/pipeline/stage-rows.sh). It runs perl where perl exists and awk where it +# does not, so the two must print the same bytes: a difference would make the facts, and so the +# graph, a function of which tools the machine happens to have. +# +# Each fixture below is staged by both, and the stdout bytes and the stderr report must match. The +# fixtures are the shapes where a line-based tab counter and awk's field splitting could part: a +# trailing tab (an empty last field), an empty line, CRLF, a last line with no newline, a header with +# no newline, an empty file, a quoted field holding a tab (both see two fields: neither parses quotes), +# bytes that are not UTF-8, and torn rows of both kinds (too few fields, too many). +# A NUL byte is left out on purpose: BSD awk ends the line at it (and drops the row as torn) while gawk +# and perl keep the whole line, so awk does not agree with itself there and is no reference. +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail +unset AXIOM_STAGE_WITH +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" + +command -v perl >/dev/null 2>&1 || { echo "stage-rows-test: FAILED (no perl on PATH; this test compares perl against awk)"; exit 1; } + +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +fail=0; checks=0 +bad(){ checks=$((checks+1)); printf ' FAIL %s\n' "$1"; fail=1; } +ok(){ checks=$((checks+1)); } + +# The tool the sourced file picks: perl here, awk when forced. +( . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = perl ] ) && ok || bad "stage-rows.sh did not pick perl although perl is on PATH" +( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = awk ] ) && ok || bad "AXIOM_STAGE_WITH=awk did not force awk" + +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6\n' > "$W/clean.csv" +printf 'a\tb\tc\n1\t2\t3\n1\t2\n1\t2\t3\t4\n7\t8\t9\n' > "$W/torn.csv" +printf 'a\tb\tc\n1\t2\t\n\t\t\n1\t2\t3\t\n' > "$W/trailing-tab.csv" +printf 'a\tb\tc\n1\t2\t3\n\n4\t5\t6\n\n' > "$W/empty-line.csv" +printf 'a\tb\tc\r\n1\t2\t3\r\n4\t5\r\n' > "$W/crlf.csv" +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6' > "$W/no-final-newline.csv" +printf 'a\tb\tc\n1\t2' > "$W/torn-last-line.csv" +printf 'a\tb\tc' > "$W/header-only-no-newline.csv" +printf 'a\tb\tc\n' > "$W/header-only.csv" +: > "$W/empty.csv" +printf 'a\tb\n"x\ty"\tz\n"p\tq"\n' > "$W/quoted-tab.csv" +printf 'a\tb\n\xff\xfe\t\xc3\x28\n' > "$W/bytes.csv" +printf 'a\n1\n\n2\n' > "$W/one-column.csv" +printf '\n\n1\n\n' > "$W/empty-header.csv" + +n=0 +for f in "$W"/*.csv; do + n=$((n+1)); name="$(basename "$f")" + ( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/p.out" 2> "$W/p.err"; prc=$? + ( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/a.out" 2> "$W/a.err"; arc=$? + [ "$prc" = 0 ] && [ "$arc" = 0 ] && ok || bad "$name: exit status perl=$prc awk=$arc" + cmp -s "$W/p.out" "$W/a.out" && ok || { bad "$name: staged rows differ"; od -c "$W/a.out" | sed 's/^/ awk /'; od -c "$W/p.out" | sed 's/^/ perl /'; } + cmp -s "$W/p.err" "$W/a.err" && ok || bad "$name: report differs: perl '$(cat "$W/p.err")' awk '$(cat "$W/a.err")'" +done + +# Not vacuous: the fixtures must hold rows that are kept and rows that are dropped. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/torn.csv" ) > "$W/t.out" 2> "$W/t.err" +[ "$(printf '1\t2\t3\n7\t8\t9\n')" = "$(cat "$W/t.out")" ] && ok || bad "torn.csv: expected the two whole rows, got '$(cat "$W/t.out")'" +grep -q "dropped 2 malformed row(s) from $W/torn.csv" "$W/t.err" && ok || bad "torn.csv: expected a report of 2 dropped rows, got '$(cat "$W/t.err")'" +# The control: a clean file comes through as its rows, header removed, nothing reported. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/clean.csv" ) > "$W/c.out" 2> "$W/c.err" +tail -n +2 "$W/clean.csv" | cmp -s - "$W/c.out" && [ ! -s "$W/c.err" ] && ok || bad "clean.csv: not passed through unchanged" + +[ "$n" -ge 14 ] || bad "only $n fixtures were staged" +[ $fail = 0 ] && echo "stage-rows-test: ok ($checks checks over $n fixtures)" || { echo "stage-rows-test: FAILED"; exit 1; } diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index 7a52d595c..28ec84d31 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -316,14 +316,16 @@ if [ -f "$OUT/corrupt" ]; then echo "rebuilding: $(head -1 "$OUT/corrupt")"; fi if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && NEWER="$(AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" newer "$REPO")"; then echo "$NEWER"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then +# AXIOMCODE_REINDEX=1 IS A REBUILD, as every message that names it promises ("rebuilds it with this one", "to force"): +# it skips both up-to-date answers below, which otherwise said "graph up to date" whenever the files matched the table. +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi set_base_files has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true echo "graph up to date (${LANGS//,/, }) at $OUT/graph.sqlite"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index index 2751c4430..b5ddd518d 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index @@ -20,8 +20,7 @@ to --src). The language's IR is read through the ADAPTER table below — header positions — mirroring graph/bundle/languages.ts. When the IR is absent the index degrades: symbols come from methods/types alone, refs/literals/comments are empty, and index_meta says so. """ -import csv, os, re, sqlite3, sys, time, glob, collections, functools -csv.field_size_limit(10**9) +import os, re, sqlite3, sys, time, glob, collections, functools INDEX_VERSION = '9' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding; # v6: TypeScript class-property arrows named after their field; @@ -273,13 +272,40 @@ def is_test_file(fp): def rows(name): p = os.path.join(IR, name) if not HAVE_IR or not os.path.exists(p): return iter(()) - f = open(p, newline='', encoding='utf-8', errors='replace') - return csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE) + return ir_rows(p) + +def ir_rows(p): + """The rows of one IR file as dicts keyed by its header, EXACTLY as csv.DictReader(f, delimiter='\\t', + quoting=csv.QUOTE_NONE) over open(p, newline='') reads them, at about half the cost (the expressions file is + most of this script's time on a large repository). That reader treats a quote as an ordinary character, so a row + is one line and a field is the text between tabs: a line ends at \\n, \\r or \\r\\n (newline='' splits on all + three and keeps them, and each ends a record), a blank line is skipped, a short row has None for its missing + columns and a long one keeps the rest as a list under the key None. The writer's RFC 4180 quoting is NOT undone + here, as it was not before (see unquoted() below). tests/index_reader.py holds the two to the same dicts.""" + with open(p, newline='', encoding='utf-8', errors='replace') as f: + head = f.readline() + if not head: return + if head[-1:] == '\n': head = head[:-1] + if head[-1:] == '\r': head = head[:-1] + keys = head.split('\t') if head else [] + n = len(keys) + for line in f: + if line[-1:] == '\n': line = line[:-1] + if line[-1:] == '\r': line = line[:-1] + if not line: continue + v = line.split('\t') + d = dict(zip(keys, v)) + if len(v) != n: + if len(v) > n: d[None] = v[n:] + else: + for k in keys[len(v):]: d[k] = None + yield d # an older augment step left `symbols` as a VIEW; a name is dropped as whichever it is +dropped = False # whether this run frees pages: it decides the VACUUM at the end for name in ('symbols', 'refs', 'literals', 'comments', 'nesting', 'type_refs', 'decorations', 'skipped', 'index_meta', 'paths', 'callers', 'callees', 'source', 'sites'): kind = c.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() - if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}") + if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}"); dropped = True c.executescript(''' DROP TABLE IF EXISTS symbols; DROP TABLE IF EXISTS refs; DROP TABLE IF EXISTS literals; DROP TABLE IF EXISTS comments; DROP TABLE IF EXISTS nesting; DROP TABLE IF EXISTS type_refs; DROP TABLE IF EXISTS decorations; DROP TABLE IF EXISTS skipped; DROP TABLE IF EXISTS index_meta; DROP TABLE IF EXISTS paths; @@ -765,11 +791,21 @@ if dec: c.executemany("INSERT INTO skipped VALUES (?,?)", ((rel(r.get('filePath') or r.get('file') or ''), r.get('reason') or r.get('skipReason') or '') for r in rows(A['skipped']))) # ── views: the joins written once ──────────────────────────────────────────────────────────────── +# Who calls a method and what it calls are looked up by call_edges(callee_method_id) and call_edges(caller_id); the +# fields a caller reads by field_access(caller_id): via_base_rows asks per single-target call site, and without that +# index each ask scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm +# → 1.4 s). The bundle indexes all three columns itself (idx__, schema.ts `indexed`), so they are made +# here only for a graph that lacks one: building the same index twice cost a second pass over the largest tables. +def has_index_on(table, cols): + for row in c.execute(f"PRAGMA index_list({table})").fetchall(): + if row[4] == 0 and [r[2] for r in c.execute(f"PRAGMA index_info({row[1]})")] == cols: return True # not partial + return False +for _name, _table, _col in (('ce_callee', 'call_edges', 'callee_method_id'), ('ce_caller', 'call_edges', 'caller_id'), ('fa_caller', 'field_access', 'caller_id')): + if not has_index_on(_table, [_col]): c.execute(f"CREATE INDEX IF NOT EXISTS {_name} ON {_table}({_col})") c.executescript(''' CREATE INDEX IF NOT EXISTS symbols_name ON symbols(name); CREATE INDEX IF NOT EXISTS symbols_display ON symbols(display); CREATE INDEX IF NOT EXISTS symbols_file ON symbols(file, line); CREATE INDEX IF NOT EXISTS symbols_mid ON symbols(method_id); CREATE INDEX IF NOT EXISTS symbols_tid ON symbols(type_id); CREATE INDEX IF NOT EXISTS refs_name ON refs(name); CREATE INDEX IF NOT EXISTS type_refs_name ON type_refs(name); CREATE INDEX IF NOT EXISTS decorations_owner ON decorations(owner_id); CREATE INDEX IF NOT EXISTS literals_value ON literals(value); CREATE INDEX IF NOT EXISTS comments_file ON comments(file); -CREATE INDEX IF NOT EXISTS ce_callee ON call_edges(callee_method_id); CREATE INDEX IF NOT EXISTS ce_caller ON call_edges(caller_id); -- lookups by id and by file + line: without them each per-site or per-test query scans the whole table CREATE INDEX IF NOT EXISTS symbols_id ON symbols(id); CREATE INDEX IF NOT EXISTS refs_file_line ON refs(file, line); CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); @@ -777,9 +813,6 @@ CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); -- the parts of a partial type (type_parts): without these each ask scanned type_refs / symbols, 1.2 s of a 1.8 s impact -- on a 1,300-file Java repository → 0.5-0.75 s, answers unchanged CREATE INDEX IF NOT EXISTS type_refs_file_line ON type_refs(file, line); CREATE INDEX IF NOT EXISTS symbols_qn ON symbols(qualified_name); --- the fields a caller reads: via_base_rows asks per single-target call site, and without the index each ask --- scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm → 1.4 s) -CREATE INDEX IF NOT EXISTS fa_caller ON field_access(caller_id); -- who calls a method: one row per (site, target), with the engine's tier. caller_id is symbols.id: a -- Java call written in a field initializer has the TYPE as its caller, and that row is kept, not dropped CREATE VIEW callers AS @@ -804,7 +837,12 @@ meta = dict(language=LANG, repo=REPO, source_dir=SRC, ir=IR if HAVE_IR else '', symbols=n('symbols'), refs=n('refs'), type_refs=n('type_refs'), decorations=n('decorations'), literals=n('literals'), comments=n('comments'), nested_types=n('nesting'), skipped_files=n('skipped')) c.executemany("INSERT INTO index_meta VALUES (?,?)", [(k, str(v)) for k, v in meta.items()]) con.commit() -con.execute("VACUUM") # the tables above are dropped and recreated on every run; without this the file only grows +# The tables above are dropped and recreated on every run, so a re-index leaves their old pages free and the file only +# grows without a VACUUM. A graph the bundle has just written has nothing to reclaim: no table of this script existed +# in it, and nothing freed a page (freelist_count). Rewriting it anyway cost 4.2 s of a 12 s index on a ~1,300-file +# Python repository, for the same rows. +if dropped or c.execute("PRAGMA freelist_count").fetchone()[0] > 0: + con.execute("VACUUM") con.close() # THE ASSERTED LINKS (ax_links.py, axiomcode-links.tsv) are applied to every graph as it is indexed, after the engine's # export and before the graph is published, so a rebuild keeps them and no query ever sees a graph without them: each is diff --git a/tests/README.md b/tests/README.md index 06150d2fd..bd5377d2d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -26,6 +26,9 @@ One check needs no graph and is its own script: signature edit is reported before it lands with nothing upstream did; `--range` from a branch left behind its remote reads from the remote's fork (indexes a small project) + python3 tests/index_reader.py axiomcode-index's IR reader returns the same rows csv.DictReader did, on quoted tabs + and newlines, CRLF, a lone CR, a short or long row and invalid UTF-8; pass IR + directories to compare them too (no graph needed) python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine) diff --git a/tests/index_reader.py b/tests/index_reader.py new file mode 100644 index 000000000..bc43878aa --- /dev/null +++ b/tests/index_reader.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""tests/index_reader.py: axiomcode-index reads the IR exactly as csv.DictReader did, row for row and key for key. + +axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced +csv.DictReader(f, delimiter='\\t', quoting=csv.QUOTE_NONE) because the DictReader was most of the index's time on a +large repository. The index's tables are made of these dicts, so any difference is a different graph. Each fixture is +read by both and the lists of dicts must be equal: the shapes where a line splitter and the csv module could part +are a quoted field holding a tab or a newline (QUOTE_NONE does not unquote, so both split it), CRLF, a lone CR, a +blank line, a short row (None for the missing columns), a long row (the rest under the key None), a repeated header +name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not UTF-8, and the +Unicode line separators that str.splitlines() would split on and the csv module does not. + + python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared as well +""" +import ast, csv, glob, itertools, os, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-index') +csv.field_size_limit(10**9) +MISSING = object() + + +def load_ir_rows(): + """ir_rows() out of the script without running it (the script indexes a graph at import).""" + tree = ast.parse(open(SCRIPT, encoding='utf-8').read()) + fn = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'ir_rows'] + if not fn: + sys.exit('FAIL: axiomcode-index defines no ir_rows()') + ns = {} + exec(compile(ast.Module(body=fn, type_ignores=[]), SCRIPT, 'exec'), ns) + return ns['ir_rows'] + + +def dictreader(p): + with open(p, newline='', encoding='utf-8', errors='replace') as f: + return list(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)) + + +FIXTURES = { + 'plain': b'a\tb\tc\n1\t2\t3\n4\t5\t6\n', + 'quoted-tab': b'a\tb\tc\n"x\ty"\t2\t3\n1\t"p\tq"\t3\n', + 'quoted-newline': b'a\tb\tc\n"line one\nline two"\t2\t3\n4\t5\t6\n', + 'quoted-crlf-inside': b'a\tb\n"x\r\ny"\tz\n', + 'doubled-quotes': b'a\tb\n"@r.post(""/orders"")"\t1\n"""Fee"""\t2\n', + 'crlf': b'a\tb\tc\r\n1\t2\t3\r\n4\t5\t6\r\n', + 'lone-cr': b'a\tb\r1\t2\r3\t4\r', + 'mixed-endings': b'a\tb\n1\t2\r\n3\t4\r5\t6\n', + 'blank-lines': b'a\tb\n\n1\t2\n\r\n\n3\t4\n\n', + 'short-row': b'a\tb\tc\n1\t2\n1\n', + 'long-row': b'a\tb\n1\t2\t3\t4\n', + 'trailing-tab': b'a\tb\n1\t\n\t\n', + 'repeated-header': b'id\tname\tid\n1\tx\t2\n3\n', + 'blank-header': b'\n1\t2\n3\n', + 'empty': b'', + 'header-only': b'a\tb\n', + 'header-only-no-newline': b'a\tb', + 'no-final-newline': b'a\tb\n1\t2\n3\t4', + 'not-utf8': b'a\tb\n\xff\xfe\t\xc3\x28\n', + 'unicode-separators': 'a\tb\nx
y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), + 'spaces': b'a\tb\n 1 \t 2 \n', +} + + +def main(argv): + ir_rows = load_ir_rows() + fails = checks = 0 + rows_compared = 0 + with tempfile.TemporaryDirectory() as w: + for name, data in FIXTURES.items(): + p = os.path.join(w, name + '.csv') + open(p, 'wb').write(data) + want, got = dictreader(p), list(ir_rows(p)) + checks += 1; rows_compared += len(want) + if want != got: + fails += 1 + print(f'FAIL {name}: csv.DictReader {want!r}\n ir_rows {got!r}') + # not vacuous: the fixtures hold rows, and the shapes the reader special-cases are among them + p = os.path.join(w, 'short-row.csv') + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', 'c': None}, {'a': '1', 'b': None, 'c': None}]: + fails += 1; print(f'FAIL short-row: {list(ir_rows(p))!r}') + p = os.path.join(w, 'long-row.csv') + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', None: ['3', '4']}]: + fails += 1; print(f'FAIL long-row: {list(ir_rows(p))!r}') + # a real IR: every file of every directory named on the command line + files = 0 + for d in argv: + for p in sorted(glob.glob(os.path.join(d, '*.csv'))): + files += 1; checks += 1 + n = 0 + with open(p, newline='', encoding='utf-8', errors='replace') as f: # streamed: an IR file can be 10^6 rows + for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE), ir_rows(p), fillvalue=MISSING): + n += 1 + if a != b: + fails += 1; print(f'FAIL {p} row {n}: csv.DictReader {a!r}\n ir_rows {b!r}'); break + rows_compared += n + if rows_compared < 20: + fails += 1; print(f'FAIL only {rows_compared} rows compared') + print(f"index_reader: {'ok' if not fails else 'FAILED'} ({checks - fails}/{checks} checks, {rows_compared} rows," + f" {len(FIXTURES)} fixtures{f', {files} IR files' if files else ''})") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tests/refresh.py b/tests/refresh.py index 2a02b7c6a..405029879 100644 --- a/tests/refresh.py +++ b/tests/refresh.py @@ -3,7 +3,7 @@ Each language gets a throwaway git repository with a real graph. Then, per language: - up to date `index` with nothing changed does not rebuild + up to date `index` with nothing changed does not rebuild; with AXIOMCODE_REINDEX=1 it does query an edit adds a function; a query verb, which starts the refresher and waits for it, finds it baseline `changed` answers the same before and after the refresh absorbed the edit: the refresh moves the graph, not the baseline edits are measured against. The answer must name the edit (a pass on two @@ -110,6 +110,13 @@ def check(ok, why, detail=''): check(m.get('refresh_reason') == 'axiomcode index' and m.get('refreshed_at', '').endswith('Z'), f'{lang}: the graph records when and why it was built', json.dumps(m)) t0 = time.time(); again = sh(repo, AX, 'index', '.', '--lang', lang, env=env); took = time.time() - t0 check('graph up to date' in again.stdout, f'{lang}: `index` with nothing changed does not rebuild ({took:.1f}s)', again.stdout + again.stderr) + # AXIOMCODE_REINDEX=1 rebuilds with nothing changed, as SKILL.md and the build's own messages promise; it used to + # answer "graph up to date" whenever the files matched the table. The control is the check above: the same + # `index` without it does not rebuild + built = meta().get('refreshed_at') + rx = sh(repo, AX, 'index', '.', '--lang', lang, env=dict(env, AXIOMCODE_REINDEX='1')) + check(rx.returncode == 0 and 'graph up to date' not in rx.stdout and 'building' in rx.stdout and meta().get('refreshed_at') != built, + f'{lang}: AXIOMCODE_REINDEX=1 `index` rebuilds although nothing changed', rx.stdout + rx.stderr) # ── built by an older axiomcode ──────────────────────────────────────────────────────────────────── # no file changed, but the table says another engine built the graph (what an update leaves behind): not up