From c5d95a284ed8a96496c19371a7e833108f5a9f65 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:27:32 -0700 Subject: [PATCH] plugin: axiomcode-index decodes quoted IR fields, so a literal with quotes is found The parser writes the IR as RFC 4180 over a tab: EntityUtils.escapeTsv turns newlines, CRs and tabs into two-character escapes first, then quotes a field that holds a double quote and doubles the quote. axiomcode-index read it with QUOTE_NONE semantics, so the bundle decoded these fields and the index did not: a literal `Order "A-1" Is Late` was stored with doubled quotes, `impact` on that string said it appears in no literal, `impact KEY` printed 'a \""b\"" c', a Java annotation was stored as @Route("""/x"""), and a C# member read on a string literal was named `Length"`. ir_rows() now decodes a field that opens with a quote exactly as the bundle's readRfc4180 does (a doubled quote is one, the closing quote ends the field, a quote elsewhere is text). A line with no quote is still a plain tab split, so the reader costs the same. A quoted field left open at a line end takes the next line, as the bundle's reader does; escapeTsv never writes one. The hand-written decoders that ran on the quoted text go: decorations' unquoted() (Python decorator text starts with @, so it can no longer fire), and the C# literal un-doubling, replaced by cs_string_text(), which reads the C# token: regular, verbatim (a doubled quote is one) and raw (as many quotes close it as open it). On decoded text the old un-doubling read the raw string """"a "" b"""" as "a " b" and left a verbatim string's doubled quotes. C# verbatim and raw strings are now literals. ax_registration's '"""' replace stays: a Python triple-quoted decorator path still needs it. INDEX_VERSION 10. On four measured projects every table but literals, comments and decorations is identical, and those now match an index read with csv QUOTE_MINIMAL row for row. Tests: tests/index_reader.py holds ir_rows to csv.DictReader and to the built bundle reader, with the decoded values written out; tests/cases quoted-literal-text for python, java and csharp, with controls. Refs #1835 --- .../skills/axiomcode/scripts/axiomcode-index | 97 ++++++++---- tests/README.md | 8 +- .../csharp/quoted-literal-text/case.json | 28 ++++ .../csharp/quoted-literal-text/src/Orders.cs | 20 +++ .../quoted-literal-text/src/Orders.csproj | 1 + .../quoted-literal-text/src/RouteAttribute.cs | 8 + .../cases/java/quoted-literal-text/case.json | 24 +++ .../quoted-literal-text/src/app/Orders.java | 20 +++ .../quoted-literal-text/src/app/Route.java | 10 ++ .../python/quoted-literal-text/case.json | 23 +++ .../quoted-literal-text/src/app/__init__.py | 0 .../quoted-literal-text/src/app/orders.py | 22 +++ tests/index_reader.py | 138 +++++++++++++++--- 13 files changed, 348 insertions(+), 51 deletions(-) create mode 100644 tests/cases/csharp/quoted-literal-text/case.json create mode 100644 tests/cases/csharp/quoted-literal-text/src/Orders.cs create mode 100644 tests/cases/csharp/quoted-literal-text/src/Orders.csproj create mode 100644 tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs create mode 100644 tests/cases/java/quoted-literal-text/case.json create mode 100644 tests/cases/java/quoted-literal-text/src/app/Orders.java create mode 100644 tests/cases/java/quoted-literal-text/src/app/Route.java create mode 100644 tests/cases/python/quoted-literal-text/case.json create mode 100644 tests/cases/python/quoted-literal-text/src/app/__init__.py create mode 100644 tests/cases/python/quoted-literal-text/src/app/orders.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index index b5ddd518d..28ac9874b 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index @@ -21,14 +21,16 @@ positions — mirroring graph/bundle/languages.ts. When the IR is absent the ind come from methods/types alone, refs/literals/comments are empty, and index_meta says so. """ import os, re, sqlite3, sys, time, glob, collections, functools -INDEX_VERSION = '9' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; +INDEX_VERSION = '10' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding; # v6: TypeScript class-property arrows named after their field; # v7: data keys of module-level const objects declared, Object.freeze seen through; # v8: literals carry numbers and booleans with a kind column, and the name of the constant # a scalar initializes — a spec's numbers were the one thing the graph could not answer; # v9: the web graph (HTML + CSS) is indexed: pages, stylesheets, .classes, #ids, custom properties (var(--x)), - # @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols); the query + # @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols; + # v10: the IR's RFC 4180 quoting is decoded, so a literal, comment, decoration or name holding a `"` + # is stored as written (`a "b" c`, not `a ""b"" c`), and C# verbatim and raw strings are literals); the query # frontend re-indexes an older graph when the IR is still there REPO = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].startswith('-') else os.environ.get('AXIOMCODE_REPO') or '.') @@ -192,7 +194,7 @@ A = { # name elsewhere is a by-name reader whatever its qualifier says, so `_r.Limit` was listed under AuditOptions.Limit memberName=dict(role='edgeRole', value='MEMBER_NAME', parent='parentExpressionHash', id='csExpressionUniqueHash'), # the literal TYPE column is literalKind here, not literalType as in java/typescript/python - litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}), + litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'VERBATIM_STRING': 'string', 'RAW_STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}), comments=dict(file='all-csharp-comments.csv', text='commentText', kind='commentKind', line='startLine', fileVia=('modules', 'csModuleLinkHash')), # type references carry no module link of their own. The owner is a type only for a base-list reference; for # `M()`, `new T()`, a parameter, a return or a field it is the expression, method, parameter or field, so @@ -275,25 +277,61 @@ def rows(name): return ir_rows(p) def ir_rows(p): - """The rows of one IR file as dicts keyed by its header, EXACTLY as csv.DictReader(f, delimiter='\\t', - quoting=csv.QUOTE_NONE) over open(p, newline='') reads them, at about half the cost (the expressions file is - most of this script's time on a large repository). That reader treats a quote as an ordinary character, so a row - is one line and a field is the text between tabs: a line ends at \\n, \\r or \\r\\n (newline='' splits on all - three and keeps them, and each ends a record), a blank line is skipped, a short row has None for its missing - columns and a long one keeps the rest as a list under the key None. The writer's RFC 4180 quoting is NOT undone - here, as it was not before (see unquoted() below). tests/index_reader.py holds the two to the same dicts.""" + """The rows of one IR file as dicts keyed by its header, its fields DECODED the way the bundle's reader + (graph/bundle/csv.ts readRfc4180) decodes them: the parser writes RFC 4180 over a tab, so a field that STARTS with + a double quote is quoted, `""` inside it is one quote, and the closing quote ends it (what follows the close, up + to the tab, is kept as written); a quote anywhere else in a field is an ordinary character. The writer + (EntityUtils.escapeTsv) turns a newline, a CR and a tab into the two-character escapes BEFORE it quotes, and quotes + only for a `"`, so a quoted field it wrote never holds a tab or a line break and a row is one line. Reading + QUOTE_NONE stored `Order "A-1"` as `Order ""A-1""`, so `impact` said the string was in no literal (#1835). + The rest is csv.DictReader(f, delimiter='\\t') over open(p, newline=''), at about half its cost (the expressions + file is most of this script's time on a large repository): a line ends at \\n, \\r or \\r\\n, a blank line is + skipped, a short row has None for its missing columns and a long one keeps the rest as a list under the key None. + A line with no quote in it is split on tabs and nothing else: that is nearly every line, and a test for a quote + that OPENS a field cost more than the decoding it saved. A quoted field left open at the end of a line, which this + writer does not produce, takes the line break and the next line, as the bundle's reader does. + tests/index_reader.py holds this to both readers.""" with open(p, newline='', encoding='utf-8', errors='replace') as f: - head = f.readline() + def fields(line): + # a line holding a quote: split on tabs, then decode each field that opens with one, rejoined across the tabs + # (and, if it is still open at the end of the line, the line breaks) it holds + out = []; parts = line.split('\t'); i = 0 + while True: + if i == len(parts): return out + s = parts[i]; i += 1 + if s[:1] != '"': out.append(s); continue + buf = []; k = 1 + while True: + j = s.find('"', k) + if j >= 0: + buf.append(s[k:j]) + if s[j + 1:j + 2] == '"': buf.append('"'); k = j + 2; continue + buf.append(s[j + 1:]); break # the closing quote; the rest of the field as written + buf.append(s[k:]); k = 0 + if i < len(parts): buf.append('\t'); s = parts[i]; i += 1; continue + buf.append(line_end[0]); nxt = f.readline() # still open at the end of the line: the break is text + if not nxt: break + body = nxt.rstrip('\r\n'); line_end[0] = nxt[len(body):] + parts = body.split('\t'); s = parts[0]; i = 1 + out.append(''.join(buf)) + def decoded(line, raw): + line_end[0] = raw[len(line):] + v = fields(line) + if v[-1][-1:] == '\r': v[-1] = v[-1][:-1] # as the bundle's reader drops a CR before the newline + return v + line_end = [''] + head = raw = f.readline() if not head: return if head[-1:] == '\n': head = head[:-1] if head[-1:] == '\r': head = head[:-1] - keys = head.split('\t') if head else [] + keys = (head.split('\t') if '"' not in head else decoded(head, raw)) if head else [] n = len(keys) - for line in f: + for raw in f: + line = raw if line[-1:] == '\n': line = line[:-1] if line[-1:] == '\r': line = line[:-1] if not line: continue - v = line.split('\t') + v = line.split('\t') if '"' not in line else decoded(line, raw) d = dict(zip(keys, v)) if len(v) != n: if len(v) > n: d[None] = v[n:] @@ -699,6 +737,20 @@ for name, od, v, k in data_keys: c.executemany("INSERT INTO symbols VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", sym) # ── references, literals, comments ─────────────────────────────────────────────────────────────── +def cs_string_text(v): + """The text of a C# string token, as ir_rows hands it over (the IR's RFC 4180 quoting already undone): `"x"` is x + with its escapes as written, as Java's and Python's are; a verbatim `@"x"` is x with a doubled quote made one, the + language's own escape; a raw string is x, closed by as many quotes as opened it (three or more: a regular string + cannot open with three, `""` is the whole empty string); a `$` is dropped. Before ir_rows decoded the IR, a C# + literal holding a quote was un-doubled here by hand; on decoded text that is a second decoding, which reads the + raw `\"\"\"\"a \"\" b\"\"\"\"` as `"a " b"`, and a verbatim string's doubled quote was never made one.""" + p = 0 + while p < len(v) and v[p] in '@$': p += 1 + t = v[p:] + q = len(t) - len(t.lstrip('"')) + if q >= 3 and len(t) >= 2 * q and t.endswith('"' * q): return t[q:-q] + if len(t) >= 2 and t[0] == '"' == t[-1]: return t[1:-1].replace('""', '"') if '@' in v[:p] else t[1:-1] + return v e = A['expr']; refs = []; lits = [] namecol = e['name'] if isinstance(e['name'], str) else dict(x.split(':') for x in e['name']) mname = e.get('memberName'); child_ek = {} @@ -727,11 +779,8 @@ for r in rows(e['file']): kind = e['litType'][1][r.get(e['litType'][0])] # 'string' | 'number' | 'bool' across every language v = (r.get(e['litValue']) or '') if kind == 'string': - # C# keeps the source token, and the IR writer RFC4180-quotes a field holding a double quote, so `"Fee"` arrived - # as `"""Fee"""` and was stored `""Fee""`: no key, route or member name written in a C# string ever matched - if LANG == 'csharp' and len(v) > 1 and v[0] == '"' == v[-1] and '""' in v: v = v[1:-1].replace('""', '"') - if LANG == 'csharp' and len(v) > 2 and v[0] in '@$' and v[-1] == '"': v = v.lstrip('@$') # @"verbatim", $"interpolated" - if len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1] # JavaScript keeps the quotes + if LANG == 'csharp': v = cs_string_text(v) # C# keeps the source token: "x", @"x", """x""" + elif len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1] # JavaScript keeps the quotes if v: lits.append((v[:200], file_of(r, e), int(r.get(e['line']) or 0), kind, r.get('edgeRole', ''))) # a scalar's NAME: `MAX_ITEMS = 5` — the one literal on a const/variable/field declaration's line IS its value, # so the row gets the symbol's display and "what is MAX_ITEMS" has an answer with a source location. Strictly: @@ -767,13 +816,6 @@ if tr: c.executemany("INSERT INTO type_refs VALUES (?,?,?,?,?)", ((r.get(tr['name'], '').split('.')[-1].split('<')[0], file_of(r, tr), int(r.get(tr['line']) or 0), r.get(tr['context'], ''), r.get(tr['ownerKind'], '')) for r in rows(tr['file']) if r.get(tr['name']))) except (KeyError, ValueError): pass # decorations: @Annotation(args) / @decorator(args) on a method or type — a route, a bean name, a validator, a marker -def unquoted(v): - """The IR is tab-separated, but the writer still RFC4180-quotes a field that contains a double quote and the - reader is QUOTE_NONE, so `@router.post("/orders")` arrives as `"@router.post(""/orders"")"`. Every decoration - whose argument is a double-quoted string is in that shape — which is exactly the decorations that carry a KEY - (a route path, a command name, a topic, a validated field), so everything downstream that reads a key out of - the text read the quoting instead: `@exporter("csv")` yielded `@exporter(` and `)` and never `csv`.""" - return v[1:-1].replace('""', '"') if len(v) > 1 and v[0] == '"' == v[-1] and '""' in v else v dec = A.get('decorations') if dec: args = collections.defaultdict(list) @@ -784,7 +826,8 @@ if dec: for r in rows(dec['file']): nm = r.get(dec['name'], '') if not nm: continue - text = unquoted(r.get(dec['text'], '')) if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else '')) + # the text as written (ir_rows undoes the writer's RFC 4180 quoting, so `@exporter("csv")` arrives as that) + text = r.get(dec['text'], '') if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else '')) try: drows.append((r.get(dec['owner'], ''), nm, text[:200], file_of(r, dec), int(r.get(dec['line']) or 0))) except ValueError: pass c.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", drows) diff --git a/tests/README.md b/tests/README.md index bd5377d2d..a9cac2520 100644 --- a/tests/README.md +++ b/tests/README.md @@ -26,9 +26,11 @@ One check needs no graph and is its own script: signature edit is reported before it lands with nothing upstream did; `--range` from a branch left behind its remote reads from the remote's fork (indexes a small project) - python3 tests/index_reader.py axiomcode-index's IR reader returns the same rows csv.DictReader did, on quoted tabs - and newlines, CRLF, a lone CR, a short or long row and invalid UTF-8; pass IR - directories to compare them too (no graph needed) + python3 tests/index_reader.py axiomcode-index's IR reader decodes the IR's RFC 4180 quoting as csv.DictReader and + the bundle's reader do (a literal, decoration or name holding a `"`, a C# raw + string), and agrees with them on quoted tabs and newlines, CRLF, a lone CR, a + short or long row and invalid UTF-8; pass IR directories to compare them too + (no graph needed; the bundle half runs node on dist/bundle/csv.js) python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine) diff --git a/tests/cases/csharp/quoted-literal-text/case.json b/tests/cases/csharp/quoted-literal-text/case.json new file mode 100644 index 000000000..c2ae33141 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/case.json @@ -0,0 +1,28 @@ +{"lang": "csharp", "src": "src", + "checks": [ + {"why": "a member read on a string literal is named by the member: the IR quotes the expression (RFC 4180) and the index kept a stray quote, `Length\"` (#1835)", + "run": ["impact", "Length"], "expect_error": true, + "want": ["[by name] Orders.LabelWidth"]}, + {"why": "a raw string is a literal, and its value is the text between the quotes that open and close it", + "run": ["impact", "Orders.RawKey"], + "want": ["const Orders.RawKey = 'order.created'"]}, + {"why": "a raw string holding a quote keeps it, once", + "run": ["impact", "Orders.Raw"], + "want": ["const Orders.Raw = 'raw \"x\" here'"]}, + {"why": "a raw string opened by four quotes may hold two, and both are kept: un-doubling quotes by hand on the decoded IR read it as `\"a \" b\"`", + "run": ["impact", "Orders.Raw4"], + "want": ["const Orders.Raw4 = 'a \"\" b'"]}, + {"why": "a verbatim string's doubled quote is one quote", + "run": ["impact", "Orders.Verbatim"], + "want": ["const Orders.Verbatim = 'say \"hi\" now'"]}, + {"why": "control: a regular string holding an escaped quote is printed as written, in single quotes", + "run": ["impact", "Orders.Key"], + "want": ["const Orders.Key = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "control: a string literal holding an escaped quote is found", + "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "Orders.Plain"], + "want": ["const Orders.Plain = 'a plain value'"]} + ]} diff --git a/tests/cases/csharp/quoted-literal-text/src/Orders.cs b/tests/cases/csharp/quoted-literal-text/src/Orders.cs new file mode 100644 index 000000000..474829f26 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/Orders.cs @@ -0,0 +1,20 @@ +namespace App; + +public class Orders +{ + public const string Key = "a \"b\" c"; + public const string Plain = "a plain value"; + public const string Verbatim = @"say ""hi"" now"; + public const string RawKey = """order.created"""; + public const string Raw = """raw "x" here"""; + public const string Raw4 = """"a "" b""""; + + string Send(string subject) => subject; + + [Route("/orders/{code}")] + public string NotifyLate() => Send("Order \"Foo\" Is Late"); + + public string Resolve() => Send("The order has shipped"); + + public int LabelWidth() => "header".Length; +} diff --git a/tests/cases/csharp/quoted-literal-text/src/Orders.csproj b/tests/cases/csharp/quoted-literal-text/src/Orders.csproj new file mode 100644 index 000000000..d3b2f0306 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/Orders.csproj @@ -0,0 +1 @@ +net8.0 diff --git a/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs b/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs new file mode 100644 index 000000000..5c155c401 --- /dev/null +++ b/tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs @@ -0,0 +1,8 @@ +namespace App; + +[System.AttributeUsage(System.AttributeTargets.Method)] +public class RouteAttribute : System.Attribute +{ + public RouteAttribute(string path) { Path = path; } + public string Path { get; } +} diff --git a/tests/cases/java/quoted-literal-text/case.json b/tests/cases/java/quoted-literal-text/case.json new file mode 100644 index 000000000..b771009ca --- /dev/null +++ b/tests/cases/java/quoted-literal-text/case.json @@ -0,0 +1,24 @@ +{"lang": "java", "src": "src", + "checks": [ + {"why": "a string literal holding an escaped double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)", + "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled", + "run": ["impact", "Orders.KEY"], + "want": ["const Orders.KEY = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "an annotation argument holding a quote is found as written: the decoration was stored with every quote doubled", + "run": ["impact", "\"/say \\\"hi\\\"\""], + "want": ["(0 literal site(s), 1 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "Orders.PLAIN"], + "want": ["const Orders.PLAIN = 'a plain value'"]}, + {"why": "control: a literal with no quote in it is found as before", + "run": ["impact", "\"The order has shipped\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "control: an annotation's quoted path still binds the method and registers it under the path", + "run": ["impact", "Orders.notifyLate"], + "want": ["registered by @Route \"/orders/{code}\""]} + ]} diff --git a/tests/cases/java/quoted-literal-text/src/app/Orders.java b/tests/cases/java/quoted-literal-text/src/app/Orders.java new file mode 100644 index 000000000..f8c987bfd --- /dev/null +++ b/tests/cases/java/quoted-literal-text/src/app/Orders.java @@ -0,0 +1,20 @@ +package app; + +public class Orders { + static final String KEY = "a \"b\" c"; + static final String PLAIN = "a plain value"; + + String send(String subject) { + return subject; + } + + @Route("/orders/{code}") + public String notifyLate() { + return send("Order \"Foo\" Is Late"); + } + + @Route("/say \"hi\"") + public String greet() { + return send("The order has shipped"); + } +} diff --git a/tests/cases/java/quoted-literal-text/src/app/Route.java b/tests/cases/java/quoted-literal-text/src/app/Route.java new file mode 100644 index 000000000..4bc1364fd --- /dev/null +++ b/tests/cases/java/quoted-literal-text/src/app/Route.java @@ -0,0 +1,10 @@ +package app; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +public @interface Route { + String value() default ""; + String name() default ""; +} diff --git a/tests/cases/python/quoted-literal-text/case.json b/tests/cases/python/quoted-literal-text/case.json new file mode 100644 index 000000000..04c622f2d --- /dev/null +++ b/tests/cases/python/quoted-literal-text/case.json @@ -0,0 +1,23 @@ +{"lang": "python", "src": "src", + "checks": [ + {"why": "a string literal holding a double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)", + "run": ["impact", "\"Order \"Foo\" Is Late\""], + "want": ["(1 literal site(s), 0 decoration site(s))"], + "avoid": ["appears in no literal"]}, + {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled", + "run": ["impact", "KEY"], + "want": ["const KEY = 'a \\\"b\\\" c'"], + "avoid": ["\"\""]}, + {"why": "control: a constant with no quote in its value is unchanged", + "run": ["impact", "PLAIN"], + "want": ["const PLAIN = 'a plain value'"]}, + {"why": "control: a literal with no quote in it is found as before", + "run": ["impact", "\"The order has shipped\""], + "want": ["(1 literal site(s), 0 decoration site(s))"]}, + {"why": "a decorator's quoted path reads cleanly: it binds the function and registers it under the path", + "run": ["impact", "\"/orders/{code}\""], + "want": ["1 decoration site(s)", "binds to it through @route"]}, + {"why": "a decorator path written as a triple-quoted string is read as the path", + "run": ["impact", "notify_triple"], + "want": ["registered by @route \"/triple/{code}\""]} + ]} diff --git a/tests/cases/python/quoted-literal-text/src/app/__init__.py b/tests/cases/python/quoted-literal-text/src/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cases/python/quoted-literal-text/src/app/orders.py b/tests/cases/python/quoted-literal-text/src/app/orders.py new file mode 100644 index 000000000..85a5aadbf --- /dev/null +++ b/tests/cases/python/quoted-literal-text/src/app/orders.py @@ -0,0 +1,22 @@ +KEY = "a \"b\" c" +PLAIN = "a plain value" + + +def route(path): + def deco(f): + return f + return deco + + +def send(subject): + return subject + + +@route("/orders/{code}") +def notify(): + return send('Order "Foo" Is Late') + + +@route("""/triple/{code}""") +def notify_triple(): + return send("The order has shipped") diff --git a/tests/index_reader.py b/tests/index_reader.py index bc43878aa..3f6508eac 100644 --- a/tests/index_reader.py +++ b/tests/index_reader.py @@ -1,21 +1,33 @@ #!/usr/bin/env python3 -"""tests/index_reader.py: axiomcode-index reads the IR exactly as csv.DictReader did, row for row and key for key. - -axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced -csv.DictReader(f, delimiter='\\t', quoting=csv.QUOTE_NONE) because the DictReader was most of the index's time on a -large repository. The index's tables are made of these dicts, so any difference is a different graph. Each fixture is -read by both and the lists of dicts must be equal: the shapes where a line splitter and the csv module could part -are a quoted field holding a tab or a newline (QUOTE_NONE does not unquote, so both split it), CRLF, a lone CR, a -blank line, a short row (None for the missing columns), a long row (the rest under the key None), a repeated header -name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not UTF-8, and the -Unicode line separators that str.splitlines() would split on and the csv module does not. - - python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared as well +"""tests/index_reader.py: axiomcode-index reads the IR as the bundle does, row for row and key for key. + +axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced csv.DictReader because the +DictReader was most of the index's time on a large repository. The index's tables are made of these dicts, so any +difference is a different graph. The parser writes RFC 4180 over a tab (EntityUtils.escapeTsv quotes a field that +holds a `"` and doubles the quote), and ir_rows decodes it: read QUOTE_NONE, as it once was, the literal +`Order "Foo"` was stored `Order ""Foo""` and `impact` said no literal held it (#1835). + +Each fixture is read three ways, and they must agree: + - csv.DictReader(f, delimiter='\\t') over open(p, newline=''), RFC 4180 quoting on: the same list of dicts; + - the bundle's own reader, graph/bundle/csv.ts readRfc4180 (the built dist/bundle/csv.js, run with node): its rows, + zipped with the header, must give the same dicts. A lone CR is left out of this half on purpose: ir_rows keeps + the csv module's rule that it ends a line, and the bundle splits on \\n only. A blank line is no row in either + (the bundle yields one empty field for it, which is dropped here); + - for the shapes the bug was about, the decoded value itself, written out, so two readers agreeing on a wrong value + is not a pass: a literal with quotes, a decoration with a quoted path, a C# raw string `\"\"\"x\"\"\"`, a C# + member access on a string (`"x".Length`), a quote inside a field, and the control, a field with no quote. +The other shapes are where a line splitter and the csv module could part: a quoted field holding a tab or a newline, +CRLF, a lone CR, a blank line, a short row (None for the missing columns), a long row (the rest under the key None), +a repeated header name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not +UTF-8, and the Unicode line separators that str.splitlines() would split on and the csv module does not. + + python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared with csv.DictReader too """ -import ast, csv, glob, itertools, os, sys, tempfile +import ast, csv, glob, itertools, json, os, shutil, subprocess, sys, tempfile ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SCRIPT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-index') +BUNDLE_CSV = os.path.join(ROOT, 'dist', 'bundle', 'csv.js') # graph/bundle/csv.ts, built csv.field_size_limit(10**9) MISSING = object() @@ -33,7 +45,46 @@ def load_ir_rows(): def dictreader(p): with open(p, newline='', encoding='utf-8', errors='replace') as f: - return list(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)) + return list(csv.DictReader(f, delimiter='\t')) + + +BUNDLE_JS = r''' +const { readRfc4180 } = require(process.argv[1]); +(async () => { + const out = {}; + for (const p of process.argv.slice(2)) { out[p] = []; for await (const r of readRfc4180(p)) out[p].push(r); } + process.stdout.write(JSON.stringify(out)); +})(); +''' + + +def bundle_rows(paths): + """{path: the bundle reader's rows}, or None when node or the built reader is not there.""" + node = shutil.which('node') + if not node or not os.path.exists(BUNDLE_CSV): + return None + r = subprocess.run([node, '-e', BUNDLE_JS, BUNDLE_CSV] + paths, capture_output=True, text=True, encoding='utf-8') + if r.returncode: + sys.exit(f'FAIL: the bundle reader did not run: {r.stderr[-400:]}') + return json.loads(r.stdout) + + +def as_dicts(rows): + """The bundle's rows as ir_rows' dicts: the first row is the header, a blank line (one empty field) is no row.""" + if not rows: + return [] + keys = rows[0] if rows[0] != [''] else [] + n, out = len(keys), [] + for v in rows[1:]: + if v == ['']: + continue + d = dict(zip(keys, v)) + if len(v) > n: + d[None] = v[n:] + for k in keys[len(v):]: + d[k] = None + out.append(d) + return out FIXTURES = { @@ -56,30 +107,71 @@ def dictreader(p): 'header-only-no-newline': b'a\tb', 'no-final-newline': b'a\tb\n1\t2\n3\t4', 'not-utf8': b'a\tb\n\xff\xfe\t\xc3\x28\n', - 'unicode-separators': 'a\tb\nx
y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), + 'unicode-separators': 'a\tb\nx y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), 'spaces': b'a\tb\n 1 \t 2 \n', + # the quoting EntityUtils.escapeTsv writes, as the parser wrote it for each shape + 'quoted-literal': b'value\tkind\n"Order ""Foo"" Is Late"\tstring\n', + 'quoted-decoration': b'name\ttext\nRequestMapping\t"@RequestMapping(""/a/{b}"")"\nRequestMapping\t"""/a/{b}"""\n', + 'csharp-raw-string': b'value\tkind\n"""""""x"""""""\tstring\n', + 'csharp-string-member': b'name\tkind\n"""x"".Length"\tMEMBER_ACCESS\n', + 'plain-control': b'value\tkind\nThe order has shipped\tstring\n', + # what RFC 4180 says about a quote that does not open a field, and about text after the closing quote + 'quote-inside-field': b'a\tb\tc\nsay "hi"\tx"\ty"z"\n', + 'text-after-close': b'a\tb\n"ab"cd\t"x""y"z\n', + 'empty-quoted': b'a\tb\tc\n""\t""""\t\n', + 'quoted-crlf-row': b'a\tb\r\n"x ""y"""\t2\r\n', + 'quoted-header': b'"a"\t"b ""c"""\n1\t2\n', + 'quoted-open-at-eof': b'a\tb\n"x\ty\n', +} + +# the decoded values, written out +DECODED = { + 'quoted-literal': [{'value': 'Order "Foo" Is Late', 'kind': 'string'}], + 'quoted-decoration': [{'name': 'RequestMapping', 'text': '@RequestMapping("/a/{b}")'}, + {'name': 'RequestMapping', 'text': '"/a/{b}"'}], + 'csharp-raw-string': [{'value': '"""x"""', 'kind': 'string'}], + 'csharp-string-member': [{'name': '"x".Length', 'kind': 'MEMBER_ACCESS'}], + 'plain-control': [{'value': 'The order has shipped', 'kind': 'string'}], + 'quote-inside-field': [{'a': 'say "hi"', 'b': 'x"', 'c': 'y"z"'}], + 'quoted-open-at-eof': [{'a': 'x\ty\n', 'b': None}], + 'text-after-close': [{'a': 'abcd', 'b': 'x"yz'}], + 'quoted-header': [{'a': '1', 'b "c"': '2'}], } +# a lone CR ends an ir_rows line and is text to the bundle's reader (the parser never writes one: escapeTsv escapes it) +NOT_BUNDLE = {'lone-cr', 'mixed-endings'} def main(argv): ir_rows = load_ir_rows() fails = checks = 0 - rows_compared = 0 + rows_compared = bundle_checked = 0 with tempfile.TemporaryDirectory() as w: + paths = {} for name, data in FIXTURES.items(): - p = os.path.join(w, name + '.csv') + p = paths[name] = os.path.join(w, name + '.csv') open(p, 'wb').write(data) + bundle = bundle_rows(list(paths.values())) + for name, p in paths.items(): want, got = dictreader(p), list(ir_rows(p)) checks += 1; rows_compared += len(want) if want != got: fails += 1 print(f'FAIL {name}: csv.DictReader {want!r}\n ir_rows {got!r}') + if name in DECODED: + checks += 1 + if got != DECODED[name]: + fails += 1; print(f'FAIL {name}: decoded {got!r}\n want {DECODED[name]!r}') + if bundle is not None and name not in NOT_BUNDLE: + checks += 1; bundle_checked += 1 + b = as_dicts(bundle[p]) + if b != got: + fails += 1; print(f'FAIL {name}: bundle readRfc4180 {b!r}\n ir_rows {got!r}') # not vacuous: the fixtures hold rows, and the shapes the reader special-cases are among them - p = os.path.join(w, 'short-row.csv') + p = paths['short-row'] checks += 1 if list(ir_rows(p)) != [{'a': '1', 'b': '2', 'c': None}, {'a': '1', 'b': None, 'c': None}]: fails += 1; print(f'FAIL short-row: {list(ir_rows(p))!r}') - p = os.path.join(w, 'long-row.csv') + p = paths['long-row'] checks += 1 if list(ir_rows(p)) != [{'a': '1', 'b': '2', None: ['3', '4']}]: fails += 1; print(f'FAIL long-row: {list(ir_rows(p))!r}') @@ -90,15 +182,19 @@ def main(argv): files += 1; checks += 1 n = 0 with open(p, newline='', encoding='utf-8', errors='replace') as f: # streamed: an IR file can be 10^6 rows - for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE), ir_rows(p), fillvalue=MISSING): + for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t'), ir_rows(p), fillvalue=MISSING): n += 1 if a != b: fails += 1; print(f'FAIL {p} row {n}: csv.DictReader {a!r}\n ir_rows {b!r}'); break rows_compared += n if rows_compared < 20: fails += 1; print(f'FAIL only {rows_compared} rows compared') + if bundle is None: + print('index_reader: the bundle half was NOT run (no node on PATH, or dist/bundle/csv.js is not built)') + elif bundle_checked < len(FIXTURES) - len(NOT_BUNDLE): + fails += 1; print(f'FAIL the bundle reader was compared on only {bundle_checked} fixtures') print(f"index_reader: {'ok' if not fails else 'FAILED'} ({checks - fails}/{checks} checks, {rows_compared} rows," - f" {len(FIXTURES)} fixtures{f', {files} IR files' if files else ''})") + f" {len(FIXTURES)} fixtures, {bundle_checked} against the bundle reader{f', {files} IR files' if files else ''})") return 1 if fails else 0