plugin: axiomcode-index decodes quoted IR fields, so a literal with quotes is found (#1835)

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index

@@ -21,14 +21,16 @@ positions — mirroring graph/bundle/languages.ts. When the IR is absent the ind
 come from methods/types alone, refs/literals/comments are empty, and index_meta says so.
 """
 import os, re, sqlite3, sys, time, glob, collections, functools
-INDEX_VERSION = '9'      # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function;
+INDEX_VERSION = '10'     # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function;
                          # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding;
                          # v6: TypeScript class-property arrows named after their field;
                          # v7: data keys of module-level const objects declared, Object.freeze seen through;
                          # v8: literals carry numbers and booleans with a kind column, and the name of the constant
                          #     a scalar initializes — a spec's numbers were the one thing the graph could not answer;
                          # v9: the web graph (HTML + CSS) is indexed: pages, stylesheets, .classes, #ids, custom properties (var(--x)),
-                         #     @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols); the query
+                         #     @keyframes/@font-face/@layer/@container, selectors and inline scripts are symbols;
+                         # v10: the IR's RFC 4180 quoting is decoded, so a literal, comment, decoration or name holding a `"`
+                         #     is stored as written (`a "b" c`, not `a ""b"" c`), and C# verbatim and raw strings are literals); the query
                          # frontend re-indexes an older graph when the IR is still there
 
 REPO = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].startswith('-') else os.environ.get('AXIOMCODE_REPO') or '.')
@@ -192,7 +194,7 @@ A = {
               # name elsewhere is a by-name reader whatever its qualifier says, so `_r.Limit` was listed under AuditOptions.Limit
               memberName=dict(role='edgeRole', value='MEMBER_NAME', parent='parentExpressionHash', id='csExpressionUniqueHash'),
               # the literal TYPE column is literalKind here, not literalType as in java/typescript/python
-              litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}),
+              litKinds={'LITERAL'}, litType=('literalKind', {'STRING': 'string', 'VERBATIM_STRING': 'string', 'RAW_STRING': 'string', 'INTEGER': 'number', 'REAL': 'number', 'BOOLEAN': 'bool'}), litValue='literalValue', litLinkRoles={'ROOT'}),
     comments=dict(file='all-csharp-comments.csv', text='commentText', kind='commentKind', line='startLine', fileVia=('modules', 'csModuleLinkHash')),
     # type references carry no module link of their own. The owner is a type only for a base-list reference; for
     # `M<T>()`, `new T()`, a parameter, a return or a field it is the expression, method, parameter or field, so
@@ -275,25 +277,61 @@ def rows(name):
     return ir_rows(p)
 
 def ir_rows(p):
-    """The rows of one IR file as dicts keyed by its header, EXACTLY as csv.DictReader(f, delimiter='\\t',
-    quoting=csv.QUOTE_NONE) over open(p, newline='') reads them, at about half the cost (the expressions file is
-    most of this script's time on a large repository). That reader treats a quote as an ordinary character, so a row
-    is one line and a field is the text between tabs: a line ends at \\n, \\r or \\r\\n (newline='' splits on all
-    three and keeps them, and each ends a record), a blank line is skipped, a short row has None for its missing
-    columns and a long one keeps the rest as a list under the key None. The writer's RFC 4180 quoting is NOT undone
-    here, as it was not before (see unquoted() below). tests/index_reader.py holds the two to the same dicts."""
+    """The rows of one IR file as dicts keyed by its header, its fields DECODED the way the bundle's reader
+    (graph/bundle/csv.ts readRfc4180) decodes them: the parser writes RFC 4180 over a tab, so a field that STARTS with
+    a double quote is quoted, `""` inside it is one quote, and the closing quote ends it (what follows the close, up
+    to the tab, is kept as written); a quote anywhere else in a field is an ordinary character. The writer
+    (EntityUtils.escapeTsv) turns a newline, a CR and a tab into the two-character escapes BEFORE it quotes, and quotes
+    only for a `"`, so a quoted field it wrote never holds a tab or a line break and a row is one line. Reading
+    QUOTE_NONE stored `Order "A-1"` as `Order ""A-1""`, so `impact` said the string was in no literal (#1835).
+    The rest is csv.DictReader(f, delimiter='\\t') over open(p, newline=''), at about half its cost (the expressions
+    file is most of this script's time on a large repository): a line ends at \\n, \\r or \\r\\n, a blank line is
+    skipped, a short row has None for its missing columns and a long one keeps the rest as a list under the key None.
+    A line with no quote in it is split on tabs and nothing else: that is nearly every line, and a test for a quote
+    that OPENS a field cost more than the decoding it saved. A quoted field left open at the end of a line, which this
+    writer does not produce, takes the line break and the next line, as the bundle's reader does.
+    tests/index_reader.py holds this to both readers."""
     with open(p, newline='', encoding='utf-8', errors='replace') as f:
-        head = f.readline()
+        def fields(line):
+            # a line holding a quote: split on tabs, then decode each field that opens with one, rejoined across the tabs
+            # (and, if it is still open at the end of the line, the line breaks) it holds
+            out = []; parts = line.split('\t'); i = 0
+            while True:
+                if i == len(parts): return out
+                s = parts[i]; i += 1
+                if s[:1] != '"': out.append(s); continue
+                buf = []; k = 1
+                while True:
+                    j = s.find('"', k)
+                    if j >= 0:
+                        buf.append(s[k:j])
+                        if s[j + 1:j + 2] == '"': buf.append('"'); k = j + 2; continue
+                        buf.append(s[j + 1:]); break                  # the closing quote; the rest of the field as written
+                    buf.append(s[k:]); k = 0
+                    if i < len(parts): buf.append('\t'); s = parts[i]; i += 1; continue
+                    buf.append(line_end[0]); nxt = f.readline()       # still open at the end of the line: the break is text
+                    if not nxt: break
+                    body = nxt.rstrip('\r\n'); line_end[0] = nxt[len(body):]
+                    parts = body.split('\t'); s = parts[0]; i = 1
+                out.append(''.join(buf))
+        def decoded(line, raw):
+            line_end[0] = raw[len(line):]
+            v = fields(line)
+            if v[-1][-1:] == '\r': v[-1] = v[-1][:-1]                 # as the bundle's reader drops a CR before the newline
+            return v
+        line_end = ['']
+        head = raw = f.readline()
         if not head: return
         if head[-1:] == '\n': head = head[:-1]
         if head[-1:] == '\r': head = head[:-1]
-        keys = head.split('\t') if head else []
+        keys = (head.split('\t') if '"' not in head else decoded(head, raw)) if head else []
         n = len(keys)
-        for line in f:
+        for raw in f:
+            line = raw
             if line[-1:] == '\n': line = line[:-1]
             if line[-1:] == '\r': line = line[:-1]
             if not line: continue
-            v = line.split('\t')
+            v = line.split('\t') if '"' not in line else decoded(line, raw)
             d = dict(zip(keys, v))
             if len(v) != n:
                 if len(v) > n: d[None] = v[n:]
@@ -699,6 +737,20 @@ for name, od, v, k in data_keys:
 c.executemany("INSERT INTO symbols VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", sym)
 
 # ── references, literals, comments ───────────────────────────────────────────────────────────────
+def cs_string_text(v):
+    """The text of a C# string token, as ir_rows hands it over (the IR's RFC 4180 quoting already undone): `"x"` is x
+    with its escapes as written, as Java's and Python's are; a verbatim `@"x"` is x with a doubled quote made one, the
+    language's own escape; a raw string is x, closed by as many quotes as opened it (three or more: a regular string
+    cannot open with three, `""` is the whole empty string); a `$` is dropped. Before ir_rows decoded the IR, a C#
+    literal holding a quote was un-doubled here by hand; on decoded text that is a second decoding, which reads the
+    raw `\"\"\"\"a \"\" b\"\"\"\"` as `"a " b"`, and a verbatim string's doubled quote was never made one."""
+    p = 0
+    while p < len(v) and v[p] in '@$': p += 1
+    t = v[p:]
+    q = len(t) - len(t.lstrip('"'))
+    if q >= 3 and len(t) >= 2 * q and t.endswith('"' * q): return t[q:-q]
+    if len(t) >= 2 and t[0] == '"' == t[-1]: return t[1:-1].replace('""', '"') if '@' in v[:p] else t[1:-1]
+    return v
 e = A['expr']; refs = []; lits = []
 namecol = e['name'] if isinstance(e['name'], str) else dict(x.split(':') for x in e['name'])
 mname = e.get('memberName'); child_ek = {}
@@ -727,11 +779,8 @@ for r in rows(e['file']):
         kind = e['litType'][1][r.get(e['litType'][0])]        # 'string' | 'number' | 'bool' across every language
         v = (r.get(e['litValue']) or '')
         if kind == 'string':
-            # C# keeps the source token, and the IR writer RFC4180-quotes a field holding a double quote, so `"Fee"` arrived
-            # as `"""Fee"""` and was stored `""Fee""`: no key, route or member name written in a C# string ever matched
-            if LANG == 'csharp' and len(v) > 1 and v[0] == '"' == v[-1] and '""' in v: v = v[1:-1].replace('""', '"')
-            if LANG == 'csharp' and len(v) > 2 and v[0] in '@$' and v[-1] == '"': v = v.lstrip('@$')   # @"verbatim", $"interpolated"
-            if len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1]   # JavaScript keeps the quotes
+            if LANG == 'csharp': v = cs_string_text(v)          # C# keeps the source token: "x", @"x", """x"""
+            elif len(v) >= 2 and v[0] in '\'"`' and v[-1] == v[0]: v = v[1:-1]   # JavaScript keeps the quotes
         if v: lits.append((v[:200], file_of(r, e), int(r.get(e['line']) or 0), kind, r.get('edgeRole', '')))
 # a scalar's NAME: `MAX_ITEMS = 5` — the one literal on a const/variable/field declaration's line IS its value,
 # so the row gets the symbol's display and "what is MAX_ITEMS" has an answer with a source location. Strictly:
@@ -767,13 +816,6 @@ if tr:
         c.executemany("INSERT INTO type_refs VALUES (?,?,?,?,?)", ((r.get(tr['name'], '').split('.')[-1].split('<')[0], file_of(r, tr), int(r.get(tr['line']) or 0), r.get(tr['context'], ''), r.get(tr['ownerKind'], '')) for r in rows(tr['file']) if r.get(tr['name'])))
     except (KeyError, ValueError): pass
 # decorations: @Annotation(args) / @decorator(args) on a method or type — a route, a bean name, a validator, a marker
-def unquoted(v):
-    """The IR is tab-separated, but the writer still RFC4180-quotes a field that contains a double quote and the
-    reader is QUOTE_NONE, so `@router.post("/orders")` arrives as `"@router.post(""/orders"")"`. Every decoration
-    whose argument is a double-quoted string is in that shape — which is exactly the decorations that carry a KEY
-    (a route path, a command name, a topic, a validated field), so everything downstream that reads a key out of
-    the text read the quoting instead: `@exporter("csv")` yielded `@exporter(` and `)` and never `csv`."""
-    return v[1:-1].replace('""', '"') if len(v) > 1 and v[0] == '"' == v[-1] and '""' in v else v
 dec = A.get('decorations')
 if dec:
     args = collections.defaultdict(list)
@@ -784,7 +826,8 @@ if dec:
     for r in rows(dec['file']):
         nm = r.get(dec['name'], '')
         if not nm: continue
-        text = unquoted(r.get(dec['text'], '')) if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else ''))
+        # the text as written (ir_rows undoes the writer's RFC 4180 quoting, so `@exporter("csv")` arrives as that)
+        text = r.get(dec['text'], '') if dec.get('text') else ('@' + nm + ('(' + ', '.join(args.get(r.get(dec['id'], ''), [])) + ')' if args.get(r.get(dec['id'], '')) else ''))
         try: drows.append((r.get(dec['owner'], ''), nm, text[:200], file_of(r, dec), int(r.get(dec['line']) or 0)))
         except ValueError: pass
     c.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", drows)

tests/README.md

@@ -26,9 +26,11 @@ One check needs no graph and is its own script:
                                          signature edit is reported before it lands with nothing upstream did; `--range`
                                          from a branch left behind its remote reads from the remote's fork (indexes a small
                                          project)
-    python3 tests/index_reader.py        axiomcode-index's IR reader returns the same rows csv.DictReader did, on quoted tabs
-                                         and newlines, CRLF, a lone CR, a short or long row and invalid UTF-8; pass IR
-                                         directories to compare them too (no graph needed)
+    python3 tests/index_reader.py        axiomcode-index's IR reader decodes the IR's RFC 4180 quoting as csv.DictReader and
+                                         the bundle's reader do (a literal, decoration or name holding a `"`, a C# raw
+                                         string), and agrees with them on quoted tabs and newlines, CRLF, a lone CR, a
+                                         short or long row and invalid UTF-8; pass IR directories to compare them too
+                                         (no graph needed; the bundle half runs node on dist/bundle/csv.js)
     python3 tests/refresh.py             the graph refreshes itself after an edit in every language: a query sees the
                                          edit, `changed` answers the same before and after, a burst costs one rebuild
                                          and queries during it answer (#1305; builds real graphs, needs the engine)

tests/cases/csharp/quoted-literal-text/case.json

@@ -0,0 +1,28 @@
+{"lang": "csharp", "src": "src",
+ "checks": [
+  {"why": "a member read on a string literal is named by the member: the IR quotes the expression (RFC 4180) and the index kept a stray quote, `Length\"` (#1835)",
+   "run": ["impact", "Length"], "expect_error": true,
+   "want": ["[by name] Orders.LabelWidth"]},
+  {"why": "a raw string is a literal, and its value is the text between the quotes that open and close it",
+   "run": ["impact", "Orders.RawKey"],
+   "want": ["const Orders.RawKey = 'order.created'"]},
+  {"why": "a raw string holding a quote keeps it, once",
+   "run": ["impact", "Orders.Raw"],
+   "want": ["const Orders.Raw = 'raw \"x\" here'"]},
+  {"why": "a raw string opened by four quotes may hold two, and both are kept: un-doubling quotes by hand on the decoded IR read it as `\"a \" b\"`",
+   "run": ["impact", "Orders.Raw4"],
+   "want": ["const Orders.Raw4 = 'a \"\" b'"]},
+  {"why": "a verbatim string's doubled quote is one quote",
+   "run": ["impact", "Orders.Verbatim"],
+   "want": ["const Orders.Verbatim = 'say \"hi\" now'"]},
+  {"why": "control: a regular string holding an escaped quote is printed as written, in single quotes",
+   "run": ["impact", "Orders.Key"],
+   "want": ["const Orders.Key = 'a \\\"b\\\" c'"],
+   "avoid": ["\"\""]},
+  {"why": "control: a string literal holding an escaped quote is found",
+   "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""],
+   "want": ["(1 literal site(s), 0 decoration site(s))"]},
+  {"why": "control: a constant with no quote in its value is unchanged",
+   "run": ["impact", "Orders.Plain"],
+   "want": ["const Orders.Plain = 'a plain value'"]}
+ ]}

tests/cases/csharp/quoted-literal-text/src/Orders.cs

@@ -0,0 +1,20 @@
+namespace App;
+
+public class Orders
+{
+    public const string Key = "a \"b\" c";
+    public const string Plain = "a plain value";
+    public const string Verbatim = @"say ""hi"" now";
+    public const string RawKey = """order.created""";
+    public const string Raw = """raw "x" here""";
+    public const string Raw4 = """"a "" b"""";
+
+    string Send(string subject) => subject;
+
+    [Route("/orders/{code}")]
+    public string NotifyLate() => Send("Order \"Foo\" Is Late");
+
+    public string Resolve() => Send("The order has shipped");
+
+    public int LabelWidth() => "header".Length;
+}

tests/cases/csharp/quoted-literal-text/src/Orders.csproj

@@ -0,0 +1 @@
+<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><TargetFramework>net8.0</TargetFramework></PropertyGroup></Project>

tests/cases/csharp/quoted-literal-text/src/RouteAttribute.cs

@@ -0,0 +1,8 @@
+namespace App;
+
+[System.AttributeUsage(System.AttributeTargets.Method)]
+public class RouteAttribute : System.Attribute
+{
+    public RouteAttribute(string path) { Path = path; }
+    public string Path { get; }
+}

tests/cases/java/quoted-literal-text/case.json

@@ -0,0 +1,24 @@
+{"lang": "java", "src": "src",
+ "checks": [
+  {"why": "a string literal holding an escaped double quote is found: the IR quotes such a field (RFC 4180) and the index stored it doubled, so `impact` said no literal held it (#1835)",
+   "run": ["impact", "\"Order \\\"Foo\\\" Is Late\""],
+   "want": ["(1 literal site(s), 0 decoration site(s))"],
+   "avoid": ["appears in no literal"]},
+  {"why": "a constant's value holding a quote is printed as written, in single quotes, not with each quote doubled",
+   "run": ["impact", "Orders.KEY"],
+   "want": ["const Orders.KEY = 'a \\\"b\\\" c'"],
+   "avoid": ["\"\""]},
+  {"why": "an annotation argument holding a quote is found as written: the decoration was stored with every quote doubled",
+   "run": ["impact", "\"/say \\\"hi\\\"\""],
+   "want": ["(0 literal site(s), 1 decoration site(s))"],
+   "avoid": ["appears in no literal"]},
+  {"why": "control: a constant with no quote in its value is unchanged",
+   "run": ["impact", "Orders.PLAIN"],
+   "want": ["const Orders.PLAIN = 'a plain value'"]},
+  {"why": "control: a literal with no quote in it is found as before",
+   "run": ["impact", "\"The order has shipped\""],
+   "want": ["(1 literal site(s), 0 decoration site(s))"]},
+  {"why": "control: an annotation's quoted path still binds the method and registers it under the path",
+   "run": ["impact", "Orders.notifyLate"],
+   "want": ["registered by @Route \"/orders/{code}\""]}
+ ]}

tests/cases/java/quoted-literal-text/src/app/Orders.java

@@ -0,0 +1,20 @@
+package app;
+
+public class Orders {
+    static final String KEY = "a \"b\" c";
+    static final String PLAIN = "a plain value";
+
+    String send(String subject) {
+        return subject;
+    }
+
+    @Route("/orders/{code}")
+    public String notifyLate() {
+        return send("Order \"Foo\" Is Late");
+    }
+
+    @Route("/say \"hi\"")
+    public String greet() {
+        return send("The order has shipped");
+    }
+}

tests/cases/java/quoted-literal-text/src/app/Route.java

@@ -0,0 +1,10 @@
+package app;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Route {
+    String value() default "";
+    String name() default "";
+}