From 1f1df0fa48e7cc7c437e9a4a44f297350ca05a24 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 12:14:16 -0700 Subject: [PATCH 01/11] WIP: split credentials (2e) --- nameparser/_pipeline/_assign.py | 22 +++++-- nameparser/_pipeline/_classify.py | 45 +++++++++++-- nameparser/_pipeline/_comma.py | 41 +++++++----- nameparser/_pipeline/_group.py | 92 +++++++++++++++++++++----- nameparser/_pipeline/_pieces.py | 79 ++++++++++++++++------ nameparser/_pipeline/_state.py | 12 ++++ nameparser/_pipeline/_vocab.py | 105 ++++++++++++++++++++++-------- 7 files changed, 308 insertions(+), 88 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index cc4d987b..43274c7c 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -543,10 +543,15 @@ def _assign_main(seg_idx: int, state: ParseState, # does). Decided here, where the trailing read has said what that # run is # `_ROMAN` first, C-level, so an ordinary name word pays no frame - if (len(name_pieces) >= 2 and (suffix_pieces or titled_tail) + if (len(name_pieces) >= 2 and len(last := pieces[name_pieces[-1]]) == 1 and _ROMAN.match((word := tokens[last[0]].text).rstrip(".")) is not None + # the trailing run, or the Ph./D. pair, which the walk + # leaves out, behind it + and (suffix_pieces or titled_tail + or name_pieces[-1] + 1 < len(pieces) + and "suffix" in ptags[name_pieces[-1] + 1]) and tuple(last) not in peeled.picks and reads_as_a_reported_numeral( word, tokens[last[0]].tags, @@ -1209,6 +1214,13 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: # credential list behind it makes the credential # reading likelier rather than less. piece = pieces[m] + # rules/S3: a split credential is "a member of the + # ambiguous class", here read as the class is with no + # writing to lean on and in silence, as group merges it + # only behind the given word + if "split" in ptags[m]: + return (m >= trailing_floor(m, titled) + and not attaches_the_lead(m, titled)) if len(piece) == 1: tok = tokens[piece[0]] if AMBIGUOUS_ACRONYM_TAG in tok.tags: @@ -1357,7 +1369,7 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: for m in range(n + 1, len(pieces)): tags = tokens[pieces[m][0]].tags if (m not in titled_idx - and ("suffix" in ptags[m] + and ("suffix" in ptags[m] or "split" in ptags[m] or (("vocab:suffix" in tags or _ROMAN.match(tokens[pieces[m][0]].text .rstrip(".")) is not None) @@ -1448,8 +1460,10 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: f"{tokens[piece[0]].text!r} ending the given " f"part is also an ordinary name word; read " f"as a credential", (piece[0],))) - elif not suffix_here and has_name_content(piece, - tokens): + # the run's own first word is the credential that + # starts it, a split one ('Pharm D') included + elif (not suffix_here and m != sticky_from + and has_name_content(piece, tokens)): ambiguities.append(_absorbed(piece, tokens)) _set_roles(tokens, piece, Role.SUFFIX) continue diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index f86b98dc..7334b0e1 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -62,14 +62,15 @@ from nameparser._lexicon import _normalize, _spells_an_initial from nameparser._policy import CapsSuffixes from nameparser._pipeline._state import ( - AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG, ParseState, PendingAmbiguity, - WorkToken, copy_with, + AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG, SPLIT_CREDENTIAL_TAG, + SPLIT_MEMBER_TAG, ParseState, PendingAmbiguity, WorkToken, copy_with, ) from nameparser._types import AmbiguityKind, Role from nameparser._pipeline._vocab import ( - BUILTIN_SUFFIX_DELIMITERS, _INITIAL, caps_shape_candidate, composed_len, - delimiter_cores, glued_stem, is_cased_paired_initials, is_initial, - is_one_case, period_joined_vocab, suffix_as_written, tag_marker_runs, + BUILTIN_SUFFIX_DELIMITERS, SPLIT_CHUNK, _INITIAL, caps_shape_candidate, + composed_len, delimiter_cores, glued_stem, is_cased_paired_initials, + is_initial, is_one_case, period_joined_vocab, split_pair, + suffix_as_written, tag_marker_runs, ) from nameparser._pipeline._pieces import own_words @@ -396,6 +397,40 @@ def classify(state: ParseState) -> ParseState: tokens = tuple( copy_with(t, tags=t.tags - _JOINING_TAGS) if i in stemmed else t for i, t in enumerate(tokens)) + # rules/S3: "A credential written split across a space reads as one + # word" -- each adjacent pair of a segment asked once, here, and + # the answer written on its first word for group's merge and the + # comma's reading of its part (`_vocab.split_pair`). The shape and + # the fold's membership are asked first, in C, so an ordinary pair + # of words pays no frame; a token carrying a role is delimited + # content, which no pair reaches. + split: dict[int, str] | None = None + acronyms = state.lexicon.suffix_acronyms + words = state.lexicon.suffix_words + for seg in state.segments: + for j in range(len(seg) - 1): + a, b = tokens[seg[j]], tokens[seg[j + 1]] + if (SPLIT_CHUNK.fullmatch(a.text) is not None + and SPLIT_CHUNK.fullmatch(b.text) is not None + and a.role is None and b.role is None + and ((fold := (a.text + b.text).replace(".", "") + .lower()) in acronyms or fold in words + or not fold.isascii()) + and (kind := split_pair(a.text, b.text, + state.lexicon)) is not None + # the Ph./D. pair outranks a pair overlapping it: + # 'Dr. Ph. D.' is no 'Dr. Ph.' ('drph') and a 'D.' + and not (kind == "split" and j + 2 < len(seg) + and split_pair(b.text, tokens[seg[j + 2]].text, + state.lexicon) == "suffix")): + if split is None: + split = {} + split[seg[j]] = (SPLIT_CREDENTIAL_TAG if kind == "suffix" + else SPLIT_MEMBER_TAG) + if split is not None: + tokens = tuple( + copy_with(t, tags=t.tags | {split[i]}) if i in split else t + for i, t in enumerate(tokens)) # Delimited content whose vocabulary cannot settle it: extract's # escape sends an UNambiguous suffix straight through ("(MBA)" -> # suffix) and keeps everything else as a nickname, so an AMBIGUOUS diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index b3a9fb85..f9e22b66 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -36,14 +36,15 @@ listed_lean, own_words, segment_suffix_reading, ) from nameparser._pipeline._state import ( - AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG, ParseState, PendingAmbiguity, - Structure, WorkToken, copy_with, + AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG, SPLIT_CREDENTIAL_TAG, + SPLIT_MEMBER_TAG, ParseState, PendingAmbiguity, Structure, WorkToken, + copy_with, ) from nameparser._pipeline._vocab import ( - ROMAN, SPLIT_PAIR_OPENS, caps_shape_candidate, claimed_as_non_name, + ROMAN, caps_shape_candidate, claimed_as_non_name, composed_len, delimiter_cores, glued_stem, glues, is_caps_word, is_paired_initials, is_single_letter_numeral, is_wholly_suffix, - name_word_count, reads_as_generation, split_pair, surname_unit_facts, + name_word_count, reads_as_generation, surname_unit_facts, unit_ends, written_as_a_name, ) from nameparser._policy import CapsSuffixes @@ -52,19 +53,24 @@ def _pieces(seg: Sequence[int], tokens: Sequence[WorkToken], ) -> tuple[list[list[int]], list[set[str]]]: - """One piece per token but a split credential (`split_pair`), - which group's first merge builds (group alone declines a pair at - the head of the string, rules/S2) -- the part as it stands before - any join. Every piece a fresh list, as `merge_pieces` requires.""" + """One piece per token but a split credential (`split_pair`, read + off the tag classify wrote on its first word), which group's first + merge builds (group alone declines a pair at the head of the + string, rules/S2) -- the part as it stands before any join: the + Ph./D. pair tagged "suffix", any other "split", a member of the + ambiguous class for rules/C4's count. Every piece a fresh list, as + `merge_pieces` requires.""" pieces: list[list[int]] = [] ptags: list[set[str]] = [] k = 0 while k < len(seg): a = seg[k] - if (k + 1 < len(seg) and SPLIT_PAIR_OPENS.fullmatch(tokens[a].text) - and split_pair(tokens[a].text, tokens[seg[k + 1]].text)): + tags = tokens[a].tags + if k + 1 < len(seg) and (SPLIT_CREDENTIAL_TAG in tags + or SPLIT_MEMBER_TAG in tags): pieces.append([a, seg[k + 1]]) - ptags.append({"suffix"}) + ptags.append({"suffix" if SPLIT_CREDENTIAL_TAG in tags + else "split"}) k += 2 else: pieces.append([a]) @@ -351,7 +357,10 @@ def decide(state: ParseState) -> ParseState: # The words of the class in the part, and the unlisted all-caps # words the caps shape could admit: only a part holding one asks # the count, so 'Smith, John' and 'John Smith, PhD' never build it. - maybe = [p[0] for p in p1 if len(p) == 1 + # A split credential is a word of the class too (rules/S3), its + # piece tag the test. + maybe = [p[0] for p, pt in zip(p1, pt1) if "split" in pt + or len(p) == 1 and (AMBIGUOUS_ACRONYM_TAG in (tags := tokens[p[0]].tags) or ("vocab:suffix" not in tags and (texts[p[0]].isupper() @@ -431,9 +440,11 @@ def decide(state: ParseState) -> ParseState: # the words not of the class are asked as C1's suffix test asks # them, the lenient word test by default ('Ma B.') others: list[str] = [] - for p in p1: - if len(p) == 1 and (AMBIGUOUS_ACRONYM_TAG in tokens[p[0]].tags - or p[0] in caps): + for p, pt in zip(p1, pt1): + if ("split" in pt + or len(p) == 1 + and (AMBIGUOUS_ACRONYM_TAG in tokens[p[0]].tags + or p[0] in caps)): licensed.add(p[0]) else: # a loop, not a comprehension: a listcomp is a frame diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 6cfb22a7..5e0cdaa8 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -75,22 +75,23 @@ trailing_start_past_titles, ) from nameparser._pipeline._state import ( - ParseState, PendingAmbiguity, Structure, - WorkToken, _AMBIGUOUS_CREDENTIAL_TAGS, copy_with, + SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, ParseState, PendingAmbiguity, + Structure, WorkToken, _AMBIGUOUS_CREDENTIAL_TAGS, copy_with, ) import nameparser._pipeline._comma as _comma from nameparser._pipeline._vocab import ( - SPLIT_PAIR_OPENS, comma_follows_marker, delimiter_cores, - reads_as_a_reported_numeral, split_pair, + comma_follows_marker, delimiter_cores, reads_as_a_reported_numeral, ) from nameparser._types import AmbiguityKind, Role -# the split credential is `_vocab.split_pair`, which is_wholly_suffix -# asks to merge the same pair, as `_comma._pieces` does -- so that -# predicate's callers read what this stage merges: `_comma.decide`'s -# reading of the part after the comma, segment's C2 flag on a tail -# part, and script_segment's decline of a wholly-suffix post-comma -# run, all of which see the merged reading +# the split credential is `_vocab.split_pair`, which classify asks of +# every adjacent pair and writes on the pair's first word, and which +# is_wholly_suffix asks to merge the same Ph./D. pair, as +# `_comma._pieces` reads the tag -- so that predicate's callers read +# what this stage merges: `_comma.decide`'s reading of the part after +# the comma, segment's C2 flag on a tail part, and script_segment's +# decline of a wholly-suffix post-comma run, all of which see the +# merged reading Piece = list[int] #: What the marker pass took out of a segment: the marker's TOKEN @@ -577,6 +578,29 @@ def _between_name_words(k: int, lo: int, hi: int, and not is_title_piece(pieces[right], ptags[right], tokens)) +# rules/S4: "a BARE ambiguous acronym is consumed only when the name +# has words to spare" -- and a split credential is a member of that +# class (rules/S3), so where the trailing read did not take one, it is +# the two words it is written as +def _settle_splits(pieces: list[Piece], ptags: list[set[str]]) -> None: + """Split every merged split credential left in `pieces` back into + its two words, in place: a pair the trailing read declined reads + as it reads unmerged. One a connective join took into a wider piece + keeps the join and loses the tag (the only shape that reaches it + is a pair beside a connective, 'Pharm D and').""" + k = 0 + while k < len(pieces): + if "split" in ptags[k]: + if len(pieces[k]) == 2 and ptags[k] == {"split"}: + a, b = pieces[k] + pieces[k:k + 1] = [[a], [b]] + ptags[k:k + 1] = [set(), set()] + k += 2 + continue + ptags[k] = ptags[k] - {"split"} + k += 1 + + def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, @@ -639,16 +663,42 @@ def _group_segment(seg: tuple[int, ...], additional: int, # here, because "the first piece of the name" cannot be computed # before this merge: `is_leading_title` is true of `Ph.` itself, # so the scan would step over the very piece being judged. + # + # Every other split credential ('Pharm. D.', `_vocab.split_pair`'s + # "split" answer) merges under the same head test as a member of + # the ambiguous class, where a rule reads such a member: in a main + # segment, where S2's read below decides it by the count, kept + # whole where the read takes it and split back into its two words + # where it does not (`_settle_splits`), so a declined pair reads + # exactly as the two words read; and in the given part after a + # family comma behind the given word, where S5's walk reads it, + # and a pair declined there is middle-name text whole or split. + # Not in the part before a family comma, which reads no member as + # a suffix ('Smith MA, John'). Classify tagged each pair's first + # word, so the test here is a tag. + splits = False + given_from: int | None = None k = 0 while k < len(pieces) - 1: a, b = pieces[k], pieces[k + 1] - if (not (opens_the_name and k == 0 and a[0] == 0) - and len(a) == 1 and len(b) == 1 - and SPLIT_PAIR_OPENS.fullmatch(tokens[a[0]].text) - and split_pair(tokens[a[0]].text, tokens[b[0]].text)): - merge_pieces(pieces, ptags, k, k + 2, add={"suffix"}) - else: - k += 1 + tags = tokens[a[0]].tags + if (len(a) == 1 and len(b) == 1 + and not (opens_the_name and k == 0 and a[0] == 0)): + if SPLIT_CREDENTIAL_TAG in tags: + merge_pieces(pieces, ptags, k, k + 2, add={"suffix"}) + continue + if SPLIT_MEMBER_TAG in tags: + if site is ClauseSite.TRAILING: + merge_pieces(pieces, ptags, k, k + 2, add={"split"}) + splits = True + continue + if site is ClauseSite.GIVEN_SLOT: + if given_from is None: + given_from = leading_titles(pieces, ptags, tokens) + if k > given_from: + merge_pieces(pieces, ptags, k, k + 2, add={"split"}) + continue + k += 1 # rules/M2: "A marker that counts takes the words after it as # the maiden name, and is itself dropped" (history: @@ -684,6 +734,10 @@ def _group_segment(seg: tuple[int, ...], additional: int, read: TailRead | None = None tail_pieces: list[Piece] = [] tail_ptags: list[set[str]] = [] + # too few pieces for the read: a split credential has no words to + # spare (rules/S4) + if splits and len(pieces) + additional < 3: + _settle_splits(pieces, ptags) if len(pieces) + additional >= 3: # rules/P3: "A connective that is also generational # vocabulary joins only where a name word stands on each side @@ -795,6 +849,10 @@ def _group_segment(seg: tuple[int, ...], additional: int, tail_pieces, tail_ptags = pieces[start:], ptags[start:] del pieces[start:] del ptags[start:] + # the read has decided every split credential: one it took + # left with the run, and one still here is two name words + if splits: + _settle_splits(pieces, ptags) # rules/S2: "a trailing word of the suffix vocabulary reads as a # suffix" -- on a main segment the run read above decides it # and is split off already; elsewhere the chain stops where diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index d6fa3e76..66c00c69 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -500,9 +500,9 @@ def starts_a_credential_run(piece: Sequence[int], ptags: Set[str], `preceding` and `one_case` that predicate's (#784): 'John Smith VI Jones' as 'John Smith III Jones'. """ - if "suffix" in ptags: - # the Ph./D. pair alone: a connective join keeps the tag on the - # wider piece ('Ph. D. and Mary'), which starts nothing, as + if "suffix" in ptags or "split" in ptags: + # a split credential alone: a connective join keeps the tag on + # the wider piece ('Ph. D. and Mary'), which starts nothing, as # 'PhD and Mary' does not return len(piece) == 2 if len(piece) != 1: @@ -897,7 +897,7 @@ def segment_suffix_reading(pieces: Sequence[Sequence[int]], # caller (`_comma.decide`) counted two name words before the # comma and found the part a credential run, which settles the # ambiguous word here before S7's anchors are asked (#613) - if len(piece) == 1 and piece[0] in licensed: + if piece[0] in licensed and (len(piece) == 1 or "split" in tags): leading = False out.append(True) continue @@ -1196,7 +1196,12 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], if (k >= 2 and len(piece) == 1 and _ROMAN.match(text := tokens[piece[0]].text.rstrip(".")) is not None and text - and (k == len(rest) or len(text) > 1)): + and (len(text) > 1 + or k == len(rest) + # the walk leaves out the Ph./D. pair, which is the + # credential behind the numeral all the same + and not (rest[k - 1] + 1 < len(ptags) + and "suffix" in ptags[rest[k - 1] + 1]))): tok = tokens[piece[0]] if reads_as_generation(tok.text, tok.tags, tokens[pieces[rest[k - 2]][0]].text, @@ -1234,6 +1239,15 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], # the vocabulary is not in doubt. bare_ambiguous = (len(piece) == 1 and AMBIGUOUS_ACRONYM_TAG in tokens[piece[0]].tags) + # rules/S3: a split credential "is a member of the ambiguous + # class" -- read by the count alone, its writing carrying no + # case to lean on, and in silence: its other reading is two + # name words, which the count already refuses it behind one + if "split" in ptags[rest[k - 1]]: + if k >= 2 and k - 2 >= second: + k -= 1 + continue + break # #516, switch off: the writing still makes this token # credential-SHAPED, and the parser is choosing the name # reading over that one -- the fork the caller asked to be @@ -1301,6 +1315,16 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], return Peel(k, tuple(numerals), tuple(picks), anchors) +def behind_the_merged_credential(rest: Sequence[int], p: int, + ptags: Sequence[Set[str]]) -> bool: + """Whether the Ph./D. pair stands right in front of `rest[p]`, + left out of the walk (`peel_walk`) though it is a credential there + all the same -- the test `run_start` asks inline, its hot loop + paying no frame for it (#827).""" + return (p > 0 and rest[p] - rest[p - 1] == 2 + and "suffix" in ptags[rest[p] - 1]) + + # rules/S8: "A credential after the name core starts a run to the # end of its part" -- applied to a finished peel (#602), so every # reader of the trailing run (assign, P5's reserve, P2's chain stop, @@ -1341,16 +1365,24 @@ def run_start(rest: Sequence[int], names: int, for p in range(names): q = rest[p] tags = tokens[pieces[q][0]].tags - if (core + lone >= 2 - and ("vocab:suffix" in tags - or _ROMAN.match(tokens[pieces[q][0]].text.rstrip(".")) - is not None) - and tags.isdisjoint(_NOT_A_RUN_START) - # p >= 2 here: two name words stand in front - and starts_a_credential_run( - pieces[q], ptags[q], tokens, - tokens[pieces[rest[p - 1]][0]].text, one_case)): - return p + if core + lone >= 2: + # the Ph./D. pair standing in front of this piece starts the + # run there, though the walk leaves it out (#827): 'John + # Smith Ph. D. Jones' as 'John Smith PhD Jones' + # (`behind_the_merged_credential`, inline) + if (q - rest[p - 1] == 2 and "suffix" in ptags[q - 1] + and starts_a_credential_run(pieces[q - 1], ptags[q - 1], + tokens)): + return p + if (("vocab:suffix" in tags or "split" in ptags[q] + or _ROMAN.match(tokens[pieces[q][0]].text.rstrip(".")) + is not None) + and tags.isdisjoint(_NOT_A_RUN_START) + # p >= 2 here: two name words stand in front + and starts_a_credential_run( + pieces[q], ptags[q], tokens, + tokens[pieces[rest[p - 1]][0]].text, one_case)): + return p # a lone particle is not yet a name: 'de Mesnil' is one surname # -- unless it opens a chain run, which is then the surname # (with no `units` every piece opens its own), or the run @@ -1458,13 +1490,17 @@ def credential_run(rest: Sequence[int], peel: Peel, p: int, # a numeral starting the run is read as the generation, which a # shape alone makes a call, reported as the peel's are (rules/S10) numerals = peel.numerals + # where the Ph./D. pair the walk leaves out starts the run + # (`run_start`), rest[p] is the first word behind it, read like the + # rest; else rest[p] is the credential + first = p if behind_the_merged_credential(rest, p, ptags) else p + 1 start = pieces[rest[p]] - if (len(start) == 1 and _ROMAN.match( + if (first > p and len(start) == 1 and _ROMAN.match( (tok := tokens[start[0]]).text.rstrip(".")) is not None and not is_listed_numeral(tok.text, tok.tags) and not numeral_settled_by_case(tok.text, one_case)): numerals = (tuple(start), *numerals) - for q in rest[p + 1:peel.names]: + for q in rest[first:peel.names]: if is_suffix_piece(pieces[q], ptags[q], tokens): continue if is_title_piece(pieces[q], ptags[q], tokens): @@ -1614,7 +1650,7 @@ def trailing_candidates(lo: int, pieces: Sequence[Sequence[int]], k = c - 1 piece = pieces[k] title = is_trailing_title_word(piece, ptags[k], tokens) - if not (title or "suffix" in ptags[k] + if not (title or "suffix" in ptags[k] or "split" in ptags[k] or is_suffix_piece(piece, ptags[k], tokens) or (len(piece) == 1 and ( not tokens[piece[0]].tags.isdisjoint(_TRAILING_WORD_TAGS) @@ -1632,7 +1668,7 @@ def trailing_candidates(lo: int, pieces: Sequence[Sequence[int]], # the tag test is the predicate's own necessary half, inline so a # name word pays no frame return next((j for j in range(lo + 1, c) - if ("suffix" in ptags[j] + if ("suffix" in ptags[j] or "split" in ptags[j] or "vocab:suffix" in tokens[pieces[j][0]].tags or _ROMAN.match(tokens[pieces[j][0]].text.rstrip(".")) is not None) @@ -1944,7 +1980,10 @@ def _chain_units(pieces: Sequence[Sequence[int]], if prefix[k]: j = chain_run_end(k, pieces, ptags, tokens, count) q = k + 1 - while q < j and not _weighed(pieces[q], tokens): + # a split credential is weighed as the class it is a + # member of (rules/S3) + while (q < j and "split" not in ptags[q] + and not _weighed(pieces[q], tokens)): units[q] = JOINED q += 1 k = j diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index fbe60d32..16a6d5b0 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -99,6 +99,18 @@ class WorkToken: _AMBIGUOUS_CREDENTIAL_TAGS = frozenset( {AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG}) +#: The tags classify writes on the FIRST word of a credential written +#: split across a space (`_vocab.split_pair`), naming the piece tag the +#: pair merges with: SPLIT_CREDENTIAL_TAG the Ph./D. pair, merged as an +#: unambiguous suffix ("suffix"), and SPLIT_MEMBER_TAG any other, merged +#: as a member of the ambiguous class ("split", rules/S4, rules/C4). +#: Written by classify because it is the one stage that sees every +#: adjacent pair with the lexicon in hand, so group's merge and +#: `_comma._pieces` read the decision off a tag, C-level, rather than +#: each asking the vocabulary of every pair. +SPLIT_CREDENTIAL_TAG = "split:credential" +SPLIT_MEMBER_TAG = "split:member" + #: The roles that make a piece a name word (rules/P6's walk), and #: the roles of a word not yet read as one: a suffix, or nothing yet #: -- what `_pieces.particle_tail` reads, asked by assign's given-part diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 221ccf08..1a77304c 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -73,36 +73,18 @@ # sync by hand. _PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) -# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), the -# two halves of `split_pair` below. +# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), +# `split_pair`'s first answer, below. _PH = re.compile(r"^ph\.?$", re.IGNORECASE) _D = re.compile(r"^d\.?$", re.IGNORECASE) -#: `split_pair`'s necessary half on its FIRST word, asked C-level by -#: each caller ahead of the call, since the split test runs per -#: adjacent pair and an ordinary name word must pay no frame for it. -#: The same compiled object the predicate asks, so the two cannot -#: drift while the predicate's first-word test is `_PH`; a change to -#: that test must change this with it, or the callers' pre-test hides -#: the new spellings from the predicate. +#: `split_pair`'s necessary half on its FIRST word for the Ph./D. +#: answer alone, asked C-level by `is_wholly_suffix` ahead of the +#: call, the one caller that asks only that answer and has no tags to +#: read. The same compiled object the predicate asks, so the two +#: cannot drift. SPLIT_PAIR_OPENS = _PH -# mechanisms.md#ONE-PREDICATE-PER-QUESTION ("exactly one predicate answers it and every other site calls that one") -# -- the split credential's one test, asked by group's merge, -# `_comma._pieces` and `is_wholly_suffix`, three sites that each kept a -# copy of the pattern pair until #752. decisions/phd-merge, -# decisions/vocabulary-model (#752) -def split_pair(first: str, second: str) -> bool: - """Whether two adjacent words are one credential written split - across a space, and so read as one unit: 'Ph.' then 'D.', with or - without their periods, in any case -- v1's fix_phd, the one split - spelling the parser merges (decisions/phd-merge). Where the merge - may happen (never at the head of the string, rules/S2) is the - caller's question, not this one's. Callers ask - `SPLIT_PAIR_OPENS` of `first` before the call.""" - return (_PH.fullmatch(first) is not None - and _D.fullmatch(second) is not None) - # The codepoint table lives in _policy beside Script -- one copy # importable from the pipeline and the locale packs alike; everything # here DERIVES from it. (single_script's sweep was first written @@ -585,6 +567,70 @@ def is_cased_paired_initials(text: str) -> bool: and text.upper() != text.lower()) +# rules/S3: "A credential written split across a space reads as one +# word" -- one chunk of up to five letters, with or without its period +# ('Pharm.', 'D', 'Sc.'), the shape both words of a split credential +# take. The C-level half classify asks of every adjacent pair before +# the fold is looked up. +SPLIT_CHUNK = re.compile(r"(?:" + _LETTER + r"){1,5}\.?") + + +# mechanisms.md#ONE-PREDICATE-PER-QUESTION ("exactly one predicate answers it and every other site calls that one") +# -- the split credential's one test, asked by classify, which tags +# the pair for group's merge and `_comma._pieces`, and by +# `is_wholly_suffix` for the Ph./D. answer. decisions/phd-merge, +# decisions/vocabulary-model (#752) +def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: + """Whether two adjacent words are one credential written split + across a space, and so read as one unit -- answered with the PIECE + TAG the merged pair carries, or None. + + "suffix": 'Ph.' then 'D.', with or without their periods, in any + case -- v1's fix_phd, merged as an unambiguous suffix wherever it + stands but the head of the string (decisions/phd-merge). + + "split": any other pair of short chunks (`SPLIT_CHUNK`) whose + period-free concatenation is unambiguous suffix vocabulary while + the two are not both suffix vocabulary on their own ('Pharm. D.', + 'Pharm D', 'MB ChB' where 'chb' is unlisted). The merged piece is + a member of the ambiguous class, which the count decides (rules/S4, + rules/C4). Its first chunk has two or more letters: a single + letter opening the pair is a middle initial far more often (FEC + writes 'SMITH, JOHN A. MS.', and 'A. MS.' folds to the listed + 'ams'), so 'M. Sc.' and 'M. D.' stay two words. Two dotted single + letters are the one opening letter asked further, and asked of + #872's predicate rather than a copy: in a script with capitals + they are paired initials (`is_cased_paired_initials`), the reading + rules/S3 gives them joined, so only a caseless script's pair can + merge, and the default vocabulary lists no such fold. + + Where the merge may happen (never at the head of the string, + rules/S2; a member pair only in a main segment and in the part + after a comma) is the caller's question, not this one's.""" + if _PH.fullmatch(first) is not None and _D.fullmatch(second) is not None: + return "suffix" + if (SPLIT_CHUNK.fullmatch(first) is None + or SPLIT_CHUNK.fullmatch(second) is None): + return None + a, b = first.rstrip("."), second.rstrip(".") + if composed_len(a) < 2 and ( + composed_len(b) != 1 or not first.endswith(".") + or not second.endswith(".") + or is_cased_paired_initials(first + second)): + return None + fold = _normalize(a + b) + if (fold in lexicon.suffix_acronyms_ambiguous + or not (fold in lexicon.suffix_acronyms + or fold in lexicon.suffix_words)): + return None + na, nb = _normalize(a), _normalize(b) + if ((na in lexicon.suffix_acronyms or na in lexicon.suffix_words) + and (nb in lexicon.suffix_acronyms + or nb in lexicon.suffix_words)): + return None + return "split" + + def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: """Counts as a suffix as written, with NO initial veto (the veto differs by caller): unambiguous suffix vocabulary, or an ambiguous @@ -1356,7 +1402,11 @@ def is_wholly_suffix(texts: Sequence[str], lexicon: Lexicon, An adjacent Ph./D. pair counts as ONE unit (v1's fix_phd extracted the credential pre-parse, so 'Smith, Ph. D.' read as suffix-comma): - `split_pair`, which group's merge and `_comma._pieces` ask too. + `split_pair`'s "suffix" answer, which group's merge and + `_comma._pieces` read too. Its "split" answer ('Pharm D') does not + count here: that pair is a member of the ambiguous class, which no + run test takes by itself, as it takes no bare 'Ma' (rules/C4's + count reads it instead). `one_case` is ParseState.one_case, and it admits the LEAN: a bare ambiguous acronym written in capitals inside a mixed-case name is @@ -1431,7 +1481,8 @@ def counts_as_suffix(text: str) -> bool: k = 0 while k < len(merged) - 1: if (SPLIT_PAIR_OPENS.fullmatch(merged[k]) - and split_pair(merged[k], merged[k + 1])): + and split_pair(merged[k], merged[k + 1], lexicon) + == "suffix"): merged[k:k + 2] = ["phd"] else: k += 1 From 4b4afa829bd15b140f0dd7cea6956772aa92f763 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 12:44:38 -0700 Subject: [PATCH 02/11] Read a credential split across a space as one word (#752, phase 2e) A pair of short words whose letters together spell unambiguous suffix vocabulary, the first of two or more letters, merges into one piece, a member of the ambiguous class that the count decides (rules/S11): S2's trailing read by S4's words to spare, C4's count after a comma, S5's reading at the end of the given part. A pair the count declines is split back into its two words (`_group._settle_splits`, and a second reading of the part in `_comma.decide`), so it reads as written. The merged Ph./D. pair now starts S8's run as `PhD` does (#827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` -> family 'V'). Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions/README.md | 1 + .../decisions/S10-trailing-roman-numeral.md | 2 + .../S11-credential-split-across-a-space.md | 28 ++++ ...S7-company-of-an-unambiguous-credential.md | 6 +- .../decisions/S8-credentials-trailing-run.md | 6 +- docs/design/decisions/phd-merge.md | 2 +- docs/design/decisions/vocabulary-model.md | 8 +- docs/design/mechanisms.md | 12 ++ ...ambiguous-credential-class-at-the-comma.md | 2 +- docs/design/rules/README.md | 1 + .../rules/S10-trailing-roman-numeral.md | 8 +- .../S11-credential-split-across-a-space.md | 83 ++++++++++++ ...S7-company-of-an-unambiguous-credential.md | 2 +- .../rules/S8-credentials-trailing-run.md | 5 +- docs/design/vocabulary-model-rfc.md | 2 +- docs/release_log.rst | 2 + nameparser/_pipeline/_assign.py | 2 +- nameparser/_pipeline/_classify.py | 23 +++- nameparser/_pipeline/_comma.py | 43 ++++-- nameparser/_pipeline/_group.py | 7 +- nameparser/_pipeline/_pieces.py | 6 +- nameparser/_pipeline/_vocab.py | 13 +- tests/v2/cases.py | 124 ++++++++++++++++-- tests/v2/pipeline/test_group.py | 15 ++- tests/v2/pipeline/test_vocab.py | 61 +++++++-- tests/v2/rules_budget.json | 9 +- tests/v2/test_cases.py | 9 +- tests/v2/test_parser.py | 10 +- tools/differential/corpus_rules.jsonl | 21 +++ tools/differential/snapshot.jsonl | 25 +++- 31 files changed, 455 insertions(+), 85 deletions(-) create mode 100644 docs/design/decisions/S11-credential-split-across-a-space.md create mode 100644 docs/design/rules/S11-credential-split-across-a-space.md diff --git a/AGENTS.md b/AGENTS.md index 1748e5d6..eb3a8b40 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,7 +186,7 @@ The library has two layers: `nameparser/config/` (data) and `nameparser/parser.p **Sweep the forms a change can reach: comma shapes, then `name_order`.** No comma, a FULL name before the comma, a ONE-WORD name before the comma and NOTHING before the comma are four paths, and the last two are the misses — #429, #430 and #432 are all the one-word path disagreeing with the full-name path on inputs the full-name path reads correctly, and #613's PR review found `, PhD` and `, Jr.` (an empty surname field, as CSV data writes it) read as given and title by a decision that returned early on an empty head, after two review rounds whose every grid had a name before the comma. Put an empty head and a nickname-only head (`(Bob), PhD`) in every comma grid. Count the one-word path the way C3 counts, which is neither tokens nor pieces: a particle run and the one word it attaches to are one word (#575), a connective join and a bound given-name pair count as their words (#613, so `Ortega y Gasset, Dr.` is a whole name), and a suffix beside the surname counts as no name word at all. #603's review found the suffix half (`Vega y Lopez Jr., PhD Jones` lost its family where `Vega y Lopez, PhD Jones` kept it); #613 then reversed the connective half, and both now read given 'Vega', middle 'y', family 'Lopez' — so test a particle surname, a connective surname and a surname with a suffix, each before a comma. For orders the sweep already exists (`tests/v2/test_cases.py` runs every row under all three) but carries one assertion, R2's family partition, so it checks nothing a new change moves; coverage there has been vacuous before (PR #394's review found the suite passed with `name_order` discarded from grouping). Parse your change's names in each comma shape and each order, and read the ones you did not predict. And read a generated grid's moves by SHAPE, not by corpus membership: #614's first design passed the suite, the five gates and a fingerprint of every corpus name, because its grid moves were sorted into 'corpus name moved' and 'garbage', and the review found the moves that mattered among the 'garbage' -- a bare salutation (`Mr. and Mrs.` lost its connective to the family name) and a name with one trailing word (`Freiherr von Berg X.Y.Z.`). Pull out the head-plus-one-word slice of the grid and a salutation with nothing after it, and read every move there. Two more were the second review's (#614): no grid put TWO words that are both a title and a particle at the head (`Freiherr St van Berg MA`, where an index into the pieces was applied to a merged view of them -- `TITLES ∩ particles_ambiguous` is `{freiherr, st}`, so put both in every head list, each in its bare and its period-marked spelling -- #637's grid had `St` and `Freiherr` alone and missed that `St.` and `Freiherr.` are declined by the bound join and then attached by P6), and the grids compared ambiguity KINDS, so a report telling the caller the wrong field (`Freiherr von Berg Ma`, 'read as a given name' of a word in the family) passed every pin. #701's review added one: put a title/suffix dual (`Ms`, `Sr.`, `Cpl`) SECOND in a title run, behind an ordinary title. A stop that asks only "is this word suffix vocabulary" passes every row that opens with the dual and fails `Rev. Sr. Smith, Mary`; it was found by comparing `T Smith, John` against `T John Smith` over every title word, and a comma-versus-comma-less comparison is the cheap check for any rule that moves a word between the two slots. Compare each report's `detail` against the field its tokens land in -- since #626 `test_a_report_names_the_field_its_word_lands_in` (tests/v2/test_cases.py) does it over every non-locale case row under all three orders (a row carrying its own policy only as declared), for the phrasings its `_FIELD_CLAIMS` lists -- so a row with the shape is the test only where the report's wording is listed: a new emitter's wording goes in that list, and segment's report on a part past the second comma names no field (#629). #626's own first grid then missed an emitter for want of one comma shape (`, Ma Dr.`: an empty head, an ambiguous word, a trailing title, which H1 moves to the family): a trailing title belongs in every empty-head comma grid too. And #620's: an exclusion list in code being replaced is a set of rules, not noise -- #614's merged copy excluded every word the trailing read weighs from its units, the first #620 prototype dropped the list as a symptom of the copy, and its review found the list had been keeping positional counting's rule that a member in front counts as a word (`Freiherr von Berg MA X.Y.Z.` lost its suffix). Before deleting one, mutate each entry out and read what moves. The converse holds for replacing a hand copy WITH the shared predicate: it is a behavior change wherever the two disagreed, and #735's /simplify shipped one as "no reading moves" because the suite and the snapshot were its only check (`St Sir de Mesnil` family-first, a token test replaced by P1's piece test); diff the two predicates' answers over a grid before calling the swap a refactor. -**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, a one-case name being counted like capitals whatever the letter -- since #721 a one-case one-letter numeral carries the `initial` tag its capital does, and since #723 every one-case bare letter does (rules/T4), and a bare letter in front of the numeral is initial-shaped as its capital is, so `john v` reads as `JOHN V` does over `test_a_one_case_name_reads_as_its_capitals`'s grid (comma shapes, orders, a maiden clause, a particle chain, a bare letter in front, a trailing title, a non-ASCII letter in front: 315 of its then 1,872 pairs disagreed at 2bfa9533, before #721, and none after). Its first version claimed "every shape" with no bare letter in front and no trailing title in the grid, and #721's reviews found both broken; a grid's claim is the grid -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). #656's sweep found the same class from the other side: C12 takes the word ending the name before a comma as EVIDENCE, and its first draft asked the suffix-word tag, so `John Smith VI, AD` missed the rule `III` reached, and then counted the whole part, so `Smith VI, AD` fired with `VI` as its own name word and `Smith V, AD` with a `V` that S10 reads as the family. A rule that weighs a numeral as a generation must ask the predicate that READS it as one (the trailing fork, `_vocab.reads_as_generation`, and its words-to-spare count), not the vocabulary that lists some of its spellings. The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, `MD/PhD` by its listed chunks and `PU-PH` by the caps shape's joined spelling, #752, each with its joiner glued, spaced and glued to one side, which are three token streams -- and a new chunk reading inherits the old one's exceptions, which #752's first draft did not (`Jack I-V` lost its family to the listed numerals, and the fix's length test counted a chunk's period, so `Jack I.-V.` did too, found by the second review), and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there, and `Mr. and Mrs.` a run P3 joins, carrying a piece tag a one-word vocabulary test never sees -- PR #735's review found C12's hand copy of C13's title test (C8's until its 2026-10-19 split) reading `John Smith Jr., Mr. and Mrs. AD` as all suffix; ask C13's predicates in C13's order, `not is_suffix_piece` then `is_title_piece`, not the token tags, since `is_title_piece` alone reads the duals `Ms` and `Sr.` as titles). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. And a separator glued to a word hides the word from every vocabulary test, not only the one being changed: the semicolon delimiter's first draft read a glued word's stem in C1's run test alone and put 365 of ClinicalTrials.gov's reversed names right, missing the ones C13, C4 and C6 read off a word's TAGS (`MBBS;`, `MA;`); tagging the stem put 412 right, and reading it again in the comma decision's shape tests, which read a word's text rather than its tags (`MBBS;`), 431 (decisions/C10, 2026-10-24). A rule over a glued spelling reads the stem everywhere the word is read, its tags and every test of its text, and is swept against the space twin -- with a declared word delimiter too, which a substring test finds inside other words ('Stand' under ' and ', the review of the same change). A full stop INSIDE a word has four spellings of its own: initials run together (`J.R.R.`), joined words each closed by a stop (`Lt.Gov.`, `Msc.Ed.`), a joined word or name closed by none (`St.John`, `J.Smith`), and an abbreviation's own ending, closed or not (Italian `Sig.ra`, Latvian `Dr.philol.`). Shape tells the last from none of the others, so a rule over the shape decides which it costs; #838 found the ending only through the snapshot's CLDR row `Dut.ra Prof.ra`, and its first narrowing traded `St.John` for `Sig.ra` while the snapshot listed the `Dr.philol.` move among moves read as intended, until the design-docs review named it. Initials run together are also how some SURNAMES are written (Nepali `B.K.`, `K.C.`), and a caseless script writes its abbreviations, degrees included, as dotted letters (Thai `พ.บ.`): #864 found S3's position reading taking the first from 2.4.0 on, visible only once #866's Nepal candidates gave the corpora a population that writes them, and its first fix moved CLDR's Thai degrees into the name. A rule over dotted letters reads both before it ships. +**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, a one-case name being counted like capitals whatever the letter -- since #721 a one-case one-letter numeral carries the `initial` tag its capital does, and since #723 every one-case bare letter does (rules/T4), and a bare letter in front of the numeral is initial-shaped as its capital is, so `john v` reads as `JOHN V` does over `test_a_one_case_name_reads_as_its_capitals`'s grid (comma shapes, orders, a maiden clause, a particle chain, a bare letter in front, a trailing title, a non-ASCII letter in front: 315 of its then 1,872 pairs disagreed at 2bfa9533, before #721, and none after). Its first version claimed "every shape" with no bare letter in front and no trailing title in the grid, and #721's reviews found both broken; a grid's claim is the grid -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). #656's sweep found the same class from the other side: C12 takes the word ending the name before a comma as EVIDENCE, and its first draft asked the suffix-word tag, so `John Smith VI, AD` missed the rule `III` reached, and then counted the whole part, so `Smith VI, AD` fired with `VI` as its own name word and `Smith V, AD` with a `V` that S10 reads as the family. A rule that weighs a numeral as a generation must ask the predicate that READS it as one (the trailing fork, `_vocab.reads_as_generation`, and its words-to-spare count), not the vocabulary that lists some of its spellings. The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, `MD/PhD` by its listed chunks and `PU-PH` by the caps shape's joined spelling, #752, each with its joiner glued, spaced and glued to one side, which are three token streams -- and a new chunk reading inherits the old one's exceptions, which #752's first draft did not (`Jack I-V` lost its family to the listed numerals, and the fix's length test counted a chunk's period, so `Jack I.-V.` did too, found by the second review), and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there, and `Mr. and Mrs.` a run P3 joins, carrying a piece tag a one-word vocabulary test never sees -- PR #735's review found C12's hand copy of C13's title test (C8's until its 2026-10-19 split) reading `John Smith Jr., Mr. and Mrs. AD` as all suffix; ask C13's predicates in C13's order, `not is_suffix_piece` then `is_title_piece`, not the token tags, since `is_title_piece` alone reads the duals `Ms` and `Sr.` as titles). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. And a separator glued to a word hides the word from every vocabulary test, not only the one being changed: the semicolon delimiter's first draft read a glued word's stem in C1's run test alone and put 365 of ClinicalTrials.gov's reversed names right, missing the ones C13, C4 and C6 read off a word's TAGS (`MBBS;`, `MA;`); tagging the stem put 412 right, and reading it again in the comma decision's shape tests, which read a word's text rather than its tags (`MBBS;`), 431 (decisions/C10, 2026-10-24). A rule over a glued spelling reads the stem everywhere the word is read, its tags and every test of its text, and is swept against the space twin -- with a declared word delimiter too, which a substring test finds inside other words ('Stand' under ' and ', the review of the same change). A full stop INSIDE a word has four spellings of its own: initials run together (`J.R.R.`), joined words each closed by a stop (`Lt.Gov.`, `Msc.Ed.`), a joined word or name closed by none (`St.John`, `J.Smith`), and an abbreviation's own ending, closed or not (Italian `Sig.ra`, Latvian `Dr.philol.`). Shape tells the last from none of the others, so a rule over the shape decides which it costs; #838 found the ending only through the snapshot's CLDR row `Dut.ra Prof.ra`, and its first narrowing traded `St.John` for `Sig.ra` while the snapshot listed the `Dr.philol.` move among moves read as intended, until the design-docs review named it. Initials run together are also how some SURNAMES are written (Nepali `B.K.`, `K.C.`), and a caseless script writes its abbreviations, degrees included, as dotted letters (Thai `พ.บ.`): #864 found S3's position reading taking the first from 2.4.0 on, visible only once #866's Nepal candidates gave the corpora a population that writes them, and its first fix moved CLDR's Thai degrees into the name. A rule over dotted letters reads both before it ships. A predicate over ADJACENT PAIRS of words meets two shapes no grid of one word holds: an overlapping pair and a joining word beside an initial. #752's split-credential test (phase 2e) read `Dr. Ph.` in `Jane Doe Dr. Ph. D.` as the listed `drph`, leaving `D.` a word of its own (found by the maiden take's candidate grid), and a particle beside a surname written short as the listed `dea` (`DE A JUNIOR`, found only in tse-donors' readings), so put a title, a particle and a connective beside each pair spelling, and a pair overlapping another, in the grid. **And the encoding: a letter class written `[^\W\d_]` or tested with `isalpha()` is blind to a combining mark.** Python's `\w` and `isalpha()` do not match Mn, so a letter written as base plus accent (NFD) ends a word early; the vocabulary path is immune because `_normalize` NFC-folds, the shape path is not. #652: `Dr. inž. Janez Novak` read `inž.` as the given name when the ž was decomposed, and `Janez Novak, ŽŠČ` reversed, in two families that share the blind spot — the letter-shape regexes (`period_abbreviation`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`), now spelled with `_vocab._LETTER` (a letter plus the general combining blocks; `test_regex_sync` compares the two `_MARKS` strings and pins `period_abbreviation` against config's copy), and the caps predicates, which NFC-compose a non-ASCII word before `isalpha()` (`is_caps_word`, behind C-level `isalpha()`/`is_normalized` tests, so already-composed text pays no frame — measured 2026-10-12, `Smith, José` was 193 frames before and after; the first version called it for every non-ASCII word and cost one frame each, and no guard saw it because the reference name is ASCII). A width test must count letters, not code points (`composed_len`): admitting a decomposed word to the caps set put `John Smith, MÁ` on the pair exception's wrong side, and the review's grid found it where the rows (all three letters long) did not — put a two-letter word in every caps grid. The initial shape (`_INITIAL`, `\w\.`, v1-verbatim and public) is not widened but asked of the COMPOSED word, in `is_initial_shaped` and in `_render`'s two copies; and the single-letter connective tests count composed letters, since a decomposed `й` is two code points (#665, where `É. y É. Smith` and `John й Smith` read differently decomposed). Compose the WHOLE predicate, not one half: `is_initial` is a shape ANDed with the #320 script veto, and composing the shape alone made a decomposed `씨.` an initial, since decomposed hangul is conjoining jamo outside every script range (#665's first version; `in_initialless_script` composes too now). Not covered, deliberately: the marks only some scripts write (Indic vowel signs, Arabic and Hebrew points), declined in decisions/nfd-shapes because widening the class would move readings nobody had measured; rules/H2's Accepted clause records the one consequence it names, that a period-marked abbreviation in Bengali or Devanagari never reads as a title by shape. The guard is `test_a_decomposed_name_renders_as_its_composed_twin` over the CLDR corpus plus `test_a_decomposed_abbreviation_reads_as_its_composed_twin` (rows × three orders × EVERYWHERE); the corpus guard reaches a shape only if some corpus name does, which is how the caps shape was missed until review, so when adding a shape rule add a row there. Four of #652's rows and two of #665's pass with every fix reverted (measured) and are parity pins, and the test compares ambiguity kinds and `initials()` too, since #665's `John й Smith` differed in its report; `_DOTTED` has no observable input and is pinned by a direct call. diff --git a/docs/design/decisions/README.md b/docs/design/decisions/README.md index f73a0000..4f84b8b7 100644 --- a/docs/design/decisions/README.md +++ b/docs/design/decisions/README.md @@ -55,6 +55,7 @@ Entry conventions: - [S8 — the credential's trailing run](S8-credentials-trailing-run.md) - [S9 — unlisted words that join the ambiguous class by shape](S9-unlisted-words-that-join-the-ambiguous-class-by-shape.md) - [S10 — the trailing roman numeral](S10-trailing-roman-numeral.md) +- [S11 — a credential split across a space](S11-credential-split-across-a-space.md) ## Nicknames & quoted names (N) diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index b018726e..13030987 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -12,6 +12,8 @@ - A bare letter in a one-case name is the initial its capital is (rules/T4), and the one-letter numerals are an instance of that rule — 2026-10-18 #723. - The one-letter reading rests on the count, not on the generation being rare, and a one-case part before a capitalized credential reads under C12 (`john j v, PhD`) — 2026-10-19 #761. +Elsewhere: a one-letter numeral before a credential split across a space reads as it reads before the one-word spelling (`John Smith V Ph. D.` → family 'V', reported), where it read suffix 'V Ph. D.' as 1.4.0 did — decisions/S11's 2026-10-25 #752 entry. + Open: none in this file. ## Log diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md new file mode 100644 index 00000000..5e70f05a --- /dev/null +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -0,0 +1,28 @@ +# S11 — a credential split across a space + +## Standing + +Examples name each field as the default order reads it; under the family-first orders the given and family names trade places. + +- Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de A Junior` → family 'de A') — 2026-10-25 #752. +- The count decides it as it decides a bare member, and where the count declines it the pair is the two words it is written as: at the end of a name by S4's words to spare (`John Pharm D` → middle 'Pharm', family 'D'), after a comma by C4's count (`John Smith, Pharm D` → suffix; `Smith, Pharm D` → given 'Pharm'), ending the given part after a family comma by S5's reading (`Smith, John Pharm D` → suffix); the part before a family comma merges none (`Smith Pharm D, John` → family 'Smith Pharm D') — 2026-10-25 #752. +- Behind two name words with no comma, or after the given word after a family comma, the pair starts S8's run, the one member of the class that does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones', reported) — 2026-10-25 #752. +- The reading reports nothing either way (`John Smith MB ChB` → suffix 'MB ChB', where `mb` started the run and reported 'ChB'; `SMITH, SAM E.` → given 'SAM', middle 'E.') — 2026-10-25 #752. +- The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. +- Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.') — 2026-10-25 #752. +- The Ph./D. pair starts S8's run from where it stands, though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, #827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` → family 'V', reported) — 2026-10-25 #752. +- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial behind two name words with no comma read as the credential their letters spell (`JOHN SMITH SAM E.` → suffix 'SAM E.') — 2026-10-25 #752. + +Open: none in this file. + +## Log + +- 2026-10-25 #752 — A CREDENTIAL WRITTEN SPLIT ACROSS A SPACE IS ONE WORD, A MEMBER OF THE AMBIGUOUS CLASS THAT THE COUNT DECIDES (phase 2e: item 6 of decisions/vocabulary-model's 2026-10-23 #752 entry, agreed by Derek on 2026-10-23, and #827's reading (A)). `_vocab.split_pair` was the Ph./D. test alone (decisions/phd-merge); it now answers any pair with the piece tag it merges with, "suffix" for Ph./D. and "split" for the rest, and classify asks it once per adjacent pair of a segment, writing the answer on the pair's first word, so group's merge and the comma's reading of its part read a tag rather than each asking the vocabulary. + THE PAIR. Each word is one to five letters with or without its period, the two together spell unambiguous suffix vocabulary with the periods removed, and they are not both suffix vocabulary on their own (`MB ChB` merges, `chb` being unlisted; `MD PhD` stays two credentials). The first word has two or more letters: FEC's donors write a middle initial before a trailing salutation (`SMITH, JOHN A. MS.`), and `A. MS.` folds to the listed `ams` (merging any such pair but two single letters moved 5,122 FEC names in the 2026-10-23 measurement, `A. MS.` 2,246 of them). Two dotted single letters are the one opening letter asked further, by #872's `is_cased_paired_initials` rather than a copy of it: in a script with capitals they are paired initials, the reading rules/S3 gives them joined, so only a caseless script's pair merges, and only where its letters spell a listed word, which no default entry does (the overlap the 2026-10-25 one-predicate entry in decisions/vocabulary-model left to this change to state). Two refusals the measurement below added: a particle or a connective written without its period is no chunk, since it joins name words (`DE A`, a particle and a surname written short, folds to the listed `dea` in two tse-donors names behind `JUNIOR`), while a letter written with its period is an initial, not the connective it spells (`SAM E.` stays a pair, which the count then declines); and where two pairs overlap the Ph./D. pair merges (`Dr. Ph. D.`, whose `Dr. Ph.` folds to the listed `drph`, the case table's `the_split_degree_outranks_a_pair_overlapping_it`). + WHERE IT IS READ. As a member, by the count: in a main segment group merges the pair and S2's trailing read decides it, the words to spare S4 asks of a bare member, and a pair the read did not take is split back into its two words before anything else reads the segment, so a declined pair reads exactly as the two words read unmerged; in the part after a comma `_comma.decide` reads the part with the pair merged, C4 counting the name words before the comma, and where that reading does not take the pair as a credential reads the part again as written, so an `MB` the pair hid opens the part again (C13) and `John Smith, MB ChB BAO` keeps the suffix reading it had (the first version left the pair merged there, which made `MB` the given name in 15 ctgov names); in the given part after a family comma group merges the pair only behind the given word and S5's walk reads it as it reads a member ending the part, its writing saying nothing else. The part before a family comma reads no member as a suffix (`Smith MA, John`), so no pair merges there. The merged pair has no case of its own to lean on, so it reads as a member admitted by shape does (`St van Berg Pharm D` keeps family 'van Berg Pharm D', as `St van Berg X.Y.Z.` does). + THE RUN (S8). S8 refuses a member of the ambiguous class as a run start, because a member is a word of the name in its other reading and nothing in front of it says otherwise. A split credential's other reading is two words of the name, a given name and an initial, and the run starts only where two name words already stand in front of it, the words to spare its own count asks: so it is the one member that starts the run, behind two name words with no comma and after the given word after a family comma, as its one-word spelling does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones'). The Ph./D. pair, which the walk leaves out because assign reads it as a suffix wherever it stands, starts the run from where it stands, in front of the first word behind it (#827, reading (A): `John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, as `John Smith PhD Jones` reads; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng', the reading 2.3.0 gave). And a one-letter numeral with the pair behind it is the name word it is before `PhD`, Derek's 2026-10-25 call on #886 for the one-word spelling (`John Smith V Ph. D.` → family 'V', reported; it read suffix 'V Ph. D.', 1.4.0's reading, which rules/S10's Accepted clause had stated). Reading the numeral as the generation before the split spelling alone was the other option; it would keep the two spellings apart for one numeral shape, and no name in the five local corpora measured below moved by it, so the committed rules example is the only reading the choice moves. + SILENT. A member reports either reading at the slots S4 names; the split pair reports neither. Where the count takes it, its letters spell a listed credential nobody hesitates over (`John Smith Pharm. D.`); where the count declines it, the two words are a given name and an initial (`SMITH, SAM E.`), read unmerged and reporting nothing, as they did before. The one reading the silence hides is Accepted (ii) below. + ACCEPTED. (i) `M. D.`, `M D` and `M. Sc.` open with a single letter and stay two words, so after a comma behind a whole name the name stays reversed: in ctgov-officials (fetched 2026-10-21), of the 205 distinct names whose whole part after the comma is one of the five spellings the 2026-10-23 tally named, 133 still read the part's first word as the given name (`M. D.` 71, `M D` 41, `M. Sc.` 20, and one `Pharm D` behind a one-word head), where 185 did before (`Pharm. D.` read title 'Pharm.' there, and is not among them). (Correcting decisions/vocabulary-model's 2026-10-23 #752 entry, item 6: its "149 of the 359" officials still reversing counted `M. D.` and `M D` alone, and `M. Sc.`, which opens with a single letter, stays unmerged too.) (ii) A given name and its initial behind two name words with no comma read as the credential their letters spell (`JOHN SMITH SAM E.` → suffix 'SAM E.'), in silence; the FEC writes such a name behind a comma and one family word, where C4's count refuses it. + MEASURED (2026-10-25, Python 3.11.17, against master 225f3d6f, every local check with `--base` that checkout, so no shared snapshot was read or recorded): the committed snapshot moves 2 names, both rules examples (`John Smith Ph. D. Jones`, `John Smith V Ph. D.`), and gains S11's 21 new examples; every CLDR, GND, Wikidata and conventions agreement record matches. ctgov-officials (fetched 2026-10-21) moves 357 entries: 229 only drop a report (`MB ChB` 179, `MB BCh` 48, `DM Sc` 2, read as one credential where `mb` or `dm` started the run and reported the word behind it), and 128 move a field, each a split credential read whole where the name had read reversed or the credential as a name word (`Pharm D` and its spellings 110, `Psy D` 5, `Dr PH` 5, `Ed D` 3, `Sc. D.` 1, and `MB ChB` or `MB BCh` behind another degree 4); its force-repaired case view writes a capitals half as it did (`MB BCH`), `Ed D` forcing to `ED D` as a bare `Ed` read as a suffix does. tse-donors moves 2 (`JUNIOR` typed `JUN IOR`, now the suffix), and its pairs view (`local_corpora.py pairs tse-donors --agreement`) is unchanged; fec-individuals moves 1 (`USN R`, now the suffix run's); nepal-candidates moves none; psc-individuals (1,039,171 records scored) matches the base: 0 fixed, 0 broken, 0 changed, 1,004,782 agreeing on every field. The spike figures in decisions/vocabulary-model's 2026-10-23 entry (352 ctgov, 7 tse, 30 FEC for the first-chunk rule; 391, 34 and 41 with the run start) were a cruder rule's, with no count, no refusal of a particle chunk and no retry after a comma; they were not re-taken. + FRAMES (`sys.setprofile` call events, py3.11): the reference name, `John Smith`, `Smith, John` and `John Smith, PhD` cost what they did (363, 148, 157, 201). Classify's pair test is C-level up to the call, so a call is made only for a pair that passes the shape, the opening letter's length and the fold's membership: `John Smith Ph. D.` 232 → 233, the 64-link benchmark row unmoved once the length joined the pre-test (`i i` folds to the listed `ii`, and the first version called the predicate for every pair of links, +188). A name holding a member pair pays for its reading: `John Smith Pharm D` 230 → 238, `John Smith, MB ChB` 250 → 266, `Smith, John Pharm D` 253 → 284. + Recompute: `uv run python tools/differential/snapshot.py --since ` for the committed snapshot; for each local corpus `snapshot.py --local --base `, and `local_agreement.py psc-individuals --base `; the accepted figure by reading `corpus.jsonl` of ctgov-officials, keeping each name holding a comma whose text after its first comma, stripped, is one of `M. D.`, `M D`, `M. Sc.`, `Pharm D` and `Pharm. D.`, and counting those whose given name opens with that part's first word, on this tree and on the parent. diff --git a/docs/design/decisions/S7-company-of-an-unambiguous-credential.md b/docs/design/decisions/S7-company-of-an-unambiguous-credential.md index 8b84246c..d5a32144 100644 --- a/docs/design/decisions/S7-company-of-an-unambiguous-credential.md +++ b/docs/design/decisions/S7-company-of-an-unambiguous-credential.md @@ -8,19 +8,19 @@ Examples name each field as the default order reads it; under the family-first o - Where no credential run starts (S8's, or C13's in a part a credential opens after a comma), as with one name word before the credential and no comma, what anchors nothing and ends the run is the walk's own leading piece (`PhD Ma` → given 'PhD', family 'Ma'), a connective (`Jane Doe nee Puig i Ma` keeps maiden 'Puig i Ma'), a single-letter roman numeral in any case (`Om Jr V Ma` keeps family 'Ma', where `Om Jr III Ma` reads suffix 'Jr III Ma'), and a word of both the particle and the unambiguous suffix vocabulary (`Jan vd Ma` → family 'vd Ma'; `Smith vd Ma, John` → family 'Smith vd Ma'), whose exclusion is of the word in front only; where either run starts it takes the numeral, the particle and a name word (`John Smith PhD V Ma` and `Smith, PhD V Ma` → suffix 'PhD V Ma'), while a connective right behind the credential is joined to it first, so S8's run never starts (`John Smith PhD y Ma` → family 'PhD y Ma', `Smith, John PhD and Ma` → middle 'PhD and Ma'), except in C13's part (`Smith, PhD and Ma` → suffix 'PhD and Ma'), and one further in is the run's (`John Smith PhD Jones and Ma` → suffix 'PhD Jones and Ma') — 2026-09-27 #544, A CREDENTIAL RUN. - A title/suffix dual in the given part's leading title run after a one-word family comma is a title there, anchors nothing and silences the company for the part (`Smith, Ms Ma` → given 'Ma'; `Smith, RA PhD Ma` → title 'RA', given 'PhD', middle 'Ma'); anywhere else a dual anchors as any suffix word (`Jane Doe, MS LAc` → suffix 'MS LAc', at the accepted cost `John Smith, Ms Ma` → suffix 'Ms Ma') — 2026-09-27 #544, A CREDENTIAL RUN. - A member the company decides reports `suffix-or-name` where it was decided (`Smith, PhD Ma`); a member its own capitals made the credential reports nothing new (`John Smith, PhD MA`) — 2026-09-27 #544, A CREDENTIAL RUN. -- Accepted: `Om Jr Ma` reads given 'Om', suffix 'Jr Ma' with no family name, and `Smith, MA PhD Ma` reads suffix 'MA PhD Ma' with no given name; the split `Ph. D.` in a main segment (no comma, or before a suffix comma) is outside the walk, so `John Smith Ph. D. MEng` keeps family 'MEng', while the comma and given-slot spellings anchor (`Smith, Ph. D. MEng` → suffix 'Ph. D. MEng') — 2026-09-27 #544, A CREDENTIAL RUN. +- Accepted: `Om Jr Ma` reads given 'Om', suffix 'Jr Ma' with no family name, and `Smith, MA PhD Ma` reads suffix 'MA PhD Ma' with no given name; the comma and given-slot spellings of the split `Ph. D.` anchor (`Smith, Ph. D. MEng` → suffix 'Ph. D. MEng') — 2026-09-27 #544, A CREDENTIAL RUN. - An unambiguous credential in front of the member overrides P6's attachment at the given slot, as the capitals do (`Doe, John PhD Do` → suffix 'PhD Do'); the one-case record with nothing in front keeps P6's reading (`NASCIMENTO, EDSON ARANTES DO` → family 'DO NASCIMENTO') — 2026-09-27 #544, THE #531 PAIRING. - `abdul Smith Jr Ma` reads given 'abdul', family 'Smith', suffix 'Jr Ma', the anchor reversing decisions/S6's accepted item (ii) — 2026-09-27 (#544). Not standing: the split of 2026-10-17 #682, carried out. -Elsewhere: C1's count reading a post-comma run as it reads one word (`John Smith, Ed Ma` → suffix 'Ed Ma') — decisions/C4's 2026-09-27 (Derek), #544 entry; a listed member's chunked dotted spelling passing the period gate (`Wang M.Eng.`) — decisions/suffix-acronym-collisions' 2026-09-27 (#544) entry; a title between the credential and the member, and a member a particle chain took, read into the credential's run (`John Smith PhD Prof. Ma` → suffix 'PhD Ma', title 'Prof.') — decisions/S8's 2026-10-04 (Derek), #602 entry; a marker behind a credential is an ordinary word the run takes (`Jane Doe Jr. nee Smith Ma` → suffix 'Jr. nee Smith Ma') — decisions/M2's 2026-10-04 (Derek), #601 entry; `om` marked ambiguous, so it anchors nothing (`John Om Ma` → family 'Ma') — decisions/suffix-acronym-collisions' 2026-10-21 entry marking 33 credential readings ambiguous. +Elsewhere: C1's count reading a post-comma run as it reads one word (`John Smith, Ed Ma` → suffix 'Ed Ma') — decisions/C4's 2026-09-27 (Derek), #544 entry; a listed member's chunked dotted spelling passing the period gate (`Wang M.Eng.`) — decisions/suffix-acronym-collisions' 2026-09-27 (#544) entry; a title between the credential and the member, and a member a particle chain took, read into the credential's run (`John Smith PhD Prof. Ma` → suffix 'PhD Ma', title 'Prof.') — decisions/S8's 2026-10-04 (Derek), #602 entry; a marker behind a credential is an ordinary word the run takes (`Jane Doe Jr. nee Smith Ma` → suffix 'Jr. nee Smith Ma') — decisions/M2's 2026-10-04 (Derek), #601 entry; `om` marked ambiguous, so it anchors nothing (`John Om Ma` → family 'Ma') — decisions/suffix-acronym-collisions' 2026-10-21 entry marking 33 credential readings ambiguous; the split `Ph. D.` in a main segment starting the credential's run, so `John Smith Ph. D. MEng` reads suffix 'Ph. D. MEng' — decisions/S11's 2026-10-25 #752 entry. Open: none in this file. ## Log -- 2026-09-27 (Derek), #544 (the `doe, jane v phd do` reading in its (7) and its 2026-09-28 addendum superseded by #721, decisions/S10: the one-case `v` is an initial and ends the run, so the name reads middle 'v', suffix 'phd do', the company still taking 'do') (its (10)'s `md` examples superseded by #853, decisions/indic-honorifics, `md` being no title: `Smith, MD PhD Ma` reads suffix 'MD PhD Ma') (its (6) and the addendum's `Jane Doe, PhD, Jr nee van Ma` superseded by #601, decisions/M2, a marker behind a credential or past a comma being an ordinary word; its ACCEPTED LIMITS' `John Smith PhD Prof. Ma`, `John Smith PhD Do Do` and `Smith, PhD Do Ma` superseded by the credential's run, #602, decisions/S8, and #603, decisions/C13, which read each as suffixes; (d)'s `John Smith PhD V Ma` superseded the same way, suffix 'PhD V Ma'; (c)'s `John Om Ma` superseded by the 2026-10-21 marking of `om` ambiguous, decisions/suffix-acronym-collisions, so it reads family 'Ma') — A CREDENTIAL RUN KEEPS ITS AMBIGUOUS MEMBERS: C1'S NAME-WORD COUNT READS A RUN (rules/C4 since #728) AS IT READS ONE WORD, AN UNAMBIGUOUS CREDENTIAL IN FRONT OF A MEMBER SPEAKS FOR IT AT EVERY TRAILING SLOT, AND A LISTED MEMBER'S CHUNKED DOTTED SPELLING PASSES THE PERIOD GATE. Three readers never looked past the member itself: C1 flipped a part to the suffix comma by the name-word count only when it was ONE token, and the trailing readers — the no-comma peel, the comma part read wholly as credentials, the given part's trailing slot — met the member first, took its name lean and stopped, so the credential in front of it was never reached. #540 marked `meng` and `lac`, whose conventional spellings lean name, which turned the one-case-row cost decisions/S6's 2026-09-15 item (ii) accepted into the ordinary spelling of two degrees. DECIDED, ten forks, (9) and (10) on 2026-09-28. (1) THE WHOLE RUN AT C1: two or more words after the comma, every one suffix vocabulary or a class candidate, at least one a candidate, behind two or more name words, is the credential run — all-member runs (`John Smith, Ed Ma`) and a leading member (`John Smith, MEng PhD`) included; offered and declined were an anchored-only rule and the issue's own "last word ambiguous" wording, which contradicted its `Ed Ma` consequence. (2) ALL TRAILING PATHS, ANCHORED IN FRONT: a listed member with an unambiguous credential in front of it, through other members, reads as the credential on the no-comma peel, in the comma part read wholly as credentials and at the given slot; a credential BEHIND it anchors nothing (`Wang Ma PhD` keeps family 'Ma'). (3) THE CHUNKED DOTTED GATE: a listed member written in two or more period-closed letter chunks (`M.Eng.`, `L.Ac.`) passes as `M.A.` does; one trailing period (`Ma.`, `Ed.`) still does not. (4) SINGLE-LETTER ROMAN NUMERALS neither anchor nor count toward a C1 run, S3's single-character retirement being the precedent; they peel as before, and the anchor's half of the exclusion is a matter of their shape, boundary (d) below. (5) NO NEW REPORT WHERE THE WRITING SETTLED IT: a run whose every member is LISTED and leans credential — S2's lean, capitals in a mixed-case name — is not flipped, the family-comma path already reading it as the credential run with the fields and the report it always had (`John Smith, PhD MA`) — narrowed 2026-10-01 by decisions/C4's #562 entry: a part holding two particles side by side is read by the count instead, and reports; the alternative, flipping such a run and suppressing only the flip's report, removes reports the family-comma path makes today, `John Smith, MA Jr` losing the one it makes for 'MA'. Because the lean is the listed set's, a by-shape member in capitals never settles a run: `John Smith, X.Y.Z. MA` flips to suffix 'X.Y.Z. MA' and reports. (6) THE ANCHOR DOES NOT REACH ACROSS A MAIDEN CLAUSE: `Jane Doe Jr. nee Smith Ma` keeps maiden 'Smith Ma' where `Jane Doe Jr. Ma` reads suffix 'Jr. Ma' — rules/M2's Accepted boundary, and the M2 bullet of this date. (7) AN ANCHOR OVERRIDES P6 AT THE GIVEN SLOT: `doe, jane v phd do` reads suffix 'v phd do' where it read family 'do doe'; the bullet below amends #531's pairing. (8) THE DUAL EXCLUSION IS SCOPED TO THE GIVEN PART'S LEADING TITLE RUN, NOT THE C1 RUN: after a full name the dual counts as the suffix vocabulary it is, as the legacy disjunct already counted it (`John Smith, MS MA`, `John Smith, MD`), and that is what reads the issue's headline `Jane Doe, MS LAc` as suffix 'MS LAc'. (9) A MEMBER THE COMPANY DECIDES REPORTS WHERE IT WAS DECIDED, in a family comma's part read wholly as credentials as at the peel: each member whose own writing declined and which the credential in front made the credential reports `suffix-or-name` at that reading. `Smith, PhD Ma`, `Smith, PhD MEng` and `Smith, Ph. D. MEng` report the member; `Smith, Dr. PhD LAc` and `John Quincy Smith, Prof. PhD LAc` report 'LAc'; `john smith, v phd ma` reports 'ma'; `John Smith, PhD Ma Prof.`, which C1 does not flip for its trailing title, reports 'Ma' where its twin `John Smith, PhD Ma` reports C1's flip once over the part. A member its own capitals made the credential stays silent (`Smith, PhD MA`), and the part's first piece, which nothing stands in front of, is the first-slot report's word and never this one's. (10) A DUAL IN THE GIVEN PART'S LEADING TITLE RUN SILENCES THE WHOLE PART: refinement (b) below reads such a dual as a title that speaks for nothing, and in a part whose leading title run holds one no credential behind the dual speaks for a member of the ambiguous class either, so the company never reads such a part wholly as credentials: the titles give it its reading, the dual a title and the next word the given name, and behind that given name the given slot reads as it reads any (`Smith, MD PhD Jr Ma` reads suffix 'Jr Ma', as `Smith, MD Jane Jr Ma` does). A part with no member to speak for, or only members whose capitals decide them, reads as it did: `Smith, MD PhD` suffix 'MD PhD', `Smith, Ms MD MA` suffix 'Ms MD MA'. `Smith, Ms MD Ma`, `Smith, MD MS Ma` and `SMITH, MD MS BA` read title 'Ms MD' / 'MD MS', given 'Ma' / 'BA'; `Smith, MD PhD Ma` reads title 'MD', given 'PhD', middle 'Ma', the given slot reporting 'Ma'; e10e83b4 read each of them so. The run is the walk's leading title run, so a dual behind a plain title stands in it too (`Smith, Prof. MD Ma` and `smith, prof. md ma` read title 'Prof. MD', given 'Ma'; `Smith, Dr. MD PhD Ma` keeps given 'PhD'), while a plain title alone silences nothing (`Smith, Dr. PhD LAc` reads title 'Dr.', suffix 'PhD LAc'). Behind a real given name the company reads as ever (`Doe, Jane MD Ma` and `Doe, Jane Ms Ma` read suffix 'MD Ma' and 'Ms Ma'), and past the title run a dual speaks like any suffix word (`Smith, PhD Ms Ma` reads suffix 'PhD Ms Ma'); `Smith, Ms Ma`, `Nguyen, Sr Ba` and `Smith, MD Do` are unchanged. +- 2026-09-27 (Derek), #544 (its ACCEPTED LIMITS' merged `Ph. D.` outside the no-comma walk superseded by #752's phase 2e, decisions/S11, the merged pair starting the credential's run) (the `doe, jane v phd do` reading in its (7) and its 2026-09-28 addendum superseded by #721, decisions/S10: the one-case `v` is an initial and ends the run, so the name reads middle 'v', suffix 'phd do', the company still taking 'do') (its (10)'s `md` examples superseded by #853, decisions/indic-honorifics, `md` being no title: `Smith, MD PhD Ma` reads suffix 'MD PhD Ma') (its (6) and the addendum's `Jane Doe, PhD, Jr nee van Ma` superseded by #601, decisions/M2, a marker behind a credential or past a comma being an ordinary word; its ACCEPTED LIMITS' `John Smith PhD Prof. Ma`, `John Smith PhD Do Do` and `Smith, PhD Do Ma` superseded by the credential's run, #602, decisions/S8, and #603, decisions/C13, which read each as suffixes; (d)'s `John Smith PhD V Ma` superseded the same way, suffix 'PhD V Ma'; (c)'s `John Om Ma` superseded by the 2026-10-21 marking of `om` ambiguous, decisions/suffix-acronym-collisions, so it reads family 'Ma') — A CREDENTIAL RUN KEEPS ITS AMBIGUOUS MEMBERS: C1'S NAME-WORD COUNT READS A RUN (rules/C4 since #728) AS IT READS ONE WORD, AN UNAMBIGUOUS CREDENTIAL IN FRONT OF A MEMBER SPEAKS FOR IT AT EVERY TRAILING SLOT, AND A LISTED MEMBER'S CHUNKED DOTTED SPELLING PASSES THE PERIOD GATE. Three readers never looked past the member itself: C1 flipped a part to the suffix comma by the name-word count only when it was ONE token, and the trailing readers — the no-comma peel, the comma part read wholly as credentials, the given part's trailing slot — met the member first, took its name lean and stopped, so the credential in front of it was never reached. #540 marked `meng` and `lac`, whose conventional spellings lean name, which turned the one-case-row cost decisions/S6's 2026-09-15 item (ii) accepted into the ordinary spelling of two degrees. DECIDED, ten forks, (9) and (10) on 2026-09-28. (1) THE WHOLE RUN AT C1: two or more words after the comma, every one suffix vocabulary or a class candidate, at least one a candidate, behind two or more name words, is the credential run — all-member runs (`John Smith, Ed Ma`) and a leading member (`John Smith, MEng PhD`) included; offered and declined were an anchored-only rule and the issue's own "last word ambiguous" wording, which contradicted its `Ed Ma` consequence. (2) ALL TRAILING PATHS, ANCHORED IN FRONT: a listed member with an unambiguous credential in front of it, through other members, reads as the credential on the no-comma peel, in the comma part read wholly as credentials and at the given slot; a credential BEHIND it anchors nothing (`Wang Ma PhD` keeps family 'Ma'). (3) THE CHUNKED DOTTED GATE: a listed member written in two or more period-closed letter chunks (`M.Eng.`, `L.Ac.`) passes as `M.A.` does; one trailing period (`Ma.`, `Ed.`) still does not. (4) SINGLE-LETTER ROMAN NUMERALS neither anchor nor count toward a C1 run, S3's single-character retirement being the precedent; they peel as before, and the anchor's half of the exclusion is a matter of their shape, boundary (d) below. (5) NO NEW REPORT WHERE THE WRITING SETTLED IT: a run whose every member is LISTED and leans credential — S2's lean, capitals in a mixed-case name — is not flipped, the family-comma path already reading it as the credential run with the fields and the report it always had (`John Smith, PhD MA`) — narrowed 2026-10-01 by decisions/C4's #562 entry: a part holding two particles side by side is read by the count instead, and reports; the alternative, flipping such a run and suppressing only the flip's report, removes reports the family-comma path makes today, `John Smith, MA Jr` losing the one it makes for 'MA'. Because the lean is the listed set's, a by-shape member in capitals never settles a run: `John Smith, X.Y.Z. MA` flips to suffix 'X.Y.Z. MA' and reports. (6) THE ANCHOR DOES NOT REACH ACROSS A MAIDEN CLAUSE: `Jane Doe Jr. nee Smith Ma` keeps maiden 'Smith Ma' where `Jane Doe Jr. Ma` reads suffix 'Jr. Ma' — rules/M2's Accepted boundary, and the M2 bullet of this date. (7) AN ANCHOR OVERRIDES P6 AT THE GIVEN SLOT: `doe, jane v phd do` reads suffix 'v phd do' where it read family 'do doe'; the bullet below amends #531's pairing. (8) THE DUAL EXCLUSION IS SCOPED TO THE GIVEN PART'S LEADING TITLE RUN, NOT THE C1 RUN: after a full name the dual counts as the suffix vocabulary it is, as the legacy disjunct already counted it (`John Smith, MS MA`, `John Smith, MD`), and that is what reads the issue's headline `Jane Doe, MS LAc` as suffix 'MS LAc'. (9) A MEMBER THE COMPANY DECIDES REPORTS WHERE IT WAS DECIDED, in a family comma's part read wholly as credentials as at the peel: each member whose own writing declined and which the credential in front made the credential reports `suffix-or-name` at that reading. `Smith, PhD Ma`, `Smith, PhD MEng` and `Smith, Ph. D. MEng` report the member; `Smith, Dr. PhD LAc` and `John Quincy Smith, Prof. PhD LAc` report 'LAc'; `john smith, v phd ma` reports 'ma'; `John Smith, PhD Ma Prof.`, which C1 does not flip for its trailing title, reports 'Ma' where its twin `John Smith, PhD Ma` reports C1's flip once over the part. A member its own capitals made the credential stays silent (`Smith, PhD MA`), and the part's first piece, which nothing stands in front of, is the first-slot report's word and never this one's. (10) A DUAL IN THE GIVEN PART'S LEADING TITLE RUN SILENCES THE WHOLE PART: refinement (b) below reads such a dual as a title that speaks for nothing, and in a part whose leading title run holds one no credential behind the dual speaks for a member of the ambiguous class either, so the company never reads such a part wholly as credentials: the titles give it its reading, the dual a title and the next word the given name, and behind that given name the given slot reads as it reads any (`Smith, MD PhD Jr Ma` reads suffix 'Jr Ma', as `Smith, MD Jane Jr Ma` does). A part with no member to speak for, or only members whose capitals decide them, reads as it did: `Smith, MD PhD` suffix 'MD PhD', `Smith, Ms MD MA` suffix 'Ms MD MA'. `Smith, Ms MD Ma`, `Smith, MD MS Ma` and `SMITH, MD MS BA` read title 'Ms MD' / 'MD MS', given 'Ma' / 'BA'; `Smith, MD PhD Ma` reads title 'MD', given 'PhD', middle 'Ma', the given slot reporting 'Ma'; e10e83b4 read each of them so. The run is the walk's leading title run, so a dual behind a plain title stands in it too (`Smith, Prof. MD Ma` and `smith, prof. md ma` read title 'Prof. MD', given 'Ma'; `Smith, Dr. MD PhD Ma` keeps given 'PhD'), while a plain title alone silences nothing (`Smith, Dr. PhD LAc` reads title 'Dr.', suffix 'PhD LAc'). Behind a real given name the company reads as ever (`Doe, Jane MD Ma` and `Doe, Jane Ms Ma` read suffix 'MD Ma' and 'Ms Ma'), and past the title run a dual speaks like any suffix word (`Smith, PhD Ms Ma` reads suffix 'PhD Ms Ma'); `Smith, Ms Ma`, `Nguyen, Sr Ba` and `Smith, MD Do` are unchanged. WHAT MAY ANCHOR, four boundaries of the company clause, each a rule. (a) A CONNECTIVE (P3) anchors nothing and ends the run: the generational `i` is also Catalan's conjunction, and `Jane Doe nee Puig i Ma` keeps its link, maiden 'Puig i Ma'. (b) A TITLE/SUFFIX DUAL IN THE GIVEN PART'S LEADING TITLE RUN after a one-word family comma is a title there and anchors nothing — and, by (10), no credential behind it speaks for a member in that part — so the given slot's pass starts past the leading title run: `Smith, Ms Ma`, `Nguyen, Sr Ba` and `Smith, MD Do` keep their given names, and `Smith, MD MA Ma` keeps middle 'Ma', no unambiguous credential standing in its run once the title is set aside. Everywhere else a dual anchors like any suffix word (`John Smith MD MEng`, and (8)). (c) THE WALK'S OWN LEADING PIECE NEVER ANCHORS. In a name with no comma before the run, the leading piece is the name word H4's carve-out keeps whatever its vocabulary, and letting it anchor spends that name: in `Om Ma` and `PhD Ma` 'Ma' stays a name word under every name order (family 'Ma' in the default order, given 'Ma' under both family-first orders), where an anchoring `PhD` would leave given 'PhD', suffix 'Ma' and no family at all. One name word in front frees the same word — `John Om Ma` reads suffix 'Om Ma'. A family comma's given part has no such position, its first piece being read by the comma part's own credential reading, so `Smith, PhD MEng` reads family 'Smith', suffix 'PhD MEng'. (d) A SINGLE-LETTER ROMAN NUMERAL, IN ANY CASE (`V`, `v`, `I.`), anchors nothing for its SHAPE — one letter is how a middle initial is written, and S3 retired single-character vocabulary matches for the same reason (`John Smith PhD V Ma` keeps family 'Ma') — not because a generation is no credential. The rule is stated by that shape and the code tests exactly it (`_vocab.is_single_letter_numeral`); "initial-shaped" was the wrong word for it, `is_initial_shaped` answering False for a lower-case `v` that the exclusion covers too. And the exclusion is not about generations: a multi-letter numeral or generational word anchors as any suffix word does (`John Smith PhD III Ma` reads suffix 'PhD III Ma', `abdul Smith Jr Ma` suffix 'Jr Ma'). ACCEPTED COSTS, each a reading this design chooses. `John Smith, Ms Ma` → suffix 'Ms Ma', reported, as `John Smith, Ms` alone already reads suffix (8). `Om Jr Ma` → given 'Om', suffix 'Jr Ma' and no family name: 'Jr' is not the leading piece, so it anchors 'Ma', and rules/S2's standing Accepted clause already consumes an unambiguous suffix that leaves no family (`Om Jr` → given 'Om', suffix 'Jr'); every release from 2.0.0 made that same role assignment. `doe, jane v phd do` reports `suffix-or-name` where it reported `particle-or-given`, the fork reported being the credential pick rather than P6's attachment (7). And a one-word family comma whose given part OPENS with a capitalised `MA` in a mixed-case name, with an unambiguous credential behind it, reads the whole part as a credential run, family 'Smith' and no given name, where the parent kept given 'MA': `Smith, MA PhD Ma` read given 'MA', middle 'Ma', suffix 'PhD' and reads suffix 'MA PhD Ma'. 'MA' leans credential on its capitals, so it is read as the credential it is written as, and what follows it is then the company's (or, for `M.Eng.`, the chunked gate's). Measured 2026-09-28 on the grid below: 138 parses (46 texts, three orders each) go from given 'MA' to no given name, 72 of them with no `M.Eng.` in the run; 12 of the 138 are the parses the attribution below leaves unattributed, the rest falling in the anchor and `M.Eng.` classes. Each reports 'MA' at the first slot, and a member behind the credential reports as the company's pick (9). ACCEPTED LIMITS, each keeping the member a name word, rules/S2's Accepted block and case rows: the merged `Ph. D.` is outside the no-comma walk (`John Smith Ph. D. MEng` family 'MEng'; the comma and given-slot spellings do anchor it, `Smith, Ph. D. MEng` and `Doe, Jane Ph. D. MEng` reading suffix 'Ph. D. MEng'), a title between the credential and the member breaks the run (`John Smith PhD Prof. Ma`), and a particle member P2 has chained before the peel is out of reach (`John Smith PhD Do Do` family 'Do Do'; `Smith, PhD Do Ma` given 'PhD', middle 'Do Ma'). diff --git a/docs/design/decisions/S8-credentials-trailing-run.md b/docs/design/decisions/S8-credentials-trailing-run.md index 0bfe24ab..cac68787 100644 --- a/docs/design/decisions/S8-credentials-trailing-run.md +++ b/docs/design/decisions/S8-credentials-trailing-run.md @@ -4,7 +4,7 @@ Examples name each field as the default order reads it; under the family-first orders the given and family names trade places. -- An unambiguous suffix word of two or more letters, in any script, other than the split `Ph. D.` in a main segment (below), standing after two name words with no comma or before a suffix comma, or after the given word in the part after a family comma, starts a run to the end of its part: every later word reads as a suffix except a title word, which reads as a title, and an absorbed name word reports `suffix-or-name` (`John Smith PhD Jones` → suffix 'PhD Jones'; `Eric H. Holder Jr. Attorney General` → suffix 'Jr.', title 'Attorney General') — 2026-10-04 #602; 2026-10-14 #679. +- An unambiguous suffix word of two or more letters, in any script, standing after two name words with no comma or before a suffix comma, or after the given word in the part after a family comma, starts a run to the end of its part: every later word reads as a suffix except a title word, which reads as a title, and an absorbed name word reports `suffix-or-name` (`John Smith PhD Jones` → suffix 'PhD Jones'; `Eric H. Holder Jr. Attorney General` → suffix 'Jr.', title 'Attorney General') — 2026-10-04 #602; 2026-10-14 #679. - A roman numeral of two or more letters that reads as the generation starts the run as a listed one does (`John Smith VI Jones` → suffix 'VI Jones', as `John Smith III Jones`), its shape alone reported — 2026-10-25 #784. - What never starts the run: a title word (`Mary Jane King Smith` keeps family 'Smith'), a member of the ambiguous class, a single letter, a connective (`Josep Carod i Rovira` keeps family 'Carod i Rovira'), and a word that is also particle or bound given-name vocabulary (`Mohamed Ali Abd Allah` → family 'Allah'); a title/suffix dual does start it — 2026-10-04 #602. - An honorific or generational word starts the run as a credential does, whatever its script (`राम शर्मा साहब मोहन` → suffix 'साहब मोहन'); the one-character honorifics stay out by the two-letter minimum — 2026-10-14 #679. @@ -14,13 +14,13 @@ Examples name each field as the default order reads it; under the family-first o Not standing: the split of 2026-10-17 #682, carried out. -Elsewhere: the split `Ph. D.` in a main segment (no comma, or before a suffix comma) starts no run, the trailing peel's walk never holding the merged piece (`John Smith Ph. D. Jones` → family 'Jones', suffix 'Ph. D.', unreported; `John Smith Ph. D. Jones, Jr.` → suffix 'Ph. D., Jr.') — decisions/C13's 2026-10-04 (Derek), #603 entry, and the #602 entry's own REVERSED paragraph; a credential opening the part after the comma, and the split `Ph. D.` starting the run as `PhD` does (`Smith, John Ph. D. Jones` → suffix 'Ph. D. Jones') — decisions/C13's 2026-10-04 (Derek), #603 entry; the run stopping short of a lone letter ending the given part ahead of a further comma (`Smith, John PhD I, Jr.` → middle 'I', suffix 'PhD, Jr.') — decisions/C9's 2026-10-25 #819 entry. +Elsewhere: the split `Ph. D.` starting the run in a main segment too, from where it stands though the walk leaves it out, and any other credential split across a space starting it as the one member of the ambiguous class that does (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported; `John Smith Pharm D Jones` → suffix 'Pharm D Jones') — decisions/S11's 2026-10-25 #752 entry; a credential opening the part after the comma, and the split `Ph. D.` starting the run as `PhD` does (`Smith, John Ph. D. Jones` → suffix 'Ph. D. Jones') — decisions/C13's 2026-10-04 (Derek), #603 entry; the run stopping short of a lone letter ending the given part ahead of a further comma (`Smith, John PhD I, Jr.` → middle 'I', suffix 'PhD, Jr.') — decisions/C9's 2026-10-25 #819 entry. Open: [#785](https://github.com/derek73/python-nameparser/issues/785) (whether capitals in a mixed-case name start the run as periods do). ## Log -- 2026-10-04 (Derek), #602 (its Latin restriction and its non-Latin honorific exclusion superseded by #679, below; its lone particle's count superseded in part by #627, below; its merged `Ph. D.` kept out of reach superseded after a comma by #603, decisions/C13) — A CREDENTIAL AFTER THE NAME CORE STARTS A RUN TO THE END OF ITS PART. An unambiguous Latin suffix word (the Latin restriction retracted by #679, below) of two or more letters standing after two name words with no comma (a lone particle not counting toward the two; qualified by #627, below), or after the given word in the part after a family comma, makes every later word of that part a suffix, except a title word, which reads as a title: `John Smith PhD Jones` → suffix 'PhD Jones', `Eric H. Holder Jr. Attorney General` → suffix 'Jr.', title 'Attorney General'. A name word the run absorbs is reported `suffix-or-name`, a reader hesitating over `Jones` there. Before this the three comma shapes disagreed — middle 'Smith PhD' with no comma, middle 'Jones' and suffix 'PhD' after a family comma, and suffix 'Jr., Jones' after a suffix comma (C2's tail) — and the no-comma reading made a credential a middle name, the one reading nobody intends. The suffix-comma path already read it this way, and the comma spelling of the Holder name always did. +- 2026-10-04 (Derek), #602 (its Latin restriction and its non-Latin honorific exclusion superseded by #679, below; its lone particle's count superseded in part by #627, below; its merged `Ph. D.` kept out of reach superseded after a comma by #603, decisions/C13, and in a main segment by #752's phase 2e, decisions/S11) — A CREDENTIAL AFTER THE NAME CORE STARTS A RUN TO THE END OF ITS PART. An unambiguous Latin suffix word (the Latin restriction retracted by #679, below) of two or more letters standing after two name words with no comma (a lone particle not counting toward the two; qualified by #627, below), or after the given word in the part after a family comma, makes every later word of that part a suffix, except a title word, which reads as a title: `John Smith PhD Jones` → suffix 'PhD Jones', `Eric H. Holder Jr. Attorney General` → suffix 'Jr.', title 'Attorney General'. A name word the run absorbs is reported `suffix-or-name`, a reader hesitating over `Jones` there. Before this the three comma shapes disagreed — middle 'Smith PhD' with no comma, middle 'Jones' and suffix 'PhD' after a family comma, and suffix 'Jr., Jones' after a suffix comma (C2's tail) — and the no-comma reading made a credential a middle name, the one reading nobody intends. The suffix-comma path already read it this way, and the comma spelling of the Holder name always did. WHAT STARTS THE RUN is one predicate, `_pieces.starts_a_credential_run`, read by the trailing peel (`credential_run`, applied to every finished peel, so assign, P5's reserve, P2's chain stop and the maiden walk see one answer) and by the given-part walk. EXCLUDED, each with its reason: a TITLE WORD — 746 title words are in no suffix set (measured 2026-10-04: `len(L.titles - L.suffix_acronyms - L.suffix_words)` for the default lexicon), many of them surnames (`King`, `Bishop`, `Judge`), so `Mary Jane King Smith` keeps family 'Smith'; a title is read from the end of a name only behind a comma or with a period (H5, Derek). The AMBIGUOUS CLASS, which carries no evidence of which it is. A SINGLE LETTER, in any case: a lowercase `i` or `v` carries no `initial` tag, and found by the v1 off-switch test, `Josep Carod i Rovira` with `i` removed from the conjunctions read suffix 'i Rovira'. A CONNECTIVE. A word that is also PARTICLE or BOUND GIVEN-NAME vocabulary — found by the suite, not foreseen: `abd` is in the credential list (ABD) and heads `Abd Allah`, and `Mohamed Ali Abd Allah` read suffix 'Abd Allah' until it was excluded. The NON-LATIN HONORIFIC words, an honorific followed by a name word being a different question from a credential followed by one (retracted by #679, below). A word that is also a title (`MD`, `Ms`, `Sr.`) does start the run: position decides, and after the name core it is the suffix. DECLINED: lifting the credential out — `John Smith PhD Jones` as given 'John', middle 'Smith', family 'Jones', suffix 'PhD', the family-comma path's old reading. It keeps every name word in a name field, but it reads a credential as something written in the middle of a name, which is the reading the rule rejects. REVERSED BY THIS ENTRY rather than edited: decisions/S5's #578 bullet (`Doe, Jane PhD vd Ma` now reads suffix 'PhD vd Ma', the credential starting the given part's run), and two of the three shapes #544's ACCEPTED LIMITS (decisions/S7) kept out of the company's reach — a title standing between the credential and the member, and a member a particle chain had taken. The case rows `a_title_between_degree_and_member_keeps_it_a_name` and `a_member_the_particle_chain_took_is_out_of_reach` keep their ids and now pin the run. The third, the merged `Ph. D.`, stays out of reach: that piece is outside the walk the run is read over. diff --git a/docs/design/decisions/phd-merge.md b/docs/design/decisions/phd-merge.md index 497ee7a6..846cc26c 100644 --- a/docs/design/decisions/phd-merge.md +++ b/docs/design/decisions/phd-merge.md @@ -6,7 +6,7 @@ - The merge declines at the first TOKEN of the input, a suffix never opening the string (rules/S2): `Ph. D. Van Johnson` reads title 'Ph.', given 'D.', family 'Van Johnson', 1.4.0's reading, while a quoted clause before it leaves it merging (`"Bob" Ph. D. John Smith` → suffix 'Ph. D.') and a run opening the part after a family comma merges too (`Smith, Ph. D. Jr.` → suffix 'Ph. D. Jr.') — 2026-08-31 #371 - Accepted: a title before the pair keeps the credential reading, so `Sir Ph. D. Van Johnson` reads given 'Van Johnson' with an empty family, 1.4.0 parity — 2026-08-31 #371 -Elsewhere: the merged pair starts no credential run in a main segment, so the two spellings part there (`John Smith Ph. D. Jones` → family 'Jones', suffix 'Ph. D.', where `John Smith PhD Jones` reads suffix 'PhD Jones'; `John Smith Ph. D. MEng` keeps family 'MEng'), decisions/S8's 2026-10-04 (Derek), #602 entry and decisions/S7's 2026-09-27 (Derek), #544 entry; the empty family belongs to a title run that addresses by the given name (`Sir`), so `Dr. Ph. D. Van Johnson` reads family 'Van Johnson', decisions/H1's 2026-09-08 #489 entry; `Parser.revise(suffix="Ph. D.")` renders 'Ph. D.', the merge still not firing in a field value but rules/R1's entry pass joining the pair, decisions/R1's 2026-09-06 #511 entry. +Elsewhere: the merged pair starts the credential's run in a main segment as `PhD` does, from where it stands though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng'), and every other credential split across a space merges by the same predicate as a member of the ambiguous class, which the count decides, while this pair stays the unambiguous credential it was and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.'), decisions/S11's 2026-10-25 #752 entry; the empty family belongs to a title run that addresses by the given name (`Sir`), so `Dr. Ph. D. Van Johnson` reads family 'Van Johnson', decisions/H1's 2026-09-08 #489 entry; `Parser.revise(suffix="Ph. D.")` renders 'Ph. D.', the merge still not firing in a field value but rules/R1's entry pass joining the pair, decisions/R1's 2026-09-06 #511 entry. Open: none in this file. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 9f6cf884..160ef068 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -16,6 +16,7 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - Bearer counts are structured provenance (source and count), never used to compute a reading — 2026-10-15/16. - Phase 2 is to put vocabulary, declared name and shape readings in one pool, derive `may_be_given` and `may_be_name` from declared name readings (a count-free one carrying a `note`, `ld` left ungated, `may_be_initial` kept), and decide every clash by one gate of four steps, floor, form, structure and count, asked at the sites that decide today; the reading not taken is to surface as `Ambiguity.alternative` in 2.x and the reading taken as `Token.reading` in 3.0 — 2026-10-23 #752. - Every numeral rule asks one predicate, `_vocab.reads_as_generation`, the vocabulary's own numerals read inside it; a numeral of two or more letters is the generation wherever the trailing run holds it, while a one-letter one before a credential stays the name word decisions/S10's #721 entry made it — 2026-10-25 #752, phase 2f. +- A credential written split across a space is one word, a member of the ambiguous class that the count decides, and the Ph./D. pair starts S8's run from where it stands (decisions/S11) — 2026-10-25 #752, phase 2e. - Phase 2's title batches go in order of lone-head breakage in PSC, `buddha`, `siddha` and `gyani` in the default gate and `guru` left to a `ne` pack — 2026-10-24 #752. - `Dean Smith` reads given `Dean`. The contested-head gate is to cover the words outside `titles` (`dean`, `justice`, `major`), and a title that is also a given name gets a declared given reading one word at a time, in measured batches — 2026-10-15/16; 2026-10-23 #752. - Order and shape are separate: a language supplies only a default order, and the shapes (where titles, suffixes, particles and commas sit) belong to each order and live in the core; vocabulary readings are to settle a clash and name readings only to keep the gate on — 2026-10-15/16. @@ -141,7 +142,7 @@ Open: #752 (phase 2, derived ambiguity; its sub-phases 2a to 2m, each its own PR 3. **A particle with no given reading still states `may_be_given = false`, and `may_be_given = true` is refused with a message naming the `given` reading to write instead**, so a new particle cannot fall to the unsafe side silently (#293's safe-by-default; decisions/vocabulary-collisions C-i). 4. **"One gate for six rules" is four clashes and two floors.** The title run's floor (rules/H3) and bound given names (rules/P5) reach every member of their category and read only a count; no second reading is weighed, so they are step 1, unchanged. The acronym gate's nine decision sites stay nine, each keeping the order it reads its cues in (C4 the count before the case, S6 the case before the count); what becomes one is the predicate each asks. 5. **Joined credentials land: a word joined by `-` or `/` whose every chunk is listed and none ambiguous reads as a credential, and an unlisted word of capitals so joined, with a chunk of two or more letters, joins the ambiguous class after a comma, gated by C6's case contrast.** The stricter chunk test is because a hyphen also joins names (`Smith-Jones`). Together they moved 1,782 distinct ctgov readings (1,254 and 759, overlapping on 231), nothing in PSC, the committed corpora or the agreement records; FEC is one case, so the contrast never admits a word there. - 6. **A split credential merges only where its first chunk has two or more letters, and the merged piece is a member of the ambiguous class, decided by C4's count and S4's words to spare; it starts S8's run (#827), landing after the numeral predicate answers the numeral before a credential.** FEC decided the opening letter: it writes a middle initial before a trailing salutation (`SMITH, JOHN A. MS.`), and `A. MS.` folds to the listed `ams` (merging any pair ending its part, but for two single letters, moved 5,122 FEC names, `A. MS.` 2,246 of them). A two-letter opening chunk needs the count: the 30 FEC merges it made all stood behind a one-word family (`SAM E.` → `same`), against 351 of 352 ctgov merges behind two or more name words. Accepted cost: `M. D.` and `M D` stay unmerged. Left to the split-credential PR: rules/S8 refuses a member of the ambiguous class as a run start, so how a merged piece can be both is that PR's to state (today's `Ph. D.` merge is unambiguous, and S8's #827 row covers it). + 6. **A split credential merges only where its first chunk has two or more letters, and the merged piece is a member of the ambiguous class, decided by C4's count and S4's words to spare; it starts S8's run (#827), landing after the numeral predicate answers the numeral before a credential.** FEC decided the opening letter: it writes a middle initial before a trailing salutation (`SMITH, JOHN A. MS.`), and `A. MS.` folds to the listed `ams` (merging any pair ending its part, but for two single letters, moved 5,122 FEC names, `A. MS.` 2,246 of them). A two-letter opening chunk needs the count: the 30 FEC merges it made all stood behind a one-word family (`SAM E.` → `same`), against 351 of 352 ctgov merges behind two or more name words. Accepted cost: `M. D.` and `M D` stay unmerged. Left to the split-credential PR: rules/S8 refuses a member of the ambiguous class as a run start, so how a merged piece can be both is that PR's to state (today's `Ph. D.` merge is unambiguous, and S8's #827 row covers it). (Carried out, and the run start stated, by the 2026-10-25 #752, phase 2e entry below, decisions/S11. Corrected there too: `M. Sc.` opens with a single letter as well and stays unmerged, so the tally's figure of officials still reversing left it out.) 7. **A word taken into the credential's run by position alone (`John Smith PhD Jones`) gets no new ambiguity kind.** It is not a clash between two readings; the surfaced reading's source tells it from a word with a declared name reading (item 13). This answers #785's fourth question; its first three (case drops the report and starts the run, the one-case row keeps its report) land through the pool, since retagging the word was measured to erase its other readings (`Doe, John DO` became family `DO Doe` once `do` stopped being an ambiguous acronym, its particle reading meeting P6). 8. (narrowed by the 2026-10-25 #752, phase 2f entry: in the given part after a family comma a one-letter numeral the vocabulary lists stays the suffix, and a one-letter numeral before a credential stays a name word) **Every numeral rule asks one predicate, answering for `i` to `x` the way S10's trailing fork does (shape, the count for one letter, an initial before it, the period, the case), with three readings stated:** a lone letter ending the given part ahead of a further comma is the initial (#819; `Smith, John PSM I, Jr.` the accepted cost), a lone letter in a given part after a family comma stays the initial, and a Title-case numeral in a name of mixed case leans to the name. It answers #784 (`John Smith VI MD` reads family `VI` where `III` starts the run) and #774. 9. **Generation readings are gated by their kind's cue, and a reading with none is `default = false`, compiled to nothing by default and turned on by a pack in phase 3.** `fils` is gated by case and the count. The Brazilian agnomens `Filho`, `Filha`, `Neto`, `Neta`, `Netto`, `Sobrinho` and `Sobrinha` become gated generation readings in `pt.toml` (`pt-BR` where only Brazil writes one), with the count as their only cue: they are written in the case of the name around them. In tse-donors, 405 filed/registry pairs differ only by a trailing agnomen; master gives both spellings the same last family word in 1, and a gate emulated as an ambiguous acronym, so carrying S6's Title-case lean beside the count, in all 405, keeping every PSC surname (`Ana Neto`) and FEC given name (`SMITH, NETO`) the plain reading broke, by the count and the lean. The count-only gate the design chooses was not measured: it is to be measured before 2h lands. They are not part of the surname: a surname is what a family shares, and an agnomen marks the bearer against a relative of the same name, as `Júnior` does. Spanish `H.` (*hijo*), German regnal `I.` and Czech `st.` are `default = false`. When 2h lands it replaces decisions/generational-words' Standing line that the agnomens stay family names. @@ -186,10 +187,10 @@ Open: #752 (phase 2, derived ambiguity; its sub-phases 2a to 2m, each its own PR - **2026-10-25 #752 — one predicate per question (neutral).** Three questions that #752's later changes widen had their answers copied between sites, and each now has a shared predicate in `_pipeline/_vocab.py` for the widening to land in (mechanisms.md#ONE-PREDICATE-PER-QUESTION). Two are whole: every site asks them. The numeral's is not yet: the bullet below names the sites found still recognizing a numeral on their own. No reading moves. - **`split_pair(first, second)`**, the split credential (`Ph.` `D.`, decisions/phd-merge): group's merge, `_comma._pieces` and `is_wholly_suffix` each held their own copy of the two-pattern test. The test runs per adjacent pair, so each caller asks `SPLIT_PAIR_OPENS`, the predicate's own first-word pattern, C-level before the call, and a name holding no `Ph.` pays no frame. The split-credential change widens both together (item 6 of the 2026-10-23 #752 entry above). - **`reads_as_generation(text, preceding, one_case)`**, the numeral: `is_trailing_numeral_suffix` renamed, the predicate every numeral rule is to ask (item 8 of the 2026-10-23 #752 entry above, #784, #774, #819). Today it is the fork's SHAPE half alone. rules/C12's evidence asks a broader question beside its call: whether the word ending the part before the comma is any suffix word, by the suffix-word tag (`John Smith Jr., AD`, `John Smith Esq., AD`), or reads as a generation by the predicate (`John Smith VI, AD`); the tag stays at the call site because `Jr.` and `Esq.` are no numerals. The listed numerals belong in the predicate too, or each numeral rule tests the vocabulary itself again, the hand copy #656's first C12 draft was (it asked the suffix-word tag and missed `VI`). The shape test already takes the listed `ii` to `iv`, so reading the vocabulary inside it adds only the one-letter `i` and `v`, and that moves readings, which this PR may not: answered yes on a grid of 3,927 readings (17 heads, 11 numerals, 7 tails, three orders; the second review, 2026-10-25 on 30dc7aac), `i` and `v` moved 375 readings at the peel's fork, 348 of them in their fields (`John J. V` from middle 'J.', family 'V' to family 'J.', suffix 'V', with a new report), and 144 reports at `reads_as_a_reported_numeral` with no field moving (`John J. V MD` gains a `suffix-or-name`), the ones those sites meet carrying the `initial` tag (`I`, `V`, and their one-case spellings; a lower-case `v` in a name of mixed case carries none, and the peel takes it as a suffix word). Adding `ii` to `iv` alone moved nothing. That half is left to the change that has every numeral rule ask the predicate. Until it lands (it did: the 2026-10-25 #752, phase 2f entry below, which asks the predicate at every site listed here, the peel taking `ii` to `iv` by the suffix test, which the predicate answers alike), these sites recognize a numeral on their own, each keeping the answer it had (found 2026-10-25 by two reviews and a grep of `_pipeline/` for the numeral pattern, the suffix-word tag and the suffix predicates; a list found by search, not a proof that no other site exists): the predicate's own missing `i` and `v`, above; rules/S8's run start asks the vocabulary alone (`John Smith III MD` → suffix 'III MD', `John Smith VI MD` → family 'VI', #784); rules/C1's listing form asks the suffix-word tag (`John Smith III, M.J.` keeps suffix 'III' where `John Smith VI, M.J.` and `John Smith V, M.J.` read family 'John Smith VI' and 'John Smith V', #774); the peel takes `ii` to `iv` through `is_suffix_piece` before its fork asks the predicate; rules/C9's walk over the given part after a family comma asks `is_suffix_lenient`, the vocabulary alone (`Smith, John V` → suffix 'V', `Smith, John VI` and `Smith, John X` → middle, where `John Smith VI` reads suffix 'VI'); and after a suffix word in that part a listed one-letter numeral is taken by its tags (`_pieces._numeral_behind_the_initial_veto`), so `Smith, John PhD V` reads suffix 'PhD V' with no report where `Smith, John PhD VI` reports `suffix-or-name`. - - **`is_cased_paired_initials(text)`**, rules/S3's paired initials in a script with capitals: classify asked the script half inline (#872, decisions/S3). This is the lift the 2026-10-24 #872 entry above names, for classify and the split merge to share. One overlap is left to the split-credential PR to state: item 6 refuses a split credential whose first chunk is a single letter, so for a split `A. B.` that length test answers before this predicate is asked. rules/C5's comma reading keeps the bare `is_paired_initials`, since after a comma a caseless pair reads as a name too (`John Smith, พ.บ.` → given 'พ.บ.'), and moving C5 onto this predicate would change that reading. + - **`is_cased_paired_initials(text)`**, rules/S3's paired initials in a script with capitals: classify asked the script half inline (#872, decisions/S3). This is the lift the 2026-10-24 #872 entry above names, for classify and the split merge to share. One overlap is left to the split-credential PR to state: item 6 refuses a split credential whose first chunk is a single letter, so for a split `A. B.` that length test answers before this predicate is asked. (Stated by the phase 2e entry below, decisions/S11: the length test refuses every other opening letter, and two dotted single letters are asked of this predicate, merging only in a script without capitals.) rules/C5's comma reading keeps the bare `is_paired_initials`, since after a comma a caseless pair reads as a name too (`John Smith, พ.บ.` → given 'พ.บ.'), and moving C5 onto this predicate would change that reading. - **Measured neutral** against the branch, first on 632685c8 and again on 2dd0414b once phase 2a had merged: a grid of 20,364 texts (22 heads covering no comma, a whole name, one word and nothing before a comma, a nickname-only head, a maiden clause, a family comma with a title-particle head and a quoted clause before the pair; 33 spellings including `MA`, `M.D.`, `X.Y.Z.`, `XYZ`, `Ph. D.` and its case variants, `V`, `VI`, `III`, `XI`, `M.J.`, `K.C.` and `พ.บ.`; each text also in lower and upper case), under all three orders, read identically (fields, ambiguity kinds and details, token roles and tags, `initials()`, `capitalized()`), and the same harness reported 45,871, 18, 3,974 and 2,986 differing readings with `split_pair`, C12's call to the numeral predicate, the cased half and the peel's call to it each knocked out. The old and new predicates gave the same answer on 19,769 direct calls. The committed snapshot, every CLDR, GND, Wikidata and conventions agreement record, psc-individuals' agreement record and the ctgov-officials, fec-individuals and tse-donors local snapshots (each recorded from the base with `--tree`) all match, on both bases. psc-individuals' record was itself stale against 632685c8 (3 regressions, 5 progress, 5 changed before this change), so it was re-recorded from each base first. - **Frames**, counted with `sys.setprofile` on both trees: `_CALL_BASELINE`'s reference name and `Smith, John` cost what they did (305 and 158). A word matching `SPLIT_PAIR_OPENS` (`Ph`, `ph.`) with a word after it pays one frame at each site that tests it, whether or not a `D.` follows (`John Smith Ph. D.` 233 → 234, `Smith, John Ph. D.` 239 → 241; the review counted `ph ph ph ph ph ph` 282 → 286), so the cost grows only with the number of such words. C12's evidence keeps its tag test ahead of the call, so it costs what it did (`John Smith Jr., AD` 372). That is a rare shape that no guard counts. -- **2026-10-25 #752, phase 2f — EVERY NUMERAL RULE ASKS ONE PREDICATE.** Carries out item 8 of the 2026-10-23 #752 entry above (#784, #774, #819), and the list of sites in the 2026-10-25 one-predicate entry. `_vocab.reads_as_generation(text, tags, preceding, one_case)` answers for every spelling from `i` to `x` as S10's trailing fork does: the shape on the word without its trailing periods; a single letter written with its period an initial (rules/T5); a numeral the vocabulary lists in two or more letters (`ii` to `iv`, read from the token's `vocab:suffix` tag, not copied at a call site) the generation whatever stands around it; for the rest, an initial in front a name word, and a Title-case numeral in a name of mixed case declined, which `numeral_leans_to_the_name` tells apart so the decline is reported. `preceding` is None where a slot asks no initial in front, the given part after a family comma, where the predicate reads a one-letter numeral by the vocabulary. The count for one letter stays the caller's, which alone knows where the name's words are. +- **2026-10-25 #752, phase 2f — EVERY NUMERAL RULE ASKS ONE PREDICATE.** (Its expectation that the split-credential sub-phase keep `John Smith V Ph. D.` as suffix 'V Ph. D.' superseded by the phase 2e entry below, decisions/S11, which reads it as `John Smith V PhD` reads, family 'V'.) Carries out item 8 of the 2026-10-23 #752 entry above (#784, #774, #819), and the list of sites in the 2026-10-25 one-predicate entry. `_vocab.reads_as_generation(text, tags, preceding, one_case)` answers for every spelling from `i` to `x` as S10's trailing fork does: the shape on the word without its trailing periods; a single letter written with its period an initial (rules/T5); a numeral the vocabulary lists in two or more letters (`ii` to `iv`, read from the token's `vocab:suffix` tag, not copied at a call site) the generation whatever stands around it; for the rest, an initial in front a name word, and a Title-case numeral in a name of mixed case declined, which `numeral_leans_to_the_name` tells apart so the decline is reported. `preceding` is None where a slot asks no initial in front, the given part after a family comma, where the predicate reads a one-letter numeral by the vocabulary. The count for one letter stays the caller's, which alone knows where the name's words are. - **The sites.** The trailing peel asks it of a numeral anywhere in the run, behind the suffix test, so `John Smith VI MD` reads suffix 'VI MD' as `III MD` does (#784); rules/S8's run start asks it of a numeral of two or more letters (`John Smith VI Jones` → suffix 'VI Jones'); rules/C12's evidence asks it with the token's tags, its broader suffix-word test still beside the call; C1's listing form asks it of a numeral ending the family part, with S10's count over that part's pieces (`John Smith V, M.J.` and `John Smith VI, M.J.` keep the suffix, #774); the given part's walk asks it of a numeral of two or more letters (`Smith, John VI` → suffix 'VI') and of the bare one-letter numeral its last-piece test reads, and a one-letter numeral behind a suffix word in a part of suffix words asks it too; the maiden take and `tail_reading` reach it through the peel, `trailing_candidates` with no case (its superset); the report on a numeral a post-nominal follows asks it, with the Title-case decline. A numeral of two or more letters read by its shape alone reports `suffix-or-name` wherever it is taken, the run start and the given part included. - **The three readings agreed on 2026-10-23.** #819: a lone letter ending the given part ahead of a further comma is the middle initial, the credential run in front stopping short of it (decisions/C9's 2026-10-25 #819 entry; `Smith, John PSM I, Jr.` the accepted cost). FEC's constraint: in that part a one-letter numeral is the generation only where listed, so `SMITH, JOHN X` keeps middle 'X'. A Title-case numeral in a mixed-case name leans to the name (`Nguyen Van Vi` → family 'Van Vi', reported); a numeral the vocabulary lists in two or more letters is not leaned (`John Smith Iii` keeps suffix 'Iii'), being unambiguous vocabulary as an unambiguous credential is. - **One reading kept rather than widened: a one-letter numeral before a credential stays a name word** (decisions/S10's #721 entry, `John Smith V PhD` → family 'V', reported). Reading it with the count instead moved 6 tse-donors names, every one wrong: a lone `V` or `I` behind a surname or a particle before `JR` or `JUNIOR` (shape `Aa A Aa de V JUNIOR` read family 'de', suffix 'V JUNIOR'), where the letter abbreviates a surname; and 3 committed names (`John Smith V PhD`, `Jane Doe nee Smith V PhD`, `Jane Doe nee Smith V MA`). A split credential behind such a numeral (`John Smith V Ph. D.`) is left to the split-credential sub-phase (item 6), which must keep its suffix 'V Ph. D.' by its own rule. @@ -197,3 +198,4 @@ Open: #752 (phase 2, derived ambiguity; its sub-phases 2a to 2m, each its own PR - **Recompute:** `uv run python tools/differential/snapshot.py` on the parent of this change's merge with its code, and for each local corpus `snapshot.py --local --record --tree ` then `snapshot.py --local ` from this tree, back to back; `uv run python tools/differential/local_agreement.py psc-individuals` the same way, `--list N` printing the moved records locally. - **2026-10-25 #752 — phase 2c: a credential joined by a hyphen or a slash reads as one where every chunk counts as a suffix as written**, carrying out the listed half of item 5 of the 2026-10-23 #752 entry. `John Smith, MD/PhD` reads suffix 'MD/PhD' where it read given 'MD/PhD', family 'John Smith'; a bare member of the ambiguous class is no chunk the test counts (`John Smith, MA/PhD` keeps its reading). The record, with what was measured and what was left unread, is decisions/S3's entry of this date (rules/S3). - **2026-10-25 #752 — phase 2d: an unlisted word of capitals joined by a hyphen or a slash is the caps shape**, carrying out the unlisted half of item 5 of the 2026-10-23 #752 entry (#797's half A). `John Smith, PU-PH` reads suffix 'PU-PH', reported, where it read given 'PU-PH', family 'John Smith'; C6's case contrast gates it after a comma as it gates the unjoined word. The record is decisions/S9's entry of this date (rules/S9, rules/C6). +- **2026-10-25 #752, phase 2e — A CREDENTIAL SPLIT ACROSS A SPACE IS ONE WORD, A MEMBER OF THE AMBIGUOUS CLASS THE COUNT DECIDES**, carrying out item 6 of the 2026-10-23 #752 entry and #827's reading (A). `John Smith, Pharm D` reads suffix 'Pharm D' where it read given 'Pharm', middle 'D', family 'John Smith'; `John Smith Ph. D. Jones` reads suffix 'Ph. D. Jones' as `John Smith PhD Jones` does; `SMITH, SAM E.` keeps given 'SAM', the count declining the pair. The record, with what was measured, is decisions/S11's entry of this date (rules/S11). diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index c86e6411..626134d2 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -116,6 +116,18 @@ This catalog converts discovery into a one-time cost — it does not remove disc **Reach for it when.** A rule is growing stops, exceptions or a release check whose job is to keep one construct from disturbing the rest of the name, and above all when that check has started to model another stage. Ask what the name reads as without the construct, and make that the rule. +## MERGE-THEN-SETTLE — merge what a count decides, and split back what it declines + +**Problem shape.** Two words may be one unit, and whether they are is a question only a later reading answers: a count of the words around them, made by a rule that reads pieces. Merged too early, a pair the reading declines is read as one piece where it should be two, and the merge hides a word that would have answered something by itself; left unmerged, the reading never sees the unit at all. + +**Contract statement.** Merge the candidate into one piece, tagged as a candidate, and let the rule that decides read it. Where the rule takes it, keep the piece; where it declines, split it back into the words it is written as before anything else reads them, so a declined candidate reads exactly as the words read with no merge. Where the deciding reading is one pass over a whole part, read the part again unmerged rather than unpicking the merged reading. + +**How it works.** rules/S11's split credential (#752, phase 2e): classify tags each pair `_vocab.split_pair` admits, group merges it as a member of the ambiguous class, S2's trailing read decides it by S4's count, and `_group._settle_splits` splits back every merged pair the read left in front of the run before the particle chain or the bound join can see it. After a comma `_comma.decide` reads the part with the pairs merged and, where that reading does not take one as a credential, reads the part again with none merged: the first version kept the merged reading there, and the pair hid an `MB` that had opened the part as a credential (C13), reversing 15 ClinicalTrials.gov names (`Aa Aa, MB ChB BAO` read given 'MB'). The settled outcome is either the merge's reading or the as-written one, never a third, which is what lets a measurement read every move as a pair the count took. + +**Lives in.** nameparser/_pipeline/_classify.py (the pair tag); nameparser/_pipeline/_group.py (the merge and `_settle_splits`); nameparser/_pipeline/_comma.py (`decide`'s retry, `_pieces(splits=...)`). + +**Reach for it when.** A unit of several words is decided by a count or a position the merge itself changes, and the safe reading of a declined unit is the reading its words already had. + ## LICENSED-BY-NEIGHBOR — a structural word is structural only beside the right neighbor **Problem shape.** A word in a structural vocabulary — a marker announcing a maiden name, a connective joining surnames — turns up where what it would announce makes no sense: behind a title alone, behind a credential, at the end of a name. Designing a reading for each such shape is endless, and the shapes are mostly junk. diff --git a/docs/design/rules/C4-ambiguous-credential-class-at-the-comma.md b/docs/design/rules/C4-ambiguous-credential-class-at-the-comma.md index c6eae763..639b7a61 100644 --- a/docs/design/rules/C4-ambiguous-credential-class-at-the-comma.md +++ b/docs/design/rules/C4-ambiguous-credential-class-at-the-comma.md @@ -1,6 +1,6 @@ # C4 — the ambiguous credential class at the comma -**When:** a word of the ambiguous credential class — a bare acronym the vocabulary marks as also an ordinary name, and a word admitted to the class by shape, which S3 defines and bounds — stands in the part after the comma the decision (C1) reads, as in "John Smith, Ma". +**When:** a word of the ambiguous credential class — a bare acronym the vocabulary marks as also an ordinary name, a word admitted to the class by shape, which S3 defines and bounds, and a credential split across a space, which S11 defines — stands in the part after the comma the decision (C1) reads, as in "John Smith, Ma". **Then:** the count before the comma is of NAME words rather than of words, and two or more of them read the part after the comma as the credential run, whatever case the name is written in. diff --git a/docs/design/rules/README.md b/docs/design/rules/README.md index 6b72b715..eb81f31e 100644 --- a/docs/design/rules/README.md +++ b/docs/design/rules/README.md @@ -148,6 +148,7 @@ Terms the rules use without defining them each time, each with the rule that def - [S8 — the credential's trailing run](S8-credentials-trailing-run.md) - [S9 — unlisted words that join the ambiguous class by shape](S9-unlisted-words-that-join-the-ambiguous-class-by-shape.md) - [S10 — the trailing roman numeral](S10-trailing-roman-numeral.md) +- [S11 — a credential split across a space](S11-credential-split-across-a-space.md) ## [Nicknames & quoted names (N)](N-nicknames-and-quoted-names.md) diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index fed6c9fe..ca4d65c7 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -113,17 +113,15 @@ **Accepted:** -- A numeral of one letter with another suffix word behind it is not the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name, and the reading is reported, as either reading of the fork is. A lone letter there abbreviates a surname as often as it numbers a generation. +- A numeral of one letter with another suffix word behind it is not the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name, and the reading is reported, as either reading of the fork is, whether the credential is written as one word or split across two (S11). A lone letter there abbreviates a surname as often as it numbers a generation. ``` "John Smith V PhD" → family="V" "John Smith V PhD" → ambiguities=("suffix-or-name",) + "John Smith V Ph. D." → family="V" + "John Smith V Ph. D." → ambiguities=("suffix-or-name",) "john i phd" → ambiguities=("suffix-or-name",) "Jane Doe nee Smith V PhD" → ambiguities=("suffix-or-name",) ``` -- A credential written split across two words is read as one with the numeral in front of it, the run taking both, as 1.4.0 read it. - ``` - "John Smith V Ph. D." → suffix="V Ph. D." - ``` - A generation written in Title case in a mixed-case name reads as a name word, the lean that keeps a surname such as 'Vi' a name taking it too. ``` "John Smith Viii" → family="Viii" diff --git a/docs/design/rules/S11-credential-split-across-a-space.md b/docs/design/rules/S11-credential-split-across-a-space.md new file mode 100644 index 00000000..cb42d3a2 --- /dev/null +++ b/docs/design/rules/S11-credential-split-across-a-space.md @@ -0,0 +1,83 @@ +# S11 — a credential split across a space + +**When:** a credential stands written as two words, as in "John Smith Pharm. D.". + +**Then:** a credential written split across a space reads as one word where its two words are short chunks whose letters together spell unambiguous suffix vocabulary and the two are not both suffix vocabulary on their own, as a member of the ambiguous class (S4) that the count decides as it decides a bare acronym, so that where the count declines it, the pair is the two words it is written as. + +- **Which pairs.** Each word is one to five letters, with or without its period, and the first has two or more: a single letter opening the pair is a middle initial far more often than the start of a credential. A particle or a connective written without a period is no chunk of one either, since it joins name words, which is what it is doing beside one. + ``` + "John Smith Pharm. D." → suffix="Pharm. D." + "John Smith Pharm D" → suffix="Pharm D" + "John Smith Psy. D." → suffix="Psy. D." + "John Smith M. D." → family="D." · boundary + "Jane Doe de A Junior" → family="de A" · boundary + ``` +- **The count decides.** At the end of a name it needs the words to spare S4 asks of a bare member; after a comma, the name words C4 counts before the comma; ending the given part after a family comma, it reads as S5 reads a member there, its writing saying nothing else. + ``` + "John Pharm D" → family="D" · boundary + "John Smith, Pharm D" → suffix="Pharm D" + "Smith, Pharm D" → given="Pharm" · boundary + "Smith, John Pharm D" → suffix="Pharm D" + ``` +- **The run.** Behind two name words where no comma divides the name, or after the given word in the part after a family comma, the pair starts the credential's run (S8) as its one-word spelling does, and a name word the run absorbs reports. + ``` + "John Smith Pharm D Jones" → suffix="Pharm D Jones" + "Smith, John Pharm D Jones" → suffix="Pharm D Jones" + ``` +- **What reports.** The reading itself reports nothing either way: where the count takes the pair its letters spell a listed credential, and where it declines the pair the two words are a given name and an initial, which no reader hesitates over. + ``` + "John Smith Pharm D" → ambiguities=() + "SMITH, SAM E." → ambiguities=() · boundary + ``` + +**Because:** a credential is often typed with a space between its parts, and its letters together are what the vocabulary lists. But two short words are also how a given name and its initial are written, so the pair buys only the ambiguous reading, and the count decides it. + +``` +"John Smith MB ChB" → suffix="MB ChB" +"SMITH, SAM E." → given="SAM" · boundary +"SMITH, SAM E." → middle="E." · boundary +``` + +**Except:** + +- **The Ph./D. pair is an unambiguous credential**, merged wherever it stands but the head of the string and read as a suffix there without the count (decisions/phd-merge), so the count does not decide it; where it overlaps another pair, it is the pair that merges. + ``` + "John Ph. D." → suffix="Ph. D." + "Jane Doe Dr. Ph. D." → title="Dr." + "Jane Doe Dr. Ph. D." → suffix="Ph. D." + ``` +- **Two dotted single letters** in a script that writes capitals are paired initials, which S3 reads as the name word they are written as, and never merge; in a script without capitals they merge where their letters spell listed vocabulary, which no default entry does. + ``` + "John Smith J. R." → family="R." · boundary + ``` +- **The part before a family comma** reads no member of the class as a suffix, so no pair merges there. + ``` + "Smith Pharm D, John" → family="Smith Pharm D" · boundary + ``` + +**Precedence:** + +- **The head of the string** ("Pharm D John Smith"): a suffix never opens the string (S2), so no pair merges there. + ``` + "Pharm D John Smith" → given="Pharm" + ``` +- **A one-letter numeral in front** ("John Smith V Pharm D"): it is the name word S10 reads before any credential, one word or split. + ``` + "John Smith V Pharm D" → family="V" + ``` + +**Accepted:** + +- A split credential opening with a single letter reads as the two words it is written as, and after a comma behind a whole name the name is reversed around it: `M. D.`, `M D` and `M. Sc.` are written so. + ``` + "John Smith, M. D." → given="M." + "John Smith, M. D." → family="John Smith" + ``` +- A given name and its initial behind two name words, where no comma divides the name, whose letters spell a listed credential read as the credential, in silence: the count reads them as it reads the credential they spell. + ``` + "JOHN SMITH SAM E." → suffix="SAM E." + ``` + +--- +**Signal:** shape: two short words whose letters together spell a listed credential, decided by the count +**history:** decisions/S11 · **implemented:** nameparser/_pipeline/_assign.py, nameparser/_pipeline/_classify.py, nameparser/_pipeline/_comma.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py diff --git a/docs/design/rules/S7-company-of-an-unambiguous-credential.md b/docs/design/rules/S7-company-of-an-unambiguous-credential.md index 508fd628..78499460 100644 --- a/docs/design/rules/S7-company-of-an-unambiguous-credential.md +++ b/docs/design/rules/S7-company-of-an-unambiguous-credential.md @@ -41,7 +41,7 @@ **Accepted:** -- A name-leaning member with nothing in front of it to speak for it ends the name, words to spare or not, and whatever stands in front of it is name text. Where a credential does stand in front, the company decides instead, except in one shape that keeps it out of reach and reads the member as the name: a credential written split across two words with no comma after the name. That reading is older than the company clause, so it is witnessed by tests/v2/cases.py's a_split_degree_in_front_is_out_of_the_walk rather than by a line here, which would promise a reading this clause did not make. A title standing between the credential and the member, and a member a particle chain (P2) had taken, were two more such shapes until the credential's run (S8) took both in: the rows a_title_between_degree_and_member_keeps_it_a_name and a_member_the_particle_chain_took_is_out_of_reach keep their ids and now pin the run. +- A name-leaning member with nothing in front of it to speak for it ends the name, words to spare or not, and whatever stands in front of it is name text. Where a credential does stand in front, the company decides instead. A title standing between the credential and the member, a member a particle chain (P2) had taken, and a credential written split across two words with no comma after the name kept it out of reach until the credential's run (S8) took all three in: the rows a_title_between_degree_and_member_keeps_it_a_name and a_member_the_particle_chain_took_is_out_of_reach keep their ids and now pin the run, and a_split_degree_in_front_starts_the_run pins the third. ``` "Jack Wei Ma" → family="Ma" "Jack Wei Ma" → ambiguities=("suffix-or-name",) diff --git a/docs/design/rules/S8-credentials-trailing-run.md b/docs/design/rules/S8-credentials-trailing-run.md index a3f89634..0db00fa9 100644 --- a/docs/design/rules/S8-credentials-trailing-run.md +++ b/docs/design/rules/S8-credentials-trailing-run.md @@ -11,15 +11,16 @@ "Smith, John PhD Jones" → suffix="PhD Jones" "John Smith PhD Prof. Ma" → suffix="PhD Ma" "Eric H. Holder Jr. Attorney General" → title="Attorney General" - "John Smith Ph. D. Jones" → suffix="Ph. D. Jones" deviates: #827 (today: suffix="Ph. D.") + "John Smith Ph. D. Jones" → suffix="Ph. D. Jones" ``` - **How the name words are counted.** A lone particle is not a name word for the count, 'de Mesnil' being one surname, unless the credential stands right behind it, leaving it nothing to join: then it counts as a name word of its own ('John von PhD Jones'). A particle run with the words the chain joins to it is one name word, 'der la' and 'la Smith Secretary' each counting once (P2). ``` "John von PhD Jones" → family="von" "John von PhD Jones" → suffix="PhD Jones" ``` -- **What is a credential here.** A title word never starts the run, and neither does a member of the ambiguous class, a single letter, a connective, a word that is also particle or bound given-name vocabulary: a credential is the writer's mark that the post-nominals have begun, and those carry no such mark. An honorific or a generational word carries it as a credential does, whatever its script. +- **What is a credential here.** A title word never starts the run, and neither does a member of the ambiguous class, a single letter, a connective, a word that is also particle or bound given-name vocabulary: a credential is the writer's mark that the post-nominals have begun, and those carry no such mark. An honorific or a generational word carries it as a credential does, whatever its script. A credential split across a space (S11) is the one member of the class that carries it, its other reading being two words of the name, which the two name words standing in front of it already leave the name without. ``` + "John Smith Pharm D Jones" → suffix="Pharm D Jones" "Mary Jane King Smith" → family="Smith" "Josep Carod i Rovira" → family="Carod i Rovira" "Mohamed Ali Abd Allah" → family="Allah" diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index fd888d7a..dcbe3728 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -112,7 +112,7 @@ A reading whose writing offers no cue is one the gate could only guess, so it is **`fils` and the agnomens show what kind-specific cues buy.** `fils` is gated by its case and the count, so `Alexandre Dumas fils` is to read suffix `fils` while `Marie Fils` keeps her family name, reported. The Brazilian agnomens are to be gated by the count alone: case would say nothing, and the plain generation reading breaks real surnames (`Ana Neto`) and given names. -**A credential's spelling is a form cue.** A credential joined by `-` or `/` is to read as one when every chunk is listed and none is ambiguous, since a hyphen also joins names (`Smith-Jones`); an unlisted word of capitals joined that way is to join the ambiguous class after a comma, gated by the name's case contrast. `John Smith, MD/PhD` read given `MD/PhD` until phase 2c (2026-10-25) read the listed half: a joined word whose every chunk counts as a suffix as written ([rules/S3](rules/S3-credentials-written-in-joined-chunks.md)). A credential split across a space (`Pharm. D.`, generalizing the `Ph. D.` merge) is to merge only where its first chunk has two or more letters, the merged piece a member of the ambiguous class, decided by the comma count and words to spare: an opening single letter is a middle initial (`SMITH, JOHN A. MS.` holds an initial and a salutation, not a credential `AMS`). #872 made the same judgment for the joined spelling (`A.B.`), in scripts with capitals and outside the slot right after a comma. The merged piece is also to start the credential's run (#827), once the numeral predicate answers the numeral before a credential; since [rules/S8](rules/S8-credentials-trailing-run.md) refuses a member of the ambiguous class as a run start, how the two combine is the split-credential PR's to state (today's `Ph. D.` merge is unambiguous, and S8 already covers it). +**A credential's spelling is a form cue.** A credential joined by `-` or `/` is to read as one when every chunk is listed and none is ambiguous, since a hyphen also joins names (`Smith-Jones`); an unlisted word of capitals joined that way is to join the ambiguous class after a comma, gated by the name's case contrast. `John Smith, MD/PhD` read given `MD/PhD` until phase 2c (2026-10-25) read the listed half: a joined word whose every chunk counts as a suffix as written ([rules/S3](rules/S3-credentials-written-in-joined-chunks.md)). A credential split across a space (`Pharm. D.`, generalizing the `Ph. D.` merge) is to merge only where its first chunk has two or more letters, the merged piece a member of the ambiguous class, decided by the comma count and words to spare: an opening single letter is a middle initial (`SMITH, JOHN A. MS.` holds an initial and a salutation, not a credential `AMS`). #872 made the same judgment for the joined spelling (`A.B.`), in scripts with capitals and outside the slot right after a comma. The merged piece is also to start the credential's run (#827), once the numeral predicate answers the numeral before a credential; since [rules/S8](rules/S8-credentials-trailing-run.md) refuses a member of the ambiguous class as a run start, how the two combine is the split-credential PR's to state (today's `Ph. D.` merge is unambiguous, and S8 already covers it). Phase 2e (2026-10-25) did both ([rules/S11](rules/S11-credential-split-across-a-space.md)): `John Smith, Pharm D` reads suffix `Pharm D`, and the split pair is the one member of the class that starts the run, behind the two name words its own count asks for. ### The front of the name diff --git a/docs/release_log.rst b/docs/release_log.rst index 5f0823ab..0d41005f 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -9,6 +9,8 @@ Release Log - **Fix roman numerals VI to X reading as names where III reads as a suffix.** ``John Smith VI MD`` gives suffix ``VI MD`` and ``Smith, John VI`` suffix ``VI``, as ``III`` does, where 1.4.0 through 2.6.0 gave last ``VI``, suffix ``MD`` and middle ``VI``. ``John Smith V, M.J.`` keeps suffix ``V`` and ``Georg VI.`` gives suffix ``VI.``, where 1.4.0 through 2.6.0 gave last ``John Smith V`` and ``VI.``. A one-letter numeral before a credential is still a name (``John Smith V PhD`` keeps last ``V``), and a Title-case ``Vi`` in a mixed-case name reads as a name (``Nguyen Van Vi`` gives last ``Van Vi``, where 2.2.0 through 2.6.0 gave suffix ``Vi``). :issue:`784`, :issue:`774` - **Fix a lone letter before a further comma reading as part of a credential.** ``Smith, John PhD I, Jr.`` gives middle ``I``, suffix ``PhD, Jr.``, as 1.4.0 through 2.3.0 did, where 2.4.0 through 2.6.0 gave suffix ``PhD I, Jr.``. :issue:`819` - **Read a credential joined by a hyphen or a slash as a suffix.** ``John Smith, MD/PhD`` gives first ``John``, last ``Smith``, suffix ``MD/PhD``, where 1.4.0 through 2.6.0 gave first ``MD/PhD``, last ``John Smith``; ``MD-PhD`` and ``PhD/MD`` read the same. An unlisted one written in capitals reads as a credential too after a comma behind two or more name words written in mixed case: ``John Smith, PU-PH`` gives suffix ``PU-PH``, where 1.4.0 through 2.6.0 gave first ``PU-PH``. Otherwise every part has to be a listed credential that is not also a name, so ``Smith-Jones`` is untouched and ``John Smith, MA/PhD`` still reverses; a hyphenated name written in Title case keeps its reading even where both parts are listed credentials (``Kim, Ae-Ra``), though its capitals spelling does not (``KIM, AE-RA`` gives suffix ``AE-RA``). :issue:`752`, :issue:`797` + - **Read a credential written with a space inside it, such as Pharm D, as one credential.** ``John Smith, Pharm D`` gives first ``John``, last ``Smith``, suffix ``Pharm D``, where 1.4.0 through 2.6.0 gave first ``Pharm``, middle ``D``, last ``John Smith``. It counts only where the name has words to spare, so ``Smith, Pharm D`` still gives first ``Pharm``, and one opening with a single letter is still two words (``John Smith, M. D.`` still gives first ``M.``). :issue:`752` + - **Fix a split Ph. D. in the middle of a name stopping short of the words after it.** ``John Smith Ph. D. Jones`` gives suffix ``Ph. D. Jones``, as ``John Smith PhD Jones`` does, where 1.4.0 through 2.6.0 gave last ``Jones``, suffix ``Ph. D.``; ``John Smith V Ph. D.`` gives last ``V``, as ``John Smith V PhD`` does, where 1.4.0 through 2.6.0 read ``V`` as a suffix. :issue:`827` * 2.6.0 - October 25, 2026 diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 43274c7c..8790a388 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -1214,7 +1214,7 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: # credential list behind it makes the credential # reading likelier rather than less. piece = pieces[m] - # rules/S3: a split credential is "a member of the + # rules/S11: a split credential reads "as a member of the # ambiguous class", here read as the class is with no # writing to lean on and in silence, as group merges it # only behind the given word diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 7334b0e1..aa844f50 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -33,8 +33,12 @@ "vocab:after-given-title", "vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous", "vocab:particle-ambiguous", "vocab:bound-given", "vocab:maiden-marker", -"vocab:maiden-marker-cont"; and, in a namespace of its own, -"shape:acronym". +"vocab:maiden-marker-cont"; in a namespace of its own, +"shape:acronym"; and in another, "split:credential" and +"split:member", written on the first word of a credential split across +a space (`_vocab.split_pair`) to name the piece the pair merges into: +the Ph./D. pair's unambiguous suffix, or any other pair's member of the +ambiguous class (rules/S11). "vocab:maiden-marker" tags the HEAD of a maiden marker, which is a whole marker whenever the marker is one word; the continuation tag carries the rest of a PHRASE marker ("z domu"), so a site asking @@ -397,7 +401,7 @@ def classify(state: ParseState) -> ParseState: tokens = tuple( copy_with(t, tags=t.tags - _JOINING_TAGS) if i in stemmed else t for i, t in enumerate(tokens)) - # rules/S3: "A credential written split across a space reads as one + # rules/S11: "a credential written split across a space reads as one # word" -- each adjacent pair of a segment asked once, here, and # the answer written on its first word for group's merge and the # comma's reading of its part (`_vocab.split_pair`). The shape and @@ -413,6 +417,11 @@ def classify(state: ParseState) -> ParseState: if (SPLIT_CHUNK.fullmatch(a.text) is not None and SPLIT_CHUNK.fullmatch(b.text) is not None and a.role is None and b.role is None + # the opening letter, the length a superset of the + # predicate's count of composed letters ('i i' folds + # to the listed 'ii' and opens with one) + and (len(a.text.rstrip(".")) >= 2 + or a.text.endswith(".") and b.text.endswith(".")) and ((fold := (a.text + b.text).replace(".", "") .lower()) in acronyms or fold in words or not fold.isascii()) @@ -428,9 +437,11 @@ def classify(state: ParseState) -> ParseState: split[seg[j]] = (SPLIT_CREDENTIAL_TAG if kind == "suffix" else SPLIT_MEMBER_TAG) if split is not None: - tokens = tuple( - copy_with(t, tags=t.tags | {split[i]}) if i in split else t - for i, t in enumerate(tokens)) + # a loop, not a generator, which is a frame per token resumed + written = list(tokens) + for i, tag in split.items(): + written[i] = copy_with(written[i], tags=written[i].tags | {tag}) + tokens = tuple(written) # Delimited content whose vocabulary cannot settle it: extract's # escape sends an UNambiguous suffix straight through ("(MBA)" -> # suffix) and keeps everything else as a nickname, so an AMBIGUOUS diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index f9e22b66..d385f6fc 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -52,14 +52,15 @@ def _pieces(seg: Sequence[int], tokens: Sequence[WorkToken], + splits: bool = True, ) -> tuple[list[list[int]], list[set[str]]]: """One piece per token but a split credential (`split_pair`, read off the tag classify wrote on its first word), which group's first merge builds (group alone declines a pair at the head of the string, rules/S2) -- the part as it stands before any join: the Ph./D. pair tagged "suffix", any other "split", a member of the - ambiguous class for rules/C4's count. Every piece a fresh list, as - `merge_pieces` requires.""" + ambiguous class for rules/C4's count, unless `splits` is False. + Every piece a fresh list, as `merge_pieces` requires.""" pieces: list[list[int]] = [] ptags: list[set[str]] = [] k = 0 @@ -67,7 +68,7 @@ def _pieces(seg: Sequence[int], tokens: Sequence[WorkToken], a = seg[k] tags = tokens[a].tags if k + 1 < len(seg) and (SPLIT_CREDENTIAL_TAG in tags - or SPLIT_MEMBER_TAG in tags): + or splits and SPLIT_MEMBER_TAG in tags): pieces.append([a, seg[k + 1]]) ptags.append({"suffix" if SPLIT_CREDENTIAL_TAG in tags else "split"}) @@ -80,10 +81,11 @@ def _pieces(seg: Sequence[int], tokens: Sequence[WorkToken], def _reading_pieces(part: Sequence[int], tokens: Sequence[WorkToken], + splits: bool = True, ) -> tuple[list[list[int]], list[set[str]]]: """The part as the reading takes it: `_pieces`, then the one join it can hold.""" - pieces, ptags = _pieces(part, tokens) + pieces, ptags = _pieces(part, tokens, splits) # The one join a postnominal part can hold, made here so the reading # sees it: rules/P3's connective joining its neighbours where the # neighbour it takes its kind from is a title, which makes the @@ -273,12 +275,33 @@ def _behind_a_generation(state: ParseState, seg0: Sequence[int], return tuple(reading) -def decide(state: ParseState) -> ParseState: +def decide(state: ParseState, splits: bool | None = True) -> ParseState: """The structure of a comma form, and the postnominal part bound. - Returns `state` unchanged where the listing form stands.""" + Returns `state` unchanged where the listing form stands. + + `splits` is for the retry below alone: None reads the part with each + split credential merged and asks nothing further, False with none + merged.""" if (state.structure is not Structure.FAMILY_COMMA or len(state.segments) < 2): return state + # rules/S11: "where the count declines it, the pair is the two words + # it is written as" -- the part read with each split credential + # merged, and read again as written where the reading did not take + # one as a credential, so a declined pair reads as it reads unmerged + # (an 'MB' it hid opens the part again, rules/C13). A loop, and the + # retry a call of this function only where a pair stands, so a part + # holding none pays no frame for it. + if splits: + heads: list[int] = [] + for i in state.segments[1]: + if SPLIT_MEMBER_TAG in state.tokens[i].tags: + heads.append(i) + if heads: + out = decide(state, None) + if all(out.tokens[i].role is Role.SUFFIX for i in heads): + return out + return decide(state, False) seg0, seg1 = state.segments[0], state.segments[1] # Nothing after the comma: nothing to read. An EMPTY part BEFORE it # is read on, being no whole name (`_whole_name` answers no), so @@ -330,7 +353,7 @@ def decide(state: ParseState) -> ParseState: and AMBIGUOUS_ACRONYM_TAG not in tags): tokens[i] = copy_with( tokens[i], tags=tags | {AMBIGUOUS_ACRONYM_TAG}) - p1, pt1 = _reading_pieces(part, tokens) + p1, pt1 = _reading_pieces(part, tokens, splits is not False) anchored: list[int] = [] absorbed: list[int] = [] # A core in the part, standing alone or glued to a word or inside @@ -551,7 +574,11 @@ def decide(state: ParseState) -> ParseState: role = Role.SUFFIX if reading[k] else Role.TITLE for i in piece: tags = tokens[i].tags - if i in caps: + if (i in caps or "split" in pt1[k] and reading[k] + and texts[i].isupper() and texts[i].isalpha() + and "vocab:suffix" not in tags): + # a split credential's capitals are a credential's too + # (rules/S11), marked as a caps word bound here is # #564: the caps shape's marks, as classify writes them # EVERYWHERE, so case repair keeps 'XYZ' (rules/R4) tags = tags | {SHAPE_ACRONYM_TAG, AMBIGUOUS_ACRONYM_TAG} diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 5e0cdaa8..908d2db6 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -578,10 +578,9 @@ def _between_name_words(k: int, lo: int, hi: int, and not is_title_piece(pieces[right], ptags[right], tokens)) -# rules/S4: "a BARE ambiguous acronym is consumed only when the name -# has words to spare" -- and a split credential is a member of that -# class (rules/S3), so where the trailing read did not take one, it is -# the two words it is written as +# rules/S11: "where the count declines it, the pair is the two words +# it is written as" -- after the trailing read, which is where the +# count is asked in a main segment def _settle_splits(pieces: list[Piece], ptags: list[set[str]]) -> None: """Split every merged split credential left in `pieces` back into its two words, in place: a pair the trailing read declined reads diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 66c00c69..259f31ba 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -1239,8 +1239,8 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], # the vocabulary is not in doubt. bare_ambiguous = (len(piece) == 1 and AMBIGUOUS_ACRONYM_TAG in tokens[piece[0]].tags) - # rules/S3: a split credential "is a member of the ambiguous - # class" -- read by the count alone, its writing carrying no + # rules/S11: a split credential reads "as a member of the + # ambiguous class" -- read by the count alone, its writing carrying no # case to lean on, and in silence: its other reading is two # name words, which the count already refuses it behind one if "split" in ptags[rest[k - 1]]: @@ -1981,7 +1981,7 @@ def _chain_units(pieces: Sequence[Sequence[int]], j = chain_run_end(k, pieces, ptags, tokens, count) q = k + 1 # a split credential is weighed as the class it is a - # member of (rules/S3) + # member of (rules/S11) while (q < j and "split" not in ptags[q] and not _weighed(pieces[q], tokens)): units[q] = JOINED diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 1a77304c..0f39704f 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -567,7 +567,7 @@ def is_cased_paired_initials(text: str) -> bool: and text.upper() != text.lower()) -# rules/S3: "A credential written split across a space reads as one +# rules/S11: "a credential written split across a space reads as one # word" -- one chunk of up to five letters, with or without its period # ('Pharm.', 'D', 'Sc.'), the shape both words of a split credential # take. The C-level half classify asks of every adjacent pair before @@ -592,7 +592,8 @@ def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: "split": any other pair of short chunks (`SPLIT_CHUNK`) whose period-free concatenation is unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own ('Pharm. D.', - 'Pharm D', 'MB ChB' where 'chb' is unlisted). The merged piece is + 'Pharm D', 'MB ChB' where 'chb' is unlisted) and neither is a + particle or a connective, which join name words. The merged piece is a member of the ambiguous class, which the count decides (rules/S4, rules/C4). Its first chunk has two or more letters: a single letter opening the pair is a middle initial far more often (FEC @@ -628,6 +629,14 @@ def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: and (nb in lexicon.suffix_acronyms or nb in lexicon.suffix_words)): return None + # a particle or a connective joins name words, which is what it is + # doing beside one ('DE A JUNIOR', the A a surname written short); + # written with its period a letter is an initial, not the connective + # it spells ('SAM E.', rules/T5) + for written, n in ((first, na), (second, nb)): + if not written.endswith(".") and (n in lexicon.particles + or n in lexicon.conjunctions): + return None return "split" diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 51166464..b391c104 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1501,19 +1501,121 @@ def _check_cjk_shape_purity(self) -> None: "John Smith, PH D", {"given": "John", "family": "Smith", "suffix": "PH D"}, classification="parity"), - Case("a_split_degree_in_front_is_out_of_the_walk", + Case("a_split_degree_in_front_starts_the_run", "John Smith Ph. D. MEng", - {"given": "John", "middle": "Smith", "family": "MEng", - "suffix": "Ph. D."}, - classification="fix(#540)", + {"given": "John", "family": "Smith", "suffix": "Ph. D. MEng"}, + classification="fix(#827)", + ambiguities=("suffix-or-name",), + notes="S8: the merged 'Ph. D.' starts the credential's " + "run as its one-word spelling 'PhD' does, though the " + "walk leaves it out, so the 'MEng' behind it is in the " + "run and reported. Until #827 the member's own writing " + "decided and it read family 'MEng'; 2.3.0 read suffix " + "'Ph. D. MEng', and 1.4.0 'Ph. D., MEng'"), + # #752, phase 2e: every other credential written split across a + # space merges as a member of the ambiguous class, the count + # deciding it (rules/S11); the controls are the spellings it + # leaves split + Case("a_split_credential_with_words_to_spare_is_the_suffix", + "John Smith Pharm. D.", + {"given": "John", "family": "Smith", "suffix": "Pharm. D."}, + classification="fix(#752)", + notes="S3: 'Pharm. D.' folds to the listed 'pharmd', " + "and neither half is suffix vocabulary on its own, so " + "the two words are one credential, read with the words " + "to spare a bare member needs (S4) and in silence. " + "1.4.0 read middle 'Smith Pharm.', last 'D.'"), + Case("a_split_credential_with_no_words_to_spare_is_two_words", + "John Pharm D", + {"given": "John", "middle": "Pharm", "family": "D"}, + notes="S4's count: one name word in front leaves " + "nothing to spare, so the pair is split back into the " + "two words it is written as and reads as they read " + "unmerged"), + Case("a_split_credential_behind_a_whole_name_after_a_comma", + "John Smith, Pharm D", + {"given": "John", "family": "Smith", "suffix": "Pharm D"}, + classification="fix(#752)", + notes="C4's count: two name words before the comma make " + "the part the credential run. 1.4.0 read first 'Pharm', " + "middle 'D', last 'John Smith'"), + Case("a_split_credential_behind_one_word_after_a_comma", + "Smith, Pharm D", + {"given": "Pharm", "middle": "D", "family": "Smith"}, + notes="C4's count: one name word before the comma " + "reads the part as the given part, where the pair is " + "the given name's own words"), + Case("a_given_name_and_initial_that_spell_a_credential", + "SMITH, SAM E.", + {"given": "SAM", "middle": "E.", "family": "SMITH"}, + notes="C4's count is what keeps this a name: 'SAM E.' " + "folds to the listed 'same', and FEC's donor list writes " + "a given name and its initial so behind a one-word " + "family (decisions/vocabulary-model, 2026-10-25 #752)"), + Case("a_split_credential_opening_with_one_letter_stays_split", + "John Smith M. Sc.", + {"given": "John", "middle": "Smith M.", "family": "Sc."}, + notes="S3's opening letter: a single letter opening the " + "pair is a middle initial far more often ('SMITH, JOHN " + "A. MS.' folds to 'ams'), so the pair never merges, the " + "accepted cost being 'M. Sc.' and 'M. D.'"), + Case("a_split_credential_ending_the_given_part", + "Smith, John Pharm D", + {"given": "John", "family": "Smith", "suffix": "Pharm D"}, + classification="fix(#752)", + notes="S5: a member ending the given part after a " + "family comma reads as the credential where its writing " + "says nothing else, and a split one says nothing. 1.4.0 " + "read middle 'Pharm D'"), + Case("a_split_credential_starts_the_run_in_the_given_part", + "Smith, John Pharm D Jones", + {"given": "John", "family": "Smith", "suffix": "Pharm D Jones"}, + classification="fix(#752)", ambiguities=("suffix-or-name",), - notes="rules/S7's Accepted limit: the merged 'Ph. D.' is " - "read as a suffix at any position and so is not in the " - "no-comma walk the company is read over, and the " - "member's writing decides. The comma and given-slot " - "spellings do reach it ('Doe, Jane Ph. D. MEng' reads " - "suffix 'Ph. D. MEng'). Unchanged by #544; 2.3.0 read " - "suffix 'Ph. D. MEng', and 1.4.0 'Ph. D., MEng'"), + notes="S8: the merged pair starts the run after the " + "given word, as 'PharmD' does, and the name word it " + "absorbs is reported. 1.4.0 read middle 'Pharm D Jones'"), + Case("a_split_credential_starts_the_run_behind_two_name_words", + "John Smith Pharm D Jones", + {"given": "John", "family": "Smith", "suffix": "Pharm D Jones"}, + classification="fix(#752)", + ambiguities=("suffix-or-name",), + notes="S8: two name words in front, the words to spare " + "S4 asks of the pair, start the run there. 1.4.0 read " + "middle 'Smith Pharm D', last 'Jones'"), + Case("a_split_credential_behind_a_particle_run_reads_as_a_shape_member", + "St van Berg Pharm D", + {"title": "St", "family": "van Berg Pharm D"}, + ambiguities=("particle-or-given",), + notes="S3: no writing to lean on, so the pair reads as a " + "member admitted by shape does ('St van Berg X.Y.Z.' " + "keeps its family): the particle chain is one name " + "word, which leaves none to spare"), + Case("a_split_credential_ends_a_maiden_clause", + "Jane Doe nee Smith Pharm D", + {"given": "Jane", "family": "Doe", "maiden": "Smith", + "suffix": "Pharm D"}, + classification="fix(#752)", + notes="M2: the clause ends at the trailing run the " + "clause-free name reads, which takes the pair as " + "'Jane Doe Pharm D' does. 1.4.0 read middle 'Doe nee " + "Smith Pharm', last 'D'"), + Case("a_split_credential_reports_no_absorbed_word", + "John Smith MB ChB", + {"given": "John", "family": "Smith", "suffix": "MB ChB"}, + classification="fix(#752)", + notes="'MB ChB' is one listed credential, 'mbchb', where 'mb' " + "started the run and reported 'ChB' as a name word it " + "absorbed. 1.4.0 read middle 'Smith MB', last 'ChB'"), + Case("the_split_degree_outranks_a_pair_overlapping_it", + "Jane Doe Dr. Ph. D.", + {"title": "Dr.", "given": "Jane", "family": "Doe", + "suffix": "Ph. D."}, + classification="fix(#752)", + notes="'Dr. Ph.' folds to the listed 'drph', which would " + "leave 'D.' a word of its own; the Ph./D. pair takes " + "precedence, keeping the tree's reading before #752's " + "widened pair test. 1.4.0 read suffix 'Ph. D., Dr.'"), Case("a_member_the_particle_chain_took_is_out_of_reach", "John Smith PhD Do Do", {"given": "John", "family": "Smith", "suffix": "PhD Do Do"}, diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index df360b56..195a8762 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -734,12 +734,17 @@ def test_the_reserve_counts_the_names_the_read_left_before_a_numeral() -> None: def test_the_numeral_is_last_among_the_pieces_assign_keeps() -> None: # assign drops a group-flagged credential piece (the Ph. D. merge) # from its walk at ANY position before the trailing peel, so a - # numeral can be last in that walk without being the last piece: - # 'abdul Smith V Ph. D.' read given 'abdul Smith', family '', - # suffix 'V, Ph. D.' where 'John Smith V Ph. D.' keeps family - # 'Smith'. Found by the code review. + # numeral can be last in that walk without being the last piece. + # A one-letter numeral with the merged credential behind it is a + # name word, as it is behind 'PhD' (rules/S10, #752), so the bound + # given-name join has the three name words it needs and joins as it + # does for 'abdul Smith V PhD'. (Until #752 the walk read the V as + # the generation here, which kept the pair apart: the first fix of + # this test's defect, found by the code review, pinned that.) out = _grouped("abdul Smith V Ph. D.") - assert _piece_texts(out) == [["abdul", "Smith", "V", "Ph. D."]] + assert _piece_texts(out) == [["abdul Smith", "V", "Ph. D."]] + assert (_piece_texts(_grouped("abdul Smith V PhD")) + == [["abdul Smith", "V", "PhD"]]) def test_the_numeral_fork_is_not_mirrored_after_a_family_comma() -> None: diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index b8aa650c..23015fec 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -11,7 +11,7 @@ ambiguous_class_candidate, ambiguous_class_member, ambiguous_lean, caps_shape_candidate, comma_follows_marker, delimiter_cores, glued_stem, effective_script, is_initial, is_initial_shaped, is_one_case, - SPLIT_PAIR_OPENS, is_cased_paired_initials, is_paired_initials, + SPLIT_CHUNK, SPLIT_PAIR_OPENS, is_cased_paired_initials, is_paired_initials, is_single_letter_numeral, split_pair, is_suffix_lenient, is_suffix_strict, is_title_shaped, is_wholly_suffix, maiden_marker_run, name_word_count, period_joined_vocab, written_as_a_name, @@ -953,19 +953,60 @@ def test_a_listed_member_written_in_period_closed_chunks_is_dotted( assert not is_suffix_strict(text, lex), text -def test_split_pair_opens_is_a_necessary_condition_of_split_pair() -> None: - # every caller asks SPLIT_PAIR_OPENS C-level before split_pair, so a - # split_pair widened past it would never be reached for the new - # opener (#752); the periods are optional and the case free +def test_split_pair_answers_with_the_piece_tag_the_pair_merges_with() -> None: + lex = Lexicon.default() + # the Ph./D. pair, its periods optional and its case free + assert all(split_pair(a, b, lex) == "suffix" + for a in ("Ph.", "Ph", "PH", "ph.") for b in ("D.", "D", "d")) + # any other pair whose fold is unambiguous suffix vocabulary, its + # halves not both listed: a member of the ambiguous class (#752) + for a, b in (("Pharm.", "D."), ("Pharm", "D"), ("MB", "ChB"), + ("SAM", "E.")): + assert split_pair(a, b, lex) == "split", (a, b) + # an opening single letter ('A. MS.' folds to 'ams'), both halves + # listed, an ambiguous or unlisted fold, a chunk of six letters, and + # the Ph./D. halves in the other order + for a, b in (("M.", "Sc."), ("A.", "MS."), ("M", "D"), ("MD", "PhD"), + ("Smith", "Jones"), ("Pharma", "D"), ("D.", "Ph."), + ("Ph.", "D..")): + assert split_pair(a, b, lex) is None, (a, b) + + +def test_two_dotted_letters_merge_only_outside_a_cased_script() -> None: + # S11: two dotted single letters are paired initials where the + # script has capitals (#872's predicate), so they merge only in a + # caseless one, and only where the fold is listed, as no default + # entry is + lex = Lexicon.default().add(suffix_acronyms={"jr", "พบ"}) + assert split_pair("J.", "R.", lex) is None + assert split_pair("พ.", "บ.", lex) == "split" + assert split_pair("พ.", "บ.", Lexicon.default()) is None + assert split_pair("พ", "บ", lex) is None + + +def test_the_c_level_pre_tests_are_necessary_conditions_of_split_pair( + ) -> None: + # classify asks SPLIT_CHUNK of both words, the opening letter's + # length and the fold's membership C-level before the call, and is_wholly_suffix SPLIT_PAIR_OPENS + # before asking for the Ph./D. answer, so a split_pair widened past + # either would never be reached for the new spelling (#752) + lex = Lexicon.default() words = ["Ph.", "Ph", "PH.", "ph", "pH", "D.", "D", "d", "Pharm.", - "M.", "MD", "Phd", "Ph..", "", "Smith"] + "Pharm", "M.", "MD", "Phd", "Ph..", "", "Smith", "MB", "ChB", + "SAM", "E.", "Pharma", "Ed", "J.", "R."] for first in words: for second in words: - if split_pair(first, second): + kind = split_pair(first, second, lex) + if kind is None: + continue + fold = (first + second).replace(".", "").lower() + assert SPLIT_CHUNK.fullmatch(first) and SPLIT_CHUNK.fullmatch( + second), (first, second) + assert (len(first.rstrip(".")) >= 2 + or first.endswith(".") and second.endswith(".")) + assert fold in lex.suffix_acronyms or fold in lex.suffix_words + if kind == "suffix": assert SPLIT_PAIR_OPENS.fullmatch(first), (first, second) - assert all(split_pair(a, b) for a in ("Ph.", "Ph", "PH", "ph.") - for b in ("D.", "D", "d")) - assert not split_pair("Ph.", "D..") and not split_pair("Phd", "D.") def test_cased_paired_initials_is_the_bare_shape_in_a_cased_script() -> None: diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index a6409cab..e319f630 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -4,7 +4,7 @@ "C1": {"words": 448, "then_words": 196, "because_sentences": 1, "except": 0, "precedence": 5, "accepted": 1, "accepted_words": 116}, "C2": {"words": 492, "then_words": 399, "because_sentences": 1, "except": 2, "precedence": 0, "accepted": 1, "accepted_words": 81}, "C3": {"words": 260, "then_words": 228, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 1, "accepted_words": 173}, - "C4": {"words": 498, "then_words": 371, "because_sentences": 1, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, + "C4": {"words": 507, "then_words": 371, "because_sentences": 1, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, "C5": {"words": 319, "then_words": 79, "because_sentences": 1, "except": 2, "precedence": 1, "accepted": 1, "accepted_words": 54}, "C6": {"words": 207, "then_words": 162, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "C7": {"words": 119, "then_words": 68, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, @@ -58,10 +58,11 @@ "S4": {"words": 298, "then_words": 156, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, "S5": {"words": 632, "then_words": 317, "because_sentences": 1, "except": 2, "precedence": 1, "accepted": 0, "accepted_words": 0}, "S6": {"words": 337, "then_words": 121, "because_sentences": 2, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, - "S7": {"words": 570, "then_words": 388, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 1, "accepted_words": 147}, - "S8": {"words": 381, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, + "S7": {"words": 570, "then_words": 388, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 1, "accepted_words": 105}, + "S8": {"words": 422, "then_words": 308, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, "S9": {"words": 463, "then_words": 262, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, - "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 4, "accepted_words": 176}, + "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 3, "accepted_words": 163}, + "S11": {"words": 494, "then_words": 278, "because_sentences": 2, "except": 3, "precedence": 2, "accepted": 2, "accepted_words": 80}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 5939a7ed..81cfae36 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -239,9 +239,12 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # 'Dr. Smith VI, John' row, which reports since, each claim under # the three orders; 1445 -> 1451 the same day: #752's two # joined-capitals rows that report (phase 2d), one suffix-or-name - # claim under each order. - assert claims == 1451, ( - f"the field sweep checks {claims} claims, recorded as 1451 on " + # claim under each order; 1451 -> 1457 the same day: #752's two + # split-credential run rows (phase 2e), 'Smith, John Pharm D Jones' + # and 'John Smith Pharm D Jones', one suffix-or-name claim under + # each order. + assert claims == 1457, ( + f"the field sweep checks {claims} claims, recorded as 1457 on " f"2026-10-25") diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 798116ab..c9286142 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -599,12 +599,12 @@ def test_the_reserve_counts_what_the_trailing_read_left() -> None: n = parse("John J. V") assert (n.middle, n.family, n.suffix) == ("J.", "V", "") # and behind a merged credential, which assign drops from its walk - # so the V is last in it, the family survives + # so the V is last in it, the numeral is the name word it is behind + # 'PhD' (rules/S10, #752), so the name reads as its one-word twin n = parse("abdul Smith V Ph. D.") - # no comma between the numeral and the merged credential, so the two - # are one entry (#436); the V-is-last-in-the-walk point above is a - # claim about the ROLES and is untouched - assert (n.given, n.family, n.suffix) == ("abdul", "Smith", "V Ph. D.") + assert (n.given, n.family, n.suffix) == ("abdul Smith", "V", "Ph. D.") + twin = parse("abdul Smith V PhD") + assert (twin.given, twin.family, twin.suffix) == ("abdul Smith", "V", "PhD") def test_the_reserve_spares_the_family_the_acronym_fork_would_take() -> None: diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 1cb14d50..054d1421 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -208,6 +208,7 @@ "JOHN SMITH MA" "JOHN SMITH NEE X X" "JOHN SMITH PH.D." +"JOHN SMITH SAM E." "JOHN SMITH VI" "JOHN SMITH, FNP-BC" "JOHN SMITH, PU-PH" @@ -234,10 +235,12 @@ "Jane Doe (nee Smith Ma)" "Jane Doe (nee Smith Prof.)" "Jane Doe (nee Smith) MA" +"Jane Doe Dr. Ph. D." "Jane Doe Jr. nee Smith" "Jane Doe MA R.T." "Jane Doe Mrs" "Jane Doe PhD nee Smith" +"Jane Doe de A Junior" "Jane Doe nee King." "Jane Doe nee MA" "Jane Doe nee MA Smith" @@ -299,7 +302,9 @@ "John J. V" "John J. VI" "John Ma" +"John Ph. D." "John PhD Smith" +"John Pharm D" "John Prof. MA" "John Quincy Smith i" "John Smith" @@ -315,6 +320,7 @@ "John Smith III" "John Smith III MD" "John Smith Iii" +"John Smith J. R." "John Smith J.u.n.i.o.r." "John Smith Jr." "John Smith Jr. Prof." @@ -328,7 +334,9 @@ "John Smith Jr., Sir AD" "John Smith Jr., þingmaður" "John Smith K.C." +"John Smith M. D." "John Smith M.A." +"John Smith MB ChB" "John Smith MD PhD" "John Smith MD-Jones" "John Smith MD-PhD" @@ -342,15 +350,20 @@ "John Smith PhD Jones" "John Smith PhD MEng" "John Smith PhD Prof. Ma" +"John Smith Pharm D" +"John Smith Pharm D Jones" +"John Smith Pharm. D." "John Smith Prof." "John Smith Prof. Dr." "John Smith Prof. Jr." +"John Smith Psy. D." "John Smith Q.W.E.R.T." "John Smith R.A.I." "John Smith Sir" "John Smith V" "John Smith V Ph. D." "John Smith V PhD" +"John Smith V Pharm D" "John Smith V, AD" "John Smith V, M.J." "John Smith V." @@ -381,6 +394,7 @@ "John Smith, Jr., abdul Dr. Smith" "John Smith, Jr., van der Berg Prof." "John Smith, LEED AP" +"John Smith, M. D." "John Smith, M.D./Ph.D." "John Smith, M.D.; Ph.D." "John Smith, MA" @@ -416,6 +430,7 @@ "John Smith, PhD XYZ" "John Smith, PhD vd DO" "John Smith, PhD-Md" +"John Smith, Pharm D" "John Smith, Phd-Md" "John Smith, Pu-Ph" "John Smith, RA Jones" @@ -542,6 +557,7 @@ "Petrov Ivan Sergeevich" "Ph. D. Van Johnson" "Ph. D., John" +"Pharm D John Smith" "Pieter Bruegel stariji Jones" "Prince of Wales" "Prince of Wales Jr" @@ -567,6 +583,7 @@ "SMITH VD MA, JOHN" "SMITH VD, JOHN" "SMITH, JOHN DO" +"SMITH, SAM E." "SMITH, VD DE LA MA" "SMITH, VD MA" "STEVEN HARDMAN, MD, DO, DDS" @@ -596,6 +613,7 @@ "Smith Jr., MA" "Smith Jr., Ma" "Smith Jr., Mr." +"Smith Pharm D, John" "Smith Prof." "Smith Sir." "Smith V, AD" @@ -625,6 +643,8 @@ "Smith, John PhD Jones" "Smith, John PhD de Jr." "Smith, John PhD É, Jr." +"Smith, John Pharm D" +"Smith, John Pharm D Jones" "Smith, John Prof." "Smith, John V" "Smith, John V MD" @@ -665,6 +685,7 @@ "Smith, PhD" "Smith, PhD MEng" "Smith, PhD Ma" +"Smith, Pharm D" "Smith, RA PhD Ma" "Smith, RN - CRNA" "Smith, Sir" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index c4132974..a587dfce 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -1338,6 +1338,7 @@ {"name": "JOHN SMITH MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} {"name": "JOHN SMITH NEE X X", "given": "JOHN", "family": "SMITH", "maiden": "X X", "initials": "J. S.", "capitalized": "John Smith (X X)"} {"name": "JOHN SMITH PH.D.", "given": "JOHN", "family": "SMITH", "suffix": "PH.D.", "initials": "J. S.", "capitalized": "John Smith Ph.D."} +{"name": "JOHN SMITH SAM E.", "given": "JOHN", "family": "SMITH", "suffix": "SAM E.", "initials": "J. S.", "capitalized": "John Smith Sam E."} {"name": "JOHN SMITH VI", "given": "JOHN", "family": "SMITH", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith VI"} {"name": "JOHN SMITH, FNP-BC", "given": "JOHN", "family": "SMITH", "suffix": "FNP-BC", "initials": "J. S.", "capitalized": "John Smith FNP-BC"} {"name": "JOHN SMITH, MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} @@ -1403,11 +1404,13 @@ {"name": "Jane Doe (nee Smith Ma)", "given": "Jane", "family": "Doe", "maiden": "Smith Ma", "initials": "J. D."} {"name": "Jane Doe (nee Smith Prof.)", "title": "Prof.", "given": "Jane", "family": "Doe", "maiden": "Smith", "initials": "J. D."} {"name": "Jane Doe (nee Smith) MA", "given": "Jane", "family": "Doe", "suffix": "MA", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} +{"name": "Jane Doe Dr. Ph. D.", "title": "Dr.", "given": "Jane", "family": "Doe", "suffix": "Ph. D.", "initials": "J. D."} {"name": "Jane Doe Jr. nee Smith", "given": "Jane", "family": "Doe", "suffix": "Jr. nee Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D.", "capitalized_force": "Jane Doe Jr. Nee Smith"} {"name": "Jane Doe Jr. nee Smith Ma", "given": "Jane", "family": "Doe", "suffix": "Jr. nee Smith Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D.", "capitalized_force": "Jane Doe Jr. Nee Smith MA"} {"name": "Jane Doe MA R.T.", "given": "Jane", "middle": "Doe MA", "family": "R.T.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D. M. R.", "capitalized_force": "Jane Doe Ma R.T."} {"name": "Jane Doe Mrs", "given": "Jane", "middle": "Doe", "family": "Mrs", "initials": "J. D. M."} {"name": "Jane Doe PhD nee Smith", "given": "Jane", "family": "Doe", "suffix": "PhD nee Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D.", "capitalized_force": "Jane Doe PhD Nee Smith"} +{"name": "Jane Doe de A Junior", "given": "Jane", "middle": "Doe", "family": "de A", "suffix": "Junior", "initials": "J. D. A."} {"name": "Jane Doe geb. Smith MA", "given": "Jane", "family": "Doe", "suffix": "MA", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee King.", "given": "Jane", "family": "Doe", "maiden": "King.", "initials": "J. D."} {"name": "Jane Doe nee MA", "given": "Jane", "family": "Doe", "maiden": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D.", "capitalized_force": "Jane Doe (Ma)"} @@ -1592,7 +1595,9 @@ {"name": "John Major. Smith", "given": "John", "middle": "Major.", "family": "Smith", "initials": "J. M. S."} {"name": "John Mkwawa", "given": "John", "family": "Mkwawa", "initials": "J. M."} {"name": "John P. Doe, CLU, CFP, LUTC", "given": "John", "middle": "P.", "family": "Doe", "suffix": "CLU, CFP, LUTC", "ambiguities": ["COMMA_STRUCTURE"], "initials": "J. P. D.", "capitalized_force": "John P. Doe CLU, CFP, Lutc"} +{"name": "John Ph. D.", "given": "John", "suffix": "Ph. D.", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "J."} {"name": "John PhD Smith", "given": "John", "middle": "PhD", "family": "Smith", "initials": "J. P. S."} +{"name": "John Pharm D", "given": "John", "middle": "Pharm", "family": "D", "initials": "J. P. D."} {"name": "John Prof. MA", "title": "Prof.", "family": "John", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} {"name": "John Q. Smith", "given": "John", "middle": "Q.", "family": "Smith", "initials": "J. Q. S."} {"name": "John Quincy Smith i", "given": "John", "middle": "Quincy", "family": "Smith", "suffix": "i", "initials": "J. Q. S.", "capitalized_force": "John Quincy Smith I"} @@ -1616,6 +1621,7 @@ {"name": "John Smith III", "given": "John", "family": "Smith", "suffix": "III", "initials": "J. S."} {"name": "John Smith III MD", "given": "John", "family": "Smith", "suffix": "III MD", "initials": "J. S."} {"name": "John Smith Iii", "given": "John", "family": "Smith", "suffix": "Iii", "initials": "J. S.", "capitalized_force": "John Smith III"} +{"name": "John Smith J. R.", "given": "John", "middle": "Smith J.", "family": "R.", "initials": "J. S. J. R."} {"name": "John Smith J.u.n.i.o.r.", "given": "John", "family": "Smith", "suffix": "J.u.n.i.o.r.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith J.U.N.I.O.R."} {"name": "John Smith Jr", "given": "John", "family": "Smith", "suffix": "Jr", "initials": "J. S."} {"name": "John Smith Jr.", "given": "John", "family": "Smith", "suffix": "Jr.", "initials": "J. S."} @@ -1631,8 +1637,10 @@ {"name": "John Smith Jr., Sir AD", "title": "Sir", "given": "John", "family": "Smith", "suffix": "Jr., AD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith Jr., þingmaður", "given": "John", "family": "Smith", "suffix": "Jr., þingmaður", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith Jr., Þingmaður"} {"name": "John Smith K.C.", "given": "John", "middle": "Smith", "family": "K.C.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. K."} +{"name": "John Smith M. D.", "given": "John", "middle": "Smith M.", "family": "D.", "initials": "J. S. M. D."} {"name": "John Smith M.A.", "given": "John", "family": "Smith", "suffix": "M.A.", "initials": "J. S."} {"name": "John Smith MA", "given": "John", "family": "Smith", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith MB ChB", "given": "John", "family": "Smith", "suffix": "MB ChB", "initials": "J. S.", "capitalized_force": "John Smith MB Chb"} {"name": "John Smith MC", "given": "John", "family": "Smith", "suffix": "MC", "initials": "J. S."} {"name": "John Smith MD PhD", "given": "John", "family": "Smith", "suffix": "MD PhD", "initials": "J. S."} {"name": "John Smith MD-Jones", "given": "John", "middle": "Smith", "family": "MD-Jones", "initials": "J. S. M.", "capitalized_force": "John Smith Md-Jones"} @@ -1644,7 +1652,7 @@ {"name": "John Smith Msc.Ed.", "given": "John", "family": "Smith", "suffix": "Msc.Ed.", "initials": "J. S."} {"name": "John Smith Ph.", "given": "John", "middle": "Smith", "family": "Ph.", "initials": "J. S. P."} {"name": "John Smith Ph. D.", "given": "John", "family": "Smith", "suffix": "Ph. D.", "initials": "J. S."} -{"name": "John Smith Ph. D. Jones", "given": "John", "middle": "Smith", "family": "Jones", "suffix": "Ph. D.", "initials": "J. S. J."} +{"name": "John Smith Ph. D. Jones", "given": "John", "family": "Smith", "suffix": "Ph. D. Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith PhD", "given": "John", "family": "Smith", "suffix": "PhD", "initials": "J. S."} {"name": "John Smith PhD (MA)", "given": "John", "family": "Smith", "suffix": "PhD", "nickname": "MA", "ambiguities": ["SUFFIX_OR_NICKNAME"], "initials": "J. S.", "capitalized_force": "John \"Ma\" Smith PhD"} {"name": "John Smith PhD Ed Ma", "given": "John", "family": "Smith", "suffix": "PhD Ed Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith PhD ED MA"} @@ -1652,17 +1660,22 @@ {"name": "John Smith PhD MEng", "given": "John", "family": "Smith", "suffix": "PhD MEng", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith PhD MENG"} {"name": "John Smith PhD Ma", "given": "John", "family": "Smith", "suffix": "PhD Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith PhD MA"} {"name": "John Smith PhD Prof. Ma", "title": "Prof.", "given": "John", "family": "Smith", "suffix": "PhD Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "Prof. John Smith PhD MA"} +{"name": "John Smith Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} +{"name": "John Smith Pharm D Jones", "given": "John", "family": "Smith", "suffix": "Pharm D Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith Pharm. D.", "given": "John", "family": "Smith", "suffix": "Pharm. D.", "initials": "J. S."} {"name": "John Smith Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Prof. Dr.", "title": "Prof. Dr.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Prof. Jr.", "title": "Prof.", "given": "John", "family": "Smith", "suffix": "Jr.", "initials": "J. S."} +{"name": "John Smith Psy. D.", "given": "John", "family": "Smith", "suffix": "Psy. D.", "initials": "J. S."} {"name": "John Smith Q.W.E.R.T.", "given": "John", "family": "Smith", "suffix": "Q.W.E.R.T.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith R.A.I.", "given": "John", "family": "Smith", "suffix": "R.A.I.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith RAI", "given": "John", "middle": "Smith", "family": "RAI", "initials": "J. S. R.", "capitalized_force": "John Smith Rai"} {"name": "John Smith Rev.", "title": "Rev.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Sir", "given": "John", "middle": "Smith", "family": "Sir", "initials": "J. S. S."} {"name": "John Smith V", "given": "John", "family": "Smith", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John Smith V Ph. D.", "given": "John", "family": "Smith", "suffix": "V Ph. D.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith V Ph. D.", "given": "John", "middle": "Smith", "family": "V", "suffix": "Ph. D.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. V."} {"name": "John Smith V PhD", "given": "John", "middle": "Smith", "family": "V", "suffix": "PhD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. V."} +{"name": "John Smith V Pharm D", "given": "John", "middle": "Smith", "family": "V", "suffix": "Pharm D", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. V."} {"name": "John Smith V, AD", "given": "John", "family": "Smith", "suffix": "V, AD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith V, M.J.", "given": "M.J.", "family": "John Smith", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. J. S."} {"name": "John Smith V.", "given": "John", "middle": "Smith", "family": "V.", "initials": "J. S. V."} @@ -1701,6 +1714,7 @@ {"name": "John Smith, Jr., abdul Dr. Smith", "title": "Dr.", "given": "John", "family": "Smith", "suffix": "Jr., abdul Smith", "ambiguities": ["COMMA_STRUCTURE"], "initials": "J. S.", "capitalized_force": "Dr. John Smith Jr., Abdul Smith"} {"name": "John Smith, Jr., van der Berg Prof.", "title": "Prof.", "given": "John", "family": "Smith", "suffix": "Jr., van der Berg", "ambiguities": ["COMMA_STRUCTURE"], "initials": "J. S.", "capitalized_force": "Prof. John Smith Jr., Van Der Berg"} {"name": "John Smith, LEED AP", "given": "John", "family": "Smith", "suffix": "LEED AP", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith, M. D.", "given": "M.", "middle": "D.", "family": "John Smith", "initials": "M. D. J. S."} {"name": "John Smith, M.D./Ph.D.", "given": "John", "family": "Smith", "suffix": "M.D./Ph.D.", "initials": "J. S."} {"name": "John Smith, M.D.; Ph.D.", "given": "John", "family": "Smith", "suffix": "M.D.; Ph.D.", "initials": "J. S."} {"name": "John Smith, MA", "given": "John", "family": "Smith", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} @@ -1742,6 +1756,7 @@ {"name": "John Smith, PhD XYZ", "given": "John", "family": "Smith", "suffix": "PhD XYZ", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, PhD vd DO", "given": "John", "family": "Smith", "suffix": "PhD vd DO", "initials": "J. S.", "capitalized_force": "John Smith PhD VD DO"} {"name": "John Smith, PhD-Md", "given": "John", "family": "Smith", "suffix": "PhD-Md", "initials": "J. S.", "capitalized_force": "John Smith PhD-MD"} +{"name": "John Smith, Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} {"name": "John Smith, Phd-Md", "given": "Phd-Md", "family": "John Smith", "initials": "P. J. S.", "capitalized_force": "PhD-Md John Smith"} {"name": "John Smith, Pu-Ph", "given": "Pu-Ph", "family": "John Smith", "initials": "P. J. S."} {"name": "John Smith, RA Jones", "title": "RA", "given": "Jones", "family": "John Smith", "initials": "J. J. S.", "capitalized_force": "Ra Jones John Smith"} @@ -2641,6 +2656,7 @@ {"name": "Ph. D., Jr.", "family": "Ph. D.", "suffix": "Jr.", "initials": "P. D.", "facade": {"initials": "P D."}} {"name": "Ph. D., Prof. Dr. Alejandro Manuel César Mamani de la Cruz Jr.", "title": "Prof. Dr.", "given": "Alejandro", "middle": "Manuel César Mamani de la Cruz", "family": "Ph. D.", "suffix": "Jr.", "initials": "A. M. C. M. C. P. D.", "facade": {"initials": "A. M. C. M. C. P D."}} {"name": "Ph. D., Prof. Dr. de la Cruz Alejandro Manuel César Mamani Jr.", "order": "FAMILY_FIRST", "title": "Prof. Dr.", "family": "Ph. D. de la Cruz Alejandro Manuel César Mamani", "suffix": "Jr.", "initials": "P. D. C. A. M. C. M."} +{"name": "Pharm D John Smith", "given": "Pharm", "middle": "D John", "family": "Smith", "initials": "P. D. J. S."} {"name": "Philipp von Breuner", "given": "Philipp", "family": "von Breuner", "initials": "P. B."} {"name": "Philips van Almonde", "given": "Philips", "family": "van Almonde", "initials": "P. A."} {"name": "Pi, Xirui", "given": "Xirui", "family": "Pi", "initials": "X. P."} @@ -2832,6 +2848,7 @@ {"name": "SMITH VD MA, JOHN", "given": "JOHN", "family": "SMITH VD MA", "initials": "J. S. M.", "capitalized": "John Smith vd Ma"} {"name": "SMITH VD, JOHN", "given": "JOHN", "family": "SMITH", "suffix": "VD", "initials": "J. S.", "capitalized": "John Smith VD"} {"name": "SMITH, JOHN DO", "given": "JOHN", "family": "DO SMITH", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "J. S.", "capitalized": "John do Smith"} +{"name": "SMITH, SAM E.", "given": "SAM", "middle": "E.", "family": "SMITH", "initials": "S. E. S.", "capitalized": "Sam E. Smith"} {"name": "SMITH, VD DE LA MA", "family": "SMITH VD DE LA MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "S. M.", "capitalized": "Smith vd de la Ma"} {"name": "SMITH, VD MA", "family": "SMITH VD MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "S. M.", "capitalized": "Smith vd Ma"} {"name": "STEVEN HARDMAN, MD, DO, DDS", "given": "STEVEN", "family": "HARDMAN", "suffix": "MD, DO, DDS", "ambiguities": ["COMMA_STRUCTURE"], "initials": "S. H.", "capitalized": "Steven Hardman MD, DO, DDS"} @@ -2995,6 +3012,7 @@ {"name": "Smith Jr., Ma", "given": "Ma", "family": "Smith", "suffix": "Jr.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. S."} {"name": "Smith Jr., Mr.", "title": "Mr.", "family": "Smith", "suffix": "Jr.", "initials": "S."} {"name": "Smith Juan Vega", "given": "Smith", "middle": "Juan", "family": "Vega", "initials": "S. J. V."} +{"name": "Smith Pharm D, John", "given": "John", "family": "Smith Pharm D", "initials": "J. S. P. D."} {"name": "Smith Prof.", "title": "Prof.", "family": "Smith", "initials": "S."} {"name": "Smith Sir.", "title": "Sir.", "given": "Smith", "initials": "S."} {"name": "Smith V, AD", "given": "AD", "family": "Smith V", "initials": "A. S. V.", "capitalized_force": "Ad Smith V"} @@ -3044,6 +3062,8 @@ {"name": "Smith, John PhD Jones", "given": "John", "family": "Smith", "suffix": "PhD Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John PhD de Jr.", "given": "John", "family": "de Smith", "suffix": "PhD, Jr.", "initials": "J. S."} {"name": "Smith, John PhD É, Jr.", "given": "John", "family": "Smith", "suffix": "PhD É, Jr.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "Smith, John Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} +{"name": "Smith, John Pharm D Jones", "given": "John", "family": "Smith", "suffix": "Pharm D Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "Smith, John V", "given": "John", "family": "Smith", "suffix": "V", "initials": "J. S."} {"name": "Smith, John V MD", "given": "John", "middle": "V", "family": "Smith", "suffix": "MD", "initials": "J. V. S."} @@ -3097,6 +3117,7 @@ {"name": "Smith, PhD Jr.", "family": "Smith", "suffix": "PhD Jr.", "initials": "S."} {"name": "Smith, PhD MEng", "family": "Smith", "suffix": "PhD MEng", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "S.", "capitalized_force": "Smith PhD MENG"} {"name": "Smith, PhD Ma", "family": "Smith", "suffix": "PhD Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "S.", "capitalized_force": "Smith PhD MA"} +{"name": "Smith, Pharm D", "given": "Pharm", "middle": "D", "family": "Smith", "initials": "P. D. S."} {"name": "Smith, Prof.", "title": "Prof.", "family": "Smith", "initials": "S."} {"name": "Smith, RA PhD Ma", "title": "RA", "given": "PhD", "middle": "Ma", "family": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "P. M. S.", "capitalized_force": "Ra PhD Ma Smith"} {"name": "Smith, RN - CRNA", "family": "Smith", "suffix": "RN - CRNA", "initials": "S."} From e2df6ec3d1d12fb46c1309eb00a0c7c1bc0228e5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:01:20 -0700 Subject: [PATCH 03/11] WIP: review fixes for split credentials --- .../S11-credential-split-across-a-space.md | 8 ++-- .../S11-credential-split-across-a-space.md | 11 ++++- nameparser/_pipeline/_assign.py | 2 +- nameparser/_pipeline/_classify.py | 27 +++++++++--- nameparser/_pipeline/_comma.py | 9 ++-- nameparser/_pipeline/_group.py | 41 ++++++++++++++----- nameparser/_pipeline/_pieces.py | 2 +- nameparser/_pipeline/_state.py | 15 ++----- nameparser/_pipeline/_vocab.py | 9 +++- nameparser/_render.py | 41 +++++++++++++++++++ nameparser/_types.py | 14 +++++++ tests/v2/cases.py | 25 +++++++++++ tests/v2/pipeline/test_vocab.py | 13 ++++-- tests/v2/test_facade.py | 12 ++++++ tests/v2/test_render.py | 13 ++++++ 15 files changed, 198 insertions(+), 44 deletions(-) diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md index 5e70f05a..50671318 100644 --- a/docs/design/decisions/S11-credential-split-across-a-space.md +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -4,14 +4,15 @@ Examples name each field as the default order reads it; under the family-first orders the given and family names trade places. -- Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de A Junior` → family 'de A') — 2026-10-25 #752. +- Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class, unless a connective borders the pair (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de A Junior` → family 'de A'; `John Smith Pharm D and Mary` → family 'D and Mary') — 2026-10-25 #752. - The count decides it as it decides a bare member, and where the count declines it the pair is the two words it is written as: at the end of a name by S4's words to spare (`John Pharm D` → middle 'Pharm', family 'D'), after a comma by C4's count (`John Smith, Pharm D` → suffix; `Smith, Pharm D` → given 'Pharm'), ending the given part after a family comma by S5's reading (`Smith, John Pharm D` → suffix); the part before a family comma merges none (`Smith Pharm D, John` → family 'Smith Pharm D') — 2026-10-25 #752. - Behind two name words with no comma, or after the given word after a family comma, the pair starts S8's run, the one member of the class that does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones', reported) — 2026-10-25 #752. - The reading reports nothing either way (`John Smith MB ChB` → suffix 'MB ChB', where `mb` started the run and reported 'ChB'; `SMITH, SAM E.` → given 'SAM', middle 'E.') — 2026-10-25 #752. -- The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. +- The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, merged by its spelling whatever the vocabulary lists, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. - Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.') — 2026-10-25 #752. - The Ph./D. pair starts S8's run from where it stands, though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, #827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` → family 'V', reported) — 2026-10-25 #752. -- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial behind two name words with no comma read as the credential their letters spell (`JOHN SMITH SAM E.` → suffix 'SAM E.') — 2026-10-25 #752. +- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial read as the credential their letters spell behind two name words with no comma and ending the given part after a family comma (`JOHN SMITH SAM E.` and `Smith, John Sam E.` → suffix) — 2026-10-25 #752. +- Case repair writes the pair by its fold's mask, as the one-word spelling is written, on every comma path (`john smith, PHARM D` → 'Pharm D', as `PHARMD` → 'PharmD') — 2026-10-25 #752, review. Open: none in this file. @@ -26,3 +27,4 @@ Open: none in this file. MEASURED (2026-10-25, Python 3.11.17, against master 225f3d6f, every local check with `--base` that checkout, so no shared snapshot was read or recorded): the committed snapshot moves 2 names, both rules examples (`John Smith Ph. D. Jones`, `John Smith V Ph. D.`), and gains S11's 21 new examples; every CLDR, GND, Wikidata and conventions agreement record matches. ctgov-officials (fetched 2026-10-21) moves 357 entries: 229 only drop a report (`MB ChB` 179, `MB BCh` 48, `DM Sc` 2, read as one credential where `mb` or `dm` started the run and reported the word behind it), and 128 move a field, each a split credential read whole where the name had read reversed or the credential as a name word (`Pharm D` and its spellings 110, `Psy D` 5, `Dr PH` 5, `Ed D` 3, `Sc. D.` 1, and `MB ChB` or `MB BCh` behind another degree 4); its force-repaired case view writes a capitals half as it did (`MB BCH`), `Ed D` forcing to `ED D` as a bare `Ed` read as a suffix does. tse-donors moves 2 (`JUNIOR` typed `JUN IOR`, now the suffix), and its pairs view (`local_corpora.py pairs tse-donors --agreement`) is unchanged; fec-individuals moves 1 (`USN R`, now the suffix run's); nepal-candidates moves none; psc-individuals (1,039,171 records scored) matches the base: 0 fixed, 0 broken, 0 changed, 1,004,782 agreeing on every field. The spike figures in decisions/vocabulary-model's 2026-10-23 entry (352 ctgov, 7 tse, 30 FEC for the first-chunk rule; 391, 34 and 41 with the run start) were a cruder rule's, with no count, no refusal of a particle chunk and no retry after a comma; they were not re-taken. FRAMES (`sys.setprofile` call events, py3.11): the reference name, `John Smith`, `Smith, John` and `John Smith, PhD` cost what they did (363, 148, 157, 201). Classify's pair test is C-level up to the call, so a call is made only for a pair that passes the shape, the opening letter's length and the fold's membership: `John Smith Ph. D.` 232 → 233, the 64-link benchmark row unmoved once the length joined the pre-test (`i i` folds to the listed `ii`, and the first version called the predicate for every pair of links, +188). A name holding a member pair pays for its reading: `John Smith Pharm D` 230 → 238, `John Smith, MB ChB` 250 → 266, `Smith, John Pharm D` 253 → 284. Recompute: `uv run python tools/differential/snapshot.py --since ` for the committed snapshot; for each local corpus `snapshot.py --local --base `, and `local_agreement.py psc-individuals --base `; the accepted figure by reading `corpus.jsonl` of ctgov-officials, keeping each name holding a comma whose text after its first comma, stripped, is one of `M. D.`, `M D`, `M. Sc.`, `Pharm D` and `Pharm. D.`, and counting those whose given name opens with that part's first word, on this tree and on the parent. +- 2026-10-25 #752, review — SEVEN DEFECTS IN THE FIRST VERSION, found by the PR's code review on a grid of 3,726 texts under the three orders compared against master. (i) A connective beside a pair joined the merged piece into a wider one, which kept its tag and was read whole: `John Smith Pharm D and Mary` read suffix 'Pharm D and Mary' and `John Pharm D and Mary` family 'Pharm D and Mary', where master reads family 'D and Mary'. A pair a connective borders no longer merges, and the two readers of the tag ask for a piece of exactly two words. (ii) Classify's pre-test asked the fold's membership of the Ph./D. pair too, so a configuration without `phd` (`Constants().suffix_acronyms.remove('phd')`) stopped merging it, where v1's fix_phd merged it by its spelling and 1.4.0 and master read suffix 'Ph. D.'; the pair's own pattern now answers ahead of the vocabulary. (iii) C3's count of the part before a family comma merged a member pair, so `MB ChB, PhD` read given 'MB', family 'ChB' where master reads family 'MB ChB'; it reads the part as written now. (iv) Case repair marked a pair's capitals only where the comma's decision bound it, so the same words repaired two ways (`john smith, PHARM D` kept 'PHARM D', `john smith PHARM D` gave 'Pharm D'); render now repairs every suffix pair by its fold's mask (`MB CHB` → 'MB ChB' under 'MBChB'), else in capitals for a listed acronym, and the comma's marks are gone. The force view of `MB BCH` is now 'MB BCh', the mask of 'mbbch', where master kept 'MB BCH' as a caps word. (v) The pre-test let every non-ASCII fold through to the call, a frame per pair of short words in any accented name (`Ngô Bảo Châu` 263 → 269); it looks the composed fold up in C now and costs what master does, and a pair of two suffix words (`MD PhD`) and an opening single letter before a dotted word of two letters (FEC's `A. MS.`) are refused by tags and lengths before the call. (vi) The maiden clause's report read the first word of the piece ending it, so `Jane Doe nee Ed. D.` reported 'Ed.' ending the maiden name; a split pair the clause keeps is read in silence, as everywhere. (vii) The comma form of the given name and initial (`Smith, John Sam E.` → suffix 'Sam E.', where master read middle 'Sam E.') was unrecorded; it is Accepted with the no-comma form, as S5 reads any member ending the given part whose writing says nothing else, and a split pair has no writing to say it. Each fix has a case row or a test that fails on the first version (tests/v2/test_render.py's `test_a_split_credential_repairs_as_its_one_word_twin`, tests/v2/test_facade.py's `test_the_ph_d_pair_merges_whatever_the_vocabulary_says`), run against it before landing. diff --git a/docs/design/rules/S11-credential-split-across-a-space.md b/docs/design/rules/S11-credential-split-across-a-space.md index cb42d3a2..22fed64b 100644 --- a/docs/design/rules/S11-credential-split-across-a-space.md +++ b/docs/design/rules/S11-credential-split-across-a-space.md @@ -4,13 +4,19 @@ **Then:** a credential written split across a space reads as one word where its two words are short chunks whose letters together spell unambiguous suffix vocabulary and the two are not both suffix vocabulary on their own, as a member of the ambiguous class (S4) that the count decides as it decides a bare acronym, so that where the count declines it, the pair is the two words it is written as. -- **Which pairs.** Each word is one to five letters, with or without its period, and the first has two or more: a single letter opening the pair is a middle initial far more often than the start of a credential. A particle or a connective written without a period is no chunk of one either, since it joins name words, which is what it is doing beside one. +- **Which pairs.** Each word is one to five letters, with or without its period, and the first has two or more: a single letter opening the pair is a middle initial far more often than the start of a credential. A particle or a connective written without a period is no chunk of one either, since it joins name words, which is what it is doing beside one, and for the same reason a pair a connective borders is no credential. ``` "John Smith Pharm. D." → suffix="Pharm. D." "John Smith Pharm D" → suffix="Pharm D" "John Smith Psy. D." → suffix="Psy. D." "John Smith M. D." → family="D." · boundary "Jane Doe de A Junior" → family="de A" · boundary + "John Smith Pharm D and Mary" → family="D and Mary" · boundary + ``` +- **Case repair** writes the pair as it writes the one-word spelling, the letters of the two words cased as their fold is. + ``` + "john smith pharm d" → capitalized="John Smith Pharm D" + "john smith, PHARM D" → capitalized="John Smith Pharm D" ``` - **The count decides.** At the end of a name it needs the words to spare S4 asks of a bare member; after a comma, the name words C4 counts before the comma; ending the given part after a family comma, it reads as S5 reads a member there, its writing saying nothing else. ``` @@ -73,9 +79,10 @@ "John Smith, M. D." → given="M." "John Smith, M. D." → family="John Smith" ``` -- A given name and its initial behind two name words, where no comma divides the name, whose letters spell a listed credential read as the credential, in silence: the count reads them as it reads the credential they spell. +- A given name and its initial whose letters spell a listed credential read as the credential, in silence, behind two name words where no comma divides the name, and ending the given part after a family comma: the count, or the given part's reading, takes them as it takes the credential they spell. ``` "JOHN SMITH SAM E." → suffix="SAM E." + "Smith, John Sam E." → suffix="Sam E." ``` --- diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 8790a388..59b9f5f6 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -1218,7 +1218,7 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: # ambiguous class", here read as the class is with no # writing to lean on and in silence, as group merges it # only behind the given word - if "split" in ptags[m]: + if len(piece) == 2 and "split" in ptags[m]: return (m >= trailing_floor(m, titled) and not attaches_the_lead(m, titled)) if len(piece) == 1: diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index aa844f50..a079f6b6 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -71,7 +71,7 @@ ) from nameparser._types import AmbiguityKind, Role from nameparser._pipeline._vocab import ( - BUILTIN_SUFFIX_DELIMITERS, SPLIT_CHUNK, _INITIAL, caps_shape_candidate, + BUILTIN_SUFFIX_DELIMITERS, SPLIT_CHUNK, SPLIT_PAIR_OPENS, _INITIAL, caps_shape_candidate, composed_len, delimiter_cores, glued_stem, is_cased_paired_initials, is_initial, is_one_case, period_joined_vocab, split_pair, suffix_as_written, tag_marker_runs, @@ -421,12 +421,29 @@ def classify(state: ParseState) -> ParseState: # predicate's count of composed letters ('i i' folds # to the listed 'ii' and opens with one) and (len(a.text.rstrip(".")) >= 2 - or a.text.endswith(".") and b.text.endswith(".")) - and ((fold := (a.text + b.text).replace(".", "") - .lower()) in acronyms or fold in words - or not fold.isascii()) + or a.text.endswith(".") and b.text.endswith(".") + and (len(b.text) == 2 or not b.text.isascii())) + # two suffix words are two credentials already + and not ("vocab:suffix" in a.tags + and "vocab:suffix" in b.tags) + # the Ph./D. pair whatever the vocabulary, as v1's + # fix_phd merged it; any other only where its fold + # is listed, composed in C where it is not ASCII + and (SPLIT_PAIR_OPENS.fullmatch(a.text) is not None + or (fold := (a.text + b.text).replace(".", "") + .lower()) in acronyms or fold in words + or not fold.isascii() + and ((nfc := unicodedata.normalize("NFC", fold)) + in acronyms or nfc in words)) and (kind := split_pair(a.text, b.text, state.lexicon)) is not None + # a member pair a connective borders is no credential: + # the connective joins the words beside it + and (kind == "suffix" or ( + j == 0 + or "conjunction" not in tokens[seg[j - 1]].tags) + and (j + 2 >= len(seg) + or "conjunction" not in tokens[seg[j + 2]].tags)) # the Ph./D. pair outranks a pair overlapping it: # 'Dr. Ph. D.' is no 'Dr. Ph.' ('drph') and a 'D.' and not (kind == "split" and j + 2 < len(seg) diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index d385f6fc..694b4464 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -179,7 +179,8 @@ def _whole_name(seg0: Sequence[int], tokens: Sequence[WorkToken]) -> bool: word to the count an unambiguous credential takes.""" # rules/C3: "a particle run and the one name word it attaches to are # one word" - pieces, ptags = _pieces(seg0, tokens) + # the part before a family comma merges no member pair (rules/S11) + pieces, ptags = _pieces(seg0, tokens, False) named: set[int] = set() for piece, tags in zip(pieces, ptags): if not is_suffix_piece(piece, tags, tokens): @@ -574,11 +575,7 @@ def decide(state: ParseState, splits: bool | None = True) -> ParseState: role = Role.SUFFIX if reading[k] else Role.TITLE for i in piece: tags = tokens[i].tags - if (i in caps or "split" in pt1[k] and reading[k] - and texts[i].isupper() and texts[i].isalpha() - and "vocab:suffix" not in tags): - # a split credential's capitals are a credential's too - # (rules/S11), marked as a caps word bound here is + if i in caps: # #564: the caps shape's marks, as classify writes them # EVERYWHERE, so case repair keeps 'XYZ' (rules/R4) tags = tags | {SHAPE_ACRONYM_TAG, AMBIGUOUS_ACRONYM_TAG} diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 908d2db6..1d2022f0 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -345,7 +345,9 @@ def _maiden_take(pieces: Sequence[Sequence[int]], # left it a name word, which is a fork too last = pieces[end - 1] word = tokens[last[0]] - if not word.tags.isdisjoint(_AMBIGUOUS_CREDENTIAL_TAGS): + # a one-word member: a split credential the clause keeps is read in + # silence (rules/S11), and its first word is not the clause's last + if len(last) == 1 and not word.tags.isdisjoint(_AMBIGUOUS_CREDENTIAL_TAGS): ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, f"{word.text!r} ending the maiden name is also a " @@ -600,6 +602,25 @@ def _settle_splits(pieces: list[Piece], ptags: list[set[str]]) -> None: k += 1 +def _merge_given_pairs(seg: Sequence[int], pieces: list[Piece], + ptags: list[set[str]], + tokens: Sequence[WorkToken]) -> None: + """Merge each split credential still standing as its two words in + the given part after a family comma, behind the given word, in + place: run after group's joins, so a pair one of them took into a + piece of its own stays as the join left it.""" + given_from = leading_titles(pieces, ptags, tokens) + following = dict(zip(seg, seg[1:])) + k = given_from + 1 + while k < len(pieces) - 1: + a, b = pieces[k], pieces[k + 1] + if (len(a) == 1 and len(b) == 1 and not ptags[k] and not ptags[k + 1] + and SPLIT_MEMBER_TAG in tokens[a[0]].tags + and following.get(a[0]) == b[0]): + merge_pieces(pieces, ptags, k, k + 2, add={"split"}) + k += 1 + + def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, @@ -670,13 +691,15 @@ def _group_segment(seg: tuple[int, ...], additional: int, # whole where the read takes it and split back into its two words # where it does not (`_settle_splits`), so a declined pair reads # exactly as the two words read; and in the given part after a - # family comma behind the given word, where S5's walk reads it, - # and a pair declined there is middle-name text whole or split. + # family comma behind the given word, where S5's walk reads it -- + # merged there only once the joins below have run, so no bound + # given name or connective joins it as it joins no credential + # ('Rahman, Abdul MB ChB', #752's review), and a pair declined + # there is middle-name text whole or split. # Not in the part before a family comma, which reads no member as # a suffix ('Smith MA, John'). Classify tagged each pair's first # word, so the test here is a tag. - splits = False - given_from: int | None = None + splits = given_pairs = False k = 0 while k < len(pieces) - 1: a, b = pieces[k], pieces[k + 1] @@ -692,11 +715,7 @@ def _group_segment(seg: tuple[int, ...], additional: int, splits = True continue if site is ClauseSite.GIVEN_SLOT: - if given_from is None: - given_from = leading_titles(pieces, ptags, tokens) - if k > given_from: - merge_pieces(pieces, ptags, k, k + 2, add={"split"}) - continue + given_pairs = True k += 1 # rules/M2: "A marker that counts takes the words after it as @@ -1259,6 +1278,8 @@ def chain(tail: int) -> None: # carried (rules/P5); joined_tags says why the # title tag is dropped. Pinned in test_group.py. merge_pieces(pieces, ptags, fk, fk + 2, drop={"title"}) + if given_pairs: + _merge_given_pairs(seg, pieces, ptags, tokens) if reads is not None: reads.append(read) pieces.extend(tail_pieces) diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 259f31ba..0b41e172 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -1243,7 +1243,7 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], # ambiguous class" -- read by the count alone, its writing carrying no # case to lean on, and in silence: its other reading is two # name words, which the count already refuses it behind one - if "split" in ptags[rest[k - 1]]: + if len(piece) == 2 and "split" in ptags[rest[k - 1]]: if k >= 2 and k - 2 >= second: k -= 1 continue diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 16a6d5b0..93d25cfe 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -22,6 +22,8 @@ from nameparser._policy import Policy from nameparser._types import (SHAPE_ACRONYM_TAG, AmbiguityKind, Role, Segmenter, Span) +from nameparser._types import SPLIT_CREDENTIAL_TAG as SPLIT_CREDENTIAL_TAG +from nameparser._types import SPLIT_MEMBER_TAG as SPLIT_MEMBER_TAG if TYPE_CHECKING: from nameparser._pipeline._pieces import TailRead @@ -99,17 +101,8 @@ class WorkToken: _AMBIGUOUS_CREDENTIAL_TAGS = frozenset( {AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG}) -#: The tags classify writes on the FIRST word of a credential written -#: split across a space (`_vocab.split_pair`), naming the piece tag the -#: pair merges with: SPLIT_CREDENTIAL_TAG the Ph./D. pair, merged as an -#: unambiguous suffix ("suffix"), and SPLIT_MEMBER_TAG any other, merged -#: as a member of the ambiguous class ("split", rules/S4, rules/C4). -#: Written by classify because it is the one stage that sees every -#: adjacent pair with the lexicon in hand, so group's merge and -#: `_comma._pieces` read the decision off a tag, C-level, rather than -#: each asking the vocabulary of every pair. -SPLIT_CREDENTIAL_TAG = "split:credential" -SPLIT_MEMBER_TAG = "split:member" +# SPLIT_CREDENTIAL_TAG and SPLIT_MEMBER_TAG are defined and explained in +# _types beside SHAPE_ACRONYM_TAG, case repair reading them too. #: The roles that make a piece a name word (rules/P6's walk), and #: the roles of a word not yet read as one: a suffix, or nothing yet diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 0f39704f..1ac38bb9 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -603,7 +603,9 @@ def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: #872's predicate rather than a copy: in a script with capitals they are paired initials (`is_cased_paired_initials`), the reading rules/S3 gives them joined, so only a caseless script's pair can - merge, and the default vocabulary lists no such fold. + merge, and not one in a script that writes no abbreviations (Han, + hangul, kana: '博. 士.' is no dotted '博士'), which leaves no + default entry such a fold. Where the merge may happen (never at the head of the string, rules/S2; a member pair only in a main segment and in the part @@ -617,7 +619,10 @@ def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: if composed_len(a) < 2 and ( composed_len(b) != 1 or not first.endswith(".") or not second.endswith(".") - or is_cased_paired_initials(first + second)): + or is_cased_paired_initials(first + second) + # a script that writes no abbreviations is not wearing an + # abbreviation's periods, as rules/S3 reads it joined + or in_initialless_script(first + second)): return None fold = _normalize(a + b) if (fold in lexicon.suffix_acronyms_ambiguous diff --git a/nameparser/_render.py b/nameparser/_render.py index c7e1b002..d79db1a7 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -19,6 +19,7 @@ from nameparser._lexicon import (FULL_STOPS, Lexicon, _normalize, _spells_an_initial) from nameparser._types import (FOLDED_TAG, SHAPE_ACRONYM_TAG, + SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, UNCLASSIFIED_TAG, UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, Ambiguity, ParsedName, Role, Token) @@ -674,6 +675,42 @@ def cap(word: str) -> str: for at, part in enumerate(parts)) +def _split_pairs(tokens: tuple[Token, ...], repaired: tuple[Token, ...], + lex: Lexicon) -> tuple[Token, ...]: + """`repaired` with each split credential read as a suffix (classify + tags its first word) recased by its fold, as the one-word spelling + is: the exceptions map's mask laid over the two words as written + ('PHARM D' under 'PharmD' is 'Pharm D'), else capitals for a listed + acronym ('MB CHB' is 'MB CHB' under 'MBChB'). A pair one of whose + words the gate kept as written is left as the gate left it. A loop + over the tags, so a name with no such pair pays no frame for it.""" + out: list[Token] | None = None + for i in range(len(tokens) - 1): + a, b = tokens[i], tokens[i + 1] + if (SPLIT_MEMBER_TAG not in a.tags + and SPLIT_CREDENTIAL_TAG not in a.tags): + continue + if (a.role is not Role.SUFFIX or b.role is not Role.SUFFIX + or repaired[i] is a or repaired[i + 1] is b): + continue + joined = a.text + b.text + fold = _normalize(joined.replace(".", "")) + mask = lex.capitalization_exceptions_map.get(fold) + cased = _apply_mask(joined, mask) if mask is not None else None + if cased is None and fold in lex.suffix_acronyms: + cased = joined.upper() + # a case mapping that changes the length ('ß') has no place to + # cut the pair back into its two words + if cased is None or len(cased) != len(joined): + continue + if out is None: + out = list(repaired) + cut = len(a.text) + out[i] = Token(cased[:cut], a.span, a.role, a.tags) + out[i + 1] = Token(cased[cut:], b.span, b.role, b.tags) + return repaired if out is None else tuple(out) + + def _in_one_case(text: str) -> bool: """R5's test: `text` is written all upper or all lower (a caseless text is both, and so passes).""" @@ -765,6 +802,10 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, else Token(_cap_text(t.text, t.role, t.tags, lex), t.span, t.role, t.tags) for t in name.tokens) + # rules/R4: "Repair changes case and nothing else" -- and a + # credential split across a space (rules/S11) is repaired as its + # one-word twin is, the pair's fold asked once + new_tokens = _split_pairs(name.tokens, new_tokens, lex) # equal tokens (possible only for synthetic span=None duplicates) # collapse to one mapping entry -- benign: the rebuilt ambiguity # references an equal token, so the subset invariant still holds diff --git a/nameparser/_types.py b/nameparser/_types.py index 8169731b..1e4c745a 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -171,6 +171,20 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: membership half, since the stages read both halves from one module. SHAPE_ACRONYM_TAG = "shape:acronym" +#: The tags classify writes on the FIRST word of a credential written +#: split across a space (`_pipeline._vocab.split_pair`), naming the +#: piece tag the pair merges with: SPLIT_CREDENTIAL_TAG the Ph./D. pair, +#: merged as an unambiguous suffix ("suffix"), and SPLIT_MEMBER_TAG any +#: other, merged as a member of the ambiguous class ("split", rules/S11). +#: Written by classify because it is the one stage that sees every +#: adjacent pair with the lexicon in hand, so group's merge and +#: `_comma._pieces` read the decision off a tag, C-level, rather than +#: each asking the vocabulary of every pair; and defined here, at the +#: bottom of the graph, because case repair reads them too, repairing +#: a split credential in the suffix as its one-word twin. +SPLIT_CREDENTIAL_TAG = "split:credential" +SPLIT_MEMBER_TAG = "split:member" + #: The one-element tag set its two producers stamp, built once. _UNCLASSIFIED = frozenset({UNCLASSIFIED_TAG}) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index b391c104..c6867eaf 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1607,6 +1607,31 @@ def _check_cjk_shape_purity(self) -> None: notes="'MB ChB' is one listed credential, 'mbchb', where 'mb' " "started the run and reported 'ChB' as a name word it " "absorbed. 1.4.0 read middle 'Smith MB', last 'ChB'"), + Case("a_split_credential_beside_a_connective_stays_two_words", + "John Smith Pharm D and Mary", + {"given": "John", "middle": "Smith Pharm", "family": "D and Mary"}, + notes="S11: a connective joins the words beside it, so a pair " + "it borders is no credential; merged, the join took the " + "pair into a wider piece the count then read whole (the " + "first version's 'Pharm D and Mary' suffix, found by " + "review)"), + Case("no_split_credential_merges_before_a_family_comma", + "MB ChB, PhD", + {"family": "MB ChB", "suffix": "PhD"}, + classification="fix(#752)", + notes="S11: the part before a family comma merges no member " + "pair, so C3's count of it reads the two words as " + "written, the tree's reading before #752 (the first " + "version read given 'MB', family 'ChB'). 1.4.0 read " + "first 'MB', last 'ChB'"), + Case("a_split_credential_ending_a_maiden_clause_reports_nothing", + "Jane Doe nee Ed. D.", + {"given": "Jane", "family": "Doe", "maiden": "Ed. D."}, + classification="fix(#752)", + notes="S11: the clause keeps the pair, read in silence; its " + "first word is a listed member, and the first version " + "reported 'Ed.' as the word ending the clause, which " + "'D.' is. 1.4.0 read middle 'Doe nee Ed.', last 'D.'"), Case("the_split_degree_outranks_a_pair_overlapping_it", "Jane Doe Dr. Ph. D.", {"title": "Dr.", "given": "Jane", "family": "Doe", diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 23015fec..ba2ce30d 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -992,6 +992,8 @@ def test_the_c_level_pre_tests_are_necessary_conditions_of_split_pair( # either would never be reached for the new spelling (#752) lex = Lexicon.default() words = ["Ph.", "Ph", "PH.", "ph", "pH", "D.", "D", "d", "Pharm.", + unicodedata.normalize("NFD", "Éd"), "É.", unicodedata.normalize( + "NFD", "É."), "Ж.", "Ш.", "Pharm", "M.", "MD", "Phd", "Ph..", "", "Smith", "MB", "ChB", "SAM", "E.", "Pharma", "Ed", "J.", "R."] for first in words: @@ -999,14 +1001,19 @@ def test_the_c_level_pre_tests_are_necessary_conditions_of_split_pair( kind = split_pair(first, second, lex) if kind is None: continue - fold = (first + second).replace(".", "").lower() + fold = unicodedata.normalize( + "NFC", (first + second).replace(".", "").lower()) assert SPLIT_CHUNK.fullmatch(first) and SPLIT_CHUNK.fullmatch( second), (first, second) assert (len(first.rstrip(".")) >= 2 - or first.endswith(".") and second.endswith(".")) - assert fold in lex.suffix_acronyms or fold in lex.suffix_words + or first.endswith(".") and second.endswith(".") + and (len(second) == 2 or not second.isascii())) if kind == "suffix": + # merged whatever the vocabulary, as v1's fix_phd did assert SPLIT_PAIR_OPENS.fullmatch(first), (first, second) + else: + assert (fold in lex.suffix_acronyms + or fold in lex.suffix_words), (first, second) def test_cased_paired_initials_is_the_bare_shape_in_a_cased_script() -> None: diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 05dd2096..46a926fc 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -1081,3 +1081,15 @@ def test_the_v1_off_switch_unjoins_the_link_only() -> None: unjoined = HumanName("Josep Carod i Rovira", constants) assert unjoined.middle == "Carod i" assert unjoined.last == "Rovira" + + +def test_the_ph_d_pair_merges_whatever_the_vocabulary_says() -> None: + # v1's fix_phd merged 'Ph. D.' by its spelling alone, so a config + # without 'phd' still read the pair as one suffix; the split + # credential's vocabulary pre-test must not stand in front of that + # answer (#752's review) + c = Constants() + c.suffix_acronyms.remove("phd") + assert HumanName("John Smith Ph. D.", constants=c).suffix == "Ph. D." + h = HumanName("John Smith, Ph. D.", constants=c) + assert (h.first, h.last, h.suffix) == ("John", "Smith", "Ph. D.") diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 25e0fe93..48924bb0 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1866,3 +1866,16 @@ def test_a_marked_particle_lowercases_only_where_the_particle_arm_would( hyphened = Parser(lexicon=Lexicon.default().add(particles={"van-der"})) assert str(hyphened.capitalized(hyphened.parse("jan van-der"))) == \ "Jan Van-Der" + + +@pytest.mark.parametrize("text", [ + "john smith, PHARM D", "john smith PHARM D", "smith, john PHARM D", + "JOHN SMITH, PHARM D", "JOHN SMITH PHARM D", "john smith pharm d"]) +def test_a_split_credential_repairs_as_its_one_word_twin(text: str) -> None: + # rules/S11's pair is repaired by its fold's mask ('pharmd' -> + # 'PharmD') whatever the comma shape, as 'PHARMD' is; the first + # version marked its capitals only where the comma's decision bound + # it, so the same words repaired two ways (#752's review) + assert parse(text).capitalized().suffix == "Pharm D" + twin = text.replace("PHARM D", "PHARMD").replace("pharm d", "pharmd") + assert parse(twin).capitalized().suffix == "PharmD" From 64a821b4a210cdab99268d2c408781e7326e8144 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:10:57 -0700 Subject: [PATCH 04/11] Fix the review findings for split credentials (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- ...credential-opening-the-part-after-the-comma.md | 2 +- .../decisions/S10-trailing-roman-numeral.md | 2 +- .../S11-credential-split-across-a-space.md | 15 ++++++++------- .../decisions/S8-credentials-trailing-run.md | 2 +- docs/design/decisions/phd-merge.md | 2 +- docs/design/mechanisms.md | 6 +++--- .../rules/S11-credential-split-across-a-space.md | 13 +++++++++++-- docs/design/rules/S8-credentials-trailing-run.md | 7 +++++-- nameparser/_pipeline/_vocab.py | 4 ++-- tests/v2/rules_budget.json | 4 ++-- tools/differential/corpus_rules.jsonl | 8 +++++++- tools/differential/snapshot.jsonl | 12 +++++++++--- 13 files changed, 52 insertions(+), 27 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index eb3a8b40..31cf4498 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,7 +186,7 @@ The library has two layers: `nameparser/config/` (data) and `nameparser/parser.p **Sweep the forms a change can reach: comma shapes, then `name_order`.** No comma, a FULL name before the comma, a ONE-WORD name before the comma and NOTHING before the comma are four paths, and the last two are the misses — #429, #430 and #432 are all the one-word path disagreeing with the full-name path on inputs the full-name path reads correctly, and #613's PR review found `, PhD` and `, Jr.` (an empty surname field, as CSV data writes it) read as given and title by a decision that returned early on an empty head, after two review rounds whose every grid had a name before the comma. Put an empty head and a nickname-only head (`(Bob), PhD`) in every comma grid. Count the one-word path the way C3 counts, which is neither tokens nor pieces: a particle run and the one word it attaches to are one word (#575), a connective join and a bound given-name pair count as their words (#613, so `Ortega y Gasset, Dr.` is a whole name), and a suffix beside the surname counts as no name word at all. #603's review found the suffix half (`Vega y Lopez Jr., PhD Jones` lost its family where `Vega y Lopez, PhD Jones` kept it); #613 then reversed the connective half, and both now read given 'Vega', middle 'y', family 'Lopez' — so test a particle surname, a connective surname and a surname with a suffix, each before a comma. For orders the sweep already exists (`tests/v2/test_cases.py` runs every row under all three) but carries one assertion, R2's family partition, so it checks nothing a new change moves; coverage there has been vacuous before (PR #394's review found the suite passed with `name_order` discarded from grouping). Parse your change's names in each comma shape and each order, and read the ones you did not predict. And read a generated grid's moves by SHAPE, not by corpus membership: #614's first design passed the suite, the five gates and a fingerprint of every corpus name, because its grid moves were sorted into 'corpus name moved' and 'garbage', and the review found the moves that mattered among the 'garbage' -- a bare salutation (`Mr. and Mrs.` lost its connective to the family name) and a name with one trailing word (`Freiherr von Berg X.Y.Z.`). Pull out the head-plus-one-word slice of the grid and a salutation with nothing after it, and read every move there. Two more were the second review's (#614): no grid put TWO words that are both a title and a particle at the head (`Freiherr St van Berg MA`, where an index into the pieces was applied to a merged view of them -- `TITLES ∩ particles_ambiguous` is `{freiherr, st}`, so put both in every head list, each in its bare and its period-marked spelling -- #637's grid had `St` and `Freiherr` alone and missed that `St.` and `Freiherr.` are declined by the bound join and then attached by P6), and the grids compared ambiguity KINDS, so a report telling the caller the wrong field (`Freiherr von Berg Ma`, 'read as a given name' of a word in the family) passed every pin. #701's review added one: put a title/suffix dual (`Ms`, `Sr.`, `Cpl`) SECOND in a title run, behind an ordinary title. A stop that asks only "is this word suffix vocabulary" passes every row that opens with the dual and fails `Rev. Sr. Smith, Mary`; it was found by comparing `T Smith, John` against `T John Smith` over every title word, and a comma-versus-comma-less comparison is the cheap check for any rule that moves a word between the two slots. Compare each report's `detail` against the field its tokens land in -- since #626 `test_a_report_names_the_field_its_word_lands_in` (tests/v2/test_cases.py) does it over every non-locale case row under all three orders (a row carrying its own policy only as declared), for the phrasings its `_FIELD_CLAIMS` lists -- so a row with the shape is the test only where the report's wording is listed: a new emitter's wording goes in that list, and segment's report on a part past the second comma names no field (#629). #626's own first grid then missed an emitter for want of one comma shape (`, Ma Dr.`: an empty head, an ambiguous word, a trailing title, which H1 moves to the family): a trailing title belongs in every empty-head comma grid too. And #620's: an exclusion list in code being replaced is a set of rules, not noise -- #614's merged copy excluded every word the trailing read weighs from its units, the first #620 prototype dropped the list as a symptom of the copy, and its review found the list had been keeping positional counting's rule that a member in front counts as a word (`Freiherr von Berg MA X.Y.Z.` lost its suffix). Before deleting one, mutate each entry out and read what moves. The converse holds for replacing a hand copy WITH the shared predicate: it is a behavior change wherever the two disagreed, and #735's /simplify shipped one as "no reading moves" because the suite and the snapshot were its only check (`St Sir de Mesnil` family-first, a token test replaced by P1's piece test); diff the two predicates' answers over a grid before calling the swap a refactor. -**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, a one-case name being counted like capitals whatever the letter -- since #721 a one-case one-letter numeral carries the `initial` tag its capital does, and since #723 every one-case bare letter does (rules/T4), and a bare letter in front of the numeral is initial-shaped as its capital is, so `john v` reads as `JOHN V` does over `test_a_one_case_name_reads_as_its_capitals`'s grid (comma shapes, orders, a maiden clause, a particle chain, a bare letter in front, a trailing title, a non-ASCII letter in front: 315 of its then 1,872 pairs disagreed at 2bfa9533, before #721, and none after). Its first version claimed "every shape" with no bare letter in front and no trailing title in the grid, and #721's reviews found both broken; a grid's claim is the grid -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). #656's sweep found the same class from the other side: C12 takes the word ending the name before a comma as EVIDENCE, and its first draft asked the suffix-word tag, so `John Smith VI, AD` missed the rule `III` reached, and then counted the whole part, so `Smith VI, AD` fired with `VI` as its own name word and `Smith V, AD` with a `V` that S10 reads as the family. A rule that weighs a numeral as a generation must ask the predicate that READS it as one (the trailing fork, `_vocab.reads_as_generation`, and its words-to-spare count), not the vocabulary that lists some of its spellings. The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, `MD/PhD` by its listed chunks and `PU-PH` by the caps shape's joined spelling, #752, each with its joiner glued, spaced and glued to one side, which are three token streams -- and a new chunk reading inherits the old one's exceptions, which #752's first draft did not (`Jack I-V` lost its family to the listed numerals, and the fix's length test counted a chunk's period, so `Jack I.-V.` did too, found by the second review), and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there, and `Mr. and Mrs.` a run P3 joins, carrying a piece tag a one-word vocabulary test never sees -- PR #735's review found C12's hand copy of C13's title test (C8's until its 2026-10-19 split) reading `John Smith Jr., Mr. and Mrs. AD` as all suffix; ask C13's predicates in C13's order, `not is_suffix_piece` then `is_title_piece`, not the token tags, since `is_title_piece` alone reads the duals `Ms` and `Sr.` as titles). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. And a separator glued to a word hides the word from every vocabulary test, not only the one being changed: the semicolon delimiter's first draft read a glued word's stem in C1's run test alone and put 365 of ClinicalTrials.gov's reversed names right, missing the ones C13, C4 and C6 read off a word's TAGS (`MBBS;`, `MA;`); tagging the stem put 412 right, and reading it again in the comma decision's shape tests, which read a word's text rather than its tags (`MBBS;`), 431 (decisions/C10, 2026-10-24). A rule over a glued spelling reads the stem everywhere the word is read, its tags and every test of its text, and is swept against the space twin -- with a declared word delimiter too, which a substring test finds inside other words ('Stand' under ' and ', the review of the same change). A full stop INSIDE a word has four spellings of its own: initials run together (`J.R.R.`), joined words each closed by a stop (`Lt.Gov.`, `Msc.Ed.`), a joined word or name closed by none (`St.John`, `J.Smith`), and an abbreviation's own ending, closed or not (Italian `Sig.ra`, Latvian `Dr.philol.`). Shape tells the last from none of the others, so a rule over the shape decides which it costs; #838 found the ending only through the snapshot's CLDR row `Dut.ra Prof.ra`, and its first narrowing traded `St.John` for `Sig.ra` while the snapshot listed the `Dr.philol.` move among moves read as intended, until the design-docs review named it. Initials run together are also how some SURNAMES are written (Nepali `B.K.`, `K.C.`), and a caseless script writes its abbreviations, degrees included, as dotted letters (Thai `พ.บ.`): #864 found S3's position reading taking the first from 2.4.0 on, visible only once #866's Nepal candidates gave the corpora a population that writes them, and its first fix moved CLDR's Thai degrees into the name. A rule over dotted letters reads both before it ships. A predicate over ADJACENT PAIRS of words meets two shapes no grid of one word holds: an overlapping pair and a joining word beside an initial. #752's split-credential test (phase 2e) read `Dr. Ph.` in `Jane Doe Dr. Ph. D.` as the listed `drph`, leaving `D.` a word of its own (found by the maiden take's candidate grid), and a particle beside a surname written short as the listed `dea` (`DE A JUNIOR`, found only in tse-donors' readings), so put a title, a particle and a connective beside each pair spelling, and a pair overlapping another, in the grid. +**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, a one-case name being counted like capitals whatever the letter -- since #721 a one-case one-letter numeral carries the `initial` tag its capital does, and since #723 every one-case bare letter does (rules/T4), and a bare letter in front of the numeral is initial-shaped as its capital is, so `john v` reads as `JOHN V` does over `test_a_one_case_name_reads_as_its_capitals`'s grid (comma shapes, orders, a maiden clause, a particle chain, a bare letter in front, a trailing title, a non-ASCII letter in front: 315 of its then 1,872 pairs disagreed at 2bfa9533, before #721, and none after). Its first version claimed "every shape" with no bare letter in front and no trailing title in the grid, and #721's reviews found both broken; a grid's claim is the grid -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). #656's sweep found the same class from the other side: C12 takes the word ending the name before a comma as EVIDENCE, and its first draft asked the suffix-word tag, so `John Smith VI, AD` missed the rule `III` reached, and then counted the whole part, so `Smith VI, AD` fired with `VI` as its own name word and `Smith V, AD` with a `V` that S10 reads as the family. A rule that weighs a numeral as a generation must ask the predicate that READS it as one (the trailing fork, `_vocab.reads_as_generation`, and its words-to-spare count), not the vocabulary that lists some of its spellings. The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, `MD/PhD` by its listed chunks and `PU-PH` by the caps shape's joined spelling, #752, each with its joiner glued, spaced and glued to one side, which are three token streams -- and a new chunk reading inherits the old one's exceptions, which #752's first draft did not (`Jack I-V` lost its family to the listed numerals, and the fix's length test counted a chunk's period, so `Jack I.-V.` did too, found by the second review), and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there, and `Mr. and Mrs.` a run P3 joins, carrying a piece tag a one-word vocabulary test never sees -- PR #735's review found C12's hand copy of C13's title test (C8's until its 2026-10-19 split) reading `John Smith Jr., Mr. and Mrs. AD` as all suffix; ask C13's predicates in C13's order, `not is_suffix_piece` then `is_title_piece`, not the token tags, since `is_title_piece` alone reads the duals `Ms` and `Sr.` as titles). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. And a separator glued to a word hides the word from every vocabulary test, not only the one being changed: the semicolon delimiter's first draft read a glued word's stem in C1's run test alone and put 365 of ClinicalTrials.gov's reversed names right, missing the ones C13, C4 and C6 read off a word's TAGS (`MBBS;`, `MA;`); tagging the stem put 412 right, and reading it again in the comma decision's shape tests, which read a word's text rather than its tags (`MBBS;`), 431 (decisions/C10, 2026-10-24). A rule over a glued spelling reads the stem everywhere the word is read, its tags and every test of its text, and is swept against the space twin -- with a declared word delimiter too, which a substring test finds inside other words ('Stand' under ' and ', the review of the same change). A full stop INSIDE a word has four spellings of its own: initials run together (`J.R.R.`), joined words each closed by a stop (`Lt.Gov.`, `Msc.Ed.`), a joined word or name closed by none (`St.John`, `J.Smith`), and an abbreviation's own ending, closed or not (Italian `Sig.ra`, Latvian `Dr.philol.`). Shape tells the last from none of the others, so a rule over the shape decides which it costs; #838 found the ending only through the snapshot's CLDR row `Dut.ra Prof.ra`, and its first narrowing traded `St.John` for `Sig.ra` while the snapshot listed the `Dr.philol.` move among moves read as intended, until the design-docs review named it. Initials run together are also how some SURNAMES are written (Nepali `B.K.`, `K.C.`), and a caseless script writes its abbreviations, degrees included, as dotted letters (Thai `พ.บ.`): #864 found S3's position reading taking the first from 2.4.0 on, visible only once #866's Nepal candidates gave the corpora a population that writes them, and its first fix moved CLDR's Thai degrees into the name. A rule over dotted letters reads both before it ships. A predicate over ADJACENT PAIRS of words meets two shapes no grid of one word holds: an overlapping pair and a joining word beside an initial. #752's split-credential test (phase 2e) read `Dr. Ph.` in `Jane Doe Dr. Ph. D.` as the listed `drph`, leaving `D.` a word of its own (found by the maiden take's candidate grid), and a particle beside a surname written short as the listed `ded` (`DE D JUNIOR`, found only in tse-donors' readings), so put a title, a particle and a connective beside each pair spelling, and a pair overlapping another, in the grid. **And the encoding: a letter class written `[^\W\d_]` or tested with `isalpha()` is blind to a combining mark.** Python's `\w` and `isalpha()` do not match Mn, so a letter written as base plus accent (NFD) ends a word early; the vocabulary path is immune because `_normalize` NFC-folds, the shape path is not. #652: `Dr. inž. Janez Novak` read `inž.` as the given name when the ž was decomposed, and `Janez Novak, ŽŠČ` reversed, in two families that share the blind spot — the letter-shape regexes (`period_abbreviation`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`), now spelled with `_vocab._LETTER` (a letter plus the general combining blocks; `test_regex_sync` compares the two `_MARKS` strings and pins `period_abbreviation` against config's copy), and the caps predicates, which NFC-compose a non-ASCII word before `isalpha()` (`is_caps_word`, behind C-level `isalpha()`/`is_normalized` tests, so already-composed text pays no frame — measured 2026-10-12, `Smith, José` was 193 frames before and after; the first version called it for every non-ASCII word and cost one frame each, and no guard saw it because the reference name is ASCII). A width test must count letters, not code points (`composed_len`): admitting a decomposed word to the caps set put `John Smith, MÁ` on the pair exception's wrong side, and the review's grid found it where the rows (all three letters long) did not — put a two-letter word in every caps grid. The initial shape (`_INITIAL`, `\w\.`, v1-verbatim and public) is not widened but asked of the COMPOSED word, in `is_initial_shaped` and in `_render`'s two copies; and the single-letter connective tests count composed letters, since a decomposed `й` is two code points (#665, where `É. y É. Smith` and `John й Smith` read differently decomposed). Compose the WHOLE predicate, not one half: `is_initial` is a shape ANDed with the #320 script veto, and composing the shape alone made a decomposed `씨.` an initial, since decomposed hangul is conjoining jamo outside every script range (#665's first version; `in_initialless_script` composes too now). Not covered, deliberately: the marks only some scripts write (Indic vowel signs, Arabic and Hebrew points), declined in decisions/nfd-shapes because widening the class would move readings nobody had measured; rules/H2's Accepted clause records the one consequence it names, that a period-marked abbreviation in Bengali or Devanagari never reads as a title by shape. The guard is `test_a_decomposed_name_renders_as_its_composed_twin` over the CLDR corpus plus `test_a_decomposed_abbreviation_reads_as_its_composed_twin` (rows × three orders × EVERYWHERE); the corpus guard reaches a shape only if some corpus name does, which is how the caps shape was missed until review, so when adding a shape rule add a row there. Four of #652's rows and two of #665's pass with every fix reverted (measured) and are parity pins, and the test compares ambiguity kinds and `initials()` too, since #665's `John й Smith` differed in its report; `_DOTTED` has no observable input and is pinned by a direct call. diff --git a/docs/design/decisions/C13-credential-opening-the-part-after-the-comma.md b/docs/design/decisions/C13-credential-opening-the-part-after-the-comma.md index 46753b75..35ae1e63 100644 --- a/docs/design/decisions/C13-credential-opening-the-part-after-the-comma.md +++ b/docs/design/decisions/C13-credential-opening-the-part-after-the-comma.md @@ -13,7 +13,7 @@ Open: none in this file. ## Log -- 2026-10-04 (Derek), #603 (decided at group's head since #613, decisions/C1; its `John Smith, MD Jones`, `MD - PhD - FACS` and `Smith, MD PhD Ma` readings and `md`'s dual status superseded by #853, decisions/indic-honorifics, `MD` being no title there) — A CREDENTIAL OPENING THE PART AFTER THE COMMA MAKES IT THE POSTNOMINAL PART, WHATEVER FOLLOWS; AND A TITLE WORD PAST THE SECOND COMMA IS A TITLE. One word that is neither suffix nor title vocabulary used to flip the whole comma reading: `John Smith, PhD Jones` read family 'John Smith', given 'PhD', where `John Smith, MD FACS` and `Eric H. Holder, Jr. Attorney General` were already read as a name and its postnominals. Now the first suffix word of the part, with only titles in front of it, opens the part when it starts S2's run (`_pieces.starts_a_credential_run`, #602's own predicate, so the two runs cannot disagree about what starts one), and every word after it reads as that run reads: a title word as a title, any other as a suffix. The mechanism was the one #296 built for a part holding no name word, extended rather than added beside; where this entry had segment deciding it, #613 (decisions/C1) decides it once at group's head. Every word the opened part takes that would otherwise have made it the given part is reported (`suffix-or-name`), as #602's run reports what it absorbs. +- 2026-10-04 (Derek), #603 (its no-comma `John Smith Ph. D. Jones` left reading family 'Jones' superseded by #752's phase 2e, decisions/S11, the merged pair starting the run there too) (decided at group's head since #613, decisions/C1; its `John Smith, MD Jones`, `MD - PhD - FACS` and `Smith, MD PhD Ma` readings and `md`'s dual status superseded by #853, decisions/indic-honorifics, `MD` being no title there) — A CREDENTIAL OPENING THE PART AFTER THE COMMA MAKES IT THE POSTNOMINAL PART, WHATEVER FOLLOWS; AND A TITLE WORD PAST THE SECOND COMMA IS A TITLE. One word that is neither suffix nor title vocabulary used to flip the whole comma reading: `John Smith, PhD Jones` read family 'John Smith', given 'PhD', where `John Smith, MD FACS` and `Eric H. Holder, Jr. Attorney General` were already read as a name and its postnominals. Now the first suffix word of the part, with only titles in front of it, opens the part when it starts S2's run (`_pieces.starts_a_credential_run`, #602's own predicate, so the two runs cannot disagree about what starts one), and every word after it reads as that run reads: a title word as a title, any other as a suffix. The mechanism was the one #296 built for a part holding no name word, extended rather than added beside; where this entry had segment deciding it, #613 (decisions/C1) decides it once at group's head. Every word the opened part takes that would otherwise have made it the given part is reported (`suffix-or-name`), as #602's run reports what it absorbs. THE ONE-WORD PATH, DECIDED (Derek): the whole part is suffixes. `Doe, PhD Jones` reads family 'Doe', suffix 'PhD Jones', no given name, as `Smith, PhD` already reads. DECLINED: reading the credential as a suffix and the next name word as the given name (`Smith, PhD Jones` → given 'Jones'), proposed by analogy with a title at the head of the given part. Asked whether that is a real format, it is not -- nobody writes `Family, Credential Given` on purpose -- and the one realistic input it served, a misplaced generation (`Smith, Jr. John` for `Smith Jr., John`), is garbage in, garbage out. One reading for both comma paths was the deciding ground: the one-word path disagreeing with the full-name path is the shape #429, #430 and #432 each found. Accepted cost: `Smith, Jr. John` loses 'John' to the suffix, where 2.3 read given 'John' with 'Jr.' misfiled as a title. A TITLE/SUFFIX DUAL OPENS NOTHING (Derek). A word in both vocabularies (`md`, `ms`, `sr`, the ranks `lt`, `cpl` and the rest) is the listing form's title in front of a given name, `Smith, Ms Jane`, and a credential behind it opens nothing either (`Smith, Ms PhD Jones` keeps given 'PhD', as `Smith, MD PhD Ma` keeps given 'Ma' under #544). DECLINED after building it: a dual opening the part behind two or more words before the comma. Derek first chose it for `John Smith, MD Jones`; the reviews found what the count cannot see -- a surname of two words (`García Márquez, Ms Gabriela`, `Smith Jones, Sr. Maria`), a title in front of a one-word surname (`Dr. Smith, Ms Jane`, the count taking the title as a word), and spaced initials (`García Márquez, Ms G. J.`) each lost the given name -- and with the word explained (a dual is a word in both lists, decided by position), Derek chose that duals never open. So `John Smith, MD Jones` and the `MD - PhD - FACS` delimiter rows keep their 2.3 reading. An ambiguous-class member opens nothing either (`John Smith, Ma Jones` keeps given 'Ma'), nor does a suffix word S2's run start refuses (`John Smith, vd Jones`). STRICT MODE: the veto refuses an initial-shaped word the suffix READING, not the run. `Smith, PSM I.` under `lenient_comma_suffixes=False` reads suffix 'PSM I.' (2.3: given 'PSM', suffix 'I.'), reporting the absorbed 'I.', which matches #602's no-comma run (`John Smith PhD V.` reports; its given-part run does not, a disagreement #602 left and this does not touch). diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index 13030987..582a0e29 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -12,7 +12,7 @@ - A bare letter in a one-case name is the initial its capital is (rules/T4), and the one-letter numerals are an instance of that rule — 2026-10-18 #723. - The one-letter reading rests on the count, not on the generation being rare, and a one-case part before a capitalized credential reads under C12 (`john j v, PhD`) — 2026-10-19 #761. -Elsewhere: a one-letter numeral before a credential split across a space reads as it reads before the one-word spelling (`John Smith V Ph. D.` → family 'V', reported), where it read suffix 'V Ph. D.' as 1.4.0 did — decisions/S11's 2026-10-25 #752 entry. +Elsewhere: a one-letter numeral before a credential split across a space reads as it reads before the one-word spelling (`John Smith V Ph. D.` → family 'V', reported), where 2.3.0 through 2.6.0 read suffix 'V Ph. D.' (1.4.0 suffix 'Ph. D., V') — decisions/S11's 2026-10-25 #752 entry. Open: none in this file. diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md index 50671318..e88b92c9 100644 --- a/docs/design/decisions/S11-credential-split-across-a-space.md +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -4,12 +4,12 @@ Examples name each field as the default order reads it; under the family-first orders the given and family names trade places. -- Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class, unless a connective borders the pair (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de A Junior` → family 'de A'; `John Smith Pharm D and Mary` → family 'D and Mary') — 2026-10-25 #752. +- Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class, unless a connective borders the pair (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de D Junior` → family 'de D'; `John Smith Pharm D and Mary` → family 'D and Mary') — 2026-10-25 #752. - The count decides it as it decides a bare member, and where the count declines it the pair is the two words it is written as: at the end of a name by S4's words to spare (`John Pharm D` → middle 'Pharm', family 'D'), after a comma by C4's count (`John Smith, Pharm D` → suffix; `Smith, Pharm D` → given 'Pharm'), ending the given part after a family comma by S5's reading (`Smith, John Pharm D` → suffix); the part before a family comma merges none (`Smith Pharm D, John` → family 'Smith Pharm D') — 2026-10-25 #752. - Behind two name words with no comma, or after the given word after a family comma, the pair starts S8's run, the one member of the class that does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones', reported) — 2026-10-25 #752. - The reading reports nothing either way (`John Smith MB ChB` → suffix 'MB ChB', where `mb` started the run and reported 'ChB'; `SMITH, SAM E.` → given 'SAM', middle 'E.') — 2026-10-25 #752. - The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, merged by its spelling whatever the vocabulary lists, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. -- Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.') — 2026-10-25 #752. +- Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, not in one that writes no abbreviations, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.'; `John Smith 博. 士.` → family '士.') — 2026-10-25 #752; 2026-10-25 #752, review. - The Ph./D. pair starts S8's run from where it stands, though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, #827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` → family 'V', reported) — 2026-10-25 #752. - Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial read as the credential their letters spell behind two name words with no comma and ending the given part after a family comma (`JOHN SMITH SAM E.` and `Smith, John Sam E.` → suffix) — 2026-10-25 #752. - Case repair writes the pair by its fold's mask, as the one-word spelling is written, on every comma path (`john smith, PHARM D` → 'Pharm D', as `PHARMD` → 'PharmD') — 2026-10-25 #752, review. @@ -19,12 +19,13 @@ Open: none in this file. ## Log - 2026-10-25 #752 — A CREDENTIAL WRITTEN SPLIT ACROSS A SPACE IS ONE WORD, A MEMBER OF THE AMBIGUOUS CLASS THAT THE COUNT DECIDES (phase 2e: item 6 of decisions/vocabulary-model's 2026-10-23 #752 entry, agreed by Derek on 2026-10-23, and #827's reading (A)). `_vocab.split_pair` was the Ph./D. test alone (decisions/phd-merge); it now answers any pair with the piece tag it merges with, "suffix" for Ph./D. and "split" for the rest, and classify asks it once per adjacent pair of a segment, writing the answer on the pair's first word, so group's merge and the comma's reading of its part read a tag rather than each asking the vocabulary. - THE PAIR. Each word is one to five letters with or without its period, the two together spell unambiguous suffix vocabulary with the periods removed, and they are not both suffix vocabulary on their own (`MB ChB` merges, `chb` being unlisted; `MD PhD` stays two credentials). The first word has two or more letters: FEC's donors write a middle initial before a trailing salutation (`SMITH, JOHN A. MS.`), and `A. MS.` folds to the listed `ams` (merging any such pair but two single letters moved 5,122 FEC names in the 2026-10-23 measurement, `A. MS.` 2,246 of them). Two dotted single letters are the one opening letter asked further, by #872's `is_cased_paired_initials` rather than a copy of it: in a script with capitals they are paired initials, the reading rules/S3 gives them joined, so only a caseless script's pair merges, and only where its letters spell a listed word, which no default entry does (the overlap the 2026-10-25 one-predicate entry in decisions/vocabulary-model left to this change to state). Two refusals the measurement below added: a particle or a connective written without its period is no chunk, since it joins name words (`DE A`, a particle and a surname written short, folds to the listed `dea` in two tse-donors names behind `JUNIOR`), while a letter written with its period is an initial, not the connective it spells (`SAM E.` stays a pair, which the count then declines); and where two pairs overlap the Ph./D. pair merges (`Dr. Ph. D.`, whose `Dr. Ph.` folds to the listed `drph`, the case table's `the_split_degree_outranks_a_pair_overlapping_it`). - WHERE IT IS READ. As a member, by the count: in a main segment group merges the pair and S2's trailing read decides it, the words to spare S4 asks of a bare member, and a pair the read did not take is split back into its two words before anything else reads the segment, so a declined pair reads exactly as the two words read unmerged; in the part after a comma `_comma.decide` reads the part with the pair merged, C4 counting the name words before the comma, and where that reading does not take the pair as a credential reads the part again as written, so an `MB` the pair hid opens the part again (C13) and `John Smith, MB ChB BAO` keeps the suffix reading it had (the first version left the pair merged there, which made `MB` the given name in 15 ctgov names); in the given part after a family comma group merges the pair only behind the given word and S5's walk reads it as it reads a member ending the part, its writing saying nothing else. The part before a family comma reads no member as a suffix (`Smith MA, John`), so no pair merges there. The merged pair has no case of its own to lean on, so it reads as a member admitted by shape does (`St van Berg Pharm D` keeps family 'van Berg Pharm D', as `St van Berg X.Y.Z.` does). - THE RUN (S8). S8 refuses a member of the ambiguous class as a run start, because a member is a word of the name in its other reading and nothing in front of it says otherwise. A split credential's other reading is two words of the name, a given name and an initial, and the run starts only where two name words already stand in front of it, the words to spare its own count asks: so it is the one member that starts the run, behind two name words with no comma and after the given word after a family comma, as its one-word spelling does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones'). The Ph./D. pair, which the walk leaves out because assign reads it as a suffix wherever it stands, starts the run from where it stands, in front of the first word behind it (#827, reading (A): `John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, as `John Smith PhD Jones` reads; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng', the reading 2.3.0 gave). And a one-letter numeral with the pair behind it is the name word it is before `PhD`, Derek's 2026-10-25 call on #886 for the one-word spelling (`John Smith V Ph. D.` → family 'V', reported; it read suffix 'V Ph. D.', 1.4.0's reading, which rules/S10's Accepted clause had stated). Reading the numeral as the generation before the split spelling alone was the other option; it would keep the two spellings apart for one numeral shape, and no name in the five local corpora measured below moved by it, so the committed rules example is the only reading the choice moves. + THE PAIR. Each word is one to five letters with or without its period, the two together spell unambiguous suffix vocabulary with the periods removed, and they are not both suffix vocabulary on their own (`MB ChB` merges, `chb` being unlisted; `MD PhD` stays two credentials). The first word has two or more letters: FEC's donors write a middle initial before a trailing salutation (`SMITH, JOHN G. MR.`), which folds to a listed credential. Measured on this change with the length test removed from the predicate and its pre-test (fec-individuals fetched 2026-10-18, against this tree), 4,441 FEC names would move, `G. MR.` reading as `gmr` in 2,577 and `C. MS.` as `cms` in 1,133, against 206 ctgov names it would put right (`M. D.` 128, `M. Sc.` 28). (`A. MS.`, the 2026-10-23 entry's example, folds to `ams`, an ambiguous credential, which the fold test refuses whatever the opening letter.) Two dotted single letters are the one opening letter asked further, by #872's `is_cased_paired_initials` rather than a copy of it: in a script with capitals they are paired initials, the reading rules/S3 gives them joined, so only a caseless script's pair merges, and only where its letters spell a listed word, which no default entry does (the overlap the 2026-10-25 one-predicate entry in decisions/vocabulary-model left to this change to state). Two refusals the measurement below added: a particle or a connective written without its period is no chunk, since it joins name words (`DE D`, a particle and a surname written short, folds to the listed `ded` in a tse-donors name behind `JUNIOR`, the case table's witness being `Jane Doe de D Junior`), while a letter written with its period is an initial, not the connective it spells (`SAM E.` stays a pair, which the count then declines); and where two pairs overlap the Ph./D. pair merges (`Dr. Ph. D.`, whose `Dr. Ph.` folds to the listed `drph`, the case table's `the_split_degree_outranks_a_pair_overlapping_it`). + WHERE IT IS READ. As a member, by the count: in a main segment group merges the pair and S2's trailing read decides it, the words to spare S4 asks of a bare member, and a pair the read did not take is split back into its two words before anything else reads the segment, so a declined pair reads exactly as the two words read unmerged; in the part after a comma `_comma.decide` reads the part with the pair merged, C4 counting the name words before the comma, and where that reading does not take the pair as a credential reads the part again as written, so an `MB` the pair hid opens the part again (C13) and `John Smith, MB ChB BAO` keeps the suffix reading it had (the first version left the pair merged there, which made `MB` the given name in 15 ctgov names); in the given part after a family comma group merges the pair only behind the given word, and only after its joins have run (the review below), and S5's walk reads it as it reads a member ending the part, its writing saying nothing else. The part before a family comma reads no member as a suffix (`Smith MA, John`), so no pair merges there. The merged pair has no case of its own to lean on, so it reads as a member admitted by shape does (`St van Berg Pharm D` keeps family 'van Berg Pharm D', as `St van Berg X.Y.Z.` does). + THE RUN (S8). S8 refuses a member of the ambiguous class as a run start, because a member is a word of the name in its other reading and nothing in front of it says otherwise. A split credential's other reading is two words of the name, a given name and an initial, and the run starts only where two name words already stand in front of it, the words to spare its own count asks: so it is the one member that starts the run, behind two name words with no comma and after the given word after a family comma, as its one-word spelling does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones'). The Ph./D. pair, which the walk leaves out because assign reads it as a suffix wherever it stands, starts the run from where it stands, in front of the first word behind it (#827, reading (A): `John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, as `John Smith PhD Jones` reads; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng', the reading 2.3.0 gave). And a one-letter numeral with the pair behind it is the name word it is before `PhD`, Derek's 2026-10-25 call on #886 for the one-word spelling (`John Smith V Ph. D.` → family 'V', reported; 2.3.0 through 2.6.0 read suffix 'V Ph. D.' (1.4.0 suffix 'Ph. D., V', the V a suffix too), which rules/S10's Accepted clause had stated). Reading the numeral as the generation before the split spelling alone was the other option; it would keep the two spellings apart for one numeral shape, and no name in the five local corpora measured below moved by it, so the committed rules example is the only reading the choice moves. SILENT. A member reports either reading at the slots S4 names; the split pair reports neither. Where the count takes it, its letters spell a listed credential nobody hesitates over (`John Smith Pharm. D.`); where the count declines it, the two words are a given name and an initial (`SMITH, SAM E.`), read unmerged and reporting nothing, as they did before. The one reading the silence hides is Accepted (ii) below. ACCEPTED. (i) `M. D.`, `M D` and `M. Sc.` open with a single letter and stay two words, so after a comma behind a whole name the name stays reversed: in ctgov-officials (fetched 2026-10-21), of the 205 distinct names whose whole part after the comma is one of the five spellings the 2026-10-23 tally named, 133 still read the part's first word as the given name (`M. D.` 71, `M D` 41, `M. Sc.` 20, and one `Pharm D` behind a one-word head), where 185 did before (`Pharm. D.` read title 'Pharm.' there, and is not among them). (Correcting decisions/vocabulary-model's 2026-10-23 #752 entry, item 6: its "149 of the 359" officials still reversing counted `M. D.` and `M D` alone, and `M. Sc.`, which opens with a single letter, stays unmerged too.) (ii) A given name and its initial behind two name words with no comma read as the credential their letters spell (`JOHN SMITH SAM E.` → suffix 'SAM E.'), in silence; the FEC writes such a name behind a comma and one family word, where C4's count refuses it. - MEASURED (2026-10-25, Python 3.11.17, against master 225f3d6f, every local check with `--base` that checkout, so no shared snapshot was read or recorded): the committed snapshot moves 2 names, both rules examples (`John Smith Ph. D. Jones`, `John Smith V Ph. D.`), and gains S11's 21 new examples; every CLDR, GND, Wikidata and conventions agreement record matches. ctgov-officials (fetched 2026-10-21) moves 357 entries: 229 only drop a report (`MB ChB` 179, `MB BCh` 48, `DM Sc` 2, read as one credential where `mb` or `dm` started the run and reported the word behind it), and 128 move a field, each a split credential read whole where the name had read reversed or the credential as a name word (`Pharm D` and its spellings 110, `Psy D` 5, `Dr PH` 5, `Ed D` 3, `Sc. D.` 1, and `MB ChB` or `MB BCh` behind another degree 4); its force-repaired case view writes a capitals half as it did (`MB BCH`), `Ed D` forcing to `ED D` as a bare `Ed` read as a suffix does. tse-donors moves 2 (`JUNIOR` typed `JUN IOR`, now the suffix), and its pairs view (`local_corpora.py pairs tse-donors --agreement`) is unchanged; fec-individuals moves 1 (`USN R`, now the suffix run's); nepal-candidates moves none; psc-individuals (1,039,171 records scored) matches the base: 0 fixed, 0 broken, 0 changed, 1,004,782 agreeing on every field. The spike figures in decisions/vocabulary-model's 2026-10-23 entry (352 ctgov, 7 tse, 30 FEC for the first-chunk rule; 391, 34 and 41 with the run start) were a cruder rule's, with no count, no refusal of a particle chunk and no retry after a comma; they were not re-taken. - FRAMES (`sys.setprofile` call events, py3.11): the reference name, `John Smith`, `Smith, John` and `John Smith, PhD` cost what they did (363, 148, 157, 201). Classify's pair test is C-level up to the call, so a call is made only for a pair that passes the shape, the opening letter's length and the fold's membership: `John Smith Ph. D.` 232 → 233, the 64-link benchmark row unmoved once the length joined the pre-test (`i i` folds to the listed `ii`, and the first version called the predicate for every pair of links, +188). A name holding a member pair pays for its reading: `John Smith Pharm D` 230 → 238, `John Smith, MB ChB` 250 → 266, `Smith, John Pharm D` 253 → 284. + MEASURED (2026-10-25, Python 3.11.17, against master 225f3d6f, every local check with `--base` that checkout, so no shared snapshot was read or recorded): the committed snapshot moves 2 names, both rules examples (`John Smith Ph. D. Jones`, `John Smith V Ph. D.`), and gains S11's 27 new examples; every CLDR, GND, Wikidata and conventions agreement record matches. ctgov-officials (fetched 2026-10-21) moves 468 entries: 229 drop a report (`MB ChB` 179, `MB BCh` 48, `DM Sc` 2, read as one credential where `mb` or `dm` started the run and reported the word behind it), 128 move a field, each a split credential read whole where the name had read reversed or the credential as a name word (`Pharm D` and its spellings 110, `Psy D` 5, `Dr PH` 5, `Ed D` 3, `Sc. D.` 1, and `MB ChB` or `MB BCh` behind another degree 4); and 111 move only the force-repaired case view; 339 in all move that view, every one an `MB ChB` or `MB BCh` repaired by its fold's mask as its one-word twin is (`MB Chb` → `MB ChB`, `MB CHB` → `MB ChB`), 228 of them among the reports, and `Ed D` forcing to `ED D` as a bare `Ed` read as a suffix does. (Before the code review's fix (iv) the figure was 357, the case view moving only where the comma's marks reached.) tse-donors moves 2 (`JUNIOR` typed `JUN IOR`, now the suffix), and its pairs view (`local_corpora.py pairs tse-donors --agreement`) is unchanged; fec-individuals moves 1 (`USN R`, now the suffix run's); nepal-candidates moves none; psc-individuals (1,039,171 records scored) matches the base: 0 fixed, 0 broken, 0 changed, 1,004,782 agreeing on every field. The spike figures in decisions/vocabulary-model's 2026-10-23 entry (352 ctgov, 7 tse, 30 FEC for the first-chunk rule; 391, 34 and 41 with the run start) were a cruder rule's, with no count, no refusal of a particle chunk and no retry after a comma; they were not re-taken. + FRAMES (`sys.setprofile` call events over a second parse, py3.11, so one below `_CALL_BASELINE`'s count): `_REFERENCE` (`Dr. Juan0001 de la Vega III`) and `Dr. Juan Q. Xavier de la Vega III`, `John Smith`, `Smith, John` and `John Smith, PhD` cost what they did (304, 363, 148, 157, 201), and so do accented names (`Ngô Bảo Châu` 262) once the review's fix (v) landed. Classify's pair test is C-level up to the call, so a call is made only for a pair that passes the shape, the opening letter's length and the fold's membership: `John Smith Ph. D.` 232 → 233, the 64-link benchmark row unmoved once the length joined the pre-test (`i i` folds to the listed `ii`, and the first version called the predicate for every pair of links, +188). A name holding a member pair pays for its reading: `John Smith Pharm D` 230 → 238, `John Smith, MB ChB` 250 → 266, `Smith, John Pharm D` 253 → 286 (284 before the design-docs review's fix (i), which merges the given part's pair after the joins). Recompute: `uv run python tools/differential/snapshot.py --since ` for the committed snapshot; for each local corpus `snapshot.py --local --base `, and `local_agreement.py psc-individuals --base `; the accepted figure by reading `corpus.jsonl` of ctgov-officials, keeping each name holding a comma whose text after its first comma, stripped, is one of `M. D.`, `M D`, `M. Sc.`, `Pharm D` and `Pharm. D.`, and counting those whose given name opens with that part's first word, on this tree and on the parent. - 2026-10-25 #752, review — SEVEN DEFECTS IN THE FIRST VERSION, found by the PR's code review on a grid of 3,726 texts under the three orders compared against master. (i) A connective beside a pair joined the merged piece into a wider one, which kept its tag and was read whole: `John Smith Pharm D and Mary` read suffix 'Pharm D and Mary' and `John Pharm D and Mary` family 'Pharm D and Mary', where master reads family 'D and Mary'. A pair a connective borders no longer merges, and the two readers of the tag ask for a piece of exactly two words. (ii) Classify's pre-test asked the fold's membership of the Ph./D. pair too, so a configuration without `phd` (`Constants().suffix_acronyms.remove('phd')`) stopped merging it, where v1's fix_phd merged it by its spelling and 1.4.0 and master read suffix 'Ph. D.'; the pair's own pattern now answers ahead of the vocabulary. (iii) C3's count of the part before a family comma merged a member pair, so `MB ChB, PhD` read given 'MB', family 'ChB' where master reads family 'MB ChB'; it reads the part as written now. (iv) Case repair marked a pair's capitals only where the comma's decision bound it, so the same words repaired two ways (`john smith, PHARM D` kept 'PHARM D', `john smith PHARM D` gave 'Pharm D'); render now repairs every suffix pair by its fold's mask (`MB CHB` → 'MB ChB' under 'MBChB'), else in capitals for a listed acronym, and the comma's marks are gone. The force view of `MB BCH` is now 'MB BCh', the mask of 'mbbch', where master kept 'MB BCH' as a caps word. (v) The pre-test let every non-ASCII fold through to the call, a frame per pair of short words in any accented name (`Ngô Bảo Châu` 263 → 269); it looks the composed fold up in C now and costs what master does, and a pair of two suffix words (`MD PhD`) and an opening single letter before a dotted word of two letters (FEC's `A. MS.`) are refused by tags and lengths before the call. (vi) The maiden clause's report read the first word of the piece ending it, so `Jane Doe nee Ed. D.` reported 'Ed.' ending the maiden name; a split pair the clause keeps is read in silence, as everywhere. (vii) The comma form of the given name and initial (`Smith, John Sam E.` → suffix 'Sam E.', where master read middle 'Sam E.') was unrecorded; it is Accepted with the no-comma form, as S5 reads any member ending the given part whose writing says nothing else, and a split pair has no writing to say it. Each fix has a case row or a test that fails on the first version (tests/v2/test_render.py's `test_a_split_credential_repairs_as_its_one_word_twin`, tests/v2/test_facade.py's `test_the_ph_d_pair_merges_whatever_the_vocabulary_says`), run against it before landing. +- 2026-10-25 #752, design-docs review — TWO MORE CODE DEFECTS AND THE RECORD'S CORRECTIONS. (i) In the given part after a family comma group merged the pair ahead of its joins, so a bound given name took it: `Rahman, Abdul MB ChB` read given 'Abdul MB ChB' where master reads given 'Abdul', suffix 'MB ChB', against rules/P5's "The join never absorbs" a suffix word. The pair is merged there only after the joins now (`_group._merge_given_pairs`), so a join that took one of its words leaves it as written (`Smith, abdul Pharm D Jones` reads as master, given 'abdul Pharm'). (ii) Two dotted single letters in a script that writes no abbreviations merged where their letters are listed: `John Smith 博. 士.` read suffix '博. 士.', `김 민준 박. 사.` suffix '박. 사.', 18 default honorifics and titles of two characters folding so. rules/S3 refuses that script's periods, and so does the pair now, which makes true the record's "no default entry". (iii) The record's corrections: the particle refusal's case is `DE D` (`ded`), not `DE A`, whose fold is unlisted, so the rule's example witnessed nothing; the opening-letter test's evidence is the measurement in THE PAIR above, not `A. MS.`; rules/S11 states its precedence over S6's case and S5's report, and rules/S8 its own over a split pair as a Precedence bullet rather than a clause of its Then; decisions/C13, decisions/phd-merge and decisions/S8 are marked where their statements moved, and mechanisms.md's ONE-PREDICATE-PER-QUESTION names the predicate's callers as they are. diff --git a/docs/design/decisions/S8-credentials-trailing-run.md b/docs/design/decisions/S8-credentials-trailing-run.md index cac68787..ec6a0cfa 100644 --- a/docs/design/decisions/S8-credentials-trailing-run.md +++ b/docs/design/decisions/S8-credentials-trailing-run.md @@ -6,7 +6,7 @@ Examples name each field as the default order reads it; under the family-first o - An unambiguous suffix word of two or more letters, in any script, standing after two name words with no comma or before a suffix comma, or after the given word in the part after a family comma, starts a run to the end of its part: every later word reads as a suffix except a title word, which reads as a title, and an absorbed name word reports `suffix-or-name` (`John Smith PhD Jones` → suffix 'PhD Jones'; `Eric H. Holder Jr. Attorney General` → suffix 'Jr.', title 'Attorney General') — 2026-10-04 #602; 2026-10-14 #679. - A roman numeral of two or more letters that reads as the generation starts the run as a listed one does (`John Smith VI Jones` → suffix 'VI Jones', as `John Smith III Jones`), its shape alone reported — 2026-10-25 #784. -- What never starts the run: a title word (`Mary Jane King Smith` keeps family 'Smith'), a member of the ambiguous class, a single letter, a connective (`Josep Carod i Rovira` keeps family 'Carod i Rovira'), and a word that is also particle or bound given-name vocabulary (`Mohamed Ali Abd Allah` → family 'Allah'); a title/suffix dual does start it — 2026-10-04 #602. +- What never starts the run: a title word (`Mary Jane King Smith` keeps family 'Smith'), a member of the ambiguous class but a credential split across a space (below), a single letter, a connective (`Josep Carod i Rovira` keeps family 'Carod i Rovira'), and a word that is also particle or bound given-name vocabulary (`Mohamed Ali Abd Allah` → family 'Allah'); a title/suffix dual does start it — 2026-10-04 #602. - An honorific or generational word starts the run as a credential does, whatever its script (`राम शर्मा साहब मोहन` → suffix 'साहब मोहन'); the one-character honorifics stay out by the two-letter minimum — 2026-10-14 #679. - A lone particle counts toward the two name words only where the credential stands right behind it (`John von PhD Jones` → family 'von', suffix 'PhD Jones', under the family-first orders given 'von'); a particle run counts once, and a particle the run took is not asked (`Freiherr von vd PhD Jones` keeps middle 'PhD') — 2026-10-10 #627. - Lifting the credential out of the name, the family-comma path's older reading (`John Smith PhD Jones` as family 'Jones', suffix 'PhD'), was declined — 2026-10-04 #602. diff --git a/docs/design/decisions/phd-merge.md b/docs/design/decisions/phd-merge.md index 846cc26c..e6c49c04 100644 --- a/docs/design/decisions/phd-merge.md +++ b/docs/design/decisions/phd-merge.md @@ -12,7 +12,7 @@ Open: none in this file. ## Log -- 2026-07 (v2 core, PR #288; recorded plan deviation #1 of the core plan) (its spellings reading alike after a name superseded in part by #544, decisions/S7: the merged pair is outside the no-comma company walk, so `John Smith Ph. D. MEng` keeps family 'MEng' where `John Smith PhD MEng` reads suffix 'PhD MEng') — "Ph. D." tokenizes as two words and is merged back by vocabulary (v1 fix_phd), so the spaced and unspaced spellings read alike at the end of a name. (Corrected 2026-10-22, #762: this said they read alike everywhere, which never held mid-name or at the head, at 1.4.0 and 2.0.0 as on the tree: mid-name the merged piece is a suffix and a bare `PhD` a name word (`John Ph. D. Smith` reads suffix 'Ph. D.', `John PhD Smith` middle 'PhD'), and at the head `Ph. D. Van Johnson` read title 'Ph.' at 1.4.0 and suffix 'Ph. D.' at 2.0.0 where `Ph.D. Van Johnson` read given 'Ph.D.'.) (The head reading is #371's, below: the merge now declines there and `Ph. D. Van Johnson` reads title 'Ph.' where `Ph.D. Van Johnson` reads given 'Ph.D.'.) +- 2026-07 (v2 core, PR #288; recorded plan deviation #1 of the core plan) (its spellings reading alike after a name superseded in part by #544, decisions/S7: the merged pair was outside the no-comma company walk, so `John Smith Ph. D. MEng` kept family 'MEng' where `John Smith PhD MEng` read suffix 'PhD MEng'; alike again since #752's phase 2e, decisions/S11, the pair starting the credential's run) — "Ph. D." tokenizes as two words and is merged back by vocabulary (v1 fix_phd), so the spaced and unspaced spellings read alike at the end of a name. (Corrected 2026-10-22, #762: this said they read alike everywhere, which never held mid-name or at the head, at 1.4.0 and 2.0.0 as on the tree: mid-name the merged piece is a suffix and a bare `PhD` a name word (`John Ph. D. Smith` reads suffix 'Ph. D.', `John PhD Smith` middle 'PhD'), and at the head `Ph. D. Van Johnson` read title 'Ph.' at 1.4.0 and suffix 'Ph. D.' at 2.0.0 where `Ph.D. Van Johnson` read given 'Ph.D.'.) (The head reading is #371's, below: the merge now declines there and `Ph. D. Van Johnson` reads title 'Ph.' where `Ph.D. Van Johnson` reads given 'Ph.D.'.) - 2026-08-31 #371 (the merge declines at the head) (its revise() rendering superseded in part by #511, decisions/R1) — a suffix never BEGINS a name: position outranks the vocabulary match, and `rules/S2` now states it. The merge is what made a leading credential possible at all — every other suffix-shaped word standing first already falls out as a title (H2's abbreviation clause, or TITLES membership: `Jr.`, `MD`, `Esq.`, `Sr.`) or as a name word (`PhD`, `III`), so the `Ph.`/`D.` pair was the only shape that reached the defect. It reached it by emptying the family: `Ph. D. Van Johnson` read given 'Van Johnson' with no surname at all, which is #371's report. THIS RESTORES 1.4.0 RATHER THAN DEVIATING FROM IT — on three of the four names, and the fourth is named below. v1's fix_phd healed the pair by regex on the raw string, `\s(ph\.?\s+d\.?)`, whose leading `\s` made a head match impossible while it fired everywhere else, mid-name as readily as trailing. (A first draft of this entry said "only where it TRAILED", which is false in both directions and would license narrowing the merge to the trailing position — `John Ph. D. Smith` reads suffix 'Ph. D.' at 1.4.0 and here.) At the head v1 split it — measured on the released wheel, `Ph. D. Van Johnson` gives title 'Ph.', first 'D.', last 'Van Johnson', which is exactly what this change gives. So `expected_since_1.4.0.toml` LOSES its fix(leading-credential) rule — and READ THE DIAGNOSIS, not just the error: the gate reported it SHADOWED, not reverted. `compare.py` separates the two ("shadowed -- an earlier rule claimed every diff it would have" versus "reverted -- matches no diffing name") because the fixes differ. Three of its four names became parity and the fourth, `Ph. D., Jr.`, still diverges under `fix(comma-precomma-family)`, which sits above it. #426 is the precedent for dropping a shadowed rule rather than keeping it. A first draft of this entry generalized EXPLAINED NOTHING into "how a parity restoration announces itself", which would teach the next maintainer to delete a shadowed rule whose behavior is still diverging and the 2.0.0 and 2.1.0 ledgers GAIN one, claiming the same four names, the same five roles and the same digest. What shipped in 2.0 was a regression from applying the merge unconditionally, not a reading anyone chose. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 626134d2..b8287bcd 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -122,9 +122,9 @@ This catalog converts discovery into a one-time cost — it does not remove disc **Contract statement.** Merge the candidate into one piece, tagged as a candidate, and let the rule that decides read it. Where the rule takes it, keep the piece; where it declines, split it back into the words it is written as before anything else reads them, so a declined candidate reads exactly as the words read with no merge. Where the deciding reading is one pass over a whole part, read the part again unmerged rather than unpicking the merged reading. -**How it works.** rules/S11's split credential (#752, phase 2e): classify tags each pair `_vocab.split_pair` admits, group merges it as a member of the ambiguous class, S2's trailing read decides it by S4's count, and `_group._settle_splits` splits back every merged pair the read left in front of the run before the particle chain or the bound join can see it. After a comma `_comma.decide` reads the part with the pairs merged and, where that reading does not take one as a credential, reads the part again with none merged: the first version kept the merged reading there, and the pair hid an `MB` that had opened the part as a credential (C13), reversing 15 ClinicalTrials.gov names (`Aa Aa, MB ChB BAO` read given 'MB'). The settled outcome is either the merge's reading or the as-written one, never a third, which is what lets a measurement read every move as a pair the count took. +**How it works.** rules/S11's split credential (#752, phase 2e): classify tags each pair `_vocab.split_pair` admits, group merges it as a member of the ambiguous class, S2's trailing read decides it by S4's count, and `_group._settle_splits` splits back every merged pair the read left in front of the run before the particle chain or the bound join can see it. In the given part after a family comma, where the deciding reading is assign's walk, the pair is merged only after group's joins have run (`_group._merge_given_pairs`), so no join takes it into a piece the walk would read whole: merged ahead of them, a bound given name took it (`Rahman, Abdul MB ChB` read given 'Abdul MB ChB', the design-docs review). After a comma `_comma.decide` reads the part with the pairs merged and, where that reading does not take one as a credential, reads the part again with none merged: the first version kept the merged reading there, and the pair hid an `MB` that had opened the part as a credential (C13), reversing 15 ClinicalTrials.gov names (`Aa Aa, MB ChB BAO` read given 'MB'). The settled outcome is either the merge's reading or the as-written one, never a third, which is what lets a measurement read every move as a pair the count took. -**Lives in.** nameparser/_pipeline/_classify.py (the pair tag); nameparser/_pipeline/_group.py (the merge and `_settle_splits`); nameparser/_pipeline/_comma.py (`decide`'s retry, `_pieces(splits=...)`). +**Lives in.** nameparser/_pipeline/_classify.py (the pair tag); nameparser/_pipeline/_group.py (the merge, `_settle_splits` and `_merge_given_pairs`); nameparser/_pipeline/_comma.py (`decide`'s retry, `_pieces(splits=...)`). **Reach for it when.** A unit of several words is decided by a count or a position the merge itself changes, and the safe reading of a declined unit is the reading its words already had. @@ -172,7 +172,7 @@ This catalog converts discovery into a one-time cost — it does not remove disc **How it works.** A hand-written mirror agrees with its original only until one of them moves, and the drift is invisible in both directions: each site keeps passing its own tests while they disagree about an input neither covers. Every instance below was found as a defect before it was found as a pattern. The destination follows the LAYER, not the topic: a predicate over token text goes to `_vocab`, one over pieces and tags to `_pieces`, both leaves the stages sit on. The piece layer got its own module only in #439; until then those predicates collected in `_group`, the one place both stages could reach, so the sharing between group and assign predates the module (decisions/parse-cost measured #439 as a pure relocation). The cost is a second evaluation of the same predicate, measured for #429 at 1.2–2.2% of a family-comma parse and 0% of every other — the right price there over plumbing a state field the two sites would not otherwise share. Folding two copies into one, take from each caller only what genuinely differs between them, and ask whether a parameter one caller had been filling with a guard is a question the other caller must answer too (#617, decisions/P3). The census of which stage calls which predicate, and how each instance was found, was kept here until #759 and is in git history (`git show fe85bb3d:docs/design/mechanisms.md`); the decisions entries carry each instance's argument. -**Lives in.** nameparser/_pipeline/_vocab.py over token text: `is_wholly_suffix` (#319, decisions/W2), `reads_as_generation` (#401/#421 as `is_trailing_numeral_suffix`, decisions/P5; since #752 the predicate every rule weighing a numeral as the generation asks, the vocabulary's own numerals read inside it; C4's run refuses a one-letter numeral by its shape alone, `is_single_letter_numeral`, a question about the shape of an initial and not a generation, decisions/vocabulary-model), `split_pair` (#752, the split credential `Ph.` `D.` that group's merge, `_comma._pieces` and `is_wholly_suffix` each tested with their own copy, each asking `SPLIT_PAIR_OPENS` C-level first since the test runs per adjacent pair, decisions/phd-merge), `is_cased_paired_initials` (#752, rules/S3's paired initials in a script with capitals, which classify asked inline; C5 asks the bare `is_paired_initials`, decisions/S3), `maiden_marker_run` (#434, decisions/M2), `delimiter_cores` (#436/#437, decisions/R1), `in_initialless_script` (#322/#323, decisions/cjk-full-stops), `caps_shape_candidate` (#516, decisions/S9). nameparser/_pipeline/_pieces.py over pieces and tags: `is_suffix_piece`, `leading_titles` and `is_leading_title` (#424), `is_title_piece`, `peel_walk`, `trailing_start`, `tail_reading` (the peel-and-chain fixed point that answers where the name pieces end, decisions/H5), `segment_suffix_reading` (#430, decisions/R1), `particle_tail` and `trailing_candidates` (#610, decisions/M2), `join_connectives` (#617, decisions/P3). nameparser/_pipeline/_post_rules.py: `suffix_entries` (#511, decisions/R1), the one instance in a stage rather than a leaf, being a pass over a whole ParseState. tests/v2/test_layering.py holds each module's contract, and a piece predicate growing a dependency on a STAGE shows up there as a widened entry. +**Lives in.** nameparser/_pipeline/_vocab.py over token text: `is_wholly_suffix` (#319, decisions/W2), `reads_as_generation` (#401/#421 as `is_trailing_numeral_suffix`, decisions/P5; since #752 the predicate every rule weighing a numeral as the generation asks, the vocabulary's own numerals read inside it; C4's run refuses a one-letter numeral by its shape alone, `is_single_letter_numeral`, a question about the shape of an initial and not a generation, decisions/vocabulary-model), `split_pair` (#752, the split credential: `Ph.` `D.` once, which group's merge, `_comma._pieces` and `is_wholly_suffix` each tested with their own copy; since phase 2e every pair rules/S11 reads, asked by classify alone behind a C-level shape, length and fold test and written as a tag that group and `_comma._pieces` read, `is_wholly_suffix` asking `SPLIT_PAIR_OPENS` for the Ph./D. answer alone; decisions/phd-merge, decisions/S11), `is_cased_paired_initials` (#752, rules/S3's paired initials in a script with capitals, which classify asked inline; C5 asks the bare `is_paired_initials`, decisions/S3), `maiden_marker_run` (#434, decisions/M2), `delimiter_cores` (#436/#437, decisions/R1), `in_initialless_script` (#322/#323, decisions/cjk-full-stops), `caps_shape_candidate` (#516, decisions/S9). nameparser/_pipeline/_pieces.py over pieces and tags: `is_suffix_piece`, `leading_titles` and `is_leading_title` (#424), `is_title_piece`, `peel_walk`, `trailing_start`, `tail_reading` (the peel-and-chain fixed point that answers where the name pieces end, decisions/H5), `segment_suffix_reading` (#430, decisions/R1), `particle_tail` and `trailing_candidates` (#610, decisions/M2), `join_connectives` (#617, decisions/P3). nameparser/_pipeline/_post_rules.py: `suffix_entries` (#511, decisions/R1), the one instance in a stage rather than a leaf, being a pass over a whole ParseState. tests/v2/test_layering.py holds each module's contract, and a piece predicate growing a dependency on a STAGE shows up there as a widened entry. **Reach for it when.** You are about to write a condition that mirrors, matches or "does what X does" — or you find a comment saying one does. Grep for the other site's predicate and call it instead. diff --git a/docs/design/rules/S11-credential-split-across-a-space.md b/docs/design/rules/S11-credential-split-across-a-space.md index 22fed64b..d54fd0f0 100644 --- a/docs/design/rules/S11-credential-split-across-a-space.md +++ b/docs/design/rules/S11-credential-split-across-a-space.md @@ -10,7 +10,7 @@ "John Smith Pharm D" → suffix="Pharm D" "John Smith Psy. D." → suffix="Psy. D." "John Smith M. D." → family="D." · boundary - "Jane Doe de A Junior" → family="de A" · boundary + "Jane Doe de D Junior" → family="de D" · boundary "John Smith Pharm D and Mary" → family="D and Mary" · boundary ``` - **Case repair** writes the pair as it writes the one-word spelling, the letters of the two words cased as their fold is. @@ -52,9 +52,10 @@ "Jane Doe Dr. Ph. D." → title="Dr." "Jane Doe Dr. Ph. D." → suffix="Ph. D." ``` -- **Two dotted single letters** in a script that writes capitals are paired initials, which S3 reads as the name word they are written as, and never merge; in a script without capitals they merge where their letters spell listed vocabulary, which no default entry does. +- **Two dotted single letters** in a script that writes capitals are paired initials, which S3 reads as the name word they are written as, and never merge, nor do they in a script that writes no abbreviations, which wears no abbreviation's periods (S3); in any other script without capitals they merge where their letters spell listed vocabulary, which no default entry does. ``` "John Smith J. R." → family="R." · boundary + "John Smith 博. 士." → family="士." · boundary ``` - **The part before a family comma** reads no member of the class as a suffix, so no pair merges there. ``` @@ -71,6 +72,14 @@ ``` "John Smith V Pharm D" → family="V" ``` +- **Written case** ("Jack PHARM D"): the pair carries no case of its own for S6 to read, its two words being two spellings, so the count alone decides it, as it decides a member admitted by shape. + ``` + "Jack PHARM D" → family="D" + ``` +- **The given part's report** ("Smith, John Pharm D"): S5 reads the pair as it reads a member ending the given part, and the pair reports nothing there either, which is this rule's silence. + ``` + "Smith, John Pharm D" → ambiguities=() + ``` **Accepted:** diff --git a/docs/design/rules/S8-credentials-trailing-run.md b/docs/design/rules/S8-credentials-trailing-run.md index 0db00fa9..6b5cd4a6 100644 --- a/docs/design/rules/S8-credentials-trailing-run.md +++ b/docs/design/rules/S8-credentials-trailing-run.md @@ -18,9 +18,8 @@ "John von PhD Jones" → family="von" "John von PhD Jones" → suffix="PhD Jones" ``` -- **What is a credential here.** A title word never starts the run, and neither does a member of the ambiguous class, a single letter, a connective, a word that is also particle or bound given-name vocabulary: a credential is the writer's mark that the post-nominals have begun, and those carry no such mark. An honorific or a generational word carries it as a credential does, whatever its script. A credential split across a space (S11) is the one member of the class that carries it, its other reading being two words of the name, which the two name words standing in front of it already leave the name without. +- **What is a credential here.** A title word never starts the run, and neither does a member of the ambiguous class, a single letter, a connective, a word that is also particle or bound given-name vocabulary: a credential is the writer's mark that the post-nominals have begun, and those carry no such mark. An honorific or a generational word carries it as a credential does, whatever its script. ``` - "John Smith Pharm D Jones" → suffix="Pharm D Jones" "Mary Jane King Smith" → family="Smith" "Josep Carod i Rovira" → family="Carod i Rovira" "Mohamed Ali Abd Allah" → family="Allah" @@ -52,6 +51,10 @@ **Precedence:** +- **A credential split across a space** ("John Smith Pharm D Jones"): it is a member of the ambiguous class and starts the run all the same, behind the two name words its own count asks for (S11). + ``` + "John Smith Pharm D Jones" → suffix="Pharm D Jones" + ``` - **An ambiguous acronym written with its periods** ("John Smith M.A. Jones"): writing the periods makes it unambiguously a credential (S3), so it is no member of the ambiguous class here and starts the run. --- diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 1ac38bb9..add6ed4f 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -597,8 +597,8 @@ def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: a member of the ambiguous class, which the count decides (rules/S4, rules/C4). Its first chunk has two or more letters: a single letter opening the pair is a middle initial far more often (FEC - writes 'SMITH, JOHN A. MS.', and 'A. MS.' folds to the listed - 'ams'), so 'M. Sc.' and 'M. D.' stay two words. Two dotted single + writes 'SMITH, JOHN G. MR.', and 'G. MR.' folds to the listed + 'gmr'), so 'M. Sc.' and 'M. D.' stay two words. Two dotted single letters are the one opening letter asked further, and asked of #872's predicate rather than a copy: in a script with capitals they are paired initials (`is_cased_paired_initials`), the reading diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index e319f630..8a2678aa 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -59,10 +59,10 @@ "S5": {"words": 632, "then_words": 317, "because_sentences": 1, "except": 2, "precedence": 1, "accepted": 0, "accepted_words": 0}, "S6": {"words": 337, "then_words": 121, "because_sentences": 2, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, "S7": {"words": 570, "then_words": 388, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 1, "accepted_words": 105}, - "S8": {"words": 422, "then_words": 308, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, + "S8": {"words": 418, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 0, "accepted_words": 0}, "S9": {"words": 463, "then_words": 262, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 3, "accepted_words": 163}, - "S11": {"words": 494, "then_words": 278, "because_sentences": 2, "except": 3, "precedence": 2, "accepted": 2, "accepted_words": 80}, + "S11": {"words": 616, "then_words": 313, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 94}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 054d1421..dd8ff928 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -223,6 +223,7 @@ "Jack MA" "Jack Ma" "Jack Ma." +"Jack PHARM D" "Jack Wei Ma" "Jack X.Y.I." "Jack XY.ZW." @@ -240,7 +241,7 @@ "Jane Doe MA R.T." "Jane Doe Mrs" "Jane Doe PhD nee Smith" -"Jane Doe de A Junior" +"Jane Doe de D Junior" "Jane Doe nee King." "Jane Doe nee MA" "Jane Doe nee MA Smith" @@ -352,6 +353,7 @@ "John Smith PhD Prof. Ma" "John Smith Pharm D" "John Smith Pharm D Jones" +"John Smith Pharm D and Mary" "John Smith Pharm. D." "John Smith Prof." "John Smith Prof. Dr." @@ -380,6 +382,7 @@ "John Smith XYZ" "John Smith Xyz." "John Smith vi" +"John Smith 博. 士." "John Smith, A-B" "John Smith, A.B." "John Smith, A.B. Ph.D." @@ -646,6 +649,7 @@ "Smith, John Pharm D" "Smith, John Pharm D Jones" "Smith, John Prof." +"Smith, John Sam E." "Smith, John V" "Smith, John V MD" "Smith, John V, Jr." @@ -785,6 +789,7 @@ "john smith pa-c" "john smith ph. d." "john smith ph.d." +"john smith pharm d" "john smith phd" "john smith psyd" "john smith v" @@ -792,6 +797,7 @@ "john smith vi" "john smith x.y.z." "john smith xi" +"john smith, PHARM D" "john smith, ms-pt" "john smith, otr/l" "john smith, v" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index a587dfce..297a84bc 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -1338,7 +1338,7 @@ {"name": "JOHN SMITH MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} {"name": "JOHN SMITH NEE X X", "given": "JOHN", "family": "SMITH", "maiden": "X X", "initials": "J. S.", "capitalized": "John Smith (X X)"} {"name": "JOHN SMITH PH.D.", "given": "JOHN", "family": "SMITH", "suffix": "PH.D.", "initials": "J. S.", "capitalized": "John Smith Ph.D."} -{"name": "JOHN SMITH SAM E.", "given": "JOHN", "family": "SMITH", "suffix": "SAM E.", "initials": "J. S.", "capitalized": "John Smith Sam E."} +{"name": "JOHN SMITH SAM E.", "given": "JOHN", "family": "SMITH", "suffix": "SAM E.", "initials": "J. S.", "capitalized": "John Smith SAM E."} {"name": "JOHN SMITH VI", "given": "JOHN", "family": "SMITH", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith VI"} {"name": "JOHN SMITH, FNP-BC", "given": "JOHN", "family": "SMITH", "suffix": "FNP-BC", "initials": "J. S.", "capitalized": "John Smith FNP-BC"} {"name": "JOHN SMITH, MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} @@ -1367,6 +1367,7 @@ {"name": "Jack Ma", "given": "Jack", "family": "Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. M."} {"name": "Jack Ma Jr", "given": "Jack", "family": "Ma", "suffix": "Jr", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. M."} {"name": "Jack Ma.", "given": "Jack", "family": "Ma.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. M."} +{"name": "Jack PHARM D", "given": "Jack", "middle": "PHARM", "family": "D", "initials": "J. P. D.", "capitalized_force": "Jack Pharm D"} {"name": "Jack Wei Ma", "given": "Jack", "middle": "Wei", "family": "Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. W. M."} {"name": "Jack X.Y.I.", "given": "Jack", "family": "X.Y.I.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. X."} {"name": "Jack X.Y.Z.", "given": "Jack", "family": "X.Y.Z.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. X."} @@ -1410,7 +1411,7 @@ {"name": "Jane Doe MA R.T.", "given": "Jane", "middle": "Doe MA", "family": "R.T.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D. M. R.", "capitalized_force": "Jane Doe Ma R.T."} {"name": "Jane Doe Mrs", "given": "Jane", "middle": "Doe", "family": "Mrs", "initials": "J. D. M."} {"name": "Jane Doe PhD nee Smith", "given": "Jane", "family": "Doe", "suffix": "PhD nee Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D.", "capitalized_force": "Jane Doe PhD Nee Smith"} -{"name": "Jane Doe de A Junior", "given": "Jane", "middle": "Doe", "family": "de A", "suffix": "Junior", "initials": "J. D. A."} +{"name": "Jane Doe de D Junior", "given": "Jane", "middle": "Doe", "family": "de D", "suffix": "Junior", "initials": "J. D. D."} {"name": "Jane Doe geb. Smith MA", "given": "Jane", "family": "Doe", "suffix": "MA", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee King.", "given": "Jane", "family": "Doe", "maiden": "King.", "initials": "J. D."} {"name": "Jane Doe nee MA", "given": "Jane", "family": "Doe", "maiden": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D.", "capitalized_force": "Jane Doe (Ma)"} @@ -1640,7 +1641,7 @@ {"name": "John Smith M. D.", "given": "John", "middle": "Smith M.", "family": "D.", "initials": "J. S. M. D."} {"name": "John Smith M.A.", "given": "John", "family": "Smith", "suffix": "M.A.", "initials": "J. S."} {"name": "John Smith MA", "given": "John", "family": "Smith", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John Smith MB ChB", "given": "John", "family": "Smith", "suffix": "MB ChB", "initials": "J. S.", "capitalized_force": "John Smith MB Chb"} +{"name": "John Smith MB ChB", "given": "John", "family": "Smith", "suffix": "MB ChB", "initials": "J. S."} {"name": "John Smith MC", "given": "John", "family": "Smith", "suffix": "MC", "initials": "J. S."} {"name": "John Smith MD PhD", "given": "John", "family": "Smith", "suffix": "MD PhD", "initials": "J. S."} {"name": "John Smith MD-Jones", "given": "John", "middle": "Smith", "family": "MD-Jones", "initials": "J. S. M.", "capitalized_force": "John Smith Md-Jones"} @@ -1662,6 +1663,7 @@ {"name": "John Smith PhD Prof. Ma", "title": "Prof.", "given": "John", "family": "Smith", "suffix": "PhD Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "Prof. John Smith PhD MA"} {"name": "John Smith Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} {"name": "John Smith Pharm D Jones", "given": "John", "family": "Smith", "suffix": "Pharm D Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith Pharm D and Mary", "given": "John", "middle": "Smith Pharm", "family": "D and Mary", "initials": "J. S. P. D. M."} {"name": "John Smith Pharm. D.", "given": "John", "family": "Smith", "suffix": "Pharm. D.", "initials": "J. S."} {"name": "John Smith Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Prof. Dr.", "title": "Prof. Dr.", "given": "John", "family": "Smith", "initials": "J. S."} @@ -1693,6 +1695,7 @@ {"name": "John Smith Xyz.", "given": "John", "middle": "Smith", "family": "Xyz.", "initials": "J. S. X."} {"name": "John Smith nee Jones R.A.I.", "given": "John", "family": "Smith", "suffix": "R.A.I.", "maiden": "Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith vi", "given": "John", "family": "Smith", "suffix": "vi", "initials": "J. S.", "capitalized_force": "John Smith VI"} +{"name": "John Smith 博. 士.", "given": "John", "middle": "Smith 博.", "family": "士.", "initials": "J. S. 博. 士."} {"name": "John Smith 田.中.", "given": "John", "middle": "Smith", "family": "田.中.", "initials": "J. S. 田."} {"name": "John Smith, 1.4", "given": "1.4", "family": "John Smith", "initials": "1. J. S."} {"name": "John Smith, A-B", "given": "A-B", "family": "John Smith", "initials": "A. J. S."} @@ -3065,6 +3068,7 @@ {"name": "Smith, John Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} {"name": "Smith, John Pharm D Jones", "given": "John", "family": "Smith", "suffix": "Pharm D Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} +{"name": "Smith, John Sam E.", "given": "John", "family": "Smith", "suffix": "Sam E.", "initials": "J. S.", "capitalized_force": "John Smith SAM E."} {"name": "Smith, John V", "given": "John", "family": "Smith", "suffix": "V", "initials": "J. S."} {"name": "Smith, John V MD", "given": "John", "middle": "V", "family": "Smith", "suffix": "MD", "initials": "J. V. S."} {"name": "Smith, John V, Jr.", "given": "John", "middle": "V", "family": "Smith", "suffix": "Jr.", "initials": "J. V. S."} @@ -3661,6 +3665,7 @@ {"name": "john smith pa-c", "given": "john", "family": "smith", "suffix": "pa-c", "initials": "j. s.", "capitalized": "John Smith PA-C"} {"name": "john smith ph. d.", "given": "john", "family": "smith", "suffix": "ph. d.", "initials": "j. s.", "capitalized": "John Smith Ph. D."} {"name": "john smith ph.d.", "given": "john", "family": "smith", "suffix": "ph.d.", "initials": "j. s.", "capitalized": "John Smith Ph.D."} +{"name": "john smith pharm d", "given": "john", "family": "smith", "suffix": "pharm d", "initials": "j. s.", "capitalized": "John Smith Pharm D"} {"name": "john smith phd", "given": "john", "family": "smith", "suffix": "phd", "initials": "j. s.", "capitalized": "John Smith PhD"} {"name": "john smith psyd", "given": "john", "family": "smith", "suffix": "psyd", "initials": "j. s.", "capitalized": "John Smith PsyD"} {"name": "john smith v", "given": "john", "family": "smith", "suffix": "v", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith V"} @@ -3668,6 +3673,7 @@ {"name": "john smith vi", "given": "john", "family": "smith", "suffix": "vi", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith VI"} {"name": "john smith x.y.z.", "given": "john", "family": "smith", "suffix": "x.y.z.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith X.Y.Z."} {"name": "john smith xi", "given": "john", "middle": "smith", "family": "xi", "initials": "j. s. x.", "capitalized": "John Smith Xi"} +{"name": "john smith, PHARM D", "given": "john", "family": "smith", "suffix": "PHARM D", "initials": "j. s.", "capitalized": "John Smith Pharm D"} {"name": "john smith, ma", "given": "john", "family": "smith", "suffix": "ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith MA"} {"name": "john smith, md ma", "given": "john", "family": "smith", "suffix": "md ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith MD MA"} {"name": "john smith, ms-pt", "given": "john", "family": "smith", "suffix": "ms-pt", "initials": "j. s.", "capitalized": "John Smith MS-PT"} From 56b17df69c115c34f4708ecee3c70e1a1736e73a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:30:03 -0700 Subject: [PATCH 05/11] Fix the second review's findings for split credentials (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- .../S11-credential-split-across-a-space.md | 9 +-- docs/design/mechanisms.md | 4 +- docs/design/rules/P5-bound-given-names.md | 3 +- .../S11-credential-split-across-a-space.md | 3 +- nameparser/_pipeline/_classify.py | 22 +++---- nameparser/_pipeline/_group.py | 46 ++++++--------- nameparser/_render.py | 59 +++++++++++-------- tests/v2/cases.py | 18 ++++++ tests/v2/pipeline/test_vocab.py | 9 +++ tests/v2/rules_budget.json | 4 +- tests/v2/test_cases.py | 6 +- tests/v2/test_render.py | 12 ++++ tools/differential/corpus_rules.jsonl | 1 + tools/differential/snapshot.jsonl | 1 + 14 files changed, 122 insertions(+), 75 deletions(-) diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md index e88b92c9..95610b2c 100644 --- a/docs/design/decisions/S11-credential-split-across-a-space.md +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -11,8 +11,8 @@ Examples name each field as the default order reads it; under the family-first o - The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, merged by its spelling whatever the vocabulary lists, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. - Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, not in one that writes no abbreviations, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.'; `John Smith 博. 士.` → family '士.') — 2026-10-25 #752; 2026-10-25 #752, review. - The Ph./D. pair starts S8's run from where it stands, though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, #827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` → family 'V', reported) — 2026-10-25 #752. -- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial read as the credential their letters spell behind two name words with no comma and ending the given part after a family comma (`JOHN SMITH SAM E.` and `Smith, John Sam E.` → suffix) — 2026-10-25 #752. -- Case repair writes the pair by its fold's mask, as the one-word spelling is written, on every comma path (`john smith, PHARM D` → 'Pharm D', as `PHARMD` → 'PharmD') — 2026-10-25 #752, review. +- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial read as the credential their letters spell behind two name words with no comma and ending the given part after a family comma (`JOHN SMITH SAM E.` and `Smith, John Sam E.` → suffix), and case repair then writes them as the acronym (`JOHN SMITH SAM E.` → 'John Smith SAM E.') — 2026-10-25 #752; 2026-10-25 #752, second review. +- Case repair writes a pair group merged by its fold's mask, as the one-word spelling is written, on every comma path (`john smith, PHARM D` → 'Pharm D', as `PHARMD` → 'PharmD'), and unforced keeps a pair written in two cases as written, as the one-word spelling is kept (`john smith pharm D`, as `pharmD`) — 2026-10-25 #752, review; 2026-10-25 #752, second review. Open: none in this file. @@ -20,7 +20,7 @@ Open: none in this file. - 2026-10-25 #752 — A CREDENTIAL WRITTEN SPLIT ACROSS A SPACE IS ONE WORD, A MEMBER OF THE AMBIGUOUS CLASS THAT THE COUNT DECIDES (phase 2e: item 6 of decisions/vocabulary-model's 2026-10-23 #752 entry, agreed by Derek on 2026-10-23, and #827's reading (A)). `_vocab.split_pair` was the Ph./D. test alone (decisions/phd-merge); it now answers any pair with the piece tag it merges with, "suffix" for Ph./D. and "split" for the rest, and classify asks it once per adjacent pair of a segment, writing the answer on the pair's first word, so group's merge and the comma's reading of its part read a tag rather than each asking the vocabulary. THE PAIR. Each word is one to five letters with or without its period, the two together spell unambiguous suffix vocabulary with the periods removed, and they are not both suffix vocabulary on their own (`MB ChB` merges, `chb` being unlisted; `MD PhD` stays two credentials). The first word has two or more letters: FEC's donors write a middle initial before a trailing salutation (`SMITH, JOHN G. MR.`), which folds to a listed credential. Measured on this change with the length test removed from the predicate and its pre-test (fec-individuals fetched 2026-10-18, against this tree), 4,441 FEC names would move, `G. MR.` reading as `gmr` in 2,577 and `C. MS.` as `cms` in 1,133, against 206 ctgov names it would put right (`M. D.` 128, `M. Sc.` 28). (`A. MS.`, the 2026-10-23 entry's example, folds to `ams`, an ambiguous credential, which the fold test refuses whatever the opening letter.) Two dotted single letters are the one opening letter asked further, by #872's `is_cased_paired_initials` rather than a copy of it: in a script with capitals they are paired initials, the reading rules/S3 gives them joined, so only a caseless script's pair merges, and only where its letters spell a listed word, which no default entry does (the overlap the 2026-10-25 one-predicate entry in decisions/vocabulary-model left to this change to state). Two refusals the measurement below added: a particle or a connective written without its period is no chunk, since it joins name words (`DE D`, a particle and a surname written short, folds to the listed `ded` in a tse-donors name behind `JUNIOR`, the case table's witness being `Jane Doe de D Junior`), while a letter written with its period is an initial, not the connective it spells (`SAM E.` stays a pair, which the count then declines); and where two pairs overlap the Ph./D. pair merges (`Dr. Ph. D.`, whose `Dr. Ph.` folds to the listed `drph`, the case table's `the_split_degree_outranks_a_pair_overlapping_it`). - WHERE IT IS READ. As a member, by the count: in a main segment group merges the pair and S2's trailing read decides it, the words to spare S4 asks of a bare member, and a pair the read did not take is split back into its two words before anything else reads the segment, so a declined pair reads exactly as the two words read unmerged; in the part after a comma `_comma.decide` reads the part with the pair merged, C4 counting the name words before the comma, and where that reading does not take the pair as a credential reads the part again as written, so an `MB` the pair hid opens the part again (C13) and `John Smith, MB ChB BAO` keeps the suffix reading it had (the first version left the pair merged there, which made `MB` the given name in 15 ctgov names); in the given part after a family comma group merges the pair only behind the given word, and only after its joins have run (the review below), and S5's walk reads it as it reads a member ending the part, its writing saying nothing else. The part before a family comma reads no member as a suffix (`Smith MA, John`), so no pair merges there. The merged pair has no case of its own to lean on, so it reads as a member admitted by shape does (`St van Berg Pharm D` keeps family 'van Berg Pharm D', as `St van Berg X.Y.Z.` does). + WHERE IT IS READ. As a member, by the count: in a main segment group merges the pair and S2's trailing read decides it, the words to spare S4 asks of a bare member, and a pair the read did not take is split back into its two words before anything else reads the segment, so a declined pair reads exactly as the two words read unmerged; in the part after a comma `_comma.decide` reads the part with the pair merged, C4 counting the name words before the comma, and where that reading does not take the pair as a credential reads the part again as written, so an `MB` the pair hid opens the part again (C13) and `John Smith, MB ChB BAO` keeps the suffix reading it had (the first version left the pair merged there, which made `MB` the given name in 15 ctgov names); in the given part after a family comma group merges the pair only behind the given word, ahead of its joins, so the particle chain stops in front of it and P5's bound join declines it as it declines a suffix word (the second review below), and S5's walk reads it as it reads a member ending the part, its writing saying nothing else. The part before a family comma reads no member as a suffix (`Smith MA, John`), so no pair merges there. The merged pair has no case of its own to lean on, so it reads as a member admitted by shape does (`St van Berg Pharm D` keeps family 'van Berg Pharm D', as `St van Berg X.Y.Z.` does). THE RUN (S8). S8 refuses a member of the ambiguous class as a run start, because a member is a word of the name in its other reading and nothing in front of it says otherwise. A split credential's other reading is two words of the name, a given name and an initial, and the run starts only where two name words already stand in front of it, the words to spare its own count asks: so it is the one member that starts the run, behind two name words with no comma and after the given word after a family comma, as its one-word spelling does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones'). The Ph./D. pair, which the walk leaves out because assign reads it as a suffix wherever it stands, starts the run from where it stands, in front of the first word behind it (#827, reading (A): `John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, as `John Smith PhD Jones` reads; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng', the reading 2.3.0 gave). And a one-letter numeral with the pair behind it is the name word it is before `PhD`, Derek's 2026-10-25 call on #886 for the one-word spelling (`John Smith V Ph. D.` → family 'V', reported; 2.3.0 through 2.6.0 read suffix 'V Ph. D.' (1.4.0 suffix 'Ph. D., V', the V a suffix too), which rules/S10's Accepted clause had stated). Reading the numeral as the generation before the split spelling alone was the other option; it would keep the two spellings apart for one numeral shape, and no name in the five local corpora measured below moved by it, so the committed rules example is the only reading the choice moves. SILENT. A member reports either reading at the slots S4 names; the split pair reports neither. Where the count takes it, its letters spell a listed credential nobody hesitates over (`John Smith Pharm. D.`); where the count declines it, the two words are a given name and an initial (`SMITH, SAM E.`), read unmerged and reporting nothing, as they did before. The one reading the silence hides is Accepted (ii) below. ACCEPTED. (i) `M. D.`, `M D` and `M. Sc.` open with a single letter and stay two words, so after a comma behind a whole name the name stays reversed: in ctgov-officials (fetched 2026-10-21), of the 205 distinct names whose whole part after the comma is one of the five spellings the 2026-10-23 tally named, 133 still read the part's first word as the given name (`M. D.` 71, `M D` 41, `M. Sc.` 20, and one `Pharm D` behind a one-word head), where 185 did before (`Pharm. D.` read title 'Pharm.' there, and is not among them). (Correcting decisions/vocabulary-model's 2026-10-23 #752 entry, item 6: its "149 of the 359" officials still reversing counted `M. D.` and `M D` alone, and `M. Sc.`, which opens with a single letter, stays unmerged too.) (ii) A given name and its initial behind two name words with no comma read as the credential their letters spell (`JOHN SMITH SAM E.` → suffix 'SAM E.'), in silence; the FEC writes such a name behind a comma and one family word, where C4's count refuses it. @@ -28,4 +28,5 @@ Open: none in this file. FRAMES (`sys.setprofile` call events over a second parse, py3.11, so one below `_CALL_BASELINE`'s count): `_REFERENCE` (`Dr. Juan0001 de la Vega III`) and `Dr. Juan Q. Xavier de la Vega III`, `John Smith`, `Smith, John` and `John Smith, PhD` cost what they did (304, 363, 148, 157, 201), and so do accented names (`Ngô Bảo Châu` 262) once the review's fix (v) landed. Classify's pair test is C-level up to the call, so a call is made only for a pair that passes the shape, the opening letter's length and the fold's membership: `John Smith Ph. D.` 232 → 233, the 64-link benchmark row unmoved once the length joined the pre-test (`i i` folds to the listed `ii`, and the first version called the predicate for every pair of links, +188). A name holding a member pair pays for its reading: `John Smith Pharm D` 230 → 238, `John Smith, MB ChB` 250 → 266, `Smith, John Pharm D` 253 → 286 (284 before the design-docs review's fix (i), which merges the given part's pair after the joins). Recompute: `uv run python tools/differential/snapshot.py --since ` for the committed snapshot; for each local corpus `snapshot.py --local --base `, and `local_agreement.py psc-individuals --base `; the accepted figure by reading `corpus.jsonl` of ctgov-officials, keeping each name holding a comma whose text after its first comma, stripped, is one of `M. D.`, `M D`, `M. Sc.`, `Pharm D` and `Pharm. D.`, and counting those whose given name opens with that part's first word, on this tree and on the parent. - 2026-10-25 #752, review — SEVEN DEFECTS IN THE FIRST VERSION, found by the PR's code review on a grid of 3,726 texts under the three orders compared against master. (i) A connective beside a pair joined the merged piece into a wider one, which kept its tag and was read whole: `John Smith Pharm D and Mary` read suffix 'Pharm D and Mary' and `John Pharm D and Mary` family 'Pharm D and Mary', where master reads family 'D and Mary'. A pair a connective borders no longer merges, and the two readers of the tag ask for a piece of exactly two words. (ii) Classify's pre-test asked the fold's membership of the Ph./D. pair too, so a configuration without `phd` (`Constants().suffix_acronyms.remove('phd')`) stopped merging it, where v1's fix_phd merged it by its spelling and 1.4.0 and master read suffix 'Ph. D.'; the pair's own pattern now answers ahead of the vocabulary. (iii) C3's count of the part before a family comma merged a member pair, so `MB ChB, PhD` read given 'MB', family 'ChB' where master reads family 'MB ChB'; it reads the part as written now. (iv) Case repair marked a pair's capitals only where the comma's decision bound it, so the same words repaired two ways (`john smith, PHARM D` kept 'PHARM D', `john smith PHARM D` gave 'Pharm D'); render now repairs every suffix pair by its fold's mask (`MB CHB` → 'MB ChB' under 'MBChB'), else in capitals for a listed acronym, and the comma's marks are gone. The force view of `MB BCH` is now 'MB BCh', the mask of 'mbbch', where master kept 'MB BCH' as a caps word. (v) The pre-test let every non-ASCII fold through to the call, a frame per pair of short words in any accented name (`Ngô Bảo Châu` 263 → 269); it looks the composed fold up in C now and costs what master does, and a pair of two suffix words (`MD PhD`) and an opening single letter before a dotted word of two letters (FEC's `A. MS.`) are refused by tags and lengths before the call. (vi) The maiden clause's report read the first word of the piece ending it, so `Jane Doe nee Ed. D.` reported 'Ed.' ending the maiden name; a split pair the clause keeps is read in silence, as everywhere. (vii) The comma form of the given name and initial (`Smith, John Sam E.` → suffix 'Sam E.', where master read middle 'Sam E.') was unrecorded; it is Accepted with the no-comma form, as S5 reads any member ending the given part whose writing says nothing else, and a split pair has no writing to say it. Each fix has a case row or a test that fails on the first version (tests/v2/test_render.py's `test_a_split_credential_repairs_as_its_one_word_twin`, tests/v2/test_facade.py's `test_the_ph_d_pair_merges_whatever_the_vocabulary_says`), run against it before landing. -- 2026-10-25 #752, design-docs review — TWO MORE CODE DEFECTS AND THE RECORD'S CORRECTIONS. (i) In the given part after a family comma group merged the pair ahead of its joins, so a bound given name took it: `Rahman, Abdul MB ChB` read given 'Abdul MB ChB' where master reads given 'Abdul', suffix 'MB ChB', against rules/P5's "The join never absorbs" a suffix word. The pair is merged there only after the joins now (`_group._merge_given_pairs`), so a join that took one of its words leaves it as written (`Smith, abdul Pharm D Jones` reads as master, given 'abdul Pharm'). (ii) Two dotted single letters in a script that writes no abbreviations merged where their letters are listed: `John Smith 博. 士.` read suffix '博. 士.', `김 민준 박. 사.` suffix '박. 사.', 18 default honorifics and titles of two characters folding so. rules/S3 refuses that script's periods, and so does the pair now, which makes true the record's "no default entry". (iii) The record's corrections: the particle refusal's case is `DE D` (`ded`), not `DE A`, whose fold is unlisted, so the rule's example witnessed nothing; the opening-letter test's evidence is the measurement in THE PAIR above, not `A. MS.`; rules/S11 states its precedence over S6's case and S5's report, and rules/S8 its own over a split pair as a Precedence bullet rather than a clause of its Then; decisions/C13, decisions/phd-merge and decisions/S8 are marked where their statements moved, and mechanisms.md's ONE-PREDICATE-PER-QUESTION names the predicate's callers as they are. +- 2026-10-25 #752, design-docs review (its fix (i), the merge after the joins, superseded by the 2026-10-25 #752, second review entry below) — TWO MORE CODE DEFECTS AND THE RECORD'S CORRECTIONS. (i) In the given part after a family comma group merged the pair ahead of its joins, so a bound given name took it: `Rahman, Abdul MB ChB` read given 'Abdul MB ChB' where master reads given 'Abdul', suffix 'MB ChB', against rules/P5's "The join never absorbs" a suffix word. The pair is merged there only after the joins now (`_group._merge_given_pairs`), so a join that took one of its words leaves it as written (`Smith, abdul Pharm D Jones` reads as master, given 'abdul Pharm'). (ii) Two dotted single letters in a script that writes no abbreviations merged where their letters are listed: `John Smith 博. 士.` read suffix '博. 士.', `김 민준 박. 사.` suffix '박. 사.', 18 default honorifics and titles of two characters folding so. rules/S3 refuses that script's periods, and so does the pair now, which makes true the record's "no default entry". (iii) The record's corrections: the particle refusal's case is `DE D` (`ded`), not `DE A`, whose fold is unlisted, so the rule's example witnessed nothing; the opening-letter test's evidence is the measurement in THE PAIR above, not `A. MS.`; rules/S11 states its precedence over S6's case and S5's report, and rules/S8 its own over a split pair as a Precedence bullet rather than a clause of its Then; decisions/C13, decisions/phd-merge and decisions/S8 are marked where their statements moved, and mechanisms.md's ONE-PREDICATE-PER-QUESTION names the predicate's callers as they are. +- 2026-10-25 #752, second review — FOUR DEFECTS IN THE FIXES, found by a review of the two fix commits. SUPERSEDES: the design-docs review entry's fix (i). (i) Merging the given part's pair after group's joins let the particle chain take its first word, since the chain then saw `Pharm` and `D` apart and its bound stopped behind them: `Smith, John van Pharm D` read middle 'van Pharm D' where `Smith, John van PharmD` reads family 'van Smith', suffix 'PharmD', and the bound join took `Pharm` the same way (`Smith, abdul Pharm D` → given 'abdul Pharm'). The pair is merged ahead of the joins again, the chain stopping in front of it, and P5's bound join declines it as it declines a suffix word (rules/P5's Except), which keeps `Rahman, Abdul MB ChB` reading suffix 'MB ChB'; the widened fix had narrowed the chain's stop, the shape AGENTS.md's review paragraph names. (ii) Classify's refusal of two suffix words stood in front of the Ph./D. answer, so a lexicon listing `ph` and `d` stopped the merge (`John Ph. D. Smith` read middle 'Ph. D.' with `Lexicon.default().add(suffix_words={"ph", "d"})`), against "merged by its spelling whatever the vocabulary lists"; it is asked of the other pairs only. (iii) Case repair recased every tagged pair, so of two overlapping pairs (`MB CHB` and `CHB A`, `chba` being listed) the second wrote over the first (`JOHN SMITH MB CHB A` → 'MB CHB A'); it reads only the pair group merged, its second word carrying the merge's `joined` mark. (iv) Unforced, case repair asked each word of a pair whether it is in one case, where the one-word spelling is asked whole, so `john smith pharm D` gave 'Pharm D' while `john smith pharmD` keeps 'pharmD'; the pair is asked whole now and kept as written (rules/R5). And the record: case repair costs no frame on a name holding no pair (`capitalized(force=True)` of `John Smith` 59, as on master; the first fix called the function on every repair, +1), and the given name and initial the count takes (`JOHN SMITH SAM E.`) repair as the acronym their letters spell, recorded with that Accepted reading. Each fix has a case row or a test that fails on the commit it fixes. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index b8287bcd..fe0e7338 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -122,9 +122,9 @@ This catalog converts discovery into a one-time cost — it does not remove disc **Contract statement.** Merge the candidate into one piece, tagged as a candidate, and let the rule that decides read it. Where the rule takes it, keep the piece; where it declines, split it back into the words it is written as before anything else reads them, so a declined candidate reads exactly as the words read with no merge. Where the deciding reading is one pass over a whole part, read the part again unmerged rather than unpicking the merged reading. -**How it works.** rules/S11's split credential (#752, phase 2e): classify tags each pair `_vocab.split_pair` admits, group merges it as a member of the ambiguous class, S2's trailing read decides it by S4's count, and `_group._settle_splits` splits back every merged pair the read left in front of the run before the particle chain or the bound join can see it. In the given part after a family comma, where the deciding reading is assign's walk, the pair is merged only after group's joins have run (`_group._merge_given_pairs`), so no join takes it into a piece the walk would read whole: merged ahead of them, a bound given name took it (`Rahman, Abdul MB ChB` read given 'Abdul MB ChB', the design-docs review). After a comma `_comma.decide` reads the part with the pairs merged and, where that reading does not take one as a credential, reads the part again with none merged: the first version kept the merged reading there, and the pair hid an `MB` that had opened the part as a credential (C13), reversing 15 ClinicalTrials.gov names (`Aa Aa, MB ChB BAO` read given 'MB'). The settled outcome is either the merge's reading or the as-written one, never a third, which is what lets a measurement read every move as a pair the count took. +**How it works.** rules/S11's split credential (#752, phase 2e): classify tags each pair `_vocab.split_pair` admits, group merges it as a member of the ambiguous class, S2's trailing read decides it by S4's count, and `_group._settle_splits` splits back every merged pair the read left in front of the run before the particle chain or the bound join can see it. In the given part after a family comma, where the deciding reading is assign's walk, the pair is merged ahead of group's joins and the joins treat it as the credential it may be: the particle chain stops in front of it and P5's bound join declines it. Moving the merge after the joins instead, so that none could take it, let the chain take its first word (`Smith, John van Pharm D` read middle 'van Pharm D'): a candidate merged later than the readers it should stop is no candidate to them. After a comma `_comma.decide` reads the part with the pairs merged and, where that reading does not take one as a credential, reads the part again with none merged: the first version kept the merged reading there, and the pair hid an `MB` that had opened the part as a credential (C13), reversing 15 ClinicalTrials.gov names (`Aa Aa, MB ChB BAO` read given 'MB'). The settled outcome is either the merge's reading or the as-written one, never a third, which is what lets a measurement read every move as a pair the count took. -**Lives in.** nameparser/_pipeline/_classify.py (the pair tag); nameparser/_pipeline/_group.py (the merge, `_settle_splits` and `_merge_given_pairs`); nameparser/_pipeline/_comma.py (`decide`'s retry, `_pieces(splits=...)`). +**Lives in.** nameparser/_pipeline/_classify.py (the pair tag); nameparser/_pipeline/_group.py (the merge, `_settle_splits`, and P5's decline of a split piece); nameparser/_pipeline/_comma.py (`decide`'s retry, `_pieces(splits=...)`). **Reach for it when.** A unit of several words is decided by a count or a position the merge itself changes, and the safe reading of a declined unit is the reading its words already had. diff --git a/docs/design/rules/P5-bound-given-names.md b/docs/design/rules/P5-bound-given-names.md index 3d33b3b2..ce9e0198 100644 --- a/docs/design/rules/P5-bound-given-names.md +++ b/docs/design/rules/P5-bound-given-names.md @@ -51,11 +51,12 @@ **Except:** -- **The join never absorbs a marker standing as a word of its own** — a marker is not a name word (M2) — nor a word of the unambiguous suffix vocabulary (S2), wherever position will then place it, nor a title word a trailing rule reads as a title: one the trailing title run (H5) takes from the given part after a family comma, read as the comma-less name reads it with the bare salutations H8 adds, or any title word in a part past the second comma (C2). +- **The join never absorbs a marker standing as a word of its own** — a marker is not a name word (M2) — nor a word of the unambiguous suffix vocabulary (S2), wherever position will then place it, nor a credential split across a space (S11) that the given part after a family comma holds, nor a title word a trailing rule reads as a title: one the trailing title run (H5) takes from the given part after a family comma, read as the comma-less name reads it with the bare salutations H8 adds, or any title word in a part past the second comma (C2). ``` "abdul Jr Smith Berg" → given="abdul" "abdul Jr Smith Berg" → middle="Jr Smith" "abdul Ph. D. Smith Berg" → suffix="Ph. D." + "Rahman, Abdul MB ChB" → suffix="MB ChB" "Berg, abdul Prof." → given="abdul" "John Smith, Jr., abdul Dr. Smith" → title="Dr." "Berg, abd née Jones" → given="abd" · boundary diff --git a/docs/design/rules/S11-credential-split-across-a-space.md b/docs/design/rules/S11-credential-split-across-a-space.md index d54fd0f0..67cba370 100644 --- a/docs/design/rules/S11-credential-split-across-a-space.md +++ b/docs/design/rules/S11-credential-split-across-a-space.md @@ -13,7 +13,7 @@ "Jane Doe de D Junior" → family="de D" · boundary "John Smith Pharm D and Mary" → family="D and Mary" · boundary ``` -- **Case repair** writes the pair as it writes the one-word spelling, the letters of the two words cased as their fold is. +- **Case repair** writes the pair as it writes the one-word spelling, the letters of the two words cased as their fold is, and a pair written in two cases is the writer's spelling as the one-word spelling would be (R5). ``` "john smith pharm d" → capitalized="John Smith Pharm D" "john smith, PHARM D" → capitalized="John Smith Pharm D" @@ -92,6 +92,7 @@ ``` "JOHN SMITH SAM E." → suffix="SAM E." "Smith, John Sam E." → suffix="Sam E." + "JOHN SMITH SAM E." → capitalized="John Smith SAM E." ``` --- diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index a079f6b6..bf5d7065 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -423,18 +423,20 @@ def classify(state: ParseState) -> ParseState: and (len(a.text.rstrip(".")) >= 2 or a.text.endswith(".") and b.text.endswith(".") and (len(b.text) == 2 or not b.text.isascii())) - # two suffix words are two credentials already - and not ("vocab:suffix" in a.tags - and "vocab:suffix" in b.tags) # the Ph./D. pair whatever the vocabulary, as v1's - # fix_phd merged it; any other only where its fold - # is listed, composed in C where it is not ASCII + # fix_phd merged it; any other only where its words + # are not two suffix words, two credentials already, + # and its fold is listed, composed in C where it is + # not ASCII and (SPLIT_PAIR_OPENS.fullmatch(a.text) is not None - or (fold := (a.text + b.text).replace(".", "") - .lower()) in acronyms or fold in words - or not fold.isascii() - and ((nfc := unicodedata.normalize("NFC", fold)) - in acronyms or nfc in words)) + or not ("vocab:suffix" in a.tags + and "vocab:suffix" in b.tags) + and ((fold := (a.text + b.text).replace(".", "") + .lower()) in acronyms or fold in words + or not fold.isascii() + and ((nfc := unicodedata.normalize( + "NFC", fold)) in acronyms + or nfc in words))) and (kind := split_pair(a.text, b.text, state.lexicon)) is not None # a member pair a connective borders is no credential: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 1d2022f0..7138490c 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -602,25 +602,6 @@ def _settle_splits(pieces: list[Piece], ptags: list[set[str]]) -> None: k += 1 -def _merge_given_pairs(seg: Sequence[int], pieces: list[Piece], - ptags: list[set[str]], - tokens: Sequence[WorkToken]) -> None: - """Merge each split credential still standing as its two words in - the given part after a family comma, behind the given word, in - place: run after group's joins, so a pair one of them took into a - piece of its own stays as the join left it.""" - given_from = leading_titles(pieces, ptags, tokens) - following = dict(zip(seg, seg[1:])) - k = given_from + 1 - while k < len(pieces) - 1: - a, b = pieces[k], pieces[k + 1] - if (len(a) == 1 and len(b) == 1 and not ptags[k] and not ptags[k + 1] - and SPLIT_MEMBER_TAG in tokens[a[0]].tags - and following.get(a[0]) == b[0]): - merge_pieces(pieces, ptags, k, k + 2, add={"split"}) - k += 1 - - def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, @@ -691,15 +672,17 @@ def _group_segment(seg: tuple[int, ...], additional: int, # whole where the read takes it and split back into its two words # where it does not (`_settle_splits`), so a declined pair reads # exactly as the two words read; and in the given part after a - # family comma behind the given word, where S5's walk reads it -- - # merged there only once the joins below have run, so no bound - # given name or connective joins it as it joins no credential - # ('Rahman, Abdul MB ChB', #752's review), and a pair declined - # there is middle-name text whole or split. + # family comma behind the given word, where S5's walk reads it, + # merged ahead of the joins so the particle chain stops in front of + # it and P5's bound join declines it as it declines a suffix word, + # and a pair declined there is middle-name text whole or split. + # (A connective beside a pair keeps it from merging at all, in + # classify.) # Not in the part before a family comma, which reads no member as # a suffix ('Smith MA, John'). Classify tagged each pair's first # word, so the test here is a tag. - splits = given_pairs = False + splits = False + given_from: int | None = None k = 0 while k < len(pieces) - 1: a, b = pieces[k], pieces[k + 1] @@ -715,7 +698,11 @@ def _group_segment(seg: tuple[int, ...], additional: int, splits = True continue if site is ClauseSite.GIVEN_SLOT: - given_pairs = True + if given_from is None: + given_from = leading_titles(pieces, ptags, tokens) + if k > given_from: + merge_pieces(pieces, ptags, k, k + 2, add={"split"}) + continue k += 1 # rules/M2: "A marker that counts takes the words after it as @@ -1164,9 +1151,14 @@ def chain(tail: int) -> None: # Smith, Jr., abdul Dr. Smith', #640). A title word standing # in the name is a name word and joins ('abdul Sir Smith # Berg', 'Berg, abdul Prof. Ali'). + # A credential split across a space (rules/S11) is declined + # as its one-word spelling is: merged in the given part + # before this join, so the particle chain stops in front of + # it too ('Rahman, Abdul MB ChB', 'Smith, John van Pharm D') nxt, nxt_tags = pieces[fk + 1], ptags[fk + 1] if (_is_maiden_marker_piece(nxt, tokens) or is_suffix_piece(nxt, nxt_tags, tokens) + or "split" in nxt_tags or ((site is ClauseSite.TAIL or bound_join is BoundJoin.LENIENT) and is_title_piece(nxt, nxt_tags, tokens) @@ -1278,8 +1270,6 @@ def chain(tail: int) -> None: # carried (rules/P5); joined_tags says why the # title tag is dropped. Pinned in test_group.py. merge_pieces(pieces, ptags, fk, fk + 2, drop={"title"}) - if given_pairs: - _merge_given_pairs(seg, pieces, ptags, tokens) if reads is not None: reads.append(read) pieces.extend(tail_pieces) diff --git a/nameparser/_render.py b/nameparser/_render.py index d79db1a7..d9fb4962 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -676,24 +676,31 @@ def cap(word: str) -> str: def _split_pairs(tokens: tuple[Token, ...], repaired: tuple[Token, ...], - lex: Lexicon) -> tuple[Token, ...]: - """`repaired` with each split credential read as a suffix (classify - tags its first word) recased by its fold, as the one-word spelling - is: the exceptions map's mask laid over the two words as written - ('PHARM D' under 'PharmD' is 'Pharm D'), else capitals for a listed - acronym ('MB CHB' is 'MB CHB' under 'MBChB'). A pair one of whose - words the gate kept as written is left as the gate left it. A loop - over the tags, so a name with no such pair pays no frame for it.""" - out: list[Token] | None = None - for i in range(len(tokens) - 1): + lex: Lexicon, force: bool) -> tuple[Token, ...]: + """`repaired` with each split credential group merged and read as a + suffix -- classify tags its first word, and the merge marks its + second `joined` -- recased as the one-word spelling is: the + exceptions map's mask laid over the two words as written ('PHARM D' + under 'PharmD' is 'Pharm D', 'MB CHB' under 'MBChB' is 'MB ChB'), + else capitals for a listed acronym. Unforced, a pair the gate keeps + as written is the pair written in more than one case, asked of the + two words together as the one-word spelling is asked ('pharm D' as + 'pharmD'), and it is left as written.""" + out = list(repaired) + i = 0 + while i < len(tokens) - 1: a, b = tokens[i], tokens[i + 1] - if (SPLIT_MEMBER_TAG not in a.tags - and SPLIT_CREDENTIAL_TAG not in a.tags): - continue - if (a.role is not Role.SUFFIX or b.role is not Role.SUFFIX - or repaired[i] is a or repaired[i + 1] is b): + if ((SPLIT_MEMBER_TAG not in a.tags + and SPLIT_CREDENTIAL_TAG not in a.tags) + or "joined" not in b.tags + or a.role is not Role.SUFFIX or b.role is not Role.SUFFIX): + i += 1 continue joined = a.text + b.text + if not force and not _in_one_case(joined): + out[i], out[i + 1] = a, b + i += 2 + continue fold = _normalize(joined.replace(".", "")) mask = lex.capitalization_exceptions_map.get(fold) cased = _apply_mask(joined, mask) if mask is not None else None @@ -701,14 +708,12 @@ def _split_pairs(tokens: tuple[Token, ...], repaired: tuple[Token, ...], cased = joined.upper() # a case mapping that changes the length ('ß') has no place to # cut the pair back into its two words - if cased is None or len(cased) != len(joined): - continue - if out is None: - out = list(repaired) - cut = len(a.text) - out[i] = Token(cased[:cut], a.span, a.role, a.tags) - out[i + 1] = Token(cased[cut:], b.span, b.role, b.tags) - return repaired if out is None else tuple(out) + if cased is not None and len(cased) == len(joined): + cut = len(a.text) + out[i] = Token(cased[:cut], a.span, a.role, a.tags) + out[i + 1] = Token(cased[cut:], b.span, b.role, b.tags) + i += 2 + return tuple(out) def _in_one_case(text: str) -> bool: @@ -804,8 +809,12 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, for t in name.tokens) # rules/R4: "Repair changes case and nothing else" -- and a # credential split across a space (rules/S11) is repaired as its - # one-word twin is, the pair's fold asked once - new_tokens = _split_pairs(name.tokens, new_tokens, lex) + # one-word twin is, the pair's fold asked once; a loop over the + # tags first, so a name with no such pair pays no frame for it + for t in name.tokens: + if SPLIT_MEMBER_TAG in t.tags or SPLIT_CREDENTIAL_TAG in t.tags: + new_tokens = _split_pairs(name.tokens, new_tokens, lex, force) + break # equal tokens (possible only for synthetic span=None duplicates) # collapse to one mapping entry -- benign: the rebuilt ambiguity # references an equal token, so the subset invariant still holds diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c6867eaf..e85eeb41 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1632,6 +1632,24 @@ def _check_cjk_shape_purity(self) -> None: "first word is a listed member, and the first version " "reported 'Ed.' as the word ending the clause, which " "'D.' is. 1.4.0 read middle 'Doe nee Ed.', last 'D.'"), + Case("a_split_credential_stops_the_particle_chain_in_the_given_part", + "Smith, John van Pharm D", + {"given": "John", "family": "van Smith", "suffix": "Pharm D"}, + classification="fix(#752)", + ambiguities=("particle-or-given",), + notes="S11: merged ahead of group's joins, the pair stops the " + "chain as 'PharmD' does, and P6 attaches the particle; " + "merged after them (the design-docs review's first fix), " + "the chain took 'Pharm' into the particle's run. 1.4.0 " + "read middle 'van Pharm D'"), + Case("a_bound_given_name_declines_a_split_credential", + "Rahman, Abdul MB ChB", + {"given": "Abdul", "family": "Rahman", "suffix": "MB ChB"}, + classification="fix(#752)", + notes="P5: the join declines the pair as it declines a suffix " + "word; the first version joined it, given 'Abdul MB ChB', " + "where master read suffix 'MB ChB'. 1.4.0 read first " + "'Abdul MB', middle 'ChB'"), Case("the_split_degree_outranks_a_pair_overlapping_it", "Jane Doe Dr. Ph. D.", {"title": "Dr.", "given": "Jane", "family": "Doe", diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index ba2ce30d..a709454d 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -1074,3 +1074,12 @@ def test_comma_follows_marker_reads_past_what_tokenize_drops( word: str, expected: bool) -> None: assert comma_follows_marker(word) is expected + + +def test_the_ph_d_pair_merges_when_both_its_words_are_listed() -> None: + # #752's second review: classify's two-suffix-words refusal stood in + # front of the Ph./D. answer, which merges whatever the vocabulary + # lists + parser = Parser(lexicon=Lexicon.default().add(suffix_words={"ph", "d"})) + assert parser.parse("John Ph. D. Smith").suffix == "Ph. D." + assert parser.parse("John Smith Ph. D. Jones").suffix == "Ph. D. Jones" diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index 8a2678aa..1adf4146 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -42,7 +42,7 @@ "P2": {"words": 270, "then_words": 234, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 3, "accepted_words": 167}, "P3": {"words": 639, "then_words": 598, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "P4": {"words": 190, "then_words": 82, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 0, "accepted_words": 0}, - "P5": {"words": 668, "then_words": 399, "because_sentences": 1, "except": 2, "precedence": 3, "accepted": 3, "accepted_words": 198}, + "P5": {"words": 685, "then_words": 399, "because_sentences": 1, "except": 2, "precedence": 3, "accepted": 3, "accepted_words": 198}, "P6": {"words": 974, "then_words": 644, "because_sentences": 2, "except": 2, "precedence": 4, "accepted": 6, "accepted_words": 459}, "P7": {"words": 138, "then_words": 64, "because_sentences": 3, "except": 0, "precedence": 0, "accepted": 1, "accepted_words": 28}, "P8": {"words": 515, "then_words": 450, "because_sentences": 2, "except": 0, "precedence": 0, "accepted": 2, "accepted_words": 309}, @@ -62,7 +62,7 @@ "S8": {"words": 418, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 0, "accepted_words": 0}, "S9": {"words": 463, "then_words": 262, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 3, "accepted_words": 163}, - "S11": {"words": 616, "then_words": 313, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 94}, + "S11": {"words": 634, "then_words": 331, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 94}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 81cfae36..a7e4dcee 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -242,9 +242,11 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # claim under each order; 1451 -> 1457 the same day: #752's two # split-credential run rows (phase 2e), 'Smith, John Pharm D Jones' # and 'John Smith Pharm D Jones', one suffix-or-name claim under + # each order; 1457 -> 1460 the same day: the second review's + # 'Smith, John van Pharm D' row, one particle-or-given claim under # each order. - assert claims == 1457, ( - f"the field sweep checks {claims} claims, recorded as 1457 on " + assert claims == 1460, ( + f"the field sweep checks {claims} claims, recorded as 1460 on " f"2026-10-25") diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 48924bb0..4f1e3a45 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1879,3 +1879,15 @@ def test_a_split_credential_repairs_as_its_one_word_twin(text: str) -> None: assert parse(text).capitalized().suffix == "Pharm D" twin = text.replace("PHARM D", "PHARMD").replace("pharm d", "pharmd") assert parse(twin).capitalized().suffix == "PharmD" + + +def test_a_split_credential_repair_reads_the_pair_group_merged() -> None: + # #752's second review: overlapping pairs ('MB CHB' and 'CHB A', + # 'chba' being listed) were both repaired, the second writing over + # the first; and a pair in two cases is kept as written unforced, + # as its one-word spelling is + assert parse("JOHN SMITH MB CHB A").capitalized().suffix == "MB ChB A" + assert parse("john smith pharm D").capitalized().suffix == "pharm D" + assert parse("john smith pharmD").capitalized().suffix == "pharmD" + assert (parse("john smith pharm D").capitalized(force=True).suffix + == "Pharm D") diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index dd8ff928..5a0d8285 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -572,6 +572,7 @@ "RA DDS" "RA PhD Jr., John" "RAM BAHADUR B.K." +"Rahman, Abdul MB ChB" "Ram Bahadur B.K." "Ram Bahadur B.K., PhD" "Ram Bahadur PhD B.K." diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index 297a84bc..45b40c2e 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -2765,6 +2765,7 @@ {"name": "Rafael Sousa dos Anjos", "given": "Rafael", "middle": "Sousa", "family": "dos Anjos", "initials": "R. S. A."} {"name": "Rafael Viciana Aráez", "given": "Rafael", "middle": "Viciana", "family": "Aráez", "initials": "R. V. A."} {"name": "Ragnar Bjarnason", "given": "Ragnar", "family": "Bjarnason", "initials": "R. B."} +{"name": "Rahman, Abdul MB ChB", "given": "Abdul", "family": "Rahman", "suffix": "MB ChB", "initials": "A. R."} {"name": "Ram Bahadur B.K.", "given": "Ram", "middle": "Bahadur", "family": "B.K.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "R. B. B."} {"name": "Ram Bahadur B.K., PhD", "given": "Ram", "middle": "Bahadur", "family": "B.K.", "suffix": "PhD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "R. B. B."} {"name": "Ram Bahadur PhD B.K.", "given": "Ram", "family": "Bahadur", "suffix": "PhD B.K.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "R. B."} From fb6029dc0a36affb1fc0dff1b5b803642617d5c9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:35:43 -0700 Subject: [PATCH 06/11] Record the final ctgov figure for split credentials (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/S11-credential-split-across-a-space.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md index 95610b2c..9e9b98c8 100644 --- a/docs/design/decisions/S11-credential-split-across-a-space.md +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -24,7 +24,7 @@ Open: none in this file. THE RUN (S8). S8 refuses a member of the ambiguous class as a run start, because a member is a word of the name in its other reading and nothing in front of it says otherwise. A split credential's other reading is two words of the name, a given name and an initial, and the run starts only where two name words already stand in front of it, the words to spare its own count asks: so it is the one member that starts the run, behind two name words with no comma and after the given word after a family comma, as its one-word spelling does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones'). The Ph./D. pair, which the walk leaves out because assign reads it as a suffix wherever it stands, starts the run from where it stands, in front of the first word behind it (#827, reading (A): `John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, as `John Smith PhD Jones` reads; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng', the reading 2.3.0 gave). And a one-letter numeral with the pair behind it is the name word it is before `PhD`, Derek's 2026-10-25 call on #886 for the one-word spelling (`John Smith V Ph. D.` → family 'V', reported; 2.3.0 through 2.6.0 read suffix 'V Ph. D.' (1.4.0 suffix 'Ph. D., V', the V a suffix too), which rules/S10's Accepted clause had stated). Reading the numeral as the generation before the split spelling alone was the other option; it would keep the two spellings apart for one numeral shape, and no name in the five local corpora measured below moved by it, so the committed rules example is the only reading the choice moves. SILENT. A member reports either reading at the slots S4 names; the split pair reports neither. Where the count takes it, its letters spell a listed credential nobody hesitates over (`John Smith Pharm. D.`); where the count declines it, the two words are a given name and an initial (`SMITH, SAM E.`), read unmerged and reporting nothing, as they did before. The one reading the silence hides is Accepted (ii) below. ACCEPTED. (i) `M. D.`, `M D` and `M. Sc.` open with a single letter and stay two words, so after a comma behind a whole name the name stays reversed: in ctgov-officials (fetched 2026-10-21), of the 205 distinct names whose whole part after the comma is one of the five spellings the 2026-10-23 tally named, 133 still read the part's first word as the given name (`M. D.` 71, `M D` 41, `M. Sc.` 20, and one `Pharm D` behind a one-word head), where 185 did before (`Pharm. D.` read title 'Pharm.' there, and is not among them). (Correcting decisions/vocabulary-model's 2026-10-23 #752 entry, item 6: its "149 of the 359" officials still reversing counted `M. D.` and `M D` alone, and `M. Sc.`, which opens with a single letter, stays unmerged too.) (ii) A given name and its initial behind two name words with no comma read as the credential their letters spell (`JOHN SMITH SAM E.` → suffix 'SAM E.'), in silence; the FEC writes such a name behind a comma and one family word, where C4's count refuses it. - MEASURED (2026-10-25, Python 3.11.17, against master 225f3d6f, every local check with `--base` that checkout, so no shared snapshot was read or recorded): the committed snapshot moves 2 names, both rules examples (`John Smith Ph. D. Jones`, `John Smith V Ph. D.`), and gains S11's 27 new examples; every CLDR, GND, Wikidata and conventions agreement record matches. ctgov-officials (fetched 2026-10-21) moves 468 entries: 229 drop a report (`MB ChB` 179, `MB BCh` 48, `DM Sc` 2, read as one credential where `mb` or `dm` started the run and reported the word behind it), 128 move a field, each a split credential read whole where the name had read reversed or the credential as a name word (`Pharm D` and its spellings 110, `Psy D` 5, `Dr PH` 5, `Ed D` 3, `Sc. D.` 1, and `MB ChB` or `MB BCh` behind another degree 4); and 111 move only the force-repaired case view; 339 in all move that view, every one an `MB ChB` or `MB BCh` repaired by its fold's mask as its one-word twin is (`MB Chb` → `MB ChB`, `MB CHB` → `MB ChB`), 228 of them among the reports, and `Ed D` forcing to `ED D` as a bare `Ed` read as a suffix does. (Before the code review's fix (iv) the figure was 357, the case view moving only where the comma's marks reached.) tse-donors moves 2 (`JUNIOR` typed `JUN IOR`, now the suffix), and its pairs view (`local_corpora.py pairs tse-donors --agreement`) is unchanged; fec-individuals moves 1 (`USN R`, now the suffix run's); nepal-candidates moves none; psc-individuals (1,039,171 records scored) matches the base: 0 fixed, 0 broken, 0 changed, 1,004,782 agreeing on every field. The spike figures in decisions/vocabulary-model's 2026-10-23 entry (352 ctgov, 7 tse, 30 FEC for the first-chunk rule; 391, 34 and 41 with the run start) were a cruder rule's, with no count, no refusal of a particle chunk and no retry after a comma; they were not re-taken. + MEASURED (2026-10-25, Python 3.11.17, against master 225f3d6f, every local check with `--base` that checkout, so no shared snapshot was read or recorded): the committed snapshot moves 2 names, both rules examples (`John Smith Ph. D. Jones`, `John Smith V Ph. D.`), and gains S11's 27 new examples; every CLDR, GND, Wikidata and conventions agreement record matches. ctgov-officials (fetched 2026-10-21) moves 470 entries: 229 drop a report (`MB ChB` 179, `MB BCh` 48, `DM Sc` 2, read as one credential where `mb` or `dm` started the run and reported the word behind it), 128 move a field, each a split credential read whole where the name had read reversed or the credential as a name word (`Pharm D` and its spellings 110, `Psy D` 5, `Dr PH` 5, `Ed D` 3, `Sc. D.` 1, and `MB ChB` or `MB BCh` behind another degree 4); 111 move only the force-repaired case view, and 2 the unforced view too, a `ph D` written in two cases now kept as written as `phD` is (rules/R5) where master repaired it to 'Ph D'; 339 in all move that view, every one an `MB ChB` or `MB BCh` repaired by its fold's mask as its one-word twin is (`MB Chb` → `MB ChB`, `MB CHB` → `MB ChB`), 228 of them among the reports, and `Ed D` forcing to `ED D` as a bare `Ed` read as a suffix does. (Before the code review's fix (iv) the figure was 357, the case view moving only where the comma's marks reached, and 468 before the second review's.) tse-donors moves 2 (`JUNIOR` typed `JUN IOR`, now the suffix), and its pairs view (`local_corpora.py pairs tse-donors --agreement`) is unchanged; fec-individuals moves 1 (`USN R`, now the suffix run's); nepal-candidates moves none; psc-individuals (1,039,171 records scored) matches the base: 0 fixed, 0 broken, 0 changed, 1,004,782 agreeing on every field. The spike figures in decisions/vocabulary-model's 2026-10-23 entry (352 ctgov, 7 tse, 30 FEC for the first-chunk rule; 391, 34 and 41 with the run start) were a cruder rule's, with no count, no refusal of a particle chunk and no retry after a comma; they were not re-taken. FRAMES (`sys.setprofile` call events over a second parse, py3.11, so one below `_CALL_BASELINE`'s count): `_REFERENCE` (`Dr. Juan0001 de la Vega III`) and `Dr. Juan Q. Xavier de la Vega III`, `John Smith`, `Smith, John` and `John Smith, PhD` cost what they did (304, 363, 148, 157, 201), and so do accented names (`Ngô Bảo Châu` 262) once the review's fix (v) landed. Classify's pair test is C-level up to the call, so a call is made only for a pair that passes the shape, the opening letter's length and the fold's membership: `John Smith Ph. D.` 232 → 233, the 64-link benchmark row unmoved once the length joined the pre-test (`i i` folds to the listed `ii`, and the first version called the predicate for every pair of links, +188). A name holding a member pair pays for its reading: `John Smith Pharm D` 230 → 238, `John Smith, MB ChB` 250 → 266, `Smith, John Pharm D` 253 → 286 (284 before the design-docs review's fix (i), which merges the given part's pair after the joins). Recompute: `uv run python tools/differential/snapshot.py --since ` for the committed snapshot; for each local corpus `snapshot.py --local --base `, and `local_agreement.py psc-individuals --base `; the accepted figure by reading `corpus.jsonl` of ctgov-officials, keeping each name holding a comma whose text after its first comma, stripped, is one of `M. D.`, `M D`, `M. Sc.`, `Pharm D` and `Pharm. D.`, and counting those whose given name opens with that part's first word, on this tree and on the parent. - 2026-10-25 #752, review — SEVEN DEFECTS IN THE FIRST VERSION, found by the PR's code review on a grid of 3,726 texts under the three orders compared against master. (i) A connective beside a pair joined the merged piece into a wider one, which kept its tag and was read whole: `John Smith Pharm D and Mary` read suffix 'Pharm D and Mary' and `John Pharm D and Mary` family 'Pharm D and Mary', where master reads family 'D and Mary'. A pair a connective borders no longer merges, and the two readers of the tag ask for a piece of exactly two words. (ii) Classify's pre-test asked the fold's membership of the Ph./D. pair too, so a configuration without `phd` (`Constants().suffix_acronyms.remove('phd')`) stopped merging it, where v1's fix_phd merged it by its spelling and 1.4.0 and master read suffix 'Ph. D.'; the pair's own pattern now answers ahead of the vocabulary. (iii) C3's count of the part before a family comma merged a member pair, so `MB ChB, PhD` read given 'MB', family 'ChB' where master reads family 'MB ChB'; it reads the part as written now. (iv) Case repair marked a pair's capitals only where the comma's decision bound it, so the same words repaired two ways (`john smith, PHARM D` kept 'PHARM D', `john smith PHARM D` gave 'Pharm D'); render now repairs every suffix pair by its fold's mask (`MB CHB` → 'MB ChB' under 'MBChB'), else in capitals for a listed acronym, and the comma's marks are gone. The force view of `MB BCH` is now 'MB BCh', the mask of 'mbbch', where master kept 'MB BCH' as a caps word. (v) The pre-test let every non-ASCII fold through to the call, a frame per pair of short words in any accented name (`Ngô Bảo Châu` 263 → 269); it looks the composed fold up in C now and costs what master does, and a pair of two suffix words (`MD PhD`) and an opening single letter before a dotted word of two letters (FEC's `A. MS.`) are refused by tags and lengths before the call. (vi) The maiden clause's report read the first word of the piece ending it, so `Jane Doe nee Ed. D.` reported 'Ed.' ending the maiden name; a split pair the clause keeps is read in silence, as everywhere. (vii) The comma form of the given name and initial (`Smith, John Sam E.` → suffix 'Sam E.', where master read middle 'Sam E.') was unrecorded; it is Accepted with the no-comma form, as S5 reads any member ending the given part whose writing says nothing else, and a split pair has no writing to say it. Each fix has a case row or a test that fails on the first version (tests/v2/test_render.py's `test_a_split_credential_repairs_as_its_one_word_twin`, tests/v2/test_facade.py's `test_the_ph_d_pair_merges_whatever_the_vocabulary_says`), run against it before landing. From 1c6564bad2b93347df66324d34cf06b86b58e5eb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:46:18 -0700 Subject: [PATCH 07/11] Report a split credential where the count takes it (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- .../S11-credential-split-across-a-space.md | 12 ++++++---- nameparser/_pipeline/_assign.py | 14 +++++++++++ nameparser/_pipeline/_comma.py | 10 ++++++++ nameparser/_pipeline/_group.py | 6 +++++ nameparser/_pipeline/_pieces.py | 14 ++++++++++- tests/v2/cases.py | 23 +++++++++++++------ tests/v2/rules_budget.json | 2 +- tests/v2/test_cases.py | 8 ++++--- tools/differential/snapshot.jsonl | 22 +++++++++--------- 9 files changed, 83 insertions(+), 28 deletions(-) diff --git a/docs/design/rules/S11-credential-split-across-a-space.md b/docs/design/rules/S11-credential-split-across-a-space.md index 67cba370..4a0d7d9b 100644 --- a/docs/design/rules/S11-credential-split-across-a-space.md +++ b/docs/design/rules/S11-credential-split-across-a-space.md @@ -30,9 +30,11 @@ "John Smith Pharm D Jones" → suffix="Pharm D Jones" "Smith, John Pharm D Jones" → suffix="Pharm D Jones" ``` -- **What reports.** The reading itself reports nothing either way: where the count takes the pair its letters spell a listed credential, and where it declines the pair the two words are a given name and an initial, which no reader hesitates over. +- **What reports.** Where the count takes the pair, the reading is reported, as either reading of a member is (S4), its words being a given name and an initial as often as a credential's chunks; where the count declines it, the two words read as they read unmerged and report what they reported. ``` - "John Smith Pharm D" → ambiguities=() + "John Smith Pharm D" → ambiguities=("suffix-or-name",) + "John Smith, Pharm D" → ambiguities=("suffix-or-name",) + "JOHN SMITH SAM E." → ambiguities=("suffix-or-name",) "SMITH, SAM E." → ambiguities=() · boundary ``` @@ -76,9 +78,9 @@ ``` "Jack PHARM D" → family="D" ``` -- **The given part's report** ("Smith, John Pharm D"): S5 reads the pair as it reads a member ending the given part, and the pair reports nothing there either, which is this rule's silence. +- **The given part's report** ("Smith, John Pharm D"): S5 reads the pair as it reads a member ending the given part, and the pair reports there as this rule says it does. ``` - "Smith, John Pharm D" → ambiguities=() + "Smith, John Pharm D" → ambiguities=("suffix-or-name",) ``` **Accepted:** @@ -88,7 +90,7 @@ "John Smith, M. D." → given="M." "John Smith, M. D." → family="John Smith" ``` -- A given name and its initial whose letters spell a listed credential read as the credential, in silence, behind two name words where no comma divides the name, and ending the given part after a family comma: the count, or the given part's reading, takes them as it takes the credential they spell. +- A given name and its initial whose letters spell a listed credential read as the credential, reported, behind two name words where no comma divides the name, and ending the given part after a family comma: the count, or the given part's reading, takes them as it takes the credential they spell. ``` "JOHN SMITH SAM E." → suffix="SAM E." "Smith, John Sam E." → suffix="Sam E." diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 59b9f5f6..8a2886b0 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -97,6 +97,7 @@ reads_as_a_reported_numeral, reads_as_generation, resolve_script_set, ) from nameparser._pipeline._pieces import ( + split_credential_report, anchor_in_reach, credential_at_the_given_slot, given_slot_anchors, _NOT_A_RUN_START, has_name_content, is_lone_never_given_particle, Peel, TailRead, is_suffix_piece, is_title_piece, leading_titles, @@ -1677,6 +1678,19 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: and not any("vocab:suffix" in tokens[i].tags for i in piece)) else Role.SUFFIX) + # rules/S11: "where the count takes the pair, the reading is reported" -- at + # the end of a name, at the end of the given part, as a run start, + # here where its roles are final (the part a comma's decision binds reports in `_comma`, a + # pair a maiden clause releases in group). A loop over the piece + # tags, so a name with no pair pays no frame for it. + for seg_idx in range(min(tail, len(state.segments))): + for piece, ptags_ in zip(state.pieces[seg_idx], + state.piece_tags[seg_idx]): + if ("split" in ptags_ and len(piece) == 2 + and tokens[piece[0]].role is Role.SUFFIX + and tokens[piece[1]].role is Role.SUFFIX): + ambiguities.append(split_credential_report(piece, tokens)) return copy_with(state, tokens=tuple(tokens), order=order, ambiguities=tuple(ambiguities)) + diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index 694b4464..33da2d19 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -32,6 +32,7 @@ from nameparser._lexicon import Lexicon, _normalize from nameparser._pipeline._pieces import ( + split_credential_report, has_name_content, is_suffix_piece, is_title_piece, join_connectives, listed_lean, own_words, segment_suffix_reading, ) @@ -618,6 +619,7 @@ def decide(state: ParseState, splits: bool | None = True) -> ParseState: and all(p[0] in member_set or is_title_piece(p, pt, tokens) for p, pt in zip(p1[:pairs[0]], pt1))) + part_reported = behind_generation if behind_generation: # rules/C11: "A decision either way at the comma C1 reads is # reported" -- here the suffix word before the comma decided it, @@ -628,6 +630,7 @@ def decide(state: ParseState, splits: bool | None = True) -> ParseState: f"follows {tokens[seg0[-1]].text!r}, so it is read as a " f"credential run; it may be a name", tuple(taken))) elif whole and run and ((listed and not settled) or caps or pair_only): + part_reported = True if len(part) == 1: what = (f"{tokens[part[0]].text!r} after the comma is also an " f"ordinary name word") @@ -668,6 +671,13 @@ def decide(state: ParseState, splits: bool | None = True) -> ParseState: AmbiguityKind.SUFFIX_OR_NAME, f"{text!r} follows a credential, so it reads as part of the " f"suffix run; it may be a name word", tuple(p1[k]))) + # rules/S11: "where the count takes the pair, the reading is reported" + # -- C4's count took it, unless the flip it took part in reported + # the part + if not part_reported: + for k, piece in enumerate(p1): + if "split" in pt1[k] and reading[k]: + ambiguities.append(split_credential_report(piece, tokens)) # Every word of the part is bound above or dropped as a core, so the # segment hands group nothing to join -- emptied here, where the # binding is, as #601's take removes what it binds, rather than diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 7138490c..1b0c1b10 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -67,6 +67,7 @@ from nameparser._lexicon import _run_addresses_by_given from nameparser._pipeline._pieces import ( + split_credential_report, Peel, TailRead, chain_lead, chain_run_end, is_conj_piece, is_leading_title, is_prefix_piece, is_suffix_piece, is_title_piece, join_connectives, joined_tags, leading_titles, merge_particle_runs, merge_pieces, peel_walk, @@ -328,6 +329,11 @@ def _maiden_take(pieces: Sequence[Sequence[int]], end = c + (e - m) released = [(j, Role.TITLE if m + (j - c) in titles else Role.SUFFIX) for j in range(end, len(pieces))] + # rules/S11: "where the count takes the pair, the reading is reported" -- + # the clause-free reading took it, and the take binds it here + for j, role in released: + if "split" in ptags[j] and role is Role.SUFFIX: + ambiguities.append(split_credential_report(pieces[j], tokens)) # mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE: "Emit at the # site that takes the branch, not where an ambiguous tag sits" # -- the take consumed the run, so the forks the reading called diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 0b41e172..5dabd3d8 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -59,7 +59,7 @@ from nameparser._pipeline._state import ( AMBIGUOUS_ACRONYM_TAG, NAME_ROLES, SHAPE_ACRONYM_TAG, SUFFIX_OR_UNREAD, - WorkToken, + AmbiguityKind, PendingAmbiguity, WorkToken, ) from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, _ROMAN, Lean, ambiguous_lean, in_initialless_script, @@ -529,6 +529,18 @@ def starts_a_credential_run(piece: Sequence[int], ptags: Set[str], "vocab:bound-given"}) +def split_credential_report(piece: Sequence[int], + tokens: Sequence[WorkToken]) -> PendingAmbiguity: + """rules/S11's report on a split credential the count took, one + spelling for the three sites that take one (assign's placement, the + comma's binding, the maiden clause's release).""" + text = " ".join(tokens[i].text for i in piece) + return PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{text!r} is two words whose letters spell a credential, so it " + f"reads as a credential; they may be name words", tuple(piece)) + + def is_lone_never_given_particle(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: """A piece that is one never-given particle (rules/P1's fold diff --git a/tests/v2/cases.py b/tests/v2/cases.py index e85eeb41..68910e5f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1520,10 +1520,12 @@ def _check_cjk_shape_purity(self) -> None: "John Smith Pharm. D.", {"given": "John", "family": "Smith", "suffix": "Pharm. D."}, classification="fix(#752)", + ambiguities=("suffix-or-name",), notes="S3: 'Pharm. D.' folds to the listed 'pharmd', " "and neither half is suffix vocabulary on its own, so " "the two words are one credential, read with the words " - "to spare a bare member needs (S4) and in silence. " + "to spare a bare member needs (S4) and reported as a " + "member is. " "1.4.0 read middle 'Smith Pharm.', last 'D.'"), Case("a_split_credential_with_no_words_to_spare_is_two_words", "John Pharm D", @@ -1536,6 +1538,7 @@ def _check_cjk_shape_purity(self) -> None: "John Smith, Pharm D", {"given": "John", "family": "Smith", "suffix": "Pharm D"}, classification="fix(#752)", + ambiguities=("suffix-or-name",), notes="C4's count: two name words before the comma make " "the part the credential run. 1.4.0 read first 'Pharm', " "middle 'D', last 'John Smith'"), @@ -1563,6 +1566,7 @@ def _check_cjk_shape_purity(self) -> None: "Smith, John Pharm D", {"given": "John", "family": "Smith", "suffix": "Pharm D"}, classification="fix(#752)", + ambiguities=("suffix-or-name",), notes="S5: a member ending the given part after a " "family comma reads as the credential where its writing " "says nothing else, and a split one says nothing. 1.4.0 " @@ -1571,7 +1575,7 @@ def _check_cjk_shape_purity(self) -> None: "Smith, John Pharm D Jones", {"given": "John", "family": "Smith", "suffix": "Pharm D Jones"}, classification="fix(#752)", - ambiguities=("suffix-or-name",), + ambiguities=("suffix-or-name", "suffix-or-name"), notes="S8: the merged pair starts the run after the " "given word, as 'PharmD' does, and the name word it " "absorbs is reported. 1.4.0 read middle 'Pharm D Jones'"), @@ -1579,7 +1583,7 @@ def _check_cjk_shape_purity(self) -> None: "John Smith Pharm D Jones", {"given": "John", "family": "Smith", "suffix": "Pharm D Jones"}, classification="fix(#752)", - ambiguities=("suffix-or-name",), + ambiguities=("suffix-or-name", "suffix-or-name"), notes="S8: two name words in front, the words to spare " "S4 asks of the pair, start the run there. 1.4.0 read " "middle 'Smith Pharm D', last 'Jones'"), @@ -1596,17 +1600,20 @@ def _check_cjk_shape_purity(self) -> None: {"given": "Jane", "family": "Doe", "maiden": "Smith", "suffix": "Pharm D"}, classification="fix(#752)", + ambiguities=("suffix-or-name",), notes="M2: the clause ends at the trailing run the " "clause-free name reads, which takes the pair as " "'Jane Doe Pharm D' does. 1.4.0 read middle 'Doe nee " "Smith Pharm', last 'D'"), - Case("a_split_credential_reports_no_absorbed_word", + Case("a_split_credential_reports_itself_not_an_absorbed_word", "John Smith MB ChB", {"given": "John", "family": "Smith", "suffix": "MB ChB"}, classification="fix(#752)", + ambiguities=("suffix-or-name",), notes="'MB ChB' is one listed credential, 'mbchb', where 'mb' " "started the run and reported 'ChB' as a name word it " - "absorbed. 1.4.0 read middle 'Smith MB', last 'ChB'"), + "absorbed; the pair reports itself instead, as a member " + "the count took. 1.4.0 read middle 'Smith MB', last 'ChB'"), Case("a_split_credential_beside_a_connective_stays_two_words", "John Smith Pharm D and Mary", {"given": "John", "middle": "Smith Pharm", "family": "D and Mary"}, @@ -1628,7 +1635,8 @@ def _check_cjk_shape_purity(self) -> None: "Jane Doe nee Ed. D.", {"given": "Jane", "family": "Doe", "maiden": "Ed. D."}, classification="fix(#752)", - notes="S11: the clause keeps the pair, read in silence; its " + notes="S11: the clause keeps the pair, which a declined pair " + "reports nothing for; its " "first word is a listed member, and the first version " "reported 'Ed.' as the word ending the clause, which " "'D.' is. 1.4.0 read middle 'Doe nee Ed.', last 'D.'"), @@ -1636,7 +1644,7 @@ def _check_cjk_shape_purity(self) -> None: "Smith, John van Pharm D", {"given": "John", "family": "van Smith", "suffix": "Pharm D"}, classification="fix(#752)", - ambiguities=("particle-or-given",), + ambiguities=("particle-or-given", "suffix-or-name"), notes="S11: merged ahead of group's joins, the pair stops the " "chain as 'PharmD' does, and P6 attaches the particle; " "merged after them (the design-docs review's first fix), " @@ -1646,6 +1654,7 @@ def _check_cjk_shape_purity(self) -> None: "Rahman, Abdul MB ChB", {"given": "Abdul", "family": "Rahman", "suffix": "MB ChB"}, classification="fix(#752)", + ambiguities=("suffix-or-name",), notes="P5: the join declines the pair as it declines a suffix " "word; the first version joined it, given 'Abdul MB ChB', " "where master read suffix 'MB ChB'. 1.4.0 read first " diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index 1adf4146..ebdfc1be 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -62,7 +62,7 @@ "S8": {"words": 418, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 0, "accepted_words": 0}, "S9": {"words": 463, "then_words": 262, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 3, "accepted_words": 163}, - "S11": {"words": 634, "then_words": 331, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 94}, + "S11": {"words": 644, "then_words": 342, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 93}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index a7e4dcee..05d7e6bc 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -244,9 +244,11 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # and 'John Smith Pharm D Jones', one suffix-or-name claim under # each order; 1457 -> 1460 the same day: the second review's # 'Smith, John van Pharm D' row, one particle-or-given claim under - # each order. - assert claims == 1460, ( - f"the field sweep checks {claims} claims, recorded as 1460 on " + # each order; 1460 -> 1487 the same day: the split credential's own + # report where the count takes it (Derek, on #891's review), nine + # rows claiming one suffix-or-name report under each order. + assert claims == 1487, ( + f"the field sweep checks {claims} claims, recorded as 1487 on " f"2026-10-25") diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index 45b40c2e..ff796977 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -1338,7 +1338,7 @@ {"name": "JOHN SMITH MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} {"name": "JOHN SMITH NEE X X", "given": "JOHN", "family": "SMITH", "maiden": "X X", "initials": "J. S.", "capitalized": "John Smith (X X)"} {"name": "JOHN SMITH PH.D.", "given": "JOHN", "family": "SMITH", "suffix": "PH.D.", "initials": "J. S.", "capitalized": "John Smith Ph.D."} -{"name": "JOHN SMITH SAM E.", "given": "JOHN", "family": "SMITH", "suffix": "SAM E.", "initials": "J. S.", "capitalized": "John Smith SAM E."} +{"name": "JOHN SMITH SAM E.", "given": "JOHN", "family": "SMITH", "suffix": "SAM E.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith SAM E."} {"name": "JOHN SMITH VI", "given": "JOHN", "family": "SMITH", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith VI"} {"name": "JOHN SMITH, FNP-BC", "given": "JOHN", "family": "SMITH", "suffix": "FNP-BC", "initials": "J. S.", "capitalized": "John Smith FNP-BC"} {"name": "JOHN SMITH, MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} @@ -1641,7 +1641,7 @@ {"name": "John Smith M. D.", "given": "John", "middle": "Smith M.", "family": "D.", "initials": "J. S. M. D."} {"name": "John Smith M.A.", "given": "John", "family": "Smith", "suffix": "M.A.", "initials": "J. S."} {"name": "John Smith MA", "given": "John", "family": "Smith", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John Smith MB ChB", "given": "John", "family": "Smith", "suffix": "MB ChB", "initials": "J. S."} +{"name": "John Smith MB ChB", "given": "John", "family": "Smith", "suffix": "MB ChB", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith MC", "given": "John", "family": "Smith", "suffix": "MC", "initials": "J. S."} {"name": "John Smith MD PhD", "given": "John", "family": "Smith", "suffix": "MD PhD", "initials": "J. S."} {"name": "John Smith MD-Jones", "given": "John", "middle": "Smith", "family": "MD-Jones", "initials": "J. S. M.", "capitalized_force": "John Smith Md-Jones"} @@ -1661,14 +1661,14 @@ {"name": "John Smith PhD MEng", "given": "John", "family": "Smith", "suffix": "PhD MEng", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith PhD MENG"} {"name": "John Smith PhD Ma", "given": "John", "family": "Smith", "suffix": "PhD Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith PhD MA"} {"name": "John Smith PhD Prof. Ma", "title": "Prof.", "given": "John", "family": "Smith", "suffix": "PhD Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "Prof. John Smith PhD MA"} -{"name": "John Smith Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} +{"name": "John Smith Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith Pharm D Jones", "given": "John", "family": "Smith", "suffix": "Pharm D Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith Pharm D and Mary", "given": "John", "middle": "Smith Pharm", "family": "D and Mary", "initials": "J. S. P. D. M."} -{"name": "John Smith Pharm. D.", "given": "John", "family": "Smith", "suffix": "Pharm. D.", "initials": "J. S."} +{"name": "John Smith Pharm. D.", "given": "John", "family": "Smith", "suffix": "Pharm. D.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Prof. Dr.", "title": "Prof. Dr.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Prof. Jr.", "title": "Prof.", "given": "John", "family": "Smith", "suffix": "Jr.", "initials": "J. S."} -{"name": "John Smith Psy. D.", "given": "John", "family": "Smith", "suffix": "Psy. D.", "initials": "J. S."} +{"name": "John Smith Psy. D.", "given": "John", "family": "Smith", "suffix": "Psy. D.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith Q.W.E.R.T.", "given": "John", "family": "Smith", "suffix": "Q.W.E.R.T.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith R.A.I.", "given": "John", "family": "Smith", "suffix": "R.A.I.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith RAI", "given": "John", "middle": "Smith", "family": "RAI", "initials": "J. S. R.", "capitalized_force": "John Smith Rai"} @@ -1759,7 +1759,7 @@ {"name": "John Smith, PhD XYZ", "given": "John", "family": "Smith", "suffix": "PhD XYZ", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, PhD vd DO", "given": "John", "family": "Smith", "suffix": "PhD vd DO", "initials": "J. S.", "capitalized_force": "John Smith PhD VD DO"} {"name": "John Smith, PhD-Md", "given": "John", "family": "Smith", "suffix": "PhD-Md", "initials": "J. S.", "capitalized_force": "John Smith PhD-MD"} -{"name": "John Smith, Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} +{"name": "John Smith, Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, Phd-Md", "given": "Phd-Md", "family": "John Smith", "initials": "P. J. S.", "capitalized_force": "PhD-Md John Smith"} {"name": "John Smith, Pu-Ph", "given": "Pu-Ph", "family": "John Smith", "initials": "P. J. S."} {"name": "John Smith, RA Jones", "title": "RA", "given": "Jones", "family": "John Smith", "initials": "J. J. S.", "capitalized_force": "Ra Jones John Smith"} @@ -2765,7 +2765,7 @@ {"name": "Rafael Sousa dos Anjos", "given": "Rafael", "middle": "Sousa", "family": "dos Anjos", "initials": "R. S. A."} {"name": "Rafael Viciana Aráez", "given": "Rafael", "middle": "Viciana", "family": "Aráez", "initials": "R. V. A."} {"name": "Ragnar Bjarnason", "given": "Ragnar", "family": "Bjarnason", "initials": "R. B."} -{"name": "Rahman, Abdul MB ChB", "given": "Abdul", "family": "Rahman", "suffix": "MB ChB", "initials": "A. R."} +{"name": "Rahman, Abdul MB ChB", "given": "Abdul", "family": "Rahman", "suffix": "MB ChB", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "A. R."} {"name": "Ram Bahadur B.K.", "given": "Ram", "middle": "Bahadur", "family": "B.K.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "R. B. B."} {"name": "Ram Bahadur B.K., PhD", "given": "Ram", "middle": "Bahadur", "family": "B.K.", "suffix": "PhD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "R. B. B."} {"name": "Ram Bahadur PhD B.K.", "given": "Ram", "family": "Bahadur", "suffix": "PhD B.K.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "R. B."} @@ -3066,10 +3066,10 @@ {"name": "Smith, John PhD Jones", "given": "John", "family": "Smith", "suffix": "PhD Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John PhD de Jr.", "given": "John", "family": "de Smith", "suffix": "PhD, Jr.", "initials": "J. S."} {"name": "Smith, John PhD É, Jr.", "given": "John", "family": "Smith", "suffix": "PhD É, Jr.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "Smith, John Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "initials": "J. S."} +{"name": "Smith, John Pharm D", "given": "John", "family": "Smith", "suffix": "Pharm D", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John Pharm D Jones", "given": "John", "family": "Smith", "suffix": "Pharm D Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} -{"name": "Smith, John Sam E.", "given": "John", "family": "Smith", "suffix": "Sam E.", "initials": "J. S.", "capitalized_force": "John Smith SAM E."} +{"name": "Smith, John Sam E.", "given": "John", "family": "Smith", "suffix": "Sam E.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith SAM E."} {"name": "Smith, John V", "given": "John", "family": "Smith", "suffix": "V", "initials": "J. S."} {"name": "Smith, John V MD", "given": "John", "middle": "V", "family": "Smith", "suffix": "MD", "initials": "J. V. S."} {"name": "Smith, John V, Jr.", "given": "John", "middle": "V", "family": "Smith", "suffix": "Jr.", "initials": "J. V. S."} @@ -3666,7 +3666,7 @@ {"name": "john smith pa-c", "given": "john", "family": "smith", "suffix": "pa-c", "initials": "j. s.", "capitalized": "John Smith PA-C"} {"name": "john smith ph. d.", "given": "john", "family": "smith", "suffix": "ph. d.", "initials": "j. s.", "capitalized": "John Smith Ph. D."} {"name": "john smith ph.d.", "given": "john", "family": "smith", "suffix": "ph.d.", "initials": "j. s.", "capitalized": "John Smith Ph.D."} -{"name": "john smith pharm d", "given": "john", "family": "smith", "suffix": "pharm d", "initials": "j. s.", "capitalized": "John Smith Pharm D"} +{"name": "john smith pharm d", "given": "john", "family": "smith", "suffix": "pharm d", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith Pharm D"} {"name": "john smith phd", "given": "john", "family": "smith", "suffix": "phd", "initials": "j. s.", "capitalized": "John Smith PhD"} {"name": "john smith psyd", "given": "john", "family": "smith", "suffix": "psyd", "initials": "j. s.", "capitalized": "John Smith PsyD"} {"name": "john smith v", "given": "john", "family": "smith", "suffix": "v", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith V"} @@ -3674,7 +3674,7 @@ {"name": "john smith vi", "given": "john", "family": "smith", "suffix": "vi", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith VI"} {"name": "john smith x.y.z.", "given": "john", "family": "smith", "suffix": "x.y.z.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith X.Y.Z."} {"name": "john smith xi", "given": "john", "middle": "smith", "family": "xi", "initials": "j. s. x.", "capitalized": "John Smith Xi"} -{"name": "john smith, PHARM D", "given": "john", "family": "smith", "suffix": "PHARM D", "initials": "j. s.", "capitalized": "John Smith Pharm D"} +{"name": "john smith, PHARM D", "given": "john", "family": "smith", "suffix": "PHARM D", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith Pharm D"} {"name": "john smith, ma", "given": "john", "family": "smith", "suffix": "ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith MA"} {"name": "john smith, md ma", "given": "john", "family": "smith", "suffix": "md ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s.", "capitalized": "John Smith MD MA"} {"name": "john smith, ms-pt", "given": "john", "family": "smith", "suffix": "ms-pt", "initials": "j. s.", "capitalized": "John Smith MS-PT"} From d1399dfd0d3e63c559b21d78c06114463c6a4059 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:50:53 -0700 Subject: [PATCH 08/11] Record the split credential's report (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- .../decisions/S11-credential-split-across-a-space.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md index 9e9b98c8..88e427ec 100644 --- a/docs/design/decisions/S11-credential-split-across-a-space.md +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -7,18 +7,18 @@ Examples name each field as the default order reads it; under the family-first o - Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class, unless a connective borders the pair (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de D Junior` → family 'de D'; `John Smith Pharm D and Mary` → family 'D and Mary') — 2026-10-25 #752. - The count decides it as it decides a bare member, and where the count declines it the pair is the two words it is written as: at the end of a name by S4's words to spare (`John Pharm D` → middle 'Pharm', family 'D'), after a comma by C4's count (`John Smith, Pharm D` → suffix; `Smith, Pharm D` → given 'Pharm'), ending the given part after a family comma by S5's reading (`Smith, John Pharm D` → suffix); the part before a family comma merges none (`Smith Pharm D, John` → family 'Smith Pharm D') — 2026-10-25 #752. - Behind two name words with no comma, or after the given word after a family comma, the pair starts S8's run, the one member of the class that does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones', reported) — 2026-10-25 #752. -- The reading reports nothing either way (`John Smith MB ChB` → suffix 'MB ChB', where `mb` started the run and reported 'ChB'; `SMITH, SAM E.` → given 'SAM', middle 'E.') — 2026-10-25 #752. +- Where the count takes the pair the reading reports `suffix-or-name`, at the end of a name, after a comma, at the end of the given part after a family comma, as a run start and where a maiden clause releases it (`John Smith Pharm D` and `John Smith MB ChB` → suffix, reported, `MB ChB` no longer reporting 'ChB' as a word `mb`'s run absorbed); where the count declines it the two words report what they reported unmerged (`SMITH, SAM E.` → given 'SAM', middle 'E.', no report) — 2026-10-25 #752 (Derek), the report. - The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, merged by its spelling whatever the vocabulary lists, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. - Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, not in one that writes no abbreviations, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.'; `John Smith 博. 士.` → family '士.') — 2026-10-25 #752; 2026-10-25 #752, review. - The Ph./D. pair starts S8's run from where it stands, though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, #827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` → family 'V', reported) — 2026-10-25 #752. -- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial read as the credential their letters spell behind two name words with no comma and ending the given part after a family comma (`JOHN SMITH SAM E.` and `Smith, John Sam E.` → suffix), and case repair then writes them as the acronym (`JOHN SMITH SAM E.` → 'John Smith SAM E.') — 2026-10-25 #752; 2026-10-25 #752, second review. +- Accepted: a pair opening with a single letter stays two words, so `John Smith, M. D.`, `M D` and `M. Sc.` keep the reversed reading; and a given name and its initial read as the credential their letters spell behind two name words with no comma and ending the given part after a family comma (`JOHN SMITH SAM E.` and `Smith, John Sam E.` → suffix, reported since the report entry below), and case repair then writes them as the acronym (`JOHN SMITH SAM E.` → 'John Smith SAM E.') — 2026-10-25 #752; 2026-10-25 #752, second review. - Case repair writes a pair group merged by its fold's mask, as the one-word spelling is written, on every comma path (`john smith, PHARM D` → 'Pharm D', as `PHARMD` → 'PharmD'), and unforced keeps a pair written in two cases as written, as the one-word spelling is kept (`john smith pharm D`, as `pharmD`) — 2026-10-25 #752, review; 2026-10-25 #752, second review. Open: none in this file. ## Log -- 2026-10-25 #752 — A CREDENTIAL WRITTEN SPLIT ACROSS A SPACE IS ONE WORD, A MEMBER OF THE AMBIGUOUS CLASS THAT THE COUNT DECIDES (phase 2e: item 6 of decisions/vocabulary-model's 2026-10-23 #752 entry, agreed by Derek on 2026-10-23, and #827's reading (A)). `_vocab.split_pair` was the Ph./D. test alone (decisions/phd-merge); it now answers any pair with the piece tag it merges with, "suffix" for Ph./D. and "split" for the rest, and classify asks it once per adjacent pair of a segment, writing the answer on the pair's first word, so group's merge and the comma's reading of its part read a tag rather than each asking the vocabulary. +- 2026-10-25 #752 (its SILENT paragraph superseded by the 2026-10-25 #752 (Derek), the report entry below) — A CREDENTIAL WRITTEN SPLIT ACROSS A SPACE IS ONE WORD, A MEMBER OF THE AMBIGUOUS CLASS THAT THE COUNT DECIDES (phase 2e: item 6 of decisions/vocabulary-model's 2026-10-23 #752 entry, agreed by Derek on 2026-10-23, and #827's reading (A)). `_vocab.split_pair` was the Ph./D. test alone (decisions/phd-merge); it now answers any pair with the piece tag it merges with, "suffix" for Ph./D. and "split" for the rest, and classify asks it once per adjacent pair of a segment, writing the answer on the pair's first word, so group's merge and the comma's reading of its part read a tag rather than each asking the vocabulary. THE PAIR. Each word is one to five letters with or without its period, the two together spell unambiguous suffix vocabulary with the periods removed, and they are not both suffix vocabulary on their own (`MB ChB` merges, `chb` being unlisted; `MD PhD` stays two credentials). The first word has two or more letters: FEC's donors write a middle initial before a trailing salutation (`SMITH, JOHN G. MR.`), which folds to a listed credential. Measured on this change with the length test removed from the predicate and its pre-test (fec-individuals fetched 2026-10-18, against this tree), 4,441 FEC names would move, `G. MR.` reading as `gmr` in 2,577 and `C. MS.` as `cms` in 1,133, against 206 ctgov names it would put right (`M. D.` 128, `M. Sc.` 28). (`A. MS.`, the 2026-10-23 entry's example, folds to `ams`, an ambiguous credential, which the fold test refuses whatever the opening letter.) Two dotted single letters are the one opening letter asked further, by #872's `is_cased_paired_initials` rather than a copy of it: in a script with capitals they are paired initials, the reading rules/S3 gives them joined, so only a caseless script's pair merges, and only where its letters spell a listed word, which no default entry does (the overlap the 2026-10-25 one-predicate entry in decisions/vocabulary-model left to this change to state). Two refusals the measurement below added: a particle or a connective written without its period is no chunk, since it joins name words (`DE D`, a particle and a surname written short, folds to the listed `ded` in a tse-donors name behind `JUNIOR`, the case table's witness being `Jane Doe de D Junior`), while a letter written with its period is an initial, not the connective it spells (`SAM E.` stays a pair, which the count then declines); and where two pairs overlap the Ph./D. pair merges (`Dr. Ph. D.`, whose `Dr. Ph.` folds to the listed `drph`, the case table's `the_split_degree_outranks_a_pair_overlapping_it`). WHERE IT IS READ. As a member, by the count: in a main segment group merges the pair and S2's trailing read decides it, the words to spare S4 asks of a bare member, and a pair the read did not take is split back into its two words before anything else reads the segment, so a declined pair reads exactly as the two words read unmerged; in the part after a comma `_comma.decide` reads the part with the pair merged, C4 counting the name words before the comma, and where that reading does not take the pair as a credential reads the part again as written, so an `MB` the pair hid opens the part again (C13) and `John Smith, MB ChB BAO` keeps the suffix reading it had (the first version left the pair merged there, which made `MB` the given name in 15 ctgov names); in the given part after a family comma group merges the pair only behind the given word, ahead of its joins, so the particle chain stops in front of it and P5's bound join declines it as it declines a suffix word (the second review below), and S5's walk reads it as it reads a member ending the part, its writing saying nothing else. The part before a family comma reads no member as a suffix (`Smith MA, John`), so no pair merges there. The merged pair has no case of its own to lean on, so it reads as a member admitted by shape does (`St van Berg Pharm D` keeps family 'van Berg Pharm D', as `St van Berg X.Y.Z.` does). THE RUN (S8). S8 refuses a member of the ambiguous class as a run start, because a member is a word of the name in its other reading and nothing in front of it says otherwise. A split credential's other reading is two words of the name, a given name and an initial, and the run starts only where two name words already stand in front of it, the words to spare its own count asks: so it is the one member that starts the run, behind two name words with no comma and after the given word after a family comma, as its one-word spelling does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones'). The Ph./D. pair, which the walk leaves out because assign reads it as a suffix wherever it stands, starts the run from where it stands, in front of the first word behind it (#827, reading (A): `John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, as `John Smith PhD Jones` reads; `John Smith Ph. D. MEng` → suffix 'Ph. D. MEng', the reading 2.3.0 gave). And a one-letter numeral with the pair behind it is the name word it is before `PhD`, Derek's 2026-10-25 call on #886 for the one-word spelling (`John Smith V Ph. D.` → family 'V', reported; 2.3.0 through 2.6.0 read suffix 'V Ph. D.' (1.4.0 suffix 'Ph. D., V', the V a suffix too), which rules/S10's Accepted clause had stated). Reading the numeral as the generation before the split spelling alone was the other option; it would keep the two spellings apart for one numeral shape, and no name in the five local corpora measured below moved by it, so the committed rules example is the only reading the choice moves. @@ -30,3 +30,4 @@ Open: none in this file. - 2026-10-25 #752, review — SEVEN DEFECTS IN THE FIRST VERSION, found by the PR's code review on a grid of 3,726 texts under the three orders compared against master. (i) A connective beside a pair joined the merged piece into a wider one, which kept its tag and was read whole: `John Smith Pharm D and Mary` read suffix 'Pharm D and Mary' and `John Pharm D and Mary` family 'Pharm D and Mary', where master reads family 'D and Mary'. A pair a connective borders no longer merges, and the two readers of the tag ask for a piece of exactly two words. (ii) Classify's pre-test asked the fold's membership of the Ph./D. pair too, so a configuration without `phd` (`Constants().suffix_acronyms.remove('phd')`) stopped merging it, where v1's fix_phd merged it by its spelling and 1.4.0 and master read suffix 'Ph. D.'; the pair's own pattern now answers ahead of the vocabulary. (iii) C3's count of the part before a family comma merged a member pair, so `MB ChB, PhD` read given 'MB', family 'ChB' where master reads family 'MB ChB'; it reads the part as written now. (iv) Case repair marked a pair's capitals only where the comma's decision bound it, so the same words repaired two ways (`john smith, PHARM D` kept 'PHARM D', `john smith PHARM D` gave 'Pharm D'); render now repairs every suffix pair by its fold's mask (`MB CHB` → 'MB ChB' under 'MBChB'), else in capitals for a listed acronym, and the comma's marks are gone. The force view of `MB BCH` is now 'MB BCh', the mask of 'mbbch', where master kept 'MB BCH' as a caps word. (v) The pre-test let every non-ASCII fold through to the call, a frame per pair of short words in any accented name (`Ngô Bảo Châu` 263 → 269); it looks the composed fold up in C now and costs what master does, and a pair of two suffix words (`MD PhD`) and an opening single letter before a dotted word of two letters (FEC's `A. MS.`) are refused by tags and lengths before the call. (vi) The maiden clause's report read the first word of the piece ending it, so `Jane Doe nee Ed. D.` reported 'Ed.' ending the maiden name; a split pair the clause keeps is read in silence, as everywhere. (vii) The comma form of the given name and initial (`Smith, John Sam E.` → suffix 'Sam E.', where master read middle 'Sam E.') was unrecorded; it is Accepted with the no-comma form, as S5 reads any member ending the given part whose writing says nothing else, and a split pair has no writing to say it. Each fix has a case row or a test that fails on the first version (tests/v2/test_render.py's `test_a_split_credential_repairs_as_its_one_word_twin`, tests/v2/test_facade.py's `test_the_ph_d_pair_merges_whatever_the_vocabulary_says`), run against it before landing. - 2026-10-25 #752, design-docs review (its fix (i), the merge after the joins, superseded by the 2026-10-25 #752, second review entry below) — TWO MORE CODE DEFECTS AND THE RECORD'S CORRECTIONS. (i) In the given part after a family comma group merged the pair ahead of its joins, so a bound given name took it: `Rahman, Abdul MB ChB` read given 'Abdul MB ChB' where master reads given 'Abdul', suffix 'MB ChB', against rules/P5's "The join never absorbs" a suffix word. The pair is merged there only after the joins now (`_group._merge_given_pairs`), so a join that took one of its words leaves it as written (`Smith, abdul Pharm D Jones` reads as master, given 'abdul Pharm'). (ii) Two dotted single letters in a script that writes no abbreviations merged where their letters are listed: `John Smith 博. 士.` read suffix '博. 士.', `김 민준 박. 사.` suffix '박. 사.', 18 default honorifics and titles of two characters folding so. rules/S3 refuses that script's periods, and so does the pair now, which makes true the record's "no default entry". (iii) The record's corrections: the particle refusal's case is `DE D` (`ded`), not `DE A`, whose fold is unlisted, so the rule's example witnessed nothing; the opening-letter test's evidence is the measurement in THE PAIR above, not `A. MS.`; rules/S11 states its precedence over S6's case and S5's report, and rules/S8 its own over a split pair as a Precedence bullet rather than a clause of its Then; decisions/C13, decisions/phd-merge and decisions/S8 are marked where their statements moved, and mechanisms.md's ONE-PREDICATE-PER-QUESTION names the predicate's callers as they are. - 2026-10-25 #752, second review — FOUR DEFECTS IN THE FIXES, found by a review of the two fix commits. SUPERSEDES: the design-docs review entry's fix (i). (i) Merging the given part's pair after group's joins let the particle chain take its first word, since the chain then saw `Pharm` and `D` apart and its bound stopped behind them: `Smith, John van Pharm D` read middle 'van Pharm D' where `Smith, John van PharmD` reads family 'van Smith', suffix 'PharmD', and the bound join took `Pharm` the same way (`Smith, abdul Pharm D` → given 'abdul Pharm'). The pair is merged ahead of the joins again, the chain stopping in front of it, and P5's bound join declines it as it declines a suffix word (rules/P5's Except), which keeps `Rahman, Abdul MB ChB` reading suffix 'MB ChB'; the widened fix had narrowed the chain's stop, the shape AGENTS.md's review paragraph names. (ii) Classify's refusal of two suffix words stood in front of the Ph./D. answer, so a lexicon listing `ph` and `d` stopped the merge (`John Ph. D. Smith` read middle 'Ph. D.' with `Lexicon.default().add(suffix_words={"ph", "d"})`), against "merged by its spelling whatever the vocabulary lists"; it is asked of the other pairs only. (iii) Case repair recased every tagged pair, so of two overlapping pairs (`MB CHB` and `CHB A`, `chba` being listed) the second wrote over the first (`JOHN SMITH MB CHB A` → 'MB CHB A'); it reads only the pair group merged, its second word carrying the merge's `joined` mark. (iv) Unforced, case repair asked each word of a pair whether it is in one case, where the one-word spelling is asked whole, so `john smith pharm D` gave 'Pharm D' while `john smith pharmD` keeps 'pharmD'; the pair is asked whole now and kept as written (rules/R5). And the record: case repair costs no frame on a name holding no pair (`capitalized(force=True)` of `John Smith` 59, as on master; the first fix called the function on every repair, +1), and the given name and initial the count takes (`JOHN SMITH SAM E.`) repair as the acronym their letters spell, recorded with that Accepted reading. Each fix has a case row or a test that fails on the commit it fixes. +- 2026-10-25 #752 (Derek), the report — A SPLIT CREDENTIAL THE COUNT TAKES IS REPORTED, AS EITHER READING OF A MEMBER IS. SUPERSEDES: the first entry's SILENT paragraph. Derek, on #891's review: the merged pair is a member of the ambiguous class, which reports either reading (`John Smith MA` reports `suffix-or-name`), and with no report a given name and its initial that spell a credential become one where no caller can find them (`John Smith Ed D`, `Smith, Mary Ed D.`, `JOHN SMITH SAM E.`). So wherever the count takes the pair it reports `suffix-or-name` with one wording ("reads as a credential", a phrasing tests/v2/test_cases.py's field sweep already checks against the field the pair lands in): at assign's placement for the end of a name, the end of the given part and a run start, at `_comma.decide`'s binding for C4's count unless the flip it took part in already reports the part, and at the maiden take's release. A pair the count declines reads as its two words did unmerged and reports what they did, which on master is nothing. The cost is a report on the pairs nobody hesitates over (`John Smith Pharm D`), which the class's report carries for `John Smith MEng` too. Measured 2026-10-25 against master 225f3d6f with `--base`, as the first entry: 353 ctgov-officials readings carry the report, 119 of them where master reported no `suffix-or-name` at all, the rest the `MB ChB` and run names whose absorbed-word report it replaces (so the 229 names that dropped a report in the first entry keep the kind, and ctgov moves 469 entries in all: 120 in the ambiguity kinds, each with a field move, 339 only in the force-repaired case view, and the rest in fields or case views without a kind move); the committed snapshot's rules examples gain it where the count takes a pair; tse-donors, fec-individuals, nepal-candidates and psc-individuals read as in the first entry (2, 1, none, matching the base). The field sweep's count goes from 1460 to 1487, nine case rows claiming the report under each order. From 77966a61364e3b543358d154a1f8148a6b76f4f6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 13:59:58 -0700 Subject: [PATCH 09/11] Report a split credential behind an opening credential once (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- .../S11-credential-split-across-a-space.md | 5 +++-- .../rules/S11-credential-split-across-a-space.md | 2 +- nameparser/_pipeline/_assign.py | 1 - nameparser/_pipeline/_comma.py | 3 ++- nameparser/_pipeline/_group.py | 5 +++-- tests/v2/cases.py | 14 ++++++++++++-- tests/v2/rules_budget.json | 2 +- tests/v2/test_cases.py | 8 +++++--- 8 files changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/design/decisions/S11-credential-split-across-a-space.md b/docs/design/decisions/S11-credential-split-across-a-space.md index 88e427ec..5db27468 100644 --- a/docs/design/decisions/S11-credential-split-across-a-space.md +++ b/docs/design/decisions/S11-credential-split-across-a-space.md @@ -7,7 +7,7 @@ Examples name each field as the default order reads it; under the family-first o - Two adjacent words of one to five letters each, the first of two or more, whose period-free letters spell unambiguous suffix vocabulary while the two are not both suffix vocabulary on their own, and neither a particle or a connective written without its period, are one credential, a member of the ambiguous class, unless a connective borders the pair (`John Smith Pharm D` → suffix 'Pharm D'; `John Smith M. Sc.` → family 'Sc.'; `Jane Doe de D Junior` → family 'de D'; `John Smith Pharm D and Mary` → family 'D and Mary') — 2026-10-25 #752. - The count decides it as it decides a bare member, and where the count declines it the pair is the two words it is written as: at the end of a name by S4's words to spare (`John Pharm D` → middle 'Pharm', family 'D'), after a comma by C4's count (`John Smith, Pharm D` → suffix; `Smith, Pharm D` → given 'Pharm'), ending the given part after a family comma by S5's reading (`Smith, John Pharm D` → suffix); the part before a family comma merges none (`Smith Pharm D, John` → family 'Smith Pharm D') — 2026-10-25 #752. - Behind two name words with no comma, or after the given word after a family comma, the pair starts S8's run, the one member of the class that does (`John Smith Pharm D Jones` → suffix 'Pharm D Jones', reported) — 2026-10-25 #752. -- Where the count takes the pair the reading reports `suffix-or-name`, at the end of a name, after a comma, at the end of the given part after a family comma, as a run start and where a maiden clause releases it (`John Smith Pharm D` and `John Smith MB ChB` → suffix, reported, `MB ChB` no longer reporting 'ChB' as a word `mb`'s run absorbed); where the count declines it the two words report what they reported unmerged (`SMITH, SAM E.` → given 'SAM', middle 'E.', no report) — 2026-10-25 #752 (Derek), the report. +- Where the count takes the pair the reading reports `suffix-or-name`, at the end of a name, after a comma (but where the part already reports as a whole, and in a part past the second comma, which C2 reads), at the end of the given part after a family comma, as a run start and where a maiden clause releases it (`John Smith Pharm D` and `John Smith MB ChB` → suffix, reported, `MB ChB` no longer reporting 'ChB' as a word `mb`'s run absorbed); where the count declines it the two words report what they reported unmerged (`SMITH, SAM E.` → given 'SAM', middle 'E.', no report) — 2026-10-25 #752 (Derek), the report. - The Ph./D. pair stays the unambiguous credential decisions/phd-merge reads, merged by its spelling whatever the vocabulary lists, and outranks a pair overlapping it (`Jane Doe Dr. Ph. D.` → title 'Dr.', suffix 'Ph. D.', where 'Dr. Ph.' folds to the listed 'drph') — 2026-10-25 #752. - Two dotted single letters merge only where #872's `is_cased_paired_initials` says they are no paired initials, in a script without capitals, not in one that writes no abbreviations, and only where their letters spell listed vocabulary, which no default entry does (`John Smith J. R.` → family 'R.'; `John Smith 博. 士.` → family '士.') — 2026-10-25 #752; 2026-10-25 #752, review. - The Ph./D. pair starts S8's run from where it stands, though the walk leaves it out (`John Smith Ph. D. Jones` → suffix 'Ph. D. Jones', reported, #827), and a one-letter numeral in front of it is the name word it is before `PhD` (`John Smith V Ph. D.` → family 'V', reported) — 2026-10-25 #752. @@ -30,4 +30,5 @@ Open: none in this file. - 2026-10-25 #752, review — SEVEN DEFECTS IN THE FIRST VERSION, found by the PR's code review on a grid of 3,726 texts under the three orders compared against master. (i) A connective beside a pair joined the merged piece into a wider one, which kept its tag and was read whole: `John Smith Pharm D and Mary` read suffix 'Pharm D and Mary' and `John Pharm D and Mary` family 'Pharm D and Mary', where master reads family 'D and Mary'. A pair a connective borders no longer merges, and the two readers of the tag ask for a piece of exactly two words. (ii) Classify's pre-test asked the fold's membership of the Ph./D. pair too, so a configuration without `phd` (`Constants().suffix_acronyms.remove('phd')`) stopped merging it, where v1's fix_phd merged it by its spelling and 1.4.0 and master read suffix 'Ph. D.'; the pair's own pattern now answers ahead of the vocabulary. (iii) C3's count of the part before a family comma merged a member pair, so `MB ChB, PhD` read given 'MB', family 'ChB' where master reads family 'MB ChB'; it reads the part as written now. (iv) Case repair marked a pair's capitals only where the comma's decision bound it, so the same words repaired two ways (`john smith, PHARM D` kept 'PHARM D', `john smith PHARM D` gave 'Pharm D'); render now repairs every suffix pair by its fold's mask (`MB CHB` → 'MB ChB' under 'MBChB'), else in capitals for a listed acronym, and the comma's marks are gone. The force view of `MB BCH` is now 'MB BCh', the mask of 'mbbch', where master kept 'MB BCH' as a caps word. (v) The pre-test let every non-ASCII fold through to the call, a frame per pair of short words in any accented name (`Ngô Bảo Châu` 263 → 269); it looks the composed fold up in C now and costs what master does, and a pair of two suffix words (`MD PhD`) and an opening single letter before a dotted word of two letters (FEC's `A. MS.`) are refused by tags and lengths before the call. (vi) The maiden clause's report read the first word of the piece ending it, so `Jane Doe nee Ed. D.` reported 'Ed.' ending the maiden name; a split pair the clause keeps is read in silence, as everywhere. (vii) The comma form of the given name and initial (`Smith, John Sam E.` → suffix 'Sam E.', where master read middle 'Sam E.') was unrecorded; it is Accepted with the no-comma form, as S5 reads any member ending the given part whose writing says nothing else, and a split pair has no writing to say it. Each fix has a case row or a test that fails on the first version (tests/v2/test_render.py's `test_a_split_credential_repairs_as_its_one_word_twin`, tests/v2/test_facade.py's `test_the_ph_d_pair_merges_whatever_the_vocabulary_says`), run against it before landing. - 2026-10-25 #752, design-docs review (its fix (i), the merge after the joins, superseded by the 2026-10-25 #752, second review entry below) — TWO MORE CODE DEFECTS AND THE RECORD'S CORRECTIONS. (i) In the given part after a family comma group merged the pair ahead of its joins, so a bound given name took it: `Rahman, Abdul MB ChB` read given 'Abdul MB ChB' where master reads given 'Abdul', suffix 'MB ChB', against rules/P5's "The join never absorbs" a suffix word. The pair is merged there only after the joins now (`_group._merge_given_pairs`), so a join that took one of its words leaves it as written (`Smith, abdul Pharm D Jones` reads as master, given 'abdul Pharm'). (ii) Two dotted single letters in a script that writes no abbreviations merged where their letters are listed: `John Smith 博. 士.` read suffix '博. 士.', `김 민준 박. 사.` suffix '박. 사.', 18 default honorifics and titles of two characters folding so. rules/S3 refuses that script's periods, and so does the pair now, which makes true the record's "no default entry". (iii) The record's corrections: the particle refusal's case is `DE D` (`ded`), not `DE A`, whose fold is unlisted, so the rule's example witnessed nothing; the opening-letter test's evidence is the measurement in THE PAIR above, not `A. MS.`; rules/S11 states its precedence over S6's case and S5's report, and rules/S8 its own over a split pair as a Precedence bullet rather than a clause of its Then; decisions/C13, decisions/phd-merge and decisions/S8 are marked where their statements moved, and mechanisms.md's ONE-PREDICATE-PER-QUESTION names the predicate's callers as they are. - 2026-10-25 #752, second review — FOUR DEFECTS IN THE FIXES, found by a review of the two fix commits. SUPERSEDES: the design-docs review entry's fix (i). (i) Merging the given part's pair after group's joins let the particle chain take its first word, since the chain then saw `Pharm` and `D` apart and its bound stopped behind them: `Smith, John van Pharm D` read middle 'van Pharm D' where `Smith, John van PharmD` reads family 'van Smith', suffix 'PharmD', and the bound join took `Pharm` the same way (`Smith, abdul Pharm D` → given 'abdul Pharm'). The pair is merged ahead of the joins again, the chain stopping in front of it, and P5's bound join declines it as it declines a suffix word (rules/P5's Except), which keeps `Rahman, Abdul MB ChB` reading suffix 'MB ChB'; the widened fix had narrowed the chain's stop, the shape AGENTS.md's review paragraph names. (ii) Classify's refusal of two suffix words stood in front of the Ph./D. answer, so a lexicon listing `ph` and `d` stopped the merge (`John Ph. D. Smith` read middle 'Ph. D.' with `Lexicon.default().add(suffix_words={"ph", "d"})`), against "merged by its spelling whatever the vocabulary lists"; it is asked of the other pairs only. (iii) Case repair recased every tagged pair, so of two overlapping pairs (`MB CHB` and `CHB A`, `chba` being listed) the second wrote over the first (`JOHN SMITH MB CHB A` → 'MB CHB A'); it reads only the pair group merged, its second word carrying the merge's `joined` mark. (iv) Unforced, case repair asked each word of a pair whether it is in one case, where the one-word spelling is asked whole, so `john smith pharm D` gave 'Pharm D' while `john smith pharmD` keeps 'pharmD'; the pair is asked whole now and kept as written (rules/R5). And the record: case repair costs no frame on a name holding no pair (`capitalized(force=True)` of `John Smith` 59, as on master; the first fix called the function on every repair, +1), and the given name and initial the count takes (`JOHN SMITH SAM E.`) repair as the acronym their letters spell, recorded with that Accepted reading. Each fix has a case row or a test that fails on the commit it fixes. -- 2026-10-25 #752 (Derek), the report — A SPLIT CREDENTIAL THE COUNT TAKES IS REPORTED, AS EITHER READING OF A MEMBER IS. SUPERSEDES: the first entry's SILENT paragraph. Derek, on #891's review: the merged pair is a member of the ambiguous class, which reports either reading (`John Smith MA` reports `suffix-or-name`), and with no report a given name and its initial that spell a credential become one where no caller can find them (`John Smith Ed D`, `Smith, Mary Ed D.`, `JOHN SMITH SAM E.`). So wherever the count takes the pair it reports `suffix-or-name` with one wording ("reads as a credential", a phrasing tests/v2/test_cases.py's field sweep already checks against the field the pair lands in): at assign's placement for the end of a name, the end of the given part and a run start, at `_comma.decide`'s binding for C4's count unless the flip it took part in already reports the part, and at the maiden take's release. A pair the count declines reads as its two words did unmerged and reports what they did, which on master is nothing. The cost is a report on the pairs nobody hesitates over (`John Smith Pharm D`), which the class's report carries for `John Smith MEng` too. Measured 2026-10-25 against master 225f3d6f with `--base`, as the first entry: 353 ctgov-officials readings carry the report, 119 of them where master reported no `suffix-or-name` at all, the rest the `MB ChB` and run names whose absorbed-word report it replaces (so the 229 names that dropped a report in the first entry keep the kind, and ctgov moves 469 entries in all: 120 in the ambiguity kinds, each with a field move, 339 only in the force-repaired case view, and the rest in fields or case views without a kind move); the committed snapshot's rules examples gain it where the count takes a pair; tse-donors, fec-individuals, nepal-candidates and psc-individuals read as in the first entry (2, 1, none, matching the base). The field sweep's count goes from 1460 to 1487, nine case rows claiming the report under each order. +- 2026-10-25 #752 (Derek), the report — A SPLIT CREDENTIAL THE COUNT TAKES IS REPORTED, AS EITHER READING OF A MEMBER IS. SUPERSEDES: the first entry's SILENT paragraph. Derek, on #891's review: the merged pair is a member of the ambiguous class, which reports either reading (`John Smith MA` reports `suffix-or-name`), and with no report a given name and its initial that spell a credential become one where no caller can find them (`John Smith Ed D`, `Smith, Mary Ed D.`, `JOHN SMITH SAM E.`). So wherever the count takes the pair it reports `suffix-or-name` with one wording ("reads as a credential", a phrasing tests/v2/test_cases.py's field sweep already checks against the field the pair lands in): at assign's placement for the end of a name, the end of the given part and a run start, at `_comma.decide`'s binding for C4's count unless the flip it took part in already reports the part, and at the maiden take's release. A pair the count declines reads as its two words did unmerged and reports what they did, which on master is nothing. The cost is a report on the pairs nobody hesitates over (`John Smith Pharm D`), which the class's report carries for `John Smith MA` too. Measured 2026-10-25 against master 225f3d6f with `--base`, as the first entry: 353 ctgov-officials readings carry the report, 119 of them where master reported no `suffix-or-name` at all, the rest the `MB ChB` and run names whose absorbed-word report it replaces (so the 229 names that dropped a report in the first entry keep the kind, and ctgov moves 469 entries in all: 120 in the ambiguity kinds, each with a field move, 339 only in the force-repaired case view, and the rest in fields or case views without a kind move); the committed snapshot's rules examples gain it where the count takes a pair; tse-donors, fec-individuals, nepal-candidates and psc-individuals read as in the first entry (2, 1, none, matching the base). The field sweep's count goes from 1460 to 1487, nine case rows claiming the report under each order. +- 2026-10-25 #752, review of the report — ONE PAIR REPORTED TWICE. After a comma, a pair behind a credential that opens the part (C13) was reported as a word the run absorbed and again as the pair (`Smith, MD Pharm D`, `, MD Pharm D`, `Smith, MD MB ChB`: 168 parses an order over the review's grid), where `John Smith MD Pharm D` reports it once. C13's absorbed report leaves a split pair out, so the pair reports itself (case row `a_split_credential_behind_an_opening_credential_reports_once`). The review also found the Standing line's "after a comma" wider than the code: a part the comma's decision already reports as a whole, and a part past the second comma, carry no pair report, as the Standing line now says. diff --git a/docs/design/rules/S11-credential-split-across-a-space.md b/docs/design/rules/S11-credential-split-across-a-space.md index 4a0d7d9b..14f090f1 100644 --- a/docs/design/rules/S11-credential-split-across-a-space.md +++ b/docs/design/rules/S11-credential-split-across-a-space.md @@ -30,7 +30,7 @@ "John Smith Pharm D Jones" → suffix="Pharm D Jones" "Smith, John Pharm D Jones" → suffix="Pharm D Jones" ``` -- **What reports.** Where the count takes the pair, the reading is reported, as either reading of a member is (S4), its words being a given name and an initial as often as a credential's chunks; where the count declines it, the two words read as they read unmerged and report what they reported. +- **What reports.** Where the count takes the pair, the reading is reported, as either reading of a member is (S4), its words being a given name and an initial as often as a credential's chunks; where the count declines it, the two words read as they read unmerged and report what they reported. After a comma the pair reports only where the comma's own report does not already cover the part (C11), and a part past the second comma reports as C2 says. ``` "John Smith Pharm D" → ambiguities=("suffix-or-name",) "John Smith, Pharm D" → ambiguities=("suffix-or-name",) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 8a2886b0..397d438b 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -1693,4 +1693,3 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: return copy_with(state, tokens=tuple(tokens), order=order, ambiguities=tuple(ambiguities)) - diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index 33da2d19..4ceaaef7 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -665,7 +665,8 @@ def decide(state: ParseState, splits: bool | None = True) -> ParseState: # makes it the postnominal part however it goes on" -- each word it takes that # would have made it the given part, as #602's run reports them for k in absorbed: - if has_name_content(p1[k], tokens): + # a split credential the run takes reports itself, below + if has_name_content(p1[k], tokens) and "split" not in pt1[k]: text = " ".join(tokens[i].text for i in p1[k]) ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 1b0c1b10..6840520e 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -351,8 +351,9 @@ def _maiden_take(pieces: Sequence[Sequence[int]], # left it a name word, which is a fork too last = pieces[end - 1] word = tokens[last[0]] - # a one-word member: a split credential the clause keeps is read in - # silence (rules/S11), and its first word is not the clause's last + # a one-word member: a split credential the clause keeps is one the + # count declined, which reports nothing (rules/S11), and its first + # word is not the clause's last if len(last) == 1 and not word.tags.isdisjoint(_AMBIGUOUS_CREDENTIAL_TAGS): ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 68910e5f..451dd729 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1635,8 +1635,8 @@ def _check_cjk_shape_purity(self) -> None: "Jane Doe nee Ed. D.", {"given": "Jane", "family": "Doe", "maiden": "Ed. D."}, classification="fix(#752)", - notes="S11: the clause keeps the pair, which a declined pair " - "reports nothing for; its " + notes="S11: the clause keeps the pair, a pair the count " + "declined, which reports nothing; its " "first word is a listed member, and the first version " "reported 'Ed.' as the word ending the clause, which " "'D.' is. 1.4.0 read middle 'Doe nee Ed.', last 'D.'"), @@ -1659,6 +1659,16 @@ def _check_cjk_shape_purity(self) -> None: "word; the first version joined it, given 'Abdul MB ChB', " "where master read suffix 'MB ChB'. 1.4.0 read first " "'Abdul MB', middle 'ChB'"), + Case("a_split_credential_behind_an_opening_credential_reports_once", + "Smith, MD Pharm D", + {"family": "Smith", "suffix": "MD Pharm D"}, + ambiguities=("suffix-or-name",), + classification="fix(#752)", + notes="S11: the pair the opening credential's run takes (C13) " + "reports itself, once, as 'John Smith MD Pharm D' does; " + "the review found it reported twice, as a word the run " + "absorbed and as the pair. Master reported 'Pharm' and " + "'D' as two absorbed words"), Case("the_split_degree_outranks_a_pair_overlapping_it", "Jane Doe Dr. Ph. D.", {"title": "Dr.", "given": "Jane", "family": "Doe", diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index ebdfc1be..c41f3290 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -62,7 +62,7 @@ "S8": {"words": 418, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 0, "accepted_words": 0}, "S9": {"words": 463, "then_words": 262, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 3, "accepted_words": 163}, - "S11": {"words": 644, "then_words": 342, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 93}, + "S11": {"words": 674, "then_words": 372, "because_sentences": 2, "except": 3, "precedence": 4, "accepted": 2, "accepted_words": 93}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 05d7e6bc..4f4494e6 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -246,9 +246,11 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # 'Smith, John van Pharm D' row, one particle-or-given claim under # each order; 1460 -> 1487 the same day: the split credential's own # report where the count takes it (Derek, on #891's review), nine - # rows claiming one suffix-or-name report under each order. - assert claims == 1487, ( - f"the field sweep checks {claims} claims, recorded as 1487 on " + # rows claiming one suffix-or-name report under each order; 1487 -> + # 1490 the same day: the review's 'Smith, MD Pharm D' row, one claim + # under each order. + assert claims == 1490, ( + f"the field sweep checks {claims} claims, recorded as 1490 on " f"2026-10-25") From 7b257a0d367eda8da80a6ce246e8ccca19b631de Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 16:40:42 -0700 Subject: [PATCH 10/11] Record the split-credential figures over phase 2h (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/S12-credential-split-across-a-space.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/design/decisions/S12-credential-split-across-a-space.md b/docs/design/decisions/S12-credential-split-across-a-space.md index cda73f54..49d1920c 100644 --- a/docs/design/decisions/S12-credential-split-across-a-space.md +++ b/docs/design/decisions/S12-credential-split-across-a-space.md @@ -32,3 +32,4 @@ Open: none in this file. - 2026-10-25 #752, second review — FOUR DEFECTS IN THE FIXES, found by a review of the two fix commits. SUPERSEDES: the design-docs review entry's fix (i). (i) Merging the given part's pair after group's joins let the particle chain take its first word, since the chain then saw `Pharm` and `D` apart and its bound stopped behind them: `Smith, John van Pharm D` read middle 'van Pharm D' where `Smith, John van PharmD` reads family 'van Smith', suffix 'PharmD', and the bound join took `Pharm` the same way (`Smith, abdul Pharm D` → given 'abdul Pharm'). The pair is merged ahead of the joins again, the chain stopping in front of it, and P5's bound join declines it as it declines a suffix word (rules/P5's Except), which keeps `Rahman, Abdul MB ChB` reading suffix 'MB ChB'; the widened fix had narrowed the chain's stop, the shape AGENTS.md's review paragraph names. (ii) Classify's refusal of two suffix words stood in front of the Ph./D. answer, so a lexicon listing `ph` and `d` stopped the merge (`John Ph. D. Smith` read middle 'Ph. D.' with `Lexicon.default().add(suffix_words={"ph", "d"})`), against "merged by its spelling whatever the vocabulary lists"; it is asked of the other pairs only. (iii) Case repair recased every tagged pair, so of two overlapping pairs (`MB CHB` and `CHB A`, `chba` being listed) the second wrote over the first (`JOHN SMITH MB CHB A` → 'MB CHB A'); it reads only the pair group merged, its second word carrying the merge's `joined` mark. (iv) Unforced, case repair asked each word of a pair whether it is in one case, where the one-word spelling is asked whole, so `john smith pharm D` gave 'Pharm D' while `john smith pharmD` keeps 'pharmD'; the pair is asked whole now and kept as written (rules/R5). And the record: case repair costs no frame on a name holding no pair (`capitalized(force=True)` of `John Smith` 59, as on master; the first fix called the function on every repair, +1), and the given name and initial the count takes (`JOHN SMITH SAM E.`) repair as the acronym their letters spell, recorded with that Accepted reading. Each fix has a case row or a test that fails on the commit it fixes. - 2026-10-25 #752 (Derek), the report — A SPLIT CREDENTIAL THE COUNT TAKES IS REPORTED, AS EITHER READING OF A MEMBER IS. SUPERSEDES: the first entry's SILENT paragraph. Derek, on #891's review: the merged pair is a member of the ambiguous class, which reports either reading (`John Smith MA` reports `suffix-or-name`), and with no report a given name and its initial that spell a credential become one where no caller can find them (`John Smith Ed D`, `Smith, Mary Ed D.`, `JOHN SMITH SAM E.`). So wherever the count takes the pair it reports `suffix-or-name` with one wording ("reads as a credential", a phrasing tests/v2/test_cases.py's field sweep already checks against the field the pair lands in): at assign's placement for the end of a name, the end of the given part and a run start, at `_comma.decide`'s binding for C4's count unless the flip it took part in already reports the part, and at the maiden take's release. A pair the count declines reads as its two words did unmerged and reports what they did, which on master is nothing. The cost is a report on the pairs nobody hesitates over (`John Smith Pharm D`), which the class's report carries for `John Smith MA` too. Measured 2026-10-25 against master 225f3d6f with `--base`, as the first entry: 353 ctgov-officials readings carry the report, 119 of them where master reported no `suffix-or-name` at all, the rest the `MB ChB` and run names whose absorbed-word report it replaces (so the 229 names that dropped a report in the first entry keep the kind, and ctgov moves 469 entries in all: 120 in the ambiguity kinds, each with a field move, 339 only in the force-repaired case view, and the rest in fields or case views without a kind move); the committed snapshot's rules examples gain it where the count takes a pair; tse-donors, fec-individuals, nepal-candidates and psc-individuals read as in the first entry (2, 1, none, matching the base). The field sweep's count goes from 1460 to 1487, nine case rows claiming the report under each order. - 2026-10-25 #752, review of the report — ONE PAIR REPORTED TWICE. After a comma, a pair behind a credential that opens the part (C13) was reported as a word the run absorbed and again as the pair (`Smith, MD Pharm D`, `, MD Pharm D`, `Smith, MD MB ChB`: 168 parses an order over the review's grid), where `John Smith MD Pharm D` reports it once. C13's absorbed report leaves a split pair out, so the pair reports itself (case row `a_split_credential_behind_an_opening_credential_reports_once`). The review also found the Standing line's "after a comma" wider than the code: a part the comma's decision already reports as a whole, and a part past the second comma, carry no pair report, as the Standing line now says. +- 2026-10-25 #752 — RE-MEASURED OVER #890, which merged first and took the ID S11, so this rule and file became S12. Against master b1e5a292 (phase 2h) with `--base`, every figure of the report entry above holds: ctgov-officials 469 entries, tse-donors 2, fec-individuals 1, nepal-candidates none, psc-individuals matching the base (1,004,785 of 1,039,171 agreeing on every field there, #890's own count), and the committed snapshot 2 moves and 28 new rules examples. From 3cdd2956ec3dc6ef177bf911cd0f430f84bec08d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 16:41:41 -0700 Subject: [PATCH 11/11] Give the re-measurement entry its own head (#752, phase 2e) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/S12-credential-split-across-a-space.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/decisions/S12-credential-split-across-a-space.md b/docs/design/decisions/S12-credential-split-across-a-space.md index 49d1920c..f71ccc66 100644 --- a/docs/design/decisions/S12-credential-split-across-a-space.md +++ b/docs/design/decisions/S12-credential-split-across-a-space.md @@ -32,4 +32,4 @@ Open: none in this file. - 2026-10-25 #752, second review — FOUR DEFECTS IN THE FIXES, found by a review of the two fix commits. SUPERSEDES: the design-docs review entry's fix (i). (i) Merging the given part's pair after group's joins let the particle chain take its first word, since the chain then saw `Pharm` and `D` apart and its bound stopped behind them: `Smith, John van Pharm D` read middle 'van Pharm D' where `Smith, John van PharmD` reads family 'van Smith', suffix 'PharmD', and the bound join took `Pharm` the same way (`Smith, abdul Pharm D` → given 'abdul Pharm'). The pair is merged ahead of the joins again, the chain stopping in front of it, and P5's bound join declines it as it declines a suffix word (rules/P5's Except), which keeps `Rahman, Abdul MB ChB` reading suffix 'MB ChB'; the widened fix had narrowed the chain's stop, the shape AGENTS.md's review paragraph names. (ii) Classify's refusal of two suffix words stood in front of the Ph./D. answer, so a lexicon listing `ph` and `d` stopped the merge (`John Ph. D. Smith` read middle 'Ph. D.' with `Lexicon.default().add(suffix_words={"ph", "d"})`), against "merged by its spelling whatever the vocabulary lists"; it is asked of the other pairs only. (iii) Case repair recased every tagged pair, so of two overlapping pairs (`MB CHB` and `CHB A`, `chba` being listed) the second wrote over the first (`JOHN SMITH MB CHB A` → 'MB CHB A'); it reads only the pair group merged, its second word carrying the merge's `joined` mark. (iv) Unforced, case repair asked each word of a pair whether it is in one case, where the one-word spelling is asked whole, so `john smith pharm D` gave 'Pharm D' while `john smith pharmD` keeps 'pharmD'; the pair is asked whole now and kept as written (rules/R5). And the record: case repair costs no frame on a name holding no pair (`capitalized(force=True)` of `John Smith` 59, as on master; the first fix called the function on every repair, +1), and the given name and initial the count takes (`JOHN SMITH SAM E.`) repair as the acronym their letters spell, recorded with that Accepted reading. Each fix has a case row or a test that fails on the commit it fixes. - 2026-10-25 #752 (Derek), the report — A SPLIT CREDENTIAL THE COUNT TAKES IS REPORTED, AS EITHER READING OF A MEMBER IS. SUPERSEDES: the first entry's SILENT paragraph. Derek, on #891's review: the merged pair is a member of the ambiguous class, which reports either reading (`John Smith MA` reports `suffix-or-name`), and with no report a given name and its initial that spell a credential become one where no caller can find them (`John Smith Ed D`, `Smith, Mary Ed D.`, `JOHN SMITH SAM E.`). So wherever the count takes the pair it reports `suffix-or-name` with one wording ("reads as a credential", a phrasing tests/v2/test_cases.py's field sweep already checks against the field the pair lands in): at assign's placement for the end of a name, the end of the given part and a run start, at `_comma.decide`'s binding for C4's count unless the flip it took part in already reports the part, and at the maiden take's release. A pair the count declines reads as its two words did unmerged and reports what they did, which on master is nothing. The cost is a report on the pairs nobody hesitates over (`John Smith Pharm D`), which the class's report carries for `John Smith MA` too. Measured 2026-10-25 against master 225f3d6f with `--base`, as the first entry: 353 ctgov-officials readings carry the report, 119 of them where master reported no `suffix-or-name` at all, the rest the `MB ChB` and run names whose absorbed-word report it replaces (so the 229 names that dropped a report in the first entry keep the kind, and ctgov moves 469 entries in all: 120 in the ambiguity kinds, each with a field move, 339 only in the force-repaired case view, and the rest in fields or case views without a kind move); the committed snapshot's rules examples gain it where the count takes a pair; tse-donors, fec-individuals, nepal-candidates and psc-individuals read as in the first entry (2, 1, none, matching the base). The field sweep's count goes from 1460 to 1487, nine case rows claiming the report under each order. - 2026-10-25 #752, review of the report — ONE PAIR REPORTED TWICE. After a comma, a pair behind a credential that opens the part (C13) was reported as a word the run absorbed and again as the pair (`Smith, MD Pharm D`, `, MD Pharm D`, `Smith, MD MB ChB`: 168 parses an order over the review's grid), where `John Smith MD Pharm D` reports it once. C13's absorbed report leaves a split pair out, so the pair reports itself (case row `a_split_credential_behind_an_opening_credential_reports_once`). The review also found the Standing line's "after a comma" wider than the code: a part the comma's decision already reports as a whole, and a part past the second comma, carry no pair report, as the Standing line now says. -- 2026-10-25 #752 — RE-MEASURED OVER #890, which merged first and took the ID S11, so this rule and file became S12. Against master b1e5a292 (phase 2h) with `--base`, every figure of the report entry above holds: ctgov-officials 469 entries, tse-donors 2, fec-individuals 1, nepal-candidates none, psc-individuals matching the base (1,004,785 of 1,039,171 agreeing on every field there, #890's own count), and the committed snapshot 2 moves and 28 new rules examples. +- 2026-10-25 #752, over #890 — RE-MEASURED: #890 merged first and took the ID S11, so this rule and file became S12. Against master b1e5a292 (phase 2h) with `--base`, every figure of the report entry above holds: ctgov-officials 469 entries, tse-donors 2, fec-individuals 1, nepal-candidates none, psc-individuals matching the base (1,004,785 of 1,039,171 agreeing on every field there, #890's own count), and the committed snapshot 2 moves and 28 new rules examples.