From 080f54c35813bdc1eaeb504555768f29a2db64f3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 02:38:58 -0700 Subject: [PATCH 1/7] Read a lone one-letter numeral by S4's words-to-spare count (#718) A capital V, I or X ending a name with no word to spare for it is the family name: 'John V' and 'Malcolm X' read family V and X, as 1.4.0 read them, where 2.0.0 through 2.4.2 gave suffix V and X and no family. 'John Smith V' keeps the suffix. A numeral in lower case in a mixed-case name is settled by its writing: still the generation, and no longer reported ('John Smith vi'). States the comma-less numeral as rules/S10 (the trailing roman numeral), with its decisions entry; amends P2's chain stop, P5's words-to-spare list and M2's examples, which the count moves. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions/README.md | 1 + .../decisions/S10-trailing-roman-numeral.md | 8 ++ docs/design/rules/M2-maiden-marker-rule.md | 5 +- .../design/rules/P2-particles-join-forward.md | 10 +- docs/design/rules/P5-bound-given-names.md | 2 +- docs/design/rules/README.md | 1 + .../rules/S10-trailing-roman-numeral.md | 51 ++++++++ ...-ends-a-name-and-never-opens-the-string.md | 2 +- docs/release_log.rst | 1 + nameparser/_pipeline/_assign.py | 8 +- nameparser/_pipeline/_group.py | 6 +- nameparser/_pipeline/_pieces.py | 16 ++- nameparser/_pipeline/_vocab.py | 21 ++- tests/v2/cases.py | 121 ++++++++++++++---- tests/v2/pipeline/test_assemble.py | 6 +- tests/v2/pipeline/test_group.py | 10 +- tests/v2/test_cases.py | 9 +- tests/v2/test_parser.py | 16 ++- tools/differential/corpus_rules.jsonl | 18 +++ tools/differential/snapshot.jsonl | 23 +++- 21 files changed, 273 insertions(+), 64 deletions(-) create mode 100644 docs/design/decisions/S10-trailing-roman-numeral.md create mode 100644 docs/design/rules/S10-trailing-roman-numeral.md diff --git a/AGENTS.md b/AGENTS.md index b686bf8c..1801d8bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,7 +173,7 @@ The library has two layers: `nameparser/config/` (data) and `nameparser/parser.p **Sweep the forms a change can reach: comma shapes, then `name_order`.** No comma, a FULL name before the comma, a ONE-WORD name before the comma and NOTHING before the comma are four paths, and the last two are the misses — #429, #430 and #432 are all the one-word path disagreeing with the full-name path on inputs the full-name path reads correctly, and #613's PR review found `, PhD` and `, Jr.` (an empty surname field, as CSV data writes it) read as given and title by a decision that returned early on an empty head, after two review rounds whose every grid had a name before the comma. Put an empty head and a nickname-only head (`(Bob), PhD`) in every comma grid. Count the one-word path the way C1 counts, which is neither tokens nor pieces: a particle run and the one word it attaches to are one word (#575), a connective join and a bound given-name pair count as their words (#613, so `Ortega y Gasset, Dr.` is a whole name), and a suffix beside the surname counts as no name word at all. #603's review found the suffix half (`Vega y Lopez Jr., PhD Jones` lost its family where `Vega y Lopez, PhD Jones` kept it); #613 then reversed the connective half, and both now read given 'Vega', middle 'y', family 'Lopez' — so test a particle surname, a connective surname and a surname with a suffix, each before a comma. For orders the sweep already exists (`tests/v2/test_cases.py` runs every row under all three) but carries one assertion, R2's family partition, so it checks nothing a new change moves; coverage there has been vacuous before (PR #394's review found the suite passed with `name_order` discarded from grouping). Parse your change's names in each comma shape and each order, and read the ones you did not predict. And read a generated grid's moves by SHAPE, not by corpus membership: #614's first design passed the suite, the five gates and a fingerprint of every corpus name, because its grid moves were sorted into 'corpus name moved' and 'garbage', and the review found the moves that mattered among the 'garbage' -- a bare salutation (`Mr. and Mrs.` lost its connective to the family name) and a name with one trailing word (`Freiherr von Berg X.Y.Z.`). Pull out the head-plus-one-word slice of the grid and a salutation with nothing after it, and read every move there. Two more were the second review's (#614): no grid put TWO words that are both a title and a particle at the head (`Freiherr St van Berg MA`, where an index into the pieces was applied to a merged view of them -- `TITLES ∩ particles_ambiguous` is `{freiherr, st}`, so put both in every head list, each in its bare and its period-marked spelling -- #637's grid had `St` and `Freiherr` alone and missed that `St.` and `Freiherr.` are declined by the bound join and then attached by P6), and the grids compared ambiguity KINDS, so a report telling the caller the wrong field (`Freiherr von Berg Ma`, 'read as a given name' of a word in the family) passed every pin. #701's review added one: put a title/suffix dual (`Ms`, `Sr.`, `Cpl`) SECOND in a title run, behind an ordinary title. A stop that asks only "is this word suffix vocabulary" passes every row that opens with the dual and fails `Rev. Sr. Smith, Mary`; it was found by comparing `T Smith, John` against `T John Smith` over every title word, and a comma-versus-comma-less comparison is the cheap check for any rule that moves a word between the two slots. Compare each report's `detail` against the field its tokens land in -- since #626 `test_a_report_names_the_field_its_word_lands_in` (tests/v2/test_cases.py) does it over every non-locale case row under all three orders (a row carrying its own policy only as declared), for the phrasings its `_FIELD_CLAIMS` lists -- so a row with the shape is the test only where the report's wording is listed: a new emitter's wording goes in that list, and segment's report on a part past the second comma names no field (#629). #626's own first grid then missed an emitter for want of one comma shape (`, Ma Dr.`: an empty head, an ambiguous word, a trailing title, which H1 moves to the family): a trailing title belongs in every empty-head comma grid too. And #620's: an exclusion list in code being replaced is a set of rules, not noise -- #614's merged copy excluded every word the trailing read weighs from its units, the first #620 prototype dropped the list as a symptom of the copy, and its review found the list had been keeping positional counting's rule that a member in front counts as a word (`Freiherr von Berg MA X.Y.Z.` lost its suffix). Before deleting one, mutate each entry out and read what moves. -**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. +**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, while the lowercase listed spellings never reach it and are not counted (`john v` keeps suffix 'v' where `john x` reads family 'x') -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. **And the encoding: a letter class written `[^\W\d_]` or tested with `isalpha()` is blind to a combining mark.** Python's `\w` and `isalpha()` do not match Mn, so a letter written as base plus accent (NFD) ends a word early; the vocabulary path is immune because `_normalize` NFC-folds, the shape path is not. #652: `Dr. inž. Janez Novak` read `inž.` as the given name when the ž was decomposed, and `Janez Novak, ŽŠČ` reversed, in two families that share the blind spot — the letter-shape regexes (`period_abbreviation`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`), now spelled with `_vocab._LETTER` (a letter plus the general combining blocks; `test_regex_sync` compares the two `_MARKS` strings and pins `period_abbreviation` against config's copy), and the caps predicates, which NFC-compose a non-ASCII word before `isalpha()` (`is_caps_word`, behind C-level `isalpha()`/`is_normalized` tests, so already-composed text pays no frame — measured 2026-10-12, `Smith, José` was 193 frames before and after; the first version called it for every non-ASCII word and cost one frame each, and no guard saw it because the reference name is ASCII). A width test must count letters, not code points (`composed_len`): admitting a decomposed word to the caps set put `John Smith, MÁ` on the pair exception's wrong side, and the review's grid found it where the rows (all three letters long) did not — put a two-letter word in every caps grid. The initial shape (`_INITIAL`, `\w\.`, v1-verbatim and public) is not widened but asked of the COMPOSED word, in `is_initial_shaped` and in `_render`'s two copies; and the single-letter connective tests count composed letters, since a decomposed `й` is two code points (#665, where `É. y É. Smith` and `John й Smith` read differently decomposed). Compose the WHOLE predicate, not one half: `is_initial` is a shape ANDed with the #320 script veto, and composing the shape alone made a decomposed `씨.` an initial, since decomposed hangul is conjoining jamo outside every script range (#665's first version; `in_initialless_script` composes too now). Not covered, deliberately: the marks only some scripts write (Indic vowel signs, Arabic and Hebrew points; rules/H2's Accepted clause). The guard is `test_a_decomposed_name_renders_as_its_composed_twin` over the CLDR corpus plus `test_a_decomposed_abbreviation_reads_as_its_composed_twin` (rows × three orders × EVERYWHERE); the corpus guard reaches a shape only if some corpus name does, which is how the caps shape was missed until review, so when adding a shape rule add a row there. Four of #652's rows and two of #665's pass with every fix reverted (measured) and are parity pins, and the test compares ambiguity kinds and `initials()` too, since #665's `John й Smith` differed in its report; `_DOTTED` has no observable input and is pinned by a direct call. diff --git a/docs/design/decisions/README.md b/docs/design/decisions/README.md index 2c7272eb..50f01a02 100644 --- a/docs/design/decisions/README.md +++ b/docs/design/decisions/README.md @@ -43,6 +43,7 @@ Entry conventions: - [S7 — the company of an unambiguous credential](S7-company-of-an-unambiguous-credential.md) - [S8 — the credential's trailing run](S8-credentials-trailing-run.md) - [S9 — unlisted words that join the ambiguous class by shape](S9-unlisted-words-that-join-the-ambiguous-class-by-shape.md) +- [S10 — the trailing roman numeral](S10-trailing-roman-numeral.md) ## Nicknames & quoted names (N) diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md new file mode 100644 index 00000000..e8aa9dbd --- /dev/null +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -0,0 +1,8 @@ +# S10 — the trailing roman numeral + +- 2026-10-17 (Derek), #718 — A ONE-LETTER NUMERAL NEEDS WORDS TO SPARE, AND THE RULE IS WRITTEN DOWN. Until this entry no rule stated the comma-less numeral at all: the behavior was pinned in tests/v2/cases.py and credited in code comments to rules/S2, whose statement covers the suffix vocabulary and never the shape fork that reads `VI` to `X`, nor the initial veto that sends `V`, `I` and `X` there. The extraction found one reading worth changing. `John V` and `Malcolm X` read suffix with an EMPTY family in every 2.x release (2.0.0 through 2.4.2 measured), alongside 2.0.0's `Johnson PhD` fix; 1.4.0 read family `V` and `X`. Derek: a generational V is mostly archaic, X rarer still and I hardly ever written without a comma (Sr. and Senior are what people write), so a capital letter standing alone behind the given name is far likelier an initial or a one-letter surname. The numeral was never what S2's Accepted clause (an unambiguous suffix is consumed even when that leaves no family) covers, since the fork has always reported itself ambiguous; S4's count is the analogue, and the numeral now asks it. + WHAT MOVES. The count is S4's own, in name UNITS, so a particle chain is one word (P2) and a title none. Measured on the shipped change against origin/master at 958d252e, the snapshot moves four corpus names: `John V` and `Mohamad X` (family V and X, as 1.4.0 read them), `John née Jones Smith V` (the clause-free name is `John V`, so the V is no suffix and the clause keeps it, maiden 'Jones Smith V' — M2's Accepted clause on the lone numeral), and `Freiherr von Richthofen V`, family 'von Richthofen V': behind a title the chain is the one name word, so the V has no word to spare and the chain takes it as P2 takes any name word, as it already took `Freiherr von Richthofen Ma`. That last is the reading a scratch prototype counting raw pieces did not show, and the one most worth a second look. Four case rows moved with it (`strict_comma_reads_the_bare_numeral_into_the_run`: `Smith V, Ma` under the strict knob reads family V, suffix Ma, as 1.4.0 did; and the rows whose purpose needed a suffix kept their ids on a spelling that still has one — `abdul Doe nee Smith V`, `John III, Dr.`, `John III, Sir` — with the old inputs re-pinned as new rows: `abdul nee Smith V` maiden 'Smith V', `John V, Dr.` and `John V, Sir` given John, family V). `Sir abdul V` reads given 'abdul V': the V is a name word, and P5's licence behind a given-name title joins it. Nothing with a word to spare moves: `John W. Ingram V`, `King John Alexander V`, `Larry James Johnson I` (a v1 test) keep the suffix and the report. A title is no word to spare, so `Dr Smith V` reads given Smith, family V, which is 1.4.0's reading too, where `Dr. von Richthofen V` keeps suffix V, the particle and the surname being two words there. + THE LOWERCASE HALF. A numeral in lower case in a mixed-case name is not how anyone writes an initial, so it is the generation and reports nothing (Derek). Measured before the change, the reports had no rationale: `John Smith v` and `John Smith i` were silent because the listed letters reach the vocabulary path, `John Smith vi` and `John Smith x` reported because the shape fork reports everything it takes, and the one-case `john smith i` reported because the connective tag (P3, #479) makes a one-case `i` an initial. Now the mixed-case lower case is uniformly silent and a one-case name is counted like capitals. `john smith i` keeps its report, by Derek's call. + Declined: dropping the shape fork for single capital letters outright. Prototyped in a scratch edit, it moved 26 readings and cost the real generational names the count keeps: `John W. Ingram V` read middle 'W. Ingram', family 'V', and `John van der Berg V` folded the numeral into the particle chain. A variant that dropped the fork for I and X only, on top of the count, moved four more, among them the v1 test `Larry James Johnson I`. + ACCEPTED, and recorded in rules/S10: a one-case lowercase `v` is read as the vocabulary reads it, so `john v` still loses its family where `JOHN V` and `john x` keep theirs — the listed lowercase letter is taken as a suffix word before the numeral fork is reached, and closing that means touching the suffix-vocabulary path, which is not this change. And a one-letter numeral with a credential behind it is not the last word and is read as a name word (`John Smith V PhD` → family V, 1.4.0 parity), where `John Smith III PhD` reads suffix 'III PhD'. + Recompute: the moved names are this commit's diff of tools/differential/snapshot.jsonl (or `uv run python tools/differential/snapshot.py` on the parent tree with this commit's code); the prototype figures in the Declined paragraph were measured over every cases.py row and snapshot name with the fork patched in a scratch process. diff --git a/docs/design/rules/M2-maiden-marker-rule.md b/docs/design/rules/M2-maiden-marker-rule.md index 377e0fd0..6293b4aa 100644 --- a/docs/design/rules/M2-maiden-marker-rule.md +++ b/docs/design/rules/M2-maiden-marker-rule.md @@ -63,8 +63,9 @@ M2. Rationale: a maiden marker announces that what follows it is the "Jane Smith née Jones" → maiden="Jones" "Maria López née García Pérez" → maiden="García Pérez" "Jane Smith née Jones PhD" → suffix="PhD" - "John née Jones Smith V" → maiden="Jones Smith" - "John née Jones Smith V" → suffix="V" + "Jane Doe née Jones Smith V" → maiden="Jones Smith" + "Jane Doe née Jones Smith V" → suffix="V" + "John née Jones Smith V" → maiden="Jones Smith V" · boundary "Doe nee Smith, Jane" → maiden="Smith" "Doe, Jane nee Smith" → maiden="" "Doe, Jane nee Smith" → middle="nee Smith" diff --git a/docs/design/rules/P2-particles-join-forward.md b/docs/design/rules/P2-particles-join-forward.md index 107945c0..9ec268b2 100644 --- a/docs/design/rules/P2-particles-join-forward.md +++ b/docs/design/rules/P2-particles-join-forward.md @@ -7,9 +7,9 @@ P2. Rationale: a particle is written as part of the surname it a trailing suffix begins — the trailing run S2 reads, once, before the chain is made and counting the chain's own run as the one name word it will be, so the chain cannot change what it - counts: a trailing roman numeral, or a - bare acronym with words to spare, ends the chain as a suffix word - does, and so does a bare ambiguous acronym written in capitals in + counts: a trailing roman numeral S10 reads as the generation, or + a bare acronym with words to spare, ends the chain as a suffix + word does, and so does a bare ambiguous acronym written in capitals in a mixed-case name, which needs no words to spare; the same acronym written Title-case in a mixed-case name ends nothing and joins the chain as any name word does, words to spare or not, @@ -48,7 +48,7 @@ P2. Rationale: a particle is written as part of the surname it "John van der J. V" → family="van der J. V" · boundary "Freiherr von Berg MA" → family="von Berg" "Freiherr von Berg MA" → suffix="MA" - "Freiherr von Richthofen V" → suffix="V" · boundary + "Freiherr von Richthofen V" → family="von Richthofen V" · boundary "SMITH VD MA, JOHN" → family="SMITH VD MA" "SMITH VD, JOHN" → suffix="VD" · boundary "John van der Berg née Jones" → family="van der Berg" @@ -76,5 +76,5 @@ P2. Rationale: a particle is written as part of the surname it (#132's ask) has it as the surnames view rather than the family field. "Vincent van Gogh van Beethoven" → surnames="van Gogh van Beethoven" - history: decisions/P2 · interacts: P1, P4, H5, M2, S2, C2, S4 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_post_rules.py + history: decisions/P2 · interacts: P1, P4, H5, M2, S2, C2, S4, S10 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_post_rules.py diff --git a/docs/design/rules/P5-bound-given-names.md b/docs/design/rules/P5-bound-given-names.md index 4e482e04..1998a2b4 100644 --- a/docs/design/rules/P5-bound-given-names.md +++ b/docs/design/rules/P5-bound-given-names.md @@ -44,7 +44,7 @@ P5. Rationale: some given-name words are incomplete alone — "abdul" each read over what the other leaves until neither takes anything more, are read once, before the join is tried, and the name words they leave are the words to spare. A trailing - roman numeral, or a bare acronym the run takes, or a trailing + roman numeral the run takes (S10), or a bare acronym it takes, or a trailing title word the run takes, is no word to spare. The join joins two name words into one and, read after the trailing run, changes no suffix reading: a word the diff --git a/docs/design/rules/README.md b/docs/design/rules/README.md index 425d70e8..5dd3969f 100644 --- a/docs/design/rules/README.md +++ b/docs/design/rules/README.md @@ -59,6 +59,7 @@ The marker's unit is the whole rule, so a clause that reaches past the tolerated - [S7 — the company of an unambiguous credential](S7-company-of-an-unambiguous-credential.md) - [S8 — the credential's trailing run](S8-credentials-trailing-run.md) - [S9 — unlisted words that join the ambiguous class by shape](S9-unlisted-words-that-join-the-ambiguous-class-by-shape.md) +- [S10 — the trailing roman numeral](S10-trailing-roman-numeral.md) ## [Nicknames & quoted names (N)](N-nicknames-and-quoted-names.md) diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md new file mode 100644 index 00000000..830a4cab --- /dev/null +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -0,0 +1,51 @@ +# S10 — the trailing roman numeral + +S10. Rationale: a roman numeral ending a name is its generation, + but one letter is also how an initial is written, and the + generation it would name is rare; a name of two words has no + word to spare for it. + A roman numeral ending a name written without a comma reads as + the generation — the numerals the vocabulary lists, and those + up to X it does not, by their shape alone — unless the word in + front of it is written as an initial, which makes it a name word + too. A single letter written with a period is an initial, and a + numeral past X is a name word. A numeral of one letter needs + words to spare, as a bare ambiguous acronym does (S4): as the + second of two name words it is the family name, a title in front + being no word to spare, and where it is read as the generation + the reading is reported, as it is for a numeral read by its + shape alone. A numeral the vocabulary lists in two or more + letters is not in question and reports nothing. Written case + settles the fork in one way only: a numeral written in lower + case in a name of mixed case is the generation and reports + nothing, since nobody writes an initial that way. A name written + wholly in one case says nothing about any letter in it. The part + before a suffix comma is read as the same name written without + the comma (C1). + "John Smith V" → suffix="V" + "John Smith V" → ambiguities=("suffix-or-name",) + "John W. Ingram V" → suffix="V" + "John Smith VI" → suffix="VI" + "John Smith VI" → ambiguities=("suffix-or-name",) + "John Smith III" → ambiguities=() + "John III" → suffix="III" + "John V" → family="V" + "Malcolm X" → family="X" + "JOHN V" → family="V" + "Dr Smith V" → family="V" + "John V, Dr." → family="V" + "John Smith vi" → ambiguities=() + "john smith i" → ambiguities=("suffix-or-name",) + "John Smith V." → family="V." · boundary + "John J. V" → family="V" · boundary + "John Smith XI" → family="XI" · boundary + Accepted: a one-letter numeral the vocabulary lists, written in + lower case in a name written wholly in lower case, is read as + the vocabulary reads it, words to spare or not, where the same + letter in capitals is counted. + "john v" → family="" + Accepted: a numeral with another suffix word behind it is not + the last word of the name and is read as any word there is, so a + one-letter numeral before a credential is the family name. + "John Smith V PhD" → family="V" + history: decisions/S10 · interacts: S2, S4, S7, C1, P2, P3, M2, H1, H3 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py diff --git a/docs/design/rules/S2-suffix-word-ends-a-name-and-never-opens-the-string.md b/docs/design/rules/S2-suffix-word-ends-a-name-and-never-opens-the-string.md index 8a1817b3..da7c5584 100644 --- a/docs/design/rules/S2-suffix-word-ends-a-name-and-never-opens-the-string.md +++ b/docs/design/rules/S2-suffix-word-ends-a-name-and-never-opens-the-string.md @@ -87,4 +87,4 @@ S2. Rationale: generational suffixes and credentials are recognized and unchanged (decisions/v1-xfail-triage: `king` stays a title, for the addressing forms). "Dr Jr" → suffix="Jr" - history: decisions/S2 · interacts: H1, H2, H3, H4, H5, H6, C1, S3, S4, S5, S6, S7, S8, P2, P3, P4, P5 · implemented: nameparser/_pipeline/_classify.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py + history: decisions/S2 · interacts: H1, H2, H3, H4, H5, H6, C1, S3, S4, S5, S6, S7, S8, S10, P2, P3, P4, P5 · implemented: nameparser/_pipeline/_classify.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py diff --git a/docs/release_log.rst b/docs/release_log.rst index 2a1fb260..d44f6463 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -14,6 +14,7 @@ Release Log - **Fix a surname particle such as d' or al- reading as a middle name when a listing puts it after the given name.** ``Ivoi, Paul d'`` gives last ``d' Ivoi``, where 1.4.0 through 2.4.2 gave middle ``d'``. The Arabic article reads the same in its plain and assimilated spellings (``al-``, ``az-``, ``aš-``). A particle written joined to the surname, as in ``Paul d'Ivoi``, is unchanged. :issue:`700` - **Fix case repair capitalizing the Dutch particle 't.** ``jacobus henricus van 't hoff`` repairs to ``Jacobus Henricus van 't Hoff``, where 1.4.0 through 2.4.2 gave ``van 'T Hoff``. :issue:`700` - **Add Gov as a title.** ``Gov Smith`` gives title ``Gov``, last ``Smith``, where 1.4.0 through 2.4.2 gave first ``Gov``, and ``Lt. Gov. Smith, John`` gives title ``Lt. Gov.`` with the change above. :issue:`701` + - **Fix a capital V, I or X after a lone first name being read as a generational suffix, leaving no last name.** ``John V`` gives first ``John``, last ``V``, and ``Malcolm X`` last ``X``, where 2.0.0 through 2.4.2 gave suffix ``V`` and ``X`` with no last name, restoring 1.4.0's reading. A name with a word to spare keeps the suffix: ``John Smith V`` still gives suffix ``V``. :issue:`718` * 2.4.2 - October 13, 2026 diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 1d678982..837a61d4 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -88,7 +88,8 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - effective_script, is_suffix_lenient, resolve_script_set, + effective_script, is_suffix_lenient, numeral_settled_by_case, + resolve_script_set, ) from nameparser._pipeline._pieces import ( anchor_in_reach, credential_at_the_given_slot, given_slot_anchors, @@ -404,7 +405,10 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], Role.TITLE) for piece in peeled.absorbed: ambiguities.append(_absorbed(piece, tokens)) - if peeled.numeral is not None: + # rules/S10: "a numeral written in lower case in a name of mixed + # case is the generation and reports nothing" + if peeled.numeral is not None and not numeral_settled_by_case( + tokens[peeled.numeral[0]].text, state.one_case): # a trailing single letter is a name part unless it happens # to be a roman numeral -- and V/X/I are ordinary middle # initials, so taking it as a suffix is a call, not a fact diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index e20428c8..8c614277 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -1261,9 +1261,9 @@ def chain(tail: int) -> None: == [tuple(pieces[j]) for j in rest[before.names:]]) chained = fk + 1 in chain_took names_after = after.names - # rules/P5: "a trailing roman numeral, or a bare - # acronym the run takes, or a trailing title word the - # run takes, is no word to spare" -- and the join joins + # rules/P5: "a trailing roman numeral the run takes + # (S10), or a bare acronym it takes, or a trailing title + # word the run takes, is no word to spare" -- and the join joins # two NAME words, so a piece the chain takes is no more # joinable than a marker or a suffix piece is: 'Sir # abdul Prof.' reads title 'Sir Prof.', given 'abdul'. diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index a653fa66..6ed3953e 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -63,7 +63,8 @@ ) from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, _ROMAN, Lean, ambiguous_lean, in_initialless_script, - is_single_letter_numeral, is_trailing_numeral_suffix, tag_marker_runs, + is_single_letter_numeral, is_trailing_numeral_suffix, + numeral_settled_by_case, tag_marker_runs, ) @@ -1137,13 +1138,18 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], if is_suffix_piece(piece, ptags[rest[k - 1]], tokens): k -= 1 continue - # a final single letter that is a roman numeral, after a piece + # a final single token that is a roman numeral, after a piece # that is not initial-shaped; the predicate's docstring carries - # the is_initial_shaped reasoning (#320) + # the is_initial_shaped reasoning (#320). rules/S10: "A numeral + # of one letter needs words to spare, as a bare ambiguous + # acronym does" -- the acronym fork's own count, + # below, so 'John V' keeps family 'V' (#718) if (k == len(rest) and k >= 2 and len(piece) == 1 and is_trailing_numeral_suffix( - tokens[piece[0]].text, - tokens[pieces[rest[k - 2]][0]].text)): + text := tokens[piece[0]].text, + tokens[pieces[rest[k - 2]][0]].text) + and (len(text) > 1 or k - 2 >= second + or numeral_settled_by_case(text, one_case))): numeral = tuple(piece) k -= 1 continue diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 70f4dce0..e152a3df 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -252,10 +252,11 @@ def is_initial_shaped(text: str) -> bool: _ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I) -# rules/S2: "a trailing word of the suffix vocabulary reads as a -# suffix" -- the roman-numeral half of that rule, which the initial -# veto would otherwise take: V, I and X are suffix vocabulary AND bare -# capitals, and a bare capital inside a name is a middle initial. +# rules/S10: "A roman numeral ending a name written without a comma +# reads as the generation" -- which the initial veto would otherwise +# take: V, I and X are suffix vocabulary AND bare capitals, and a bare +# capital inside a name is a middle initial. Whether a one-letter +# numeral has the words to spare is the caller's count (`peel_trailing`). def is_trailing_numeral_suffix(text: str, preceding: str) -> bool: """assign's roman-numeral fork, shared with group's bound-given reserve (#401): a FINAL single-token piece that is a roman numeral @@ -273,6 +274,18 @@ def is_trailing_numeral_suffix(text: str, preceding: str) -> bool: and not is_initial_shaped(preceding)) +# rules/S10: "a numeral written in lower case in a name of mixed case is +# the generation and reports nothing" -- the one writing that settles +# the fork, since nobody writes an initial in lower case beside +# capitals. Asked by the walk, which then needs no words to spare, and +# by assign, which then reports nothing, so the two cannot drift. +def numeral_settled_by_case(text: str, one_case: bool | None) -> bool: + """Whether a numeral the trailing fork reads is lower case in a + name written in more than one case. `one_case` is + ParseState.one_case; None (nobody asked) settles nothing.""" + return one_case is False and text.islower() + + # #544: a single-letter roman numeral ('V', 'v', 'I.'), in any case, # is excluded for its SHAPE -- one letter is how a middle initial is # written, and rules/S3 retires single-character matches for the diff --git a/tests/v2/cases.py b/tests/v2/cases.py index a7bef8b2..19331e74 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -3962,16 +3962,17 @@ def _check_cjk_shape_purity(self) -> None: "own reading of 'V.' and not the credential class's"), Case("strict_comma_reads_the_bare_numeral_into_the_run", "Smith V, Ma", - {"given": "Smith", "suffix": "V, Ma"}, + {"given": "Smith", "family": "V", "suffix": "Ma"}, policy=Policy(lenient_comma_suffixes=False), - ambiguities=("suffix-or-name", "suffix-or-name", - "given-or-family"), + ambiguities=("suffix-or-name",), classification="fix(#718)", notes="the period is the whole difference from the row " "above: bare 'V' is still initial-shaped and still " "counts as a name word here, so the structure flips " - "the same way -- but the trailing peel then takes it " - "as the roman numeral, leaving 'Smith' the only name " - "word and reporting all three forks"), + "the same way. Until #718 the trailing peel then took " + "it as the roman numeral, leaving 'Smith' the only " + "name word and reporting all three forks; a one-letter " + "numeral now needs words to spare (S10, the trailing roman numeral), so 'V' " + "is the family, as 1.4.0 read it"), Case("caps_run_declines_a_bound_given_head", "John Smith, ABDUL AP", {"given": "ABDUL AP", "family": "John Smith"}, policy=Policy(unlisted_caps_suffixes=CapsSuffixes.EVERYWHERE), @@ -5469,8 +5470,9 @@ def _check_cjk_shape_purity(self) -> None: "behind it still leaves the clause. 2.3.0 read maiden 'Smith V " "Prof.', suffix 'PhD', with no title at all"), Case("the_bound_given_join_is_the_given_slots_alone", - "abdul nee Smith V", - {"given": "abdul", "suffix": "V", "maiden": "Smith"}, + "abdul Doe nee Smith V", + {"given": "abdul", "family": "Doe", "suffix": "V", + "maiden": "Smith"}, ambiguities=("suffix-or-name",), notes="P5's bound-given join is LENIENT only after a family " "comma; before one the STRICT reserve declines a join " @@ -5479,7 +5481,17 @@ def _check_cjk_shape_purity(self) -> None: "(The contrast this note drew, 'Berg, abdul nee Smith " "V', has made no clause since #601: a marker after a " "family comma is an ordinary word.) Unchanged from " - "2.3.0"), + "2.3.0. 'Doe' since #718, which made the 'V' of " + "'abdul nee Smith V' a name word (the next row)"), + Case("a_numeral_with_no_word_to_spare_stays_in_the_clause", + "abdul nee Smith V", + {"given": "abdul", "maiden": "Smith V"}, + classification="fix(#718)", + notes="the row above without its family word: read without " + "the clause the name is 'abdul V', and a one-letter " + "numeral needs words to spare (S10, the trailing roman numeral), so 'V' is " + "no suffix and the clause keeps it as it keeps any " + "name word, by M2 (the maiden-marker rule)"), Case("a_title_released_from_the_bound_given_pair", "abdul nee Smith Dr.", {"title": "Dr.", "family": "abdul", "maiden": "Smith"}, @@ -7413,6 +7425,51 @@ def _check_cjk_shape_purity(self) -> None: notes="a trailing single letter is a name part unless it is a " "roman numeral; V/X/I are also ordinary middle initials, " "so the reading is reported"), + Case("a_lone_numeral_with_no_word_to_spare_is_the_family", + "John V", {"given": "John", "family": "V"}, + classification="fix(#718)", + notes="S10 (the trailing roman numeral): a one-letter numeral needs words to spare, " + "as a bare ambiguous acronym does, and 'John' is the " + "only other word -- the contrast with 'John Smith V' " + "above is the count. 2.0.0 to 2.4.2 read suffix 'V' " + "with no family; 1.4.0 read last 'V'"), + Case("a_lone_shape_numeral_with_no_word_to_spare_is_the_family", + "Malcolm X", {"given": "Malcolm", "family": "X"}, + classification="fix(#718)", + notes="the shape-only spelling of the row above: 'x' is in " + "no vocabulary and reaches the fork by its shape"), + Case("a_lone_numeral_in_a_one_case_name_is_counted", + "JOHN V", {"given": "JOHN", "family": "V"}, + classification="fix(#718)", + notes="a name written in one case says nothing about the " + "letter, so the count decides as it does for 'John V'"), + Case("a_title_is_no_word_to_spare_for_a_lone_numeral", + "Dr Smith V", {"title": "Dr", "given": "Smith", "family": "V"}, + classification="fix(#718)", + notes="S10's title clause (the trailing roman numeral); 1.4.0 read first 'Smith', " + "last 'V' too"), + Case("a_multi_letter_numeral_needs_no_word_to_spare", + "John III", {"given": "John", "suffix": "III"}, + ambiguities=("given-or-family",), + notes="the boundary of the count: a listed numeral of two " + "letters is not initial-shaped and reads as any suffix " + "word, so the lone 'John' is what is reported (O5)"), + Case("a_lowercase_numeral_in_a_mixed_case_name_reports_nothing", + "John Smith vi", {"given": "John", "family": "Smith", + "suffix": "vi"}, + classification="fix(#718)", + notes="S10 (the trailing roman numeral): nobody writes an initial in lower case " + "beside capitals, so the writing settles the fork. It " + "reported until #718, the shape fork reporting all it " + "took, while the listed 'John Smith v' never did"), + Case("a_lowercase_numeral_in_a_one_case_name_still_reports", + "john smith i", {"given": "john", "family": "smith", + "suffix": "i"}, + ambiguities=("suffix-or-name",), + notes="the contrast that makes the silence above a decision " + "about case: written wholly in lower case the letter " + "could be an initial, and the fork reports (Derek, " + "#718)"), Case("initial_not_suffix", "John V. Smith", {"given": "John", "middle": "V.", "family": "Smith"}, notes="shape 1's Middle slot filled by an initial-shaped " @@ -8655,29 +8712,45 @@ def _check_cjk_shape_purity(self) -> None: "silences O5's report and not H4's. 1.4.0 had no maiden " "SUPPORT and read title 'Lord Chancellor', first 'née', " "last 'Jones'"), - Case("comma_path_title_decides_the_lone_word", "John V, Dr.", - {"title": "Dr.", "family": "John", "suffix": "V"}, - ambiguities=("suffix-or-name",), classification="fix(#296)", + Case("comma_path_title_decides_the_lone_word", "John III, Dr.", + {"title": "Dr.", "family": "John", "suffix": "III"}, + classification="fix(#296)", notes="a comma with no name word after it hands segment 0 " "back to the positional read, and the title deciding " "the field stands in the OTHER segment where the " "leading-title peel cannot count it -- so the read is " "told (`titled`) and O5 stays silent, where it named " "`given` for a word H1 then wrote to `family` (#449 " - "review round). The roman numeral still reports, that " - "fork being untouched. 1.4.0 read first 'John', last " - "'V', suffix 'Dr.', 'dr' having been suffix vocabulary " - "before the audit"), + "review round). 'John III' since #718, which made the " + "'V' of 'John V, Dr.' a name word (the next row) and " + "left no lone word to decide"), + Case("a_lone_numeral_before_a_suffix_comma_is_the_family", + "John V, Dr.", + {"title": "Dr.", "given": "John", "family": "V"}, + classification="fix(#718)", + notes="the part before a suffix comma reads as the comma-less " + "name, and 'John V' has no word to spare for a " + "one-letter numeral (S10, the trailing roman numeral). 1.4.0 read first " + "'John', last 'V', suffix 'Dr.', 'dr' having been " + "suffix vocabulary before the audit"), Case("comma_path_given_name_title_decides_the_lone_word", + "John III, Sir", + {"title": "Sir", "family": "John", "suffix": "III"}, + classification="fix(#296)", + notes="the title row above with a GIVEN-NAME title. O5 is " + "silenced all the same: what the report is about is a " + "field NOTHING decided, and this one a title decided. " + "Written 'John V, Sir' until #718, which made the 'V' " + "a name word (the next row); with 'III' the word lands " + "in `family`, as it does on master before #718, where " + "'V' had left it in `given`"), + Case("a_lone_numeral_before_a_given_name_title_comma", "John V, Sir", - {"title": "Sir", "given": "John", "suffix": "V"}, - ambiguities=("suffix-or-name",), classification="fix(#296)", - notes="the row above with a GIVEN-NAME title, which leaves " - "the word in `given` -- the field O5 would have named. " - "Silenced all the same: what the report is about is a " - "field NOTHING decided, and this one a title decided, " - "so the two agreeing is not the report being right. " - "1.4.0 read title 'Sir', last 'John V'"), + {"title": "Sir", "given": "John", "family": "V"}, + classification="fix(#718)", + notes="the row above with a one-letter numeral, which has no " + "word to spare (S10, the trailing roman numeral). 1.4.0 read title 'Sir', " + "last 'John V'"), Case("marker_led_clause_in_a_quote_pair", 'Jane Smith "née Jones"', {"given": "Jane", "family": "Smith", "maiden": "Jones"}, diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index ee2a7168..425e67d9 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -154,7 +154,7 @@ def test_a_connective_or_initial_report_is_withdrawn_where_the_role_is_a_suffix( "JOHN QUINCY SMITH I": ["suffix-or-name"], "john smith i": ["suffix-or-name"], "SMITH, JOHN I": [], - "HENRY I": ["suffix-or-name", "given-or-family"], + "HENRY SMITH I": ["suffix-or-name"], } for text, kinds in withdrawn.items(): name = parse(text) @@ -164,8 +164,10 @@ def test_a_connective_or_initial_report_is_withdrawn_where_the_role_is_a_suffix( # the contrast that keeps the withdrawal honest: where the parse # roles the same letter a NAME word the fork DID resolve to one # of its branches, and the report stands + # -- 'HENRY I' among them since #718: with no word to spare the + # letter is the family name (rules/S10), where it was the suffix for text in ("JOSEP CAROD I ROVIRA", "josep carod i rovira", - "JOHN I SMITH", "JOSEP CAROD I ROVIRA III"): + "JOHN I SMITH", "JOSEP CAROD I ROVIRA III", "HENRY I"): name = parse(text) assert "conjunction-or-initial" in [ a.kind.value for a in name.ambiguities], text diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index ce353215..7ff2a2da 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -958,11 +958,17 @@ def test_the_maiden_walk_stops_before_the_numeral_too() -> None: # Since #601 the take consumes the numeral it ends at, already a # suffix -- and finds it by the fork's SHAPE test, which asks no # vocabulary: this module's lexicon lists no 'v' at all. - out = _grouped("John née Jones Smith V") + # 'Jane Doe' rather than 'John' since #718: a one-letter numeral + # needs words to spare (rules/S10), and 'John V' has none, so + # there the V is no suffix and the clause keeps it (rules/M2). + out = _grouped("Jane Doe née Jones Smith V") assert [t.text for t in out.tokens if t.role is Role.MAIDEN] == \ ["Jones", "Smith"] assert [t.text for t in out.tokens if t.role is Role.SUFFIX] == ["V"] - assert _piece_texts(out) == [["John"]] + assert _piece_texts(out) == [["Jane", "Doe"]] + out = _grouped("John née Jones Smith V") + assert [t.text for t in out.tokens if t.role is Role.MAIDEN] == \ + ["Jones", "Smith", "V"] def test_the_maiden_walk_keeps_the_acronym_its_writing_declines( diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 8def295d..9a2b9d71 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -183,9 +183,12 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # given-or-family claim under each order); 1162 -> 1171 on # 2026-10-17: three #701 rows reporting suffix-or-name, under each # order; 1171 -> 1174 on 2026-10-17: the エリザベス二世 row (#707, - # one given-or-family claim under each order). - assert claims == 1174, ( - f"the field sweep checks {claims} claims, recorded as 1174 on " + # one given-or-family claim under each order); 1174 -> 1172 on + # 2026-10-17: #718's re-pinned and added numeral rows (the lone + # numeral rows lose their given-or-family claims; 'John III' and + # 'Malcolm X' rows bring some back). + assert claims == 1172, ( + f"the field sweep checks {claims} claims, recorded as 1172 on " f"2026-10-17") diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 12872ec6..da3610f6 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -559,9 +559,12 @@ def test_the_bound_given_reserve_spares_the_family_assign_will_keep() -> None: n = parse("abdul Smith V") assert (n.given, n.family, n.suffix) == ("abdul", "Smith", "V") assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SUFFIX_OR_NAME] + # Since #718 a one-letter numeral needs words to spare (rules/S10) + # and a title is none, so 'V' is a name word there, and the + # licence joins it into the given pair as it would any word licensed = parse("Sir abdul V") assert (licensed.given, licensed.family, licensed.suffix) == \ - ("abdul", "", "V") + ("abdul V", "", "") def test_the_bound_given_join_leaves_a_suffix_where_it_stands() -> None: @@ -698,7 +701,7 @@ def test_the_chain_and_the_walk_stop_where_the_peel_begins() -> None: ("John van der Berg Ma", "van der Berg Ma", "")): n = parse(text) assert (n.family, n.suffix) == (family, suffix), text - n = parse("John née Jones Smith V") + n = parse("Jane Doe née Jones Smith V") assert (n.maiden, n.suffix) == ("Jones Smith", "V") n = parse("Jane Smith née Jones V") assert (n.family, n.maiden, n.suffix) == ("Smith", "Jones", "V") @@ -730,11 +733,14 @@ def test_the_chain_and_the_walk_stop_where_the_peel_begins() -> None: n = parse("Freiherr von Berg MA") assert (n.title, n.family, n.suffix) == ("Freiherr", "von Berg", "MA") # the numeral keeps its three pieces behind the same word, and the - # chain, now the one name piece, reads as 'Dr. Smith V' reads + # chain is the one name piece. Since #718 a one-letter numeral + # needs words to spare (rules/S10) and a title is none, so behind + # one name word the V is a name word: the chain takes it as P2 + # takes any, where 'Dr. Smith V' gives it the family n = parse("Freiherr von Richthofen V") - assert (n.given, n.family, n.suffix) == ("", "von Richthofen", "V") + assert (n.given, n.family, n.suffix) == ("", "von Richthofen V", "") n = parse("Dr. Smith V") - assert (n.given, n.family, n.suffix) == ("", "Smith", "V") + assert (n.given, n.family, n.suffix) == ("Smith", "V", "") # both stops read the TRAILING word, so an acronym with the # numeral behind it is not the word either fork asks about and # stays maiden text -- and so does a numeral with an acronym diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index dbacbfef..99eb9ee4 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -68,6 +68,7 @@ "Doe, PhD Jones" "Dr Jr" "Dr King Jr" +"Dr Smith V" "Dr." "Dr. Freiherr von Berg, Ed" "Dr. John Smith Prof." @@ -136,6 +137,7 @@ "JOHN SMITH MA" "JOHN SMITH PH.D." "JOHN SMITH, VD MA" +"JOHN V" "JOSE ORTEGA-Y-GASSET" "JOSÉ GARCÍA" "JUAN GARCIA Jr." @@ -172,6 +174,7 @@ "Jane Doe nee Smith V Prof." "Jane Doe nee Smith VI Prof." "Jane Doe nee Smith X.Y.Z." +"Jane Doe née Jones Smith V" "Jane Doe, MS LAc" "Jane Smith (Nee)" "Jane Smith (Nee) (Jones)" @@ -200,6 +203,8 @@ "Jiří Dienstbier ml." "John . Smith" "John Doctor Smith" +"John III" +"John J. V" "John Ma" "John PhD Smith" "John Prof. MA" @@ -207,6 +212,7 @@ "John Smith" "John Smith 1.4" "John Smith Esq." +"John Smith III" "John Smith J.u.n.i.o.r." "John Smith Jr." "John Smith Jr. Prof." @@ -226,9 +232,15 @@ "John Smith Q.W.E.R.T." "John Smith R.A.I." "John Smith Sir" +"John Smith V" +"John Smith V PhD" +"John Smith V." +"John Smith VI" "John Smith X.Y.Z." +"John Smith XI" "John Smith XYZ" "John Smith Xyz." +"John Smith vi" "John Smith, A.B." "John Smith, A.B. Ph.D." "John Smith, Ed" @@ -265,6 +277,9 @@ "John Smith, X.Y.Z." "John Smith, XYZ" "John Smith, vd Ma" +"John V" +"John V, Dr." +"John W. Ingram V" "John and Jane Smith" "John nee Prof. ba MA" "John née Jones Smith Ma" @@ -316,6 +331,7 @@ "MD PhD Jr., John" "MSc Dr. med. univ." "Mai Le née Nguyen" +"Malcolm X" "Mari' Aube'" "Maria Kowalska (z domu Nowak)" "Maria Kowalska (z domu)" @@ -498,6 +514,7 @@ "john smith MEng" "john smith bsc" "john smith ed" +"john smith i" "john smith iii" "john smith iii." "john smith ix" @@ -517,6 +534,7 @@ "john smith xi" "john smith, v" "john smith, vi" +"john v" "john van der berg ma" "jose e maria santos" "jose ortega-y-gasset" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index a7601d47..d1ebf5a3 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -678,6 +678,7 @@ {"name": "Dr Jr", "given": "Dr", "suffix": "Jr", "ambiguities": ["TITLE_OR_NAME"], "initials": "D."} {"name": "Dr King Jr", "title": "Dr", "family": "King", "suffix": "Jr", "ambiguities": ["TITLE_OR_NAME"], "initials": "K."} {"name": "Dr Martin Luther King, Jr.", "title": "Dr", "given": "Martin", "middle": "Luther", "family": "King", "suffix": "Jr.", "initials": "M. L. K."} +{"name": "Dr Smith V", "title": "Dr", "given": "Smith", "family": "V", "initials": "S. V."} {"name": "Dr 田中さん, V.", "title": "Dr", "family": "田中", "suffix": "さん, V.", "initials": "田."} {"name": "Dr 김민준, Jr.", "title": "Dr", "given": "민준", "family": "김", "suffix": "Jr.", "initials": "민. 김."} {"name": "Dr 김민준, MA", "title": "Dr", "given": "민준", "family": "김", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "민. 김."} @@ -936,7 +937,7 @@ {"name": "Freiherr von Berg MA", "title": "Freiherr", "family": "von Berg", "suffix": "MA", "ambiguities": ["PARTICLE_OR_GIVEN", "SUFFIX_OR_NAME"], "initials": "B."} {"name": "Freiherr von Berg, Ed", "given": "Ed", "family": "Freiherr von Berg", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "E. B."} {"name": "Freiherr von Richthofen", "title": "Freiherr", "family": "von Richthofen", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "R."} -{"name": "Freiherr von Richthofen V", "title": "Freiherr", "family": "von Richthofen", "suffix": "V", "ambiguities": ["PARTICLE_OR_GIVEN", "SUFFIX_OR_NAME"], "initials": "R."} +{"name": "Freiherr von Richthofen V", "title": "Freiherr", "family": "von Richthofen V", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "R. V."} {"name": "Freire Xiana", "order": "FAMILY_FIRST", "given": "Xiana", "family": "Freire", "initials": "X. F."} {"name": "Freire, Xiana", "given": "Xiana", "family": "Freire", "initials": "X. F."} {"name": "Friedrich IV.", "given": "Friedrich", "suffix": "IV.", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "F."} @@ -1270,6 +1271,7 @@ {"name": "JOHN SMITH PH.D.", "given": "JOHN", "family": "SMITH", "suffix": "PH.D.", "initials": "J. S."} {"name": "JOHN SMITH, MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "JOHN SMITH, VD MA", "given": "JOHN", "family": "SMITH", "suffix": "VD MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "JOHN V", "given": "JOHN", "family": "V", "initials": "J. V."} {"name": "JOSE E MARIA SANTOS", "given": "JOSE", "middle": "E MARIA", "family": "SANTOS", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "J. E. M. S."} {"name": "JOSE ORTEGA-Y-GASSET", "given": "JOSE", "family": "ORTEGA-Y-GASSET", "initials": "J. O."} {"name": "JOSEP CAROD I ROVIRA", "given": "JOSEP", "middle": "CAROD I", "family": "ROVIRA", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "J. C. I. R."} @@ -1360,6 +1362,7 @@ {"name": "Jane Doe nee Smith XYZ", "given": "Jane", "family": "Doe", "maiden": "Smith XYZ", "initials": "J. D."} {"name": "Jane Doe nee Smith do", "given": "Jane", "family": "Doe", "suffix": "do", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee Yo-Yo Ma", "given": "Jane", "family": "Doe", "maiden": "Yo-Yo Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} +{"name": "Jane Doe née Jones Smith V", "given": "Jane", "family": "Doe", "suffix": "V", "maiden": "Jones Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe, MS LAc", "given": "Jane", "family": "Doe", "suffix": "MS LAc", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Mac Beth", "given": "Jane", "family": "Mac Beth", "initials": "J. B."} {"name": "Jane Smith", "given": "Jane", "family": "Smith", "initials": "J. S."} @@ -1486,6 +1489,8 @@ {"name": "John Hamish Watson", "given": "John", "middle": "Hamish", "family": "Watson", "initials": "J. H. W."} {"name": "John Hámish Watson", "given": "John", "middle": "Hámish", "family": "Watson", "initials": "J. H. W."} {"name": "John I Smith", "given": "John", "middle": "I", "family": "Smith", "initials": "J. I. S."} +{"name": "John III", "given": "John", "suffix": "III", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "J."} +{"name": "John J. V", "given": "John", "middle": "J.", "family": "V", "initials": "J. J. V."} {"name": "John Jack Andrew Kennedy", "given": "John", "middle": "Jack Andrew", "family": "Kennedy", "initials": "J. J. A. K."} {"name": "John Jones (Google Docs)", "given": "John", "family": "Jones", "nickname": "Google Docs", "initials": "J. J."} {"name": "John Jones (Google Docs), Jr. (Unknown)", "given": "John", "family": "Jones", "suffix": "Jr.", "nickname": "Google Docs Unknown", "initials": "J. J."} @@ -1510,6 +1515,7 @@ {"name": "John Smith Esq.", "given": "John", "family": "Smith", "suffix": "Esq.", "initials": "J. S."} {"name": "John Smith Esquire", "given": "John", "family": "Smith", "suffix": "Esquire", "initials": "J. S."} {"name": "John Smith I", "given": "John", "family": "Smith", "suffix": "I", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith III", "given": "John", "family": "Smith", "suffix": "III", "initials": "J. S."} {"name": "John Smith J.u.n.i.o.r.", "given": "John", "family": "Smith", "suffix": "J.u.n.i.o.r.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith Jr", "given": "John", "family": "Smith", "suffix": "Jr", "initials": "J. S."} {"name": "John Smith Jr.", "given": "John", "family": "Smith", "suffix": "Jr.", "initials": "J. S."} @@ -1540,11 +1546,16 @@ {"name": "John Smith RAI", "given": "John", "middle": "Smith", "family": "RAI", "initials": "J. S. R."} {"name": "John Smith Rev.", "title": "Rev.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "John Smith Sir", "given": "John", "middle": "Smith", "family": "Sir", "initials": "J. S. S."} +{"name": "John Smith V", "given": "John", "family": "Smith", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith V PhD", "given": "John", "middle": "Smith", "family": "V", "suffix": "PhD", "initials": "J. S. V."} +{"name": "John Smith V.", "given": "John", "middle": "Smith", "family": "V.", "initials": "J. S. V."} {"name": "John Smith VI", "given": "John", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith X.Y.Z.", "given": "John", "family": "Smith", "suffix": "X.Y.Z.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith XI", "given": "John", "middle": "Smith", "family": "XI", "initials": "J. S. X."} {"name": "John Smith XYZ", "given": "John", "middle": "Smith", "family": "XYZ", "initials": "J. S. X."} {"name": "John Smith Xyz.", "given": "John", "middle": "Smith", "family": "Xyz.", "initials": "J. S. X."} {"name": "John Smith nee Jones R.A.I.", "given": "John", "family": "Smith", "suffix": "R.A.I.", "maiden": "Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith vi", "given": "John", "family": "Smith", "suffix": "vi", "initials": "J. S."} {"name": "John Smith 田.中.", "given": "John", "middle": "Smith", "family": "田.中.", "initials": "J. S. 田."} {"name": "John Smith, 1.4", "given": "1.4", "family": "John Smith", "initials": "1. J. S."} {"name": "John Smith, A.B.", "given": "A.B.", "family": "John Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "A. J. S."} @@ -1598,7 +1609,8 @@ {"name": "John Smith, X.Y.Z. MA", "given": "John", "family": "Smith", "suffix": "X.Y.Z. MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, XYZ", "given": "John", "family": "Smith", "suffix": "XYZ", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, vd Ma", "given": "John", "family": "Smith", "suffix": "vd Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John V", "given": "John", "suffix": "V", "ambiguities": ["GIVEN_OR_FAMILY", "SUFFIX_OR_NAME"], "initials": "J."} +{"name": "John V", "given": "John", "family": "V", "initials": "J. V."} +{"name": "John V, Dr.", "title": "Dr.", "given": "John", "family": "V", "initials": "J. V."} {"name": "John V. Smith", "given": "John", "middle": "V.", "family": "Smith", "initials": "J. V. S."} {"name": "John W. Ingram", "given": "John", "middle": "W.", "family": "Ingram", "initials": "J. W. I."} {"name": "John W. Ingram V", "given": "John", "middle": "W.", "family": "Ingram", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. W. I."} @@ -1613,7 +1625,7 @@ {"name": "John e Smith, III", "given": "John", "middle": "e", "family": "Smith", "suffix": "III", "initials": "J. e. S."} {"name": "John nee Prof. ba MA", "family": "John", "suffix": "MA", "maiden": "Prof. ba", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} {"name": "John née Jones Smith Ma", "family": "John", "maiden": "Jones Smith Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} -{"name": "John née Jones Smith V", "family": "John", "suffix": "V", "maiden": "Jones Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} +{"name": "John née Jones Smith V", "family": "John", "maiden": "Jones Smith V", "initials": "J."} {"name": "John of the Doe", "given": "John of the Doe", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "J. D."} {"name": "John van Buren, Ed", "given": "John", "family": "van Buren", "suffix": "Ed", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. B."} {"name": "John van Mc", "given": "John", "family": "van Mc", "initials": "J. v. M."} @@ -2076,6 +2088,7 @@ {"name": "Makonnen Haile Selassie", "given": "Makonnen", "middle": "Haile", "family": "Selassie", "initials": "M. H. S."} {"name": "Malam Shehu Bello Modibbo MP", "given": "Malam", "middle": "Shehu Bello", "family": "Modibbo", "suffix": "MP", "initials": "M. S. B. M."} {"name": "Malas Zénāwi", "given": "Malas", "family": "Zénāwi", "initials": "M. Z."} +{"name": "Malcolm X", "given": "Malcolm", "family": "X", "initials": "M. X."} {"name": "Maleingreau, Paul de", "given": "Paul", "family": "de Maleingreau", "initials": "P. M."} {"name": "Malik Ambar", "given": "Malik", "family": "Ambar", "initials": "M. A."} {"name": "Malik al-ʿĀdil al I.", "given": "Malik", "middle": "al-ʿĀdil", "family": "al I.", "initials": "M. a. I."} @@ -2250,7 +2263,7 @@ {"name": "Mohamad Ali", "given": "Mohamad", "family": "Ali", "initials": "M. A."} {"name": "Mohamad Ali Khalil", "given": "Mohamad", "middle": "Ali", "family": "Khalil", "initials": "M. A. K."} {"name": "Mohamad Asri Muda", "given": "Mohamad", "middle": "Asri", "family": "Muda", "initials": "M. A. M."} -{"name": "Mohamad X", "given": "Mohamad", "suffix": "X", "ambiguities": ["GIVEN_OR_FAMILY", "SUFFIX_OR_NAME"], "initials": "M."} +{"name": "Mohamad X", "given": "Mohamad", "family": "X", "initials": "M. X."} {"name": "Mohamad X Surname", "given": "Mohamad", "middle": "X", "family": "Surname", "initials": "M. X. S."} {"name": "Mohamad Zein El Dine", "given": "Mohamad", "middle": "Zein El", "family": "Dine", "initials": "M. Z. E. D."} {"name": "Mohamad Zeineddine", "given": "Mohamad", "family": "Zeineddine", "initials": "M. Z."} @@ -3372,6 +3385,7 @@ {"name": "john smith MEng", "given": "john", "middle": "smith", "family": "MEng", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s. M."} {"name": "john smith bsc", "given": "john", "family": "smith", "suffix": "bsc", "initials": "j. s."} {"name": "john smith ed", "given": "john", "family": "smith", "suffix": "ed", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} +{"name": "john smith i", "given": "john", "family": "smith", "suffix": "i", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} {"name": "john smith iii", "given": "john", "family": "smith", "suffix": "iii", "initials": "j. s."} {"name": "john smith iii.", "given": "john", "family": "smith", "suffix": "iii.", "initials": "j. s."} {"name": "john smith ix", "given": "john", "family": "smith", "suffix": "ix", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} @@ -3394,6 +3408,7 @@ {"name": "john smith, phd meng", "given": "john", "family": "smith", "suffix": "phd meng", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} {"name": "john smith, v", "given": "john", "family": "smith", "suffix": "v", "initials": "j. s."} {"name": "john smith, vi", "given": "vi", "family": "john smith", "initials": "v. j. s."} +{"name": "john v", "given": "john", "suffix": "v", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "j."} {"name": "john van der berg ma", "given": "john", "family": "van der berg", "suffix": "ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. b."} {"name": "johnny y", "given": "johnny", "family": "y", "initials": "j. y."} {"name": "jones, john e", "given": "john", "middle": "e", "family": "jones", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "j. e. j."} From bb35048b93d92d1ab5e0cac192de941c3986e740 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 02:48:00 -0700 Subject: [PATCH 2/7] Report a declined numeral both ways, as S4's fork reports (#718) 'John V' now reports suffix-or-name while reading family V, as 'Jack Ma' reports keeping its family. The declined numeral is a pick of the trailing walk, and three sites take its branches, so three report it: assign (a lone name word), the particle chain ('Freiherr von Richthofen V'), and a maiden clause that keeps it ('abdul nee Smith V'). A letter classify already reported as an initial reports nothing new (INV10 found 138 one-case rows, 'HENRY I', carrying both kinds). Co-Authored-By: Claude Opus 5.5 --- .../decisions/S10-trailing-roman-numeral.md | 2 + .../rules/S10-trailing-roman-numeral.md | 13 +++-- nameparser/_pipeline/_assign.py | 19 +++++- nameparser/_pipeline/_group.py | 38 +++++++++++- nameparser/_pipeline/_pieces.py | 40 ++++++++++--- tests/v2/cases.py | 58 ++++++++++--------- tests/v2/test_cases.py | 10 ++-- tools/differential/snapshot.jsonl | 16 ++--- 8 files changed, 141 insertions(+), 55 deletions(-) diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index e8aa9dbd..bf41386c 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -6,3 +6,5 @@ Declined: dropping the shape fork for single capital letters outright. Prototyped in a scratch edit, it moved 26 readings and cost the real generational names the count keeps: `John W. Ingram V` read middle 'W. Ingram', family 'V', and `John van der Berg V` folded the numeral into the particle chain. A variant that dropped the fork for I and X only, on top of the count, moved four more, among them the v1 test `Larry James Johnson I`. ACCEPTED, and recorded in rules/S10: a one-case lowercase `v` is read as the vocabulary reads it, so `john v` still loses its family where `JOHN V` and `john x` keep theirs — the listed lowercase letter is taken as a suffix word before the numeral fork is reached, and closing that means touching the suffix-vocabulary path, which is not this change. And a one-letter numeral with a credential behind it is not the last word and is read as a name word (`John Smith V PhD` → family V, 1.4.0 parity), where `John Smith III PhD` reads suffix 'III PhD'. Recompute: the moved names are this commit's diff of tools/differential/snapshot.jsonl (or `uv run python tools/differential/snapshot.py` on the parent tree with this commit's code); the prototype figures in the Declined paragraph were measured over every cases.py row and snapshot name with the fork patched in a scratch process. +- 2026-10-18 (Derek), #718 — THE DECLINED NUMERAL REPORTS TOO, BOTH DIRECTIONS AS S4'S FORK DOES. The count's first form reported only the generation, so `John V` read family V in silence while `Jack Ma` reports keeping its family; Derek: report both, as `Jack Ma` does. A declined numeral is a pick of the walk, and its branches are taken in three places, so three sites report it (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): assign where it stays a lone name word (`John V` — "read as a family name rather than a generational suffix"), the particle chain where it joins the run (`Freiherr von Richthofen V`), and a maiden clause that keeps it as its last word (`John née Jones Smith V`, `abdul nee Smith V`). Each wording is its own, since the acronym pick's ("both a post-nominal and an ordinary name") misdescribes a numeral. + ONE STAND-DOWN, FOUND BY INV10 rather than designed. In a one-case name the connective `i` reaches the fork as a letter classify has already reported as `conjunction-or-initial`, read as an initial, and the new report put a second report on the same token: 138 rows of the connective grid in tests/v2/test_properties.py (`AND ROVIRA I`, `HENRY I`) tripped INV10's kind pair. The two reports did not contradict each other — both read the letter as a name word — but they named one reading twice, so the numeral's report stands down where the initial reading is already reported, the shape of rules/S7's "a member its own capitals already made the credential reports nothing new". The guard was left untouched: its kind pair still means "two readings" everywhere else, and narrowing it to suit this change would have retaken its recorded control. `HENRY I` therefore reports `conjunction-or-initial` alone, and moved in test_assemble's contrast list from the withdrawn side to the side where the fork resolved to a name word. diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index 830a4cab..33984c2c 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -12,10 +12,12 @@ S10. Rationale: a roman numeral ending a name is its generation, numeral past X is a name word. A numeral of one letter needs words to spare, as a bare ambiguous acronym does (S4): as the second of two name words it is the family name, a title in front - being no word to spare, and where it is read as the generation - the reading is reported, as it is for a numeral read by its - shape alone. A numeral the vocabulary lists in two or more - letters is not in question and reports nothing. Written case + being no word to spare, and either reading is reported, as + either reading of an ambiguous acronym is. A numeral read by its + shape alone reports too; a numeral the vocabulary lists in two + or more letters is not in question and reports nothing, and a + letter already reported as read as an initial reports nothing + new. Written case settles the fork in one way only: a numeral written in lower case in a name of mixed case is the generation and reports nothing, since nobody writes an initial that way. A name written @@ -30,6 +32,7 @@ S10. Rationale: a roman numeral ending a name is its generation, "John Smith III" → ambiguities=() "John III" → suffix="III" "John V" → family="V" + "John V" → ambiguities=("suffix-or-name",) "Malcolm X" → family="X" "JOHN V" → family="V" "Dr Smith V" → family="V" @@ -48,4 +51,4 @@ S10. Rationale: a roman numeral ending a name is its generation, the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name. "John Smith V PhD" → family="V" - history: decisions/S10 · interacts: S2, S4, S7, C1, P2, P3, M2, H1, H3 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py + history: decisions/S10 · interacts: S2, S4, S7, C1, P2, P3, M2, H1, H3 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 837a61d4..ea017ad0 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -95,7 +95,8 @@ anchor_in_reach, credential_at_the_given_slot, given_slot_anchors, _NOT_A_RUN_START, has_name_content, is_lone_never_given_particle, Peel, TailRead, is_suffix_piece, is_title_piece, leading_titles, - particle_tail, listed_lean, peel_walk, starts_a_credential_run, + particle_tail, listed_lean, peel_walk, reported_as_initial, + starts_a_credential_run, tail_reading, trailing_titles, ) from nameparser._pipeline._state import ( @@ -105,6 +106,10 @@ from nameparser._policy import Policy, Script from nameparser._types import FOLDED_TAG, SHAPE_ACRONYM_TAG, AmbiguityKind, Role +#: either tag marks a pick of the ambiguous-acronym fork; a pick with +#: neither is the numeral fork's (#718) +_ACRONYM_TAGS = frozenset({AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG}) + def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], role: Role) -> None: for i in piece: @@ -533,6 +538,18 @@ def _assign_main(seg_idx: int, state: ParseState, # it was declined as, and which field assemble names (#626) token = tokens[piece[0]] assert token.role is not None + # rules/S10: "either reading is reported" -- the numeral fork's + # declined pick (#718), which carries neither acronym tag and + # is never a suffix + if token.tags.isdisjoint(_ACRONYM_TAGS): + if reported_as_initial(ambiguities, piece[0]): + continue + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{token.text!r} is a roman numeral, but the name has " + f"no other word to spare for it; read as ", piece, + field_tail=" rather than a generational suffix")) + continue declined = ("a name part" if token.role is Role.SUFFIX else "a post-nominal") # rules/S9: "An unlisted word joins the ambiguous class (S4) by diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 8c614277..5d0f8e97 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -67,7 +67,8 @@ Peel, TailRead, chain_lead, chain_run_end, is_conj_piece, is_leading_title, is_prefix_piece, is_suffix_piece, is_title_piece, join_connectives, joined_tags, leading_titles, merge_particle_runs, merge_pieces, peel_walk, - read_trailing_run, tail_reading, trailing_candidates, + read_trailing_run, reported_as_initial, tail_reading, + trailing_candidates, trailing_run_start, trailing_start, trailing_start_past_titles, ) @@ -303,6 +304,8 @@ def _maiden_take(pieces: Sequence[Sequence[int]], c = trailing_candidates(lo, pieces, ptags, tokens) end = len(pieces) released: list[tuple[int, Role]] = [] + # the view's picks, for the kept member below (#718) + picked: tuple[tuple[int, ...], ...] = () # nothing behind the clause that a trailing rule could take: the # view would be the head alone, and the whole rest is the clause if c < len(pieces): @@ -312,6 +315,7 @@ def _maiden_take(pieces: Sequence[Sequence[int]], vrest, vchained, vpeel = tail_reading( peel_walk(leading_titles(view, vtags, tokens), vtags), view, vtags, tokens, one_case) + picked = vpeel.picks titles = set(vchained) | set(vpeel.run_titles) in_run = titles | set(vrest[vpeel.names:]) | { q for q in range(m, len(view)) if "suffix" in vtags[q]} @@ -345,6 +349,16 @@ def _maiden_take(pieces: Sequence[Sequence[int]], f"{word.text!r} ending the maiden name is also a " f"post-nominal; the maiden marker's clause keeps it rather " f"than reading it as one", tuple(last))) + elif (tuple(last) in picked + and not reported_as_initial(ambiguities, last[0])): + # rules/S10: "either reading is reported" -- a numeral the read + # declined for want of words to spare, which the clause keeps + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{word.text!r} ending the maiden name is a roman numeral, " + f"but the name has no other word to spare for it; the " + f"maiden marker's clause keeps it rather than reading it as " + f"a generational suffix", tuple(last))) return (list(range(m, lo)), [(j, Role.MAIDEN) for j in range(lo, end)] + released) @@ -1098,7 +1112,27 @@ def chain(tail: int) -> None: # the given role the fold then moves ('Freiherr von # Berg Ma' was told 'given name'). last = pieces[j - 1] - if (j > k + 1 and len(last) == 1 + # ...and a numeral S2's read declined (rules/S10, #718), + # a pick with no credential tag: the read's picks are + # where it is recorded, and `tuple` is C-level, so a + # chained name still pays no frame + numeral = (j > k + 1 and len(last) == 1 + and read is not None + and tuple(last) in read.peel.picks + and tokens[last[0]].tags.isdisjoint( + _AMBIGUOUS_CREDENTIAL_TAGS)) + if numeral: + if not reported_as_initial(ambiguities, last[0]): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{tokens[last[0]].text!r} is a roman " + f"numeral, but the name has no other word to " + f"spare for it; the particle chain took it " + f"into the name rather than reading it as a " + f"generational suffix", + tuple(last))) + took.add(tuple(last)) + elif (j > k + 1 and len(last) == 1 and not tokens[last[0]].tags.isdisjoint( _AMBIGUOUS_CREDENTIAL_TAGS) and not is_prefix_piece(last, ptags[j - 1], diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 6ed3953e..60daca89 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -59,7 +59,7 @@ from nameparser._pipeline._state import ( AMBIGUOUS_ACRONYM_TAG, NAME_ROLES, SHAPE_ACRONYM_TAG, SUFFIX_OR_UNREAD, - WorkToken, + AmbiguityKind, PendingAmbiguity, WorkToken, ) from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, _ROMAN, Lean, ambiguous_lean, in_initialless_script, @@ -919,7 +919,9 @@ class Peel(NamedTuple): one token long: `numeral` is the piece the roman-numeral fork took (None when it did not fire; always the walk's last piece), `picks` the bare ambiguous acronyms the peel had to resolve, in - peel order, either way (the last may sit at rest[names - 1]). + peel order, either way (the last may sit at rest[names - 1]), and + a one-letter numeral the fork declined for want of words to spare + (#718), always the walk's last piece and a name. `anchors` is `peel_trailing`'s working state for `tail_reading` alone -- the walk's `credential_anchors` pass over `rest` where a member needed one, handed to the next pass of the fixed point @@ -1143,16 +1145,20 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], # the is_initial_shaped reasoning (#320). rules/S10: "A numeral # of one letter needs words to spare, as a bare ambiguous # acronym does" -- the acronym fork's own count, - # below, so 'John V' keeps family 'V' (#718) + # below, so 'John V' keeps family 'V' (#718). Declined, it is a + # pick, since "either reading is reported", and no branch below + # can take a numeral if (k == len(rest) and k >= 2 and len(piece) == 1 and is_trailing_numeral_suffix( text := tokens[piece[0]].text, - tokens[pieces[rest[k - 2]][0]].text) - and (len(text) > 1 or k - 2 >= second - or numeral_settled_by_case(text, one_case))): - numeral = tuple(piece) - k -= 1 - continue + tokens[pieces[rest[k - 2]][0]].text)): + if (len(text) > 1 or k - 2 >= second + or numeral_settled_by_case(text, one_case)): + numeral = tuple(piece) + k -= 1 + continue + picks.append(tuple(piece)) + break # A bare ambiguous acronym ("MA", not "M.A.") is a credential # only when peeling it still leaves a given AND a family name. # With two pieces, "one of them is a credential" is the less @@ -1351,6 +1357,22 @@ def particle_tail(seg: Sequence[Sequence[int]], return lo, hi +# rules/S10: "a letter already reported as read as an initial reports +# nothing new" -- the one question the numeral fork's three declined- +# pick emitters (assign's, the particle chain's and the maiden +# clause's, #718) all ask, so it has one answer +def reported_as_initial(ambiguities: Iterable[PendingAmbiguity], + index: int) -> bool: + """Whether classify's connective-or-initial fork already reported + the token at `index`, which it reads as an initial when it reaches + the trailing peel as a lone piece. Declining the numeral reads the + letter as that same name word, so a second report would name one + reading twice. Asked only of a declined numeral pick, a rare + shape, so the scan costs ordinary names nothing.""" + return any(a.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL + and index in a.indices for a in ambiguities) + + def has_name_content(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: """Whether a piece holds a letter or digit: a piece with none is diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 19331e74..063ea503 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -3964,15 +3964,17 @@ def _check_cjk_shape_purity(self) -> None: "Smith V, Ma", {"given": "Smith", "family": "V", "suffix": "Ma"}, policy=Policy(lenient_comma_suffixes=False), - ambiguities=("suffix-or-name",), classification="fix(#718)", + ambiguities=("suffix-or-name", "suffix-or-name"), + classification="fix(#718)", notes="the period is the whole difference from the row " "above: bare 'V' is still initial-shaped and still " "counts as a name word here, so the structure flips " "the same way. Until #718 the trailing peel then took " "it as the roman numeral, leaving 'Smith' the only " "name word and reporting all three forks; a one-letter " - "numeral now needs words to spare (S10, the trailing roman numeral), so 'V' " - "is the family, as 1.4.0 read it"), + "numeral now needs words to spare (S10, the trailing " + "roman numeral), so 'V' is the family, as 1.4.0 read " + "it, and both forks, 'V' and 'Ma', report"), Case("caps_run_declines_a_bound_given_head", "John Smith, ABDUL AP", {"given": "ABDUL AP", "family": "John Smith"}, policy=Policy(unlisted_caps_suffixes=CapsSuffixes.EVERYWHERE), @@ -5486,12 +5488,13 @@ def _check_cjk_shape_purity(self) -> None: Case("a_numeral_with_no_word_to_spare_stays_in_the_clause", "abdul nee Smith V", {"given": "abdul", "maiden": "Smith V"}, - classification="fix(#718)", + ambiguities=("suffix-or-name",), classification="fix(#718)", notes="the row above without its family word: read without " "the clause the name is 'abdul V', and a one-letter " - "numeral needs words to spare (S10, the trailing roman numeral), so 'V' is " - "no suffix and the clause keeps it as it keeps any " - "name word, by M2 (the maiden-marker rule)"), + "numeral needs words to spare (S10, the trailing " + "roman numeral), so 'V' is no suffix and the clause " + "keeps it as it keeps any name word, by M2 (the " + "maiden-marker rule), reporting the numeral it kept"), Case("a_title_released_from_the_bound_given_pair", "abdul nee Smith Dr.", {"title": "Dr.", "family": "abdul", "maiden": "Smith"}, @@ -7427,27 +7430,28 @@ def _check_cjk_shape_purity(self) -> None: "so the reading is reported"), Case("a_lone_numeral_with_no_word_to_spare_is_the_family", "John V", {"given": "John", "family": "V"}, - classification="fix(#718)", - notes="S10 (the trailing roman numeral): a one-letter numeral needs words to spare, " - "as a bare ambiguous acronym does, and 'John' is the " - "only other word -- the contrast with 'John Smith V' " - "above is the count. 2.0.0 to 2.4.2 read suffix 'V' " - "with no family; 1.4.0 read last 'V'"), + ambiguities=("suffix-or-name",), classification="fix(#718)", + notes="S10 (the trailing roman numeral): a one-letter " + "numeral needs words to spare, as a bare ambiguous " + "acronym does, and 'John' is the only other word -- the " + "contrast with 'John Smith V' above is the count, and " + "either reading reports. 2.0.0 to 2.4.2 read suffix " + "'V' with no family; 1.4.0 read last 'V'"), Case("a_lone_shape_numeral_with_no_word_to_spare_is_the_family", "Malcolm X", {"given": "Malcolm", "family": "X"}, - classification="fix(#718)", + ambiguities=("suffix-or-name",), classification="fix(#718)", notes="the shape-only spelling of the row above: 'x' is in " "no vocabulary and reaches the fork by its shape"), Case("a_lone_numeral_in_a_one_case_name_is_counted", "JOHN V", {"given": "JOHN", "family": "V"}, - classification="fix(#718)", + ambiguities=("suffix-or-name",), classification="fix(#718)", notes="a name written in one case says nothing about the " "letter, so the count decides as it does for 'John V'"), Case("a_title_is_no_word_to_spare_for_a_lone_numeral", "Dr Smith V", {"title": "Dr", "given": "Smith", "family": "V"}, - classification="fix(#718)", - notes="S10's title clause (the trailing roman numeral); 1.4.0 read first 'Smith', " - "last 'V' too"), + ambiguities=("suffix-or-name",), classification="fix(#718)", + notes="S10's title clause (the trailing roman numeral); " + "1.4.0 read first 'Smith', last 'V' too"), Case("a_multi_letter_numeral_needs_no_word_to_spare", "John III", {"given": "John", "suffix": "III"}, ambiguities=("given-or-family",), @@ -7458,8 +7462,9 @@ def _check_cjk_shape_purity(self) -> None: "John Smith vi", {"given": "John", "family": "Smith", "suffix": "vi"}, classification="fix(#718)", - notes="S10 (the trailing roman numeral): nobody writes an initial in lower case " - "beside capitals, so the writing settles the fork. It " + notes="S10 (the trailing roman numeral): nobody writes an " + "initial in lower case beside capitals, so the writing " + "settles the fork. It " "reported until #718, the shape fork reporting all it " "took, while the listed 'John Smith v' never did"), Case("a_lowercase_numeral_in_a_one_case_name_still_reports", @@ -8727,11 +8732,12 @@ def _check_cjk_shape_purity(self) -> None: Case("a_lone_numeral_before_a_suffix_comma_is_the_family", "John V, Dr.", {"title": "Dr.", "given": "John", "family": "V"}, - classification="fix(#718)", + ambiguities=("suffix-or-name",), classification="fix(#718)", notes="the part before a suffix comma reads as the comma-less " "name, and 'John V' has no word to spare for a " - "one-letter numeral (S10, the trailing roman numeral). 1.4.0 read first " - "'John', last 'V', suffix 'Dr.', 'dr' having been " + "one-letter numeral (S10, the trailing roman " + "numeral). 1.4.0 read first 'John', last 'V', suffix " + "'Dr.', 'dr' having been " "suffix vocabulary before the audit"), Case("comma_path_given_name_title_decides_the_lone_word", "John III, Sir", @@ -8747,10 +8753,10 @@ def _check_cjk_shape_purity(self) -> None: Case("a_lone_numeral_before_a_given_name_title_comma", "John V, Sir", {"title": "Sir", "given": "John", "family": "V"}, - classification="fix(#718)", + ambiguities=("suffix-or-name",), classification="fix(#718)", notes="the row above with a one-letter numeral, which has no " - "word to spare (S10, the trailing roman numeral). 1.4.0 read title 'Sir', " - "last 'John V'"), + "word to spare (S10, the trailing roman numeral). " + "1.4.0 read title 'Sir', last 'John V'"), Case("marker_led_clause_in_a_quote_pair", 'Jane Smith "née Jones"', {"given": "Jane", "family": "Smith", "maiden": "Jones"}, diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 9a2b9d71..b01d3abb 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -186,10 +186,12 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # one given-or-family claim under each order); 1174 -> 1172 on # 2026-10-17: #718's re-pinned and added numeral rows (the lone # numeral rows lose their given-or-family claims; 'John III' and - # 'Malcolm X' rows bring some back). - assert claims == 1172, ( - f"the field sweep checks {claims} claims, recorded as 1172 on " - f"2026-10-17") + # 'Malcolm X' rows bring some back); 1172 -> 1191 on 2026-10-18: + # #718's declined-numeral report, which names the field the letter + # was read into, on those rows under each order. + assert claims == 1191, ( + f"the field sweep checks {claims} claims, recorded as 1191 on " + f"2026-10-18") #: Case.__post_init__'s shape checks, each probed for the message that diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index d1ebf5a3..1e3c054c 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -678,7 +678,7 @@ {"name": "Dr Jr", "given": "Dr", "suffix": "Jr", "ambiguities": ["TITLE_OR_NAME"], "initials": "D."} {"name": "Dr King Jr", "title": "Dr", "family": "King", "suffix": "Jr", "ambiguities": ["TITLE_OR_NAME"], "initials": "K."} {"name": "Dr Martin Luther King, Jr.", "title": "Dr", "given": "Martin", "middle": "Luther", "family": "King", "suffix": "Jr.", "initials": "M. L. K."} -{"name": "Dr Smith V", "title": "Dr", "given": "Smith", "family": "V", "initials": "S. V."} +{"name": "Dr Smith V", "title": "Dr", "given": "Smith", "family": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "S. V."} {"name": "Dr 田中さん, V.", "title": "Dr", "family": "田中", "suffix": "さん, V.", "initials": "田."} {"name": "Dr 김민준, Jr.", "title": "Dr", "given": "민준", "family": "김", "suffix": "Jr.", "initials": "민. 김."} {"name": "Dr 김민준, MA", "title": "Dr", "given": "민준", "family": "김", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "민. 김."} @@ -937,7 +937,7 @@ {"name": "Freiherr von Berg MA", "title": "Freiherr", "family": "von Berg", "suffix": "MA", "ambiguities": ["PARTICLE_OR_GIVEN", "SUFFIX_OR_NAME"], "initials": "B."} {"name": "Freiherr von Berg, Ed", "given": "Ed", "family": "Freiherr von Berg", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "E. B."} {"name": "Freiherr von Richthofen", "title": "Freiherr", "family": "von Richthofen", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "R."} -{"name": "Freiherr von Richthofen V", "title": "Freiherr", "family": "von Richthofen V", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "R. V."} +{"name": "Freiherr von Richthofen V", "title": "Freiherr", "family": "von Richthofen V", "ambiguities": ["PARTICLE_OR_GIVEN", "SUFFIX_OR_NAME"], "initials": "R. V."} {"name": "Freire Xiana", "order": "FAMILY_FIRST", "given": "Xiana", "family": "Freire", "initials": "X. F."} {"name": "Freire, Xiana", "given": "Xiana", "family": "Freire", "initials": "X. F."} {"name": "Friedrich IV.", "given": "Friedrich", "suffix": "IV.", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "F."} @@ -1271,7 +1271,7 @@ {"name": "JOHN SMITH PH.D.", "given": "JOHN", "family": "SMITH", "suffix": "PH.D.", "initials": "J. S."} {"name": "JOHN SMITH, MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "JOHN SMITH, VD MA", "given": "JOHN", "family": "SMITH", "suffix": "VD MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "JOHN V", "given": "JOHN", "family": "V", "initials": "J. V."} +{"name": "JOHN V", "given": "JOHN", "family": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. V."} {"name": "JOSE E MARIA SANTOS", "given": "JOSE", "middle": "E MARIA", "family": "SANTOS", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "J. E. M. S."} {"name": "JOSE ORTEGA-Y-GASSET", "given": "JOSE", "family": "ORTEGA-Y-GASSET", "initials": "J. O."} {"name": "JOSEP CAROD I ROVIRA", "given": "JOSEP", "middle": "CAROD I", "family": "ROVIRA", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "J. C. I. R."} @@ -1609,8 +1609,8 @@ {"name": "John Smith, X.Y.Z. MA", "given": "John", "family": "Smith", "suffix": "X.Y.Z. MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, XYZ", "given": "John", "family": "Smith", "suffix": "XYZ", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith, vd Ma", "given": "John", "family": "Smith", "suffix": "vd Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John V", "given": "John", "family": "V", "initials": "J. V."} -{"name": "John V, Dr.", "title": "Dr.", "given": "John", "family": "V", "initials": "J. V."} +{"name": "John V", "given": "John", "family": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. V."} +{"name": "John V, Dr.", "title": "Dr.", "given": "John", "family": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. V."} {"name": "John V. Smith", "given": "John", "middle": "V.", "family": "Smith", "initials": "J. V. S."} {"name": "John W. Ingram", "given": "John", "middle": "W.", "family": "Ingram", "initials": "J. W. I."} {"name": "John W. Ingram V", "given": "John", "middle": "W.", "family": "Ingram", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. W. I."} @@ -1625,7 +1625,7 @@ {"name": "John e Smith, III", "given": "John", "middle": "e", "family": "Smith", "suffix": "III", "initials": "J. e. S."} {"name": "John nee Prof. ba MA", "family": "John", "suffix": "MA", "maiden": "Prof. ba", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} {"name": "John née Jones Smith Ma", "family": "John", "maiden": "Jones Smith Ma", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} -{"name": "John née Jones Smith V", "family": "John", "maiden": "Jones Smith V", "initials": "J."} +{"name": "John née Jones Smith V", "family": "John", "maiden": "Jones Smith V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J."} {"name": "John of the Doe", "given": "John of the Doe", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "J. D."} {"name": "John van Buren, Ed", "given": "John", "family": "van Buren", "suffix": "Ed", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. B."} {"name": "John van Mc", "given": "John", "family": "van Mc", "initials": "J. v. M."} @@ -2088,7 +2088,7 @@ {"name": "Makonnen Haile Selassie", "given": "Makonnen", "middle": "Haile", "family": "Selassie", "initials": "M. H. S."} {"name": "Malam Shehu Bello Modibbo MP", "given": "Malam", "middle": "Shehu Bello", "family": "Modibbo", "suffix": "MP", "initials": "M. S. B. M."} {"name": "Malas Zénāwi", "given": "Malas", "family": "Zénāwi", "initials": "M. Z."} -{"name": "Malcolm X", "given": "Malcolm", "family": "X", "initials": "M. X."} +{"name": "Malcolm X", "given": "Malcolm", "family": "X", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. X."} {"name": "Maleingreau, Paul de", "given": "Paul", "family": "de Maleingreau", "initials": "P. M."} {"name": "Malik Ambar", "given": "Malik", "family": "Ambar", "initials": "M. A."} {"name": "Malik al-ʿĀdil al I.", "given": "Malik", "middle": "al-ʿĀdil", "family": "al I.", "initials": "M. a. I."} @@ -2263,7 +2263,7 @@ {"name": "Mohamad Ali", "given": "Mohamad", "family": "Ali", "initials": "M. A."} {"name": "Mohamad Ali Khalil", "given": "Mohamad", "middle": "Ali", "family": "Khalil", "initials": "M. A. K."} {"name": "Mohamad Asri Muda", "given": "Mohamad", "middle": "Asri", "family": "Muda", "initials": "M. A. M."} -{"name": "Mohamad X", "given": "Mohamad", "family": "X", "initials": "M. X."} +{"name": "Mohamad X", "given": "Mohamad", "family": "X", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. X."} {"name": "Mohamad X Surname", "given": "Mohamad", "middle": "X", "family": "Surname", "initials": "M. X. S."} {"name": "Mohamad Zein El Dine", "given": "Mohamad", "middle": "Zein El", "family": "Dine", "initials": "M. Z. E. D."} {"name": "Mohamad Zeineddine", "given": "Mohamad", "family": "Zeineddine", "initials": "M. Z."} From 98f75f500e717290c5fb35f30ae1d84f09852b0c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 02:59:24 -0700 Subject: [PATCH 3/7] Fix the docs the numeral count moved, from the design-docs review (#718) S10's Accepted clause named every listed lowercase letter where only 'v' bypasses the count ('john i' is counted, as P3's connective read as an initial), and left out that 'john smith v' reports nothing. P3's "is the generation it also spells" now defers to S10; O5's example is 'John III, Dr.', 'John V, Dr.' no longer being a lone word; the H3 contrast comment and the release bullet ('Dr. Smith V' moves the surname into first) catch up. The case row that lost its H1 reading is renamed for what it pins, and says that nothing pins H1 behind a comma. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions/S10-trailing-roman-numeral.md | 10 +++++----- ...5-lone-name-words-convention-and-reporting-it.md | 2 +- docs/design/rules/P3-connectives.md | 2 +- docs/design/rules/S10-trailing-roman-numeral.md | 11 +++++++---- docs/release_log.rst | 2 +- nameparser/_pipeline/_pieces.py | 5 +++-- tests/v2/cases.py | 13 ++++++++----- tools/differential/corpus_rules.jsonl | 2 ++ tools/differential/snapshot.jsonl | 2 ++ 10 files changed, 31 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1801d8bf..36eb55ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,7 +173,7 @@ The library has two layers: `nameparser/config/` (data) and `nameparser/parser.p **Sweep the forms a change can reach: comma shapes, then `name_order`.** No comma, a FULL name before the comma, a ONE-WORD name before the comma and NOTHING before the comma are four paths, and the last two are the misses — #429, #430 and #432 are all the one-word path disagreeing with the full-name path on inputs the full-name path reads correctly, and #613's PR review found `, PhD` and `, Jr.` (an empty surname field, as CSV data writes it) read as given and title by a decision that returned early on an empty head, after two review rounds whose every grid had a name before the comma. Put an empty head and a nickname-only head (`(Bob), PhD`) in every comma grid. Count the one-word path the way C1 counts, which is neither tokens nor pieces: a particle run and the one word it attaches to are one word (#575), a connective join and a bound given-name pair count as their words (#613, so `Ortega y Gasset, Dr.` is a whole name), and a suffix beside the surname counts as no name word at all. #603's review found the suffix half (`Vega y Lopez Jr., PhD Jones` lost its family where `Vega y Lopez, PhD Jones` kept it); #613 then reversed the connective half, and both now read given 'Vega', middle 'y', family 'Lopez' — so test a particle surname, a connective surname and a surname with a suffix, each before a comma. For orders the sweep already exists (`tests/v2/test_cases.py` runs every row under all three) but carries one assertion, R2's family partition, so it checks nothing a new change moves; coverage there has been vacuous before (PR #394's review found the suite passed with `name_order` discarded from grouping). Parse your change's names in each comma shape and each order, and read the ones you did not predict. And read a generated grid's moves by SHAPE, not by corpus membership: #614's first design passed the suite, the five gates and a fingerprint of every corpus name, because its grid moves were sorted into 'corpus name moved' and 'garbage', and the review found the moves that mattered among the 'garbage' -- a bare salutation (`Mr. and Mrs.` lost its connective to the family name) and a name with one trailing word (`Freiherr von Berg X.Y.Z.`). Pull out the head-plus-one-word slice of the grid and a salutation with nothing after it, and read every move there. Two more were the second review's (#614): no grid put TWO words that are both a title and a particle at the head (`Freiherr St van Berg MA`, where an index into the pieces was applied to a merged view of them -- `TITLES ∩ particles_ambiguous` is `{freiherr, st}`, so put both in every head list, each in its bare and its period-marked spelling -- #637's grid had `St` and `Freiherr` alone and missed that `St.` and `Freiherr.` are declined by the bound join and then attached by P6), and the grids compared ambiguity KINDS, so a report telling the caller the wrong field (`Freiherr von Berg Ma`, 'read as a given name' of a word in the family) passed every pin. #701's review added one: put a title/suffix dual (`Ms`, `Sr.`, `Cpl`) SECOND in a title run, behind an ordinary title. A stop that asks only "is this word suffix vocabulary" passes every row that opens with the dual and fails `Rev. Sr. Smith, Mary`; it was found by comparing `T Smith, John` against `T John Smith` over every title word, and a comma-versus-comma-less comparison is the cheap check for any rule that moves a word between the two slots. Compare each report's `detail` against the field its tokens land in -- since #626 `test_a_report_names_the_field_its_word_lands_in` (tests/v2/test_cases.py) does it over every non-locale case row under all three orders (a row carrying its own policy only as declared), for the phrasings its `_FIELD_CLAIMS` lists -- so a row with the shape is the test only where the report's wording is listed: a new emitter's wording goes in that list, and segment's report on a part past the second comma names no field (#629). #626's own first grid then missed an emitter for want of one comma shape (`, Ma Dr.`: an empty head, an ambiguous word, a trailing title, which H1 moves to the family): a trailing title belongs in every empty-head comma grid too. And #620's: an exclusion list in code being replaced is a set of rules, not noise -- #614's merged copy excluded every word the trailing read weighs from its units, the first #620 prototype dropped the list as a symptom of the copy, and its review found the list had been keeping positional counting's rule that a member in front counts as a word (`Freiherr von Berg MA X.Y.Z.` lost its suffix). Before deleting one, mutate each entry out and read what moves. -**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, while the lowercase listed spellings never reach it and are not counted (`john v` keeps suffix 'v' where `john x` reads family 'x') -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. +**Then sweep the spellings: a class reached by vocabulary is also reached by SHAPE.** A test that uses only the listed spelling of a word class walks only the vocabulary path, and the shape path is a separate branch that can be missing while every test passes. Roman numerals are the sharpest case: `ii`, `iii` and `iv` are suffix vocabulary; `i` and `v` are vocabulary too but initial-shaped, so `is_suffix_piece` vetoes them and only the numeral fork reads them; and `vi` through `x` are in no list at all, read by the fork's shape test alone (checked 2026-10-04 against `Lexicon.default()`) -- which stops at `x`: `_vocab._ROMAN` matches I to X, so `John Smith XI` reads `XI` as a name word (re-checked 2026-10-07, #614's /simplify, which found a docstring claiming "any length"). And the fork counts a ONE-letter numeral besides, since #718 (rules/S10): it is the suffix only where the name has a word to spare, so `John V` and `John Smith V` take different branches of one fork, while a one-case lowercase `v`, a listed suffix word, never reaches it and is not counted (`john v` keeps suffix 'v' where `john x` and `john i` read family -- `i` arrives as P3's connective, read as an initial) -- put a head-plus-numeral row beside every words-to-spare row. So a change that handles `V` and `III` can still miss `VI` — #610 found exactly that, the maiden take's candidate loop admitting a shape-only numeral in one position where the peel reads it in two (`Jane Doe nee Smith VI Prof.` kept maiden `Smith VI` while `V` and `III` worked). The same split runs through the credentials (`MA` listed, `X.Y.Z.` by the dotted shape, `XYZ` by the caps shape, and `Ph. D.` merged by group into one flagged piece the peel's walk never holds -- #603 found #602's run-start predicate refusing it too, so `Smith, John Ph. D. Jones` kept middle 'Jones' where `Smith, John PhD Jones` read suffix 'PhD Jones') and the titles (`Prof.` period-marked and read at the end of a name, `Prof` bare and a name word there). When a change touches a word class, test one spelling from each path. The same holds for a separator's SPACING: a comma (or any punctuation) can stand alone, glue to the word before it or glue to the word after, and each is a different token stream. #653's first fix swept `(née, Jones)` and missed `(née , Jones)`, where the kept comma became a word after the marker and M3 read the clause as maiden ', Jones'; its review found it. Put all three spacings in any grid around a separator. **And the encoding: a letter class written `[^\W\d_]` or tested with `isalpha()` is blind to a combining mark.** Python's `\w` and `isalpha()` do not match Mn, so a letter written as base plus accent (NFD) ends a word early; the vocabulary path is immune because `_normalize` NFC-folds, the shape path is not. #652: `Dr. inž. Janez Novak` read `inž.` as the given name when the ž was decomposed, and `Janez Novak, ŽŠČ` reversed, in two families that share the blind spot — the letter-shape regexes (`period_abbreviation`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`), now spelled with `_vocab._LETTER` (a letter plus the general combining blocks; `test_regex_sync` compares the two `_MARKS` strings and pins `period_abbreviation` against config's copy), and the caps predicates, which NFC-compose a non-ASCII word before `isalpha()` (`is_caps_word`, behind C-level `isalpha()`/`is_normalized` tests, so already-composed text pays no frame — measured 2026-10-12, `Smith, José` was 193 frames before and after; the first version called it for every non-ASCII word and cost one frame each, and no guard saw it because the reference name is ASCII). A width test must count letters, not code points (`composed_len`): admitting a decomposed word to the caps set put `John Smith, MÁ` on the pair exception's wrong side, and the review's grid found it where the rows (all three letters long) did not — put a two-letter word in every caps grid. The initial shape (`_INITIAL`, `\w\.`, v1-verbatim and public) is not widened but asked of the COMPOSED word, in `is_initial_shaped` and in `_render`'s two copies; and the single-letter connective tests count composed letters, since a decomposed `й` is two code points (#665, where `É. y É. Smith` and `John й Smith` read differently decomposed). Compose the WHOLE predicate, not one half: `is_initial` is a shape ANDed with the #320 script veto, and composing the shape alone made a decomposed `씨.` an initial, since decomposed hangul is conjoining jamo outside every script range (#665's first version; `in_initialless_script` composes too now). Not covered, deliberately: the marks only some scripts write (Indic vowel signs, Arabic and Hebrew points; rules/H2's Accepted clause). The guard is `test_a_decomposed_name_renders_as_its_composed_twin` over the CLDR corpus plus `test_a_decomposed_abbreviation_reads_as_its_composed_twin` (rows × three orders × EVERYWHERE); the corpus guard reaches a shape only if some corpus name does, which is how the caps shape was missed until review, so when adding a shape rule add a row there. Four of #652's rows and two of #665's pass with every fix reverted (measured) and are parity pins, and the test compares ambiguity kinds and `initials()` too, since #665's `John й Smith` differed in its report; `_DOTTED` has no observable input and is pinned by a direct call. diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index bf41386c..103bfab8 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -1,10 +1,10 @@ # S10 — the trailing roman numeral - 2026-10-17 (Derek), #718 — A ONE-LETTER NUMERAL NEEDS WORDS TO SPARE, AND THE RULE IS WRITTEN DOWN. Until this entry no rule stated the comma-less numeral at all: the behavior was pinned in tests/v2/cases.py and credited in code comments to rules/S2, whose statement covers the suffix vocabulary and never the shape fork that reads `VI` to `X`, nor the initial veto that sends `V`, `I` and `X` there. The extraction found one reading worth changing. `John V` and `Malcolm X` read suffix with an EMPTY family in every 2.x release (2.0.0 through 2.4.2 measured), alongside 2.0.0's `Johnson PhD` fix; 1.4.0 read family `V` and `X`. Derek: a generational V is mostly archaic, X rarer still and I hardly ever written without a comma (Sr. and Senior are what people write), so a capital letter standing alone behind the given name is far likelier an initial or a one-letter surname. The numeral was never what S2's Accepted clause (an unambiguous suffix is consumed even when that leaves no family) covers, since the fork has always reported itself ambiguous; S4's count is the analogue, and the numeral now asks it. - WHAT MOVES. The count is S4's own, in name UNITS, so a particle chain is one word (P2) and a title none. Measured on the shipped change against origin/master at 958d252e, the snapshot moves four corpus names: `John V` and `Mohamad X` (family V and X, as 1.4.0 read them), `John née Jones Smith V` (the clause-free name is `John V`, so the V is no suffix and the clause keeps it, maiden 'Jones Smith V' — M2's Accepted clause on the lone numeral), and `Freiherr von Richthofen V`, family 'von Richthofen V': behind a title the chain is the one name word, so the V has no word to spare and the chain takes it as P2 takes any name word, as it already took `Freiherr von Richthofen Ma`. That last is the reading a scratch prototype counting raw pieces did not show, and the one most worth a second look. Four case rows moved with it (`strict_comma_reads_the_bare_numeral_into_the_run`: `Smith V, Ma` under the strict knob reads family V, suffix Ma, as 1.4.0 did; and the rows whose purpose needed a suffix kept their ids on a spelling that still has one — `abdul Doe nee Smith V`, `John III, Dr.`, `John III, Sir` — with the old inputs re-pinned as new rows: `abdul nee Smith V` maiden 'Smith V', `John V, Dr.` and `John V, Sir` given John, family V). `Sir abdul V` reads given 'abdul V': the V is a name word, and P5's licence behind a given-name title joins it. Nothing with a word to spare moves: `John W. Ingram V`, `King John Alexander V`, `Larry James Johnson I` (a v1 test) keep the suffix and the report. A title is no word to spare, so `Dr Smith V` reads given Smith, family V, which is 1.4.0's reading too, where `Dr. von Richthofen V` keeps suffix V, the particle and the surname being two words there. - THE LOWERCASE HALF. A numeral in lower case in a mixed-case name is not how anyone writes an initial, so it is the generation and reports nothing (Derek). Measured before the change, the reports had no rationale: `John Smith v` and `John Smith i` were silent because the listed letters reach the vocabulary path, `John Smith vi` and `John Smith x` reported because the shape fork reports everything it takes, and the one-case `john smith i` reported because the connective tag (P3, #479) makes a one-case `i` an initial. Now the mixed-case lower case is uniformly silent and a one-case name is counted like capitals. `john smith i` keeps its report, by Derek's call. + WHAT MOVES. The count is S4's own, in name UNITS, so a particle chain is one word (P2) and a title none. Measured on the shipped change against origin/master at 958d252e, the snapshot moves four corpus names: `John V` and `Mohamad X` (family V and X, as 1.4.0 read them), `John née Jones Smith V` (the clause-free name is `John V`, so the V is no suffix and the clause, which M2's statement takes up to the trailing run the clause-free name reads, keeps it: maiden 'Jones Smith V'), and `Freiherr von Richthofen V`, family 'von Richthofen V': behind a title the chain is the one name word, so the V has no word to spare and the chain takes it as P2 takes any name word, as it already took `Freiherr von Richthofen Ma`. That last is the reading a scratch prototype counting raw pieces did not show, and the one most worth a second look. Four case rows moved with it (`strict_comma_reads_the_bare_numeral_into_the_run`: `Smith V, Ma` under the strict knob reads family V, suffix Ma, as 1.4.0 did; and the rows whose purpose needed a suffix kept their ids on a spelling that still has one — `abdul Doe nee Smith V`, `John III, Dr.`, `John III, Sir` — with the old inputs re-pinned as new rows: `abdul nee Smith V` maiden 'Smith V', `John V, Dr.` and `John V, Sir` given John, family V). `Sir abdul V` reads given 'abdul V': the V is a name word, and P5's licence behind a given-name title joins it. Nothing with a word to spare moves: `John W. Ingram V`, `King John Alexander V`, `Larry James Johnson I` (a v1 test) keep the suffix and the report. A title is no word to spare, so `Dr Smith V` reads given Smith, family V, which is 1.4.0's reading too, where `Dr. von Richthofen V` keeps suffix V, the particle and the surname being two words there. + THE LOWERCASE HALF. A numeral in lower case in a mixed-case name is not how anyone writes an initial, so it is the generation and reports nothing (Derek). Measured before the change, the reports had no rationale: `John Smith v` and `John Smith i` were silent because the listed letters reach the vocabulary path, `John Smith vi` and `John Smith x` reported because the shape fork reports everything it takes, and the one-case `john smith i` reported because the connective tag (P3, #479) makes a one-case `i` an initial. Now the mixed-case lower case is uniformly silent and a one-case name is counted like capitals, `v` excepted (the ACCEPTED paragraph below). `john smith i` keeps its report, by Derek's call. Declined: dropping the shape fork for single capital letters outright. Prototyped in a scratch edit, it moved 26 readings and cost the real generational names the count keeps: `John W. Ingram V` read middle 'W. Ingram', family 'V', and `John van der Berg V` folded the numeral into the particle chain. A variant that dropped the fork for I and X only, on top of the count, moved four more, among them the v1 test `Larry James Johnson I`. - ACCEPTED, and recorded in rules/S10: a one-case lowercase `v` is read as the vocabulary reads it, so `john v` still loses its family where `JOHN V` and `john x` keep theirs — the listed lowercase letter is taken as a suffix word before the numeral fork is reached, and closing that means touching the suffix-vocabulary path, which is not this change. And a one-letter numeral with a credential behind it is not the last word and is read as a name word (`John Smith V PhD` → family V, 1.4.0 parity), where `John Smith III PhD` reads suffix 'III PhD'. - Recompute: the moved names are this commit's diff of tools/differential/snapshot.jsonl (or `uv run python tools/differential/snapshot.py` on the parent tree with this commit's code); the prototype figures in the Declined paragraph were measured over every cases.py row and snapshot name with the fork patched in a scratch process. + ACCEPTED, and recorded in rules/S10: a one-case lowercase `v` is read as the vocabulary reads it, so `john v` still loses its family and `john smith v` reports nothing, where `JOHN V` and `john x` keep their family and `JOHN SMITH V` reports — the listed lowercase letter is taken as a suffix word before the numeral fork is reached, and closing that means touching the suffix-vocabulary path, which is not this change. Not `i`, though it is listed too: a one-case `i` is P3's connective, which classify reads as an initial, so it reaches the fork and is counted (`john i` → family 'i'; `john smith i` reports). And a one-letter numeral with a credential behind it is not the last word and is read as a name word (`John Smith V PhD` → family V, 1.4.0 parity), where `John Smith III PhD` reads suffix 'III PhD'. + Recompute: the moved names are this commit's diff of tools/differential/snapshot.jsonl (or `uv run python tools/differential/snapshot.py` on the parent tree with this commit's code). The Declined paragraph's figures are a prototype's, over every cases.py row and snapshot name parsed before and after the patch in one process, on master at 958d252e: its 26 replace `_pieces.is_trailing_numeral_suffix` with a wrapper returning False for a text that is one capital `I`, `V` or `X` (periods stripped) and the original's answer otherwise; its "four more" apply that wrapper to `I` and `X` alone on top of the shipped count. Both are dated snapshots by Counting claims; nothing re-runs them. - 2026-10-18 (Derek), #718 — THE DECLINED NUMERAL REPORTS TOO, BOTH DIRECTIONS AS S4'S FORK DOES. The count's first form reported only the generation, so `John V` read family V in silence while `Jack Ma` reports keeping its family; Derek: report both, as `Jack Ma` does. A declined numeral is a pick of the walk, and its branches are taken in three places, so three sites report it (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): assign where it stays a lone name word (`John V` — "read as a family name rather than a generational suffix"), the particle chain where it joins the run (`Freiherr von Richthofen V`), and a maiden clause that keeps it as its last word (`John née Jones Smith V`, `abdul nee Smith V`). Each wording is its own, since the acronym pick's ("both a post-nominal and an ordinary name") misdescribes a numeral. - ONE STAND-DOWN, FOUND BY INV10 rather than designed. In a one-case name the connective `i` reaches the fork as a letter classify has already reported as `conjunction-or-initial`, read as an initial, and the new report put a second report on the same token: 138 rows of the connective grid in tests/v2/test_properties.py (`AND ROVIRA I`, `HENRY I`) tripped INV10's kind pair. The two reports did not contradict each other — both read the letter as a name word — but they named one reading twice, so the numeral's report stands down where the initial reading is already reported, the shape of rules/S7's "a member its own capitals already made the credential reports nothing new". The guard was left untouched: its kind pair still means "two readings" everywhere else, and narrowing it to suit this change would have retaken its recorded control. `HENRY I` therefore reports `conjunction-or-initial` alone, and moved in test_assemble's contrast list from the withdrawn side to the side where the fork resolved to a name word. + ONE STAND-DOWN, FOUND BY INV10 rather than designed. In a one-case name the connective `i` reaches the fork as a letter classify has already reported as `conjunction-or-initial`, read as an initial, and the new report put a second report on the same token: 138 rows of the connective grid (measured on this branch with the `reported_as_initial` check out of assign's numeral report, a dated snapshot) in tests/v2/test_properties.py (`AND ROVIRA I`, `HENRY I`) tripped INV10's kind pair. The two reports did not contradict each other — both read the letter as a name word — but they named one reading twice, so the numeral's report stands down where the initial reading is already reported, the shape of rules/S7's "a member its own capitals already made the credential reports nothing new". The guard was left untouched: its kind pair still means "two readings" everywhere else, and narrowing it to suit this change would have retaken its recorded control. `HENRY I` therefore reports `conjunction-or-initial` alone, and moved in test_assemble's contrast list from the withdrawn side to the side where the fork resolved to a name word. diff --git a/docs/design/rules/O5-lone-name-words-convention-and-reporting-it.md b/docs/design/rules/O5-lone-name-words-convention-and-reporting-it.md index 582f7484..35296db3 100644 --- a/docs/design/rules/O5-lone-name-words-convention-and-reporting-it.md +++ b/docs/design/rules/O5-lone-name-words-convention-and-reporting-it.md @@ -31,7 +31,7 @@ O5. Rationale: O4 reads a name by comparing where its words stand, nothing after it names no family and silences nothing, so `سلمان،` and `Smith,` report where `Smith, Andrew` does not. A comma with only a TITLE after it names no family either, but the - title still decides the field wherever it stands, so `John V, Dr.` + title still decides the field wherever it stands, so `John III, Dr.` is silent for H1's reason and not the comma's. A word with no letter or digit in it is no name word and reports nothing (A2). Where the one name unit carries title diff --git a/docs/design/rules/P3-connectives.md b/docs/design/rules/P3-connectives.md index 873e9ee4..505fcb69 100644 --- a/docs/design/rules/P3-connectives.md +++ b/docs/design/rules/P3-connectives.md @@ -38,7 +38,7 @@ P3. Rationale: connective words ("y", "of the") bind name words into connectives standing between. A connective with nothing to its right is connecting nothing, and a word of that vocabulary ending a name, or standing before the credential a name ends with, is the generation it also - spells. + spells wherever S10 reads one there. A separator the caller declared, standing in a trailing suffix part, ends the search there as a comma would, so a connective never joins across it (C1); anywhere else it is a word. diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index 33984c2c..c45f34de 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -42,11 +42,14 @@ S10. Rationale: a roman numeral ending a name is its generation, "John Smith V." → family="V." · boundary "John J. V" → family="V" · boundary "John Smith XI" → family="XI" · boundary - Accepted: a one-letter numeral the vocabulary lists, written in - lower case in a name written wholly in lower case, is read as - the vocabulary reads it, words to spare or not, where the same - letter in capitals is counted. + Accepted: a lower-case `v` in a name written wholly in lower + case is read as the suffix word the vocabulary lists, words to + spare or not, and reports nothing, where the same letter in + capitals is counted and reported; `i`, which is also a + connective, is read as an initial there (P3) and counted. "john v" → family="" + "john smith v" → ambiguities=() + "john i" → family="i" Accepted: a numeral with another suffix word behind it is not the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name. diff --git a/docs/release_log.rst b/docs/release_log.rst index d44f6463..96d99047 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -14,7 +14,7 @@ Release Log - **Fix a surname particle such as d' or al- reading as a middle name when a listing puts it after the given name.** ``Ivoi, Paul d'`` gives last ``d' Ivoi``, where 1.4.0 through 2.4.2 gave middle ``d'``. The Arabic article reads the same in its plain and assimilated spellings (``al-``, ``az-``, ``aš-``). A particle written joined to the surname, as in ``Paul d'Ivoi``, is unchanged. :issue:`700` - **Fix case repair capitalizing the Dutch particle 't.** ``jacobus henricus van 't hoff`` repairs to ``Jacobus Henricus van 't Hoff``, where 1.4.0 through 2.4.2 gave ``van 'T Hoff``. :issue:`700` - **Add Gov as a title.** ``Gov Smith`` gives title ``Gov``, last ``Smith``, where 1.4.0 through 2.4.2 gave first ``Gov``, and ``Lt. Gov. Smith, John`` gives title ``Lt. Gov.`` with the change above. :issue:`701` - - **Fix a capital V, I or X after a lone first name being read as a generational suffix, leaving no last name.** ``John V`` gives first ``John``, last ``V``, and ``Malcolm X`` last ``X``, where 2.0.0 through 2.4.2 gave suffix ``V`` and ``X`` with no last name, restoring 1.4.0's reading. A name with a word to spare keeps the suffix: ``John Smith V`` still gives suffix ``V``. :issue:`718` + - **Fix a capital V, I or X after a lone first name being read as a generational suffix, leaving no last name.** ``John V`` gives first ``John``, last ``V``, and ``Malcolm X`` last ``X``, where 2.0.0 through 2.4.2 gave suffix ``V`` and ``X`` with no last name, restoring 1.4.0's reading. A title is no word to spare, so ``Dr. Smith V`` gives first ``Smith``, last ``V`` (1.4.0's reading too) where 2.0.0 through 2.4.2 gave last ``Smith``, suffix ``V``; a name with a word to spare keeps the suffix, and ``John Smith V`` still gives suffix ``V``. :issue:`718` * 2.4.2 - October 13, 2026 diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 60daca89..45603e46 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -423,8 +423,9 @@ def leading_titles(pieces: Sequence[Sequence[int]], # bare initial-shaped numeral, so 'Dr King V' keeps the whole run # as the title and reads the numeral as the name -- given 'V', # 'king' being a given-name title and the run's last word, where - # 'Dr Smith V' reads suffix 'V'. That numeral fork is outside this - # floor (decisions/H3). + # 'Dr Smith V' reads family 'V' (a title being no word to spare + # for a one-letter numeral, #718). That numeral fork is outside + # this floor (decisions/H3). # # The two inline tag reads are the cheapest NECESSARY condition for # the piece behind the run to be a suffix piece at all -- diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 063ea503..e0e83b46 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -8739,17 +8739,20 @@ def _check_cjk_shape_purity(self) -> None: "numeral). 1.4.0 read first 'John', last 'V', suffix " "'Dr.', 'dr' having been " "suffix vocabulary before the audit"), - Case("comma_path_given_name_title_decides_the_lone_word", + Case("comma_path_given_name_title_behind_a_listed_numeral", "John III, Sir", {"title": "Sir", "family": "John", "suffix": "III"}, classification="fix(#296)", notes="the title row above with a GIVEN-NAME title. O5 is " "silenced all the same: what the report is about is a " "field NOTHING decided, and this one a title decided. " - "Written 'John V, Sir' until #718, which made the 'V' " - "a name word (the next row); with 'III' the word lands " - "in `family`, as it does on master before #718, where " - "'V' had left it in `given`"), + "The word lands in `family`, AGAINST H1's given-name " + "title clause, as it did before #718 ('John Jr., Sir' " + "and 'John PhD, Sir' too). Until #718 this row was " + "'comma_path_given_name_title_decides_the_lone_word' on " + "'John V, Sir', the one spelling that read H1's given " + "name, and #718 made that 'V' a name word (the next " + "row), so nothing pins H1's reading behind a comma now"), Case("a_lone_numeral_before_a_given_name_title_comma", "John V, Sir", {"title": "Sir", "given": "John", "family": "V"}, diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 99eb9ee4..04414943 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -510,6 +510,7 @@ "jacobus henricus van 't hoff" "john doe q.c." "john e smith" +"john i" "john smith EdD" "john smith MEng" "john smith bsc" @@ -529,6 +530,7 @@ "john smith ph.d." "john smith phd" "john smith psyd" +"john smith v" "john smith vi" "john smith x.y.z." "john smith xi" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index 1e3c054c..b6ea1593 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -3379,6 +3379,7 @@ {"name": "john e jones, III", "given": "john", "middle": "e", "family": "jones", "suffix": "III", "initials": "j. e. j."} {"name": "john e smith", "given": "john", "middle": "e", "family": "smith", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "j. e. s."} {"name": "john e. smith", "given": "john", "middle": "e.", "family": "smith", "initials": "j. e. s."} +{"name": "john i", "given": "john", "family": "i", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "j. i."} {"name": "john smith", "given": "john", "family": "smith", "initials": "j. s."} {"name": "john smith EdD", "given": "john", "family": "smith", "suffix": "EdD", "initials": "j. s."} {"name": "john smith MA", "given": "john", "family": "smith", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} @@ -3400,6 +3401,7 @@ {"name": "john smith ph.d.", "given": "john", "family": "smith", "suffix": "ph.d.", "initials": "j. s."} {"name": "john smith phd", "given": "john", "family": "smith", "suffix": "phd", "initials": "j. s."} {"name": "john smith psyd", "given": "john", "family": "smith", "suffix": "psyd", "initials": "j. s."} +{"name": "john smith v", "given": "john", "family": "smith", "suffix": "v", "initials": "j. s."} {"name": "john smith vi", "given": "john", "family": "smith", "suffix": "vi", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} {"name": "john smith x.y.z.", "given": "john", "family": "smith", "suffix": "x.y.z.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} {"name": "john smith xi", "given": "john", "middle": "smith", "family": "xi", "initials": "j. s. x."} From ede4d255ec6cbbfac2316c91b19d2c7ab2aa8ccc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 03:06:37 -0700 Subject: [PATCH 4/7] Keep the numeral count out of the given part after a family comma (#718) Code review found the chain's bound in a family comma's given part asking the count through tail_reading, which assign's walk there never asks (S4: "read without it"), so the chain took the V of 'Smith, Freiherr von Berg V' into the given name, unreported. The bound now reads with count_numeral=False and the four shapes read as on master. Two case rows pin it; AGENTS.md's group-side-predictor gotcha records the from-the-peel side of the class. Also: assign uses the shared _AMBIGUOUS_CREDENTIAL_TAGS rather than a second spelling of it, and group's Produces: line lists the chain's and clause's reports. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../decisions/S10-trailing-roman-numeral.md | 1 + .../rules/S10-trailing-roman-numeral.md | 3 ++- ...-ambiguous-acronym-needs-words-to-spare.md | 2 +- nameparser/_pipeline/_assign.py | 10 +++------ nameparser/_pipeline/_group.py | 5 ++++- nameparser/_pipeline/_pieces.py | 22 ++++++++++++++----- tests/v2/cases.py | 19 ++++++++++++++++ 8 files changed, 47 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 36eb55ee..25ce3de2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,7 +285,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Comparing against v1 needs `PYTHONSAFEPATH=1` AND a directory outside the worktree** — `uv run --isolated --no-project --with 'nameparser==1.4.0'` still puts the checkout's `nameparser/` ahead of the pinned wheel on `sys.path`, so the "v1" side silently imports the branch and every comparison reports parity. Run it as `cd && PYTHONSAFEPATH=1 uv run --isolated --no-project --with 'nameparser==1.4.0' python -c "..."`. This produces false confidence rather than an error, so it invalidates results without ever looking wrong. **A `tests/v2/cases.py` row's `parity` means 1.4.0, and nothing checks it** -- the table runs every row against the tree, so a row labelled `parity` that only matches master passes for good. #614 added eight such rows, compared against master; #620 found them by running the wheel this way and relabelled each with the change its reading came from, found by `git bisect run` in a scratch worktree (`git worktree add --detach`, never this one). Run the wheel before writing `parity`. -**A group-side reading of a family comma's given part that asks `tail_reading` disagrees with the given part's own walk, and the disagreement is accepted.** `tail_reading` reads the part as the comma-less name reads it; assign's walk over the given part after a family comma reads by its own predicates -- a shape-only numeral VI to X stays a name word there (`Smith, John van der Berg VI` reads middle 'van der Berg VI'), V before a third comma part is read strictly (#144), and P6 attaches a period-marked title-particle (`St.`) the comma-less reading takes as a title. So a bound or a decline group computes with `tail_reading` acts where the walk then reads no title: #623's chain bound (`Smith, John Mc Prof. VI`) and #637's join decline (`Smith, abdul Prof. VI` → middle 'Prof. VI', `Smith, abdul St.` → family 'St. Smith'). Both were recorded rather than narrowed (Derek: garbage input; decisions/H5 and #P5, rules/P5's Accepted clause). Before writing another group-side predictor of the given part, sweep those three shapes and decide, rather than rediscovering the gap in review. +**A group-side reading of a family comma's given part that asks `tail_reading` disagrees with the given part's own walk, and the disagreement is accepted.** `tail_reading` reads the part as the comma-less name reads it; assign's walk over the given part after a family comma reads by its own predicates -- a shape-only numeral VI to X stays a name word there (`Smith, John van der Berg VI` reads middle 'van der Berg VI'), V before a third comma part is read strictly (#144), and P6 attaches a period-marked title-particle (`St.`) the comma-less reading takes as a title. So a bound or a decline group computes with `tail_reading` acts where the walk then reads no title: #623's chain bound (`Smith, John Mc Prof. VI`) and #637's join decline (`Smith, abdul Prof. VI` → middle 'Prof. VI', `Smith, abdul St.` → family 'St. Smith'). Both were recorded rather than narrowed (Derek: garbage input; decisions/H5 and #P5, rules/P5's Accepted clause). Before writing another group-side predictor of the given part, sweep those three shapes and decide, rather than rediscovering the gap in review. And a change to the trailing peel ITSELF reaches the given part through those predictors without touching them: #718's words-to-spare count for a one-letter numeral, asked through `trailing_run_start`, made the chain take the V in `Smith, Freiherr von Berg V` (given 'von Berg V', unreported) while assign's walk, which never counts there (S4: "read without it"), read it as the suffix -- found by review on a grid of family-comma and empty-head shapes behind a title-particle; the predictor now passes `count_numeral=False`. When a change moves what the peel takes, put a family comma with a title-particle head in its grid. **A comparison that runs both sides in one tree reports 0 differences, which is what success looks like.** `tools/differential/compare.py` has been hardened against this — it generates its baseline worker into a temp dir, strips `PYTHONPATH` from the child, and aborts unless both the version AND the resolved path check out on each side; its README's three invocation traps are the analysis behind that design, worth reading before changing the harness. **The exposure is ad-hoc comparisons you write yourself**, where the same collapse has a cause the harness cannot disarm for you: **the shell's working directory persists between tool calls**, so a two-tree comparison written as two `cd`s silently runs both halves in whichever tree it landed in. Pin absolute paths, and assert `nameparser.__file__` on both sides the way `compare.py` does. The general rule outlives any particular trap: before believing a null result, prove the harness can report a difference — a clean run and a broken harness are the same output. The proof has to come from the input class the null result is about: #683 moved `tools/name_usage.py` to a POST and checked it with `shehu`, which is ASCII and so cannot show that the server decodes a Cyrillic body. A misdecoded query still runs and matches nothing, so the Cyrillic zeros it was meant to vouch for proved nothing until `Иван` (17,119 given) went through the same path. The converse trap is a mutation that moves nothing: it shows the code is unexercised by your grids, not that it is dead. #638 was filed calling group's chain re-ask dead on a grid with no title-particle head before a particle run and a dotted acronym; the next review found 400 moves (`Smith, Freiherr van Berg X.Y.Z.`) and the issue closed the same day. Before calling code dead, look for the input that reaches it. diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index 103bfab8..d0208773 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -8,3 +8,4 @@ Recompute: the moved names are this commit's diff of tools/differential/snapshot.jsonl (or `uv run python tools/differential/snapshot.py` on the parent tree with this commit's code). The Declined paragraph's figures are a prototype's, over every cases.py row and snapshot name parsed before and after the patch in one process, on master at 958d252e: its 26 replace `_pieces.is_trailing_numeral_suffix` with a wrapper returning False for a text that is one capital `I`, `V` or `X` (periods stripped) and the original's answer otherwise; its "four more" apply that wrapper to `I` and `X` alone on top of the shipped count. Both are dated snapshots by Counting claims; nothing re-runs them. - 2026-10-18 (Derek), #718 — THE DECLINED NUMERAL REPORTS TOO, BOTH DIRECTIONS AS S4'S FORK DOES. The count's first form reported only the generation, so `John V` read family V in silence while `Jack Ma` reports keeping its family; Derek: report both, as `Jack Ma` does. A declined numeral is a pick of the walk, and its branches are taken in three places, so three sites report it (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): assign where it stays a lone name word (`John V` — "read as a family name rather than a generational suffix"), the particle chain where it joins the run (`Freiherr von Richthofen V`), and a maiden clause that keeps it as its last word (`John née Jones Smith V`, `abdul nee Smith V`). Each wording is its own, since the acronym pick's ("both a post-nominal and an ordinary name") misdescribes a numeral. ONE STAND-DOWN, FOUND BY INV10 rather than designed. In a one-case name the connective `i` reaches the fork as a letter classify has already reported as `conjunction-or-initial`, read as an initial, and the new report put a second report on the same token: 138 rows of the connective grid (measured on this branch with the `reported_as_initial` check out of assign's numeral report, a dated snapshot) in tests/v2/test_properties.py (`AND ROVIRA I`, `HENRY I`) tripped INV10's kind pair. The two reports did not contradict each other — both read the letter as a name word — but they named one reading twice, so the numeral's report stands down where the initial reading is already reported, the shape of rules/S7's "a member its own capitals already made the credential reports nothing new". The guard was left untouched: its kind pair still means "two readings" everywhere else, and narrowing it to suit this change would have retaken its recorded control. `HENRY I` therefore reports `conjunction-or-initial` alone, and moved in test_assemble's contrast list from the withdrawn side to the side where the fork resolved to a name word. +- 2026-10-18, #718 review — THE COUNT IS NOT ASKED IN THE GIVEN PART AFTER A FAMILY COMMA. The code review's grid (family-comma and empty-head shapes behind a title-particle, about 558,000 parses) found 2,880 parses where the first form moved a numeral from the suffix into the given or family name with no report: `Smith, Freiherr von Berg V` read given 'von Berg V', `, Freiherr von V` family 'von V', `Smith, St van V` given 'van V', where `Smith, von V` and `Smith, Freiherr John V` kept suffix V. The cause was not a choice: assign's walk over that part never counts the numeral (rules/S4: the given part's trailing slot "is read without it (S5)"), but the particle chain's bound there asks `tail_reading`, which the count had reached, so the chain took a word assign would have read as the suffix. The bound now reads with the count off (`trailing_run_start`, `count_numeral=False`), and all four read as they did on master (2.2.0 onward, since #424; 1.4.0 read the numeral into the name). The class is the one the AGENTS.md gotcha on group-side predictors of the given part names, met from the other side: the predictor did not change, the peel it asks did. The S4 analogue, `Smith, Freiherr von Berg Ma` → given 'von Berg Ma', is master's reading and is left alone here. diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index c45f34de..45ccf4e1 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -23,7 +23,8 @@ S10. Rationale: a roman numeral ending a name is its generation, nothing, since nobody writes an initial that way. A name written wholly in one case says nothing about any letter in it. The part before a suffix comma is read as the same name written without - the comma (C1). + the comma (C1); the given part after a family comma is not + counted, the comma having settled the count there (S5). "John Smith V" → suffix="V" "John Smith V" → ambiguities=("suffix-or-name",) "John W. Ingram V" → suffix="V" diff --git a/docs/design/rules/S4-ambiguous-acronym-needs-words-to-spare.md b/docs/design/rules/S4-ambiguous-acronym-needs-words-to-spare.md index 5290a79f..5b330c1c 100644 --- a/docs/design/rules/S4-ambiguous-acronym-needs-words-to-spare.md +++ b/docs/design/rules/S4-ambiguous-acronym-needs-words-to-spare.md @@ -28,4 +28,4 @@ S4. Rationale: an acronym that is also an ordinary name (MA, Ma) "john smith meng" → suffix="meng" "JOHN SMITH MA" → suffix="MA" "Nguyen Van Lac" → family="Van Lac" - history: decisions/S4 · interacts: C1, C2, M2, S2, S3, S5, S6, S7, S9 · implemented: nameparser/_pipeline/_group.py + history: decisions/S4 · interacts: C1, C2, M2, S2, S3, S5, S6, S7, S9 · implemented: nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index ea017ad0..e0b1813e 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -106,10 +106,6 @@ from nameparser._policy import Policy, Script from nameparser._types import FOLDED_TAG, SHAPE_ACRONYM_TAG, AmbiguityKind, Role -#: either tag marks a pick of the ambiguous-acronym fork; a pick with -#: neither is the numeral fork's (#718) -_ACRONYM_TAGS = frozenset({AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG}) - def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], role: Role) -> None: for i in piece: @@ -539,9 +535,9 @@ def _assign_main(seg_idx: int, state: ParseState, token = tokens[piece[0]] assert token.role is not None # rules/S10: "either reading is reported" -- the numeral fork's - # declined pick (#718), which carries neither acronym tag and - # is never a suffix - if token.tags.isdisjoint(_ACRONYM_TAGS): + # declined pick (#718), which carries neither credential tag + # (the acronym fork's picks carry one) and is never a suffix + if token.tags.isdisjoint(_AMBIGUOUS_CREDENTIAL_TAGS): if reported_as_initial(ambiguities, piece[0]): continue ambiguities.append(PendingAmbiguity( diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 5d0f8e97..b0a38f0a 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -9,7 +9,10 @@ are tagged (rules/C1, #613, `_comma.decide`), binding the SUFFIX and TITLE roles of a postnominal part after the comma, emptying its segment and dropping its delimiter cores before any join, plus the SUFFIX_OR_NAME -ambiguities that decision reports and the caps shape's acronym tags on +ambiguities that decision reports, those the particle chain and a +maiden clause report on a trailing member they take into the name +(an ambiguous acronym, or since #718 a declined numeral), and the +caps shape's acronym tags on a bound all-caps word (#564); pieces + piece_tags per segment (runs of token indices -- tokens are NEVER joined into strings: the anti-#100 invariant); maiden tail tokens get role=MAIDEN, and the diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 45603e46..bea599f0 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -1094,7 +1094,8 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], start: int | None = None, anchors: list[bool] | None = None, units: Sequence[int] | None = None, - second: int = 1) -> Peel: + second: int = 1, + count_numeral: bool = True) -> Peel: """The S2 trailing peel over `rest`, a peel_walk list. In the piece layer rather than in assign because group's bound-given reserve asks the same question of the view the join would leave @@ -1153,7 +1154,7 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], and is_trailing_numeral_suffix( text := tokens[piece[0]].text, tokens[pieces[rest[k - 2]][0]].text)): - if (len(text) > 1 or k - 2 >= second + if (len(text) > 1 or not count_numeral or k - 2 >= second or numeral_settled_by_case(text, one_case)): numeral = tuple(piece) k -= 1 @@ -1567,6 +1568,7 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], tokens: Sequence[WorkToken], one_case: bool | None, units: Sequence[int] | None = None, + count_numeral: bool = True, ) -> tuple[list[int], tuple[int, ...], Peel]: """The S2 peel and the H5 chain read together to a FIXED POINT: peel, chain, splice the chained pieces out, peel again over what @@ -1651,7 +1653,8 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], # /simplify, 'Freiherr von Berg Berg ... MA Dr. MA Dr. ...') second = 1 if units is None else second_unit(rest, units) peeled = peel_trailing(rest, pieces, ptags, tokens, one_case, - units=units, second=second) + units=units, second=second, + count_numeral=count_numeral) kept = trailing_titles(rest, pieces, ptags, tokens, end=peeled.names) if kept == peeled.names: @@ -1688,7 +1691,8 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], if kept >= 2 and behind_count >= 2: peeled = peel_trailing(rest, pieces, ptags, tokens, one_case, start=hi, anchors=peeled.anchors, - units=units, second=second) + units=units, second=second, + count_numeral=count_numeral) picks.extend(peeled.picks) else: rest = rest[:hi] + [j for run in reversed(behind) for j in run] @@ -1697,7 +1701,8 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], hi = len(rest) behind, behind_count = [], 0 peeled = peel_trailing(rest, pieces, ptags, tokens, one_case, - units=units, second=second) + units=units, second=second, + count_numeral=count_numeral) picks = list(peeled.picks) numeral = peeled.numeral kept = trailing_titles(rest, pieces, ptags, tokens, @@ -1948,8 +1953,13 @@ def trailing_run_start(start: int, pieces: Sequence[Sequence[int]], suffixes, as assign's walk does in the given part after a family comma (#623), where `trailing_start` sees only the suffixes and `trailing_start_past_titles` leaves the titles inside the bound.""" + # rules/S4: "the given part's trailing slot after a family comma is + # read without it (S5)" -- the count, which assign's walk there + # never asks of the numeral either, so the chain this bounds stops + # where that walk will read the suffix (#718's review) rest, titled, peel = tail_reading(peel_walk(start, ptags), pieces, - ptags, tokens, one_case) + ptags, tokens, one_case, + count_numeral=False) first = len(pieces) for k in rest[peel.names:]: first = min(first, k) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index e0e83b46..23956e5a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -7467,6 +7467,25 @@ def _check_cjk_shape_purity(self) -> None: "settles the fork. It " "reported until #718, the shape fork reporting all it " "took, while the listed 'John Smith v' never did"), + Case("the_given_part_after_a_family_comma_does_not_count_a_numeral", + "Smith, Freiherr von Berg V", + {"title": "Freiherr", "given": "von Berg", "family": "Smith", + "suffix": "V"}, + classification="fix(#424)", + notes="S4: the given part's trailing slot after a family comma " + "is read without the count, so the one-letter numeral " + "S10 counts in a comma-less name is the suffix here " + "behind a title-particle's chain, as 'Smith, von V' " + "reads. #718's first form asked the count through the " + "chain's bound and read given 'von Berg V', unreported; " + "review found it. 1.4.0 read first 'von Berg V'"), + Case("an_empty_head_comma_does_not_count_a_numeral_either", + ", Freiherr von V", + {"title": "Freiherr", "family": "von", "suffix": "V"}, + classification="fix(#424)", + notes="the empty-head shape of the row above, which #718's " + "first form read family 'von V'; 1.4.0 read last " + "'von V'"), Case("a_lowercase_numeral_in_a_one_case_name_still_reports", "john smith i", {"given": "john", "family": "smith", "suffix": "i"}, From ed64d5499147ef57b2ce36d76949eb8ab6a5d5c9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 03:11:16 -0700 Subject: [PATCH 5/7] Document peel_trailing's and tail_reading's count_numeral (#718) Co-Authored-By: Claude Opus 5.5 --- nameparser/_pipeline/_pieces.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index bea599f0..81450c10 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -1127,6 +1127,10 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], weighing it is a name word the walk stops at. The numeral fork still reads the piece in front of the numeral as written: an initial there keeps it a name word whatever unit the initial is in ('John van der J. V', rules/P2). + + `count_numeral` False takes a one-letter numeral without asking + for words to spare: the given part after a family comma is read + without the count, and its chain bound must not ask it (#718). """ picks: list[tuple[int, ...]] = [] numeral: tuple[int, ...] | None = None @@ -1643,6 +1647,7 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], spliced pieces. The fuzz that checked this against the re-peeling loop, the three conditions each shown load-bearing by a weakened copy failing it, is recorded on the PR closing #558. + `count_numeral` is `peel_trailing`'s, passed to every pass. The splice itself is never materialized in between: the runs are collected back to front and joined once. """ From 24691d01b09397cd50ac492d0b66cf5613bb782e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 12:35:39 -0700 Subject: [PATCH 6/7] Address the PR review: one numeral report site, case-aware clause (#718) Comment review: fix labels that contradict 1.4.0 ('John III' is no parity row, 'John III, Sir' is one, 'abdul Doe nee Smith V' is #274's), and comments this change made stale (assign's deferral note and module docstring, group's 'abdul V' reserve example, _vocab's claim that X is suffix vocabulary). Test review (mutation of each emitter in a copy): the particle chain's numeral report only chose a sentence -- assign reports the same kind from the final role -- so it is gone, as is the maiden clause's stand-down, which could never fire. The clause's report on a numeral it gives up now asks the case, so 'Jane Doe nee Smith vi' is silent as every other path is. New rows pin the chain-taken numeral, its already-an-initial stand-down, 'John VI', family-first 'John V', the lowercase pair 'John x'/'john x' and the maiden-clause case contrast; five rows restoring 1.4.0 field for field are relabelled parity. Co-Authored-By: Claude Opus 5.5 --- .../decisions/S10-trailing-roman-numeral.md | 1 + .../rules/S10-trailing-roman-numeral.md | 5 + nameparser/_pipeline/_assign.py | 8 +- nameparser/_pipeline/_group.py | 53 ++++------ nameparser/_pipeline/_pieces.py | 7 +- nameparser/_pipeline/_vocab.py | 8 +- tests/v2/cases.py | 96 ++++++++++++++++--- tests/v2/test_cases.py | 12 ++- tests/v2/test_facade_cases.py | 3 + tools/differential/corpus_rules.jsonl | 1 + tools/differential/snapshot.jsonl | 1 + 11 files changed, 135 insertions(+), 60 deletions(-) diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index d0208773..c9a3d796 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -9,3 +9,4 @@ - 2026-10-18 (Derek), #718 — THE DECLINED NUMERAL REPORTS TOO, BOTH DIRECTIONS AS S4'S FORK DOES. The count's first form reported only the generation, so `John V` read family V in silence while `Jack Ma` reports keeping its family; Derek: report both, as `Jack Ma` does. A declined numeral is a pick of the walk, and its branches are taken in three places, so three sites report it (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): assign where it stays a lone name word (`John V` — "read as a family name rather than a generational suffix"), the particle chain where it joins the run (`Freiherr von Richthofen V`), and a maiden clause that keeps it as its last word (`John née Jones Smith V`, `abdul nee Smith V`). Each wording is its own, since the acronym pick's ("both a post-nominal and an ordinary name") misdescribes a numeral. ONE STAND-DOWN, FOUND BY INV10 rather than designed. In a one-case name the connective `i` reaches the fork as a letter classify has already reported as `conjunction-or-initial`, read as an initial, and the new report put a second report on the same token: 138 rows of the connective grid (measured on this branch with the `reported_as_initial` check out of assign's numeral report, a dated snapshot) in tests/v2/test_properties.py (`AND ROVIRA I`, `HENRY I`) tripped INV10's kind pair. The two reports did not contradict each other — both read the letter as a name word — but they named one reading twice, so the numeral's report stands down where the initial reading is already reported, the shape of rules/S7's "a member its own capitals already made the credential reports nothing new". The guard was left untouched: its kind pair still means "two readings" everywhere else, and narrowing it to suit this change would have retaken its recorded control. `HENRY I` therefore reports `conjunction-or-initial` alone, and moved in test_assemble's contrast list from the withdrawn side to the side where the fork resolved to a name word. - 2026-10-18, #718 review — THE COUNT IS NOT ASKED IN THE GIVEN PART AFTER A FAMILY COMMA. The code review's grid (family-comma and empty-head shapes behind a title-particle, about 558,000 parses) found 2,880 parses where the first form moved a numeral from the suffix into the given or family name with no report: `Smith, Freiherr von Berg V` read given 'von Berg V', `, Freiherr von V` family 'von V', `Smith, St van V` given 'van V', where `Smith, von V` and `Smith, Freiherr John V` kept suffix V. The cause was not a choice: assign's walk over that part never counts the numeral (rules/S4: the given part's trailing slot "is read without it (S5)"), but the particle chain's bound there asks `tail_reading`, which the count had reached, so the chain took a word assign would have read as the suffix. The bound now reads with the count off (`trailing_run_start`, `count_numeral=False`), and all four read as they did on master (2.2.0 onward, since #424; 1.4.0 read the numeral into the name). The class is the one the AGENTS.md gotcha on group-side predictors of the given part names, met from the other side: the predictor did not change, the peel it asks did. The S4 analogue, `Smith, Freiherr von Berg Ma` → given 'von Berg Ma', is master's reading and is left alone here. +- 2026-10-18, #718's PR review — TWO OF THE THREE REPORT SITES ARE ONE, AND THE CLAUSE ASKS THE CASE. The test-coverage review mutated each new emitter out in a copy of the tree. Removing the particle chain's numeral report moved no field and no kind — assign's pick loop reported the same `suffix-or-name`, worded from the final role ('read as a family name') — so the chain's branch was only choosing a sentence, and it is gone: a declined numeral stays one of the read's picks, and assign alone reports it, the decision having been the read's and not the chain's. The maiden clause's stand-down on a letter already reported as an initial could not fire at all (classify reads no connective inside a clause), so that call is gone too. And the clause's report on a numeral its take gave up did not ask the case: `Jane Doe nee Smith vi` and `x` reported where every other path had gone silent, the drift mechanisms.md#ONE-PREDICATE-PER-QUESTION names; the take now asks `numeral_settled_by_case` as assign does, `VI` still reporting beside them. Five rows the first draft labelled `fix(#718)` read field for field as 1.4.0 does and are `parity`, as are the two Freiherr rows added here; a restored 1.4.0 reading is parity by the table's own definition (rules/S4's `bare_surname_is_not_a_credential` is the precedent). diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index 45ccf4e1..a6ba308f 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -51,6 +51,11 @@ S10. Rationale: a roman numeral ending a name is its generation, "john v" → family="" "john smith v" → ambiguities=() "john i" → family="i" + "john i" → ambiguities=("conjunction-or-initial",) + Accepted: inside a maiden marker's clause the connective is not + read (M2), so a one-case `i` there is the suffix word the + vocabulary lists, as `v` is. + "john nee smith i" → suffix="i" Accepted: a numeral with another suffix word behind it is not the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name. diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index e0b1813e..3c63dbc0 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -59,7 +59,8 @@ particle out of that position) -- whatever role name_order assigns -- and, since #613, at P6's attachment after a family comma where the run holds an ambiguous particle ("Beethoven, Ludwig van"). -Emits SUFFIX_OR_NAME at these sites: the trailing roman numeral, each +Emits SUFFIX_OR_NAME at these sites: the trailing roman numeral, taken +or (since #718) declined for want of words to spare, each ambiguous acronym the trailing peel had to resolve, the bare-suffix carve-out where an input that is nothing but post-nominal vocabulary gets its first word made into the name (H4's suffix half, #491), @@ -371,8 +372,9 @@ def _assign_main(seg_idx: int, state: ParseState, # rather than overwrites. Deferred to after assignment because what # a pick is declined as depends on the role it took, and which role # "not peeled" means depends on name_order; the field itself is - # worded by assemble (#626). (The roman-numeral fork needs no such - # deferral and is reported here.) + # worded by assemble (#626). (The roman-numeral fork's TAKEN branch + # needs no such deferral and is reported here; its declined branch + # is a pick, #718, and is reported with the others.) # # Read once in group, after P3's connective joins and ahead of the # particle chain, on a main segment of three or more pieces whose diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index b0a38f0a..b2920a8e 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -70,8 +70,7 @@ Peel, TailRead, chain_lead, chain_run_end, is_conj_piece, is_leading_title, is_prefix_piece, is_suffix_piece, is_title_piece, join_connectives, joined_tags, leading_titles, merge_particle_runs, merge_pieces, peel_walk, - read_trailing_run, reported_as_initial, tail_reading, - trailing_candidates, + read_trailing_run, tail_reading, trailing_candidates, trailing_run_start, trailing_start, trailing_start_past_titles, ) @@ -82,7 +81,7 @@ import nameparser._pipeline._comma as _comma from nameparser._pipeline._vocab import D, PH from nameparser._pipeline._vocab import ( - comma_follows_marker, delimiter_cores, + comma_follows_marker, delimiter_cores, numeral_settled_by_case, ) from nameparser._types import AmbiguityKind, Role @@ -333,9 +332,14 @@ def _maiden_take(pieces: Sequence[Sequence[int]], # -- the take consumed the run, so the forks the reading called # on it are reported here, where assign will never see them consumed = {i for j in range(end, len(pieces)) for i in pieces[j]} - for piece in (*vpeel.picks, *vpeel.absorbed, - *(() if vpeel.numeral is None - else (vpeel.numeral,))): + # rules/S10: "a numeral written in lower case in a name of mixed + # case is the generation and reports nothing" -- here as at + # assign's numeral report (#718's review) + numeral = (() if vpeel.numeral is None + or numeral_settled_by_case( + tokens[vpeel.numeral[0]].text, one_case) + else (vpeel.numeral,)) + for piece in (*vpeel.picks, *vpeel.absorbed, *numeral): if piece[0] in consumed: ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, @@ -352,8 +356,7 @@ def _maiden_take(pieces: Sequence[Sequence[int]], f"{word.text!r} ending the maiden name is also a " f"post-nominal; the maiden marker's clause keeps it rather " f"than reading it as one", tuple(last))) - elif (tuple(last) in picked - and not reported_as_initial(ambiguities, last[0])): + elif tuple(last) in picked: # rules/S10: "either reading is reported" -- a numeral the read # declined for want of words to spare, which the clause keeps ambiguities.append(PendingAmbiguity( @@ -1115,27 +1118,12 @@ def chain(tail: int) -> None: # the given role the fold then moves ('Freiherr von # Berg Ma' was told 'given name'). last = pieces[j - 1] - # ...and a numeral S2's read declined (rules/S10, #718), - # a pick with no credential tag: the read's picks are - # where it is recorded, and `tuple` is C-level, so a - # chained name still pays no frame - numeral = (j > k + 1 and len(last) == 1 - and read is not None - and tuple(last) in read.peel.picks - and tokens[last[0]].tags.isdisjoint( - _AMBIGUOUS_CREDENTIAL_TAGS)) - if numeral: - if not reported_as_initial(ambiguities, last[0]): - ambiguities.append(PendingAmbiguity( - AmbiguityKind.SUFFIX_OR_NAME, - f"{tokens[last[0]].text!r} is a roman " - f"numeral, but the name has no other word to " - f"spare for it; the particle chain took it " - f"into the name rather than reading it as a " - f"generational suffix", - tuple(last))) - took.add(tuple(last)) - elif (j > k + 1 and len(last) == 1 + # A numeral S2's read declined (rules/S10, #718) is not + # reported here: it stays one of the read's picks, and + # assign words it from the role it ends in ('Freiherr + # von Richthofen V', 'a family name'), the chain taking + # nothing the read had not already given up + if (j > k + 1 and len(last) == 1 and not tokens[last[0]].tags.isdisjoint( _AMBIGUOUS_CREDENTIAL_TAGS) and not is_prefix_piece(last, ptags[j - 1], @@ -1275,9 +1263,10 @@ def chain(tail: int) -> None: # before #614: the join tried on the pieces as it would # leave them, the reading assign runs over each (the # ONE function, _pieces.tail_reading), the suffixes - # compared as the peeled pieces themselves -- 'abdul V' - # peels the V unjoined and nothing joined, 'abdul Smith - # Ma' peels the acronym unjoined and keeps it joined. + # compared as the peeled pieces themselves -- 'abdul + # Smith V' peels the V unjoined and nothing joined (no + # word to spare there, rules/S10), 'abdul Smith Ma' + # peels the acronym unjoined and keeps it joined. if read is not None: same_suffixes = True chained = False diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 81450c10..1389f79f 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -1364,9 +1364,10 @@ def particle_tail(seg: Sequence[Sequence[int]], # rules/S10: "a letter already reported as read as an initial reports -# nothing new" -- the one question the numeral fork's three declined- -# pick emitters (assign's, the particle chain's and the maiden -# clause's, #718) all ask, so it has one answer +# nothing new" -- asked by assign's report of the numeral fork's +# declined pick (#718). The maiden clause's report on the numeral it +# keeps cannot meet the case: classify reads no connective inside a +# clause, so a clause word never carries that report def reported_as_initial(ambiguities: Iterable[PendingAmbiguity], index: int) -> bool: """Whether classify's connective-or-initial fork already reported diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index e152a3df..9fffa568 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -254,14 +254,16 @@ def is_initial_shaped(text: str) -> bool: # rules/S10: "A roman numeral ending a name written without a comma # reads as the generation" -- which the initial veto would otherwise -# take: V, I and X are suffix vocabulary AND bare capitals, and a bare -# capital inside a name is a middle initial. Whether a one-letter +# take: V and I are suffix vocabulary, X a numeral by shape alone, and +# all three are bare capitals, which inside a name are middle initials. Whether a one-letter # numeral has the words to spare is the caller's count (`peel_trailing`). def is_trailing_numeral_suffix(text: str, preceding: str) -> bool: """assign's roman-numeral fork, shared with group's bound-given reserve (#401): a FINAL single-token piece that is a roman numeral reads as the suffix when the piece before it does not look like - part of an initial run. `preceding` is that piece's first token; + part of an initial run -- for a one-letter numeral a necessary + condition only, the caller's words-to-spare count deciding + (rules/S10, #718). `preceding` is that piece's first token; the callers establish that `text` is last and that a name piece precedes it. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 23956e5a..2d3bf16f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -3965,7 +3965,6 @@ def _check_cjk_shape_purity(self) -> None: {"given": "Smith", "family": "V", "suffix": "Ma"}, policy=Policy(lenient_comma_suffixes=False), ambiguities=("suffix-or-name", "suffix-or-name"), - classification="fix(#718)", notes="the period is the whole difference from the row " "above: bare 'V' is still initial-shaped and still " "counts as a name word here, so the structure flips " @@ -3973,8 +3972,9 @@ def _check_cjk_shape_purity(self) -> None: "it as the roman numeral, leaving 'Smith' the only " "name word and reporting all three forks; a one-letter " "numeral now needs words to spare (S10, the trailing " - "roman numeral), so 'V' is the family, as 1.4.0 read " - "it, and both forks, 'V' and 'Ma', report"), + "roman numeral), so 'V' is the family -- 1.4.0's " + "reading field for field, which #718 restored -- and " + "both forks, 'V' and 'Ma', report"), Case("caps_run_declines_a_bound_given_head", "John Smith, ABDUL AP", {"given": "ABDUL AP", "family": "John Smith"}, policy=Policy(unlisted_caps_suffixes=CapsSuffixes.EVERYWHERE), @@ -5475,7 +5475,7 @@ def _check_cjk_shape_purity(self) -> None: "abdul Doe nee Smith V", {"given": "abdul", "family": "Doe", "suffix": "V", "maiden": "Smith"}, - ambiguities=("suffix-or-name",), + ambiguities=("suffix-or-name",), classification="fix(#274)", notes="P5's bound-given join is LENIENT only after a family " "comma; before one the STRICT reserve declines a join " "that would change a suffix reading, so the release " @@ -5484,7 +5484,9 @@ def _check_cjk_shape_purity(self) -> None: "V', has made no clause since #601: a marker after a " "family comma is an ordinary word.) Unchanged from " "2.3.0. 'Doe' since #718, which made the 'V' of " - "'abdul nee Smith V' a name word (the next row)"), + "'abdul nee Smith V' a name word (the next row); 1.4.0 " + "read first 'abdul Doe', middle 'nee', having no " + "maiden markers"), Case("a_numeral_with_no_word_to_spare_stays_in_the_clause", "abdul nee Smith V", {"given": "abdul", "maiden": "Smith V"}, @@ -7430,7 +7432,7 @@ def _check_cjk_shape_purity(self) -> None: "so the reading is reported"), Case("a_lone_numeral_with_no_word_to_spare_is_the_family", "John V", {"given": "John", "family": "V"}, - ambiguities=("suffix-or-name",), classification="fix(#718)", + ambiguities=("suffix-or-name",), notes="S10 (the trailing roman numeral): a one-letter " "numeral needs words to spare, as a bare ambiguous " "acronym does, and 'John' is the only other word -- the " @@ -7439,29 +7441,30 @@ def _check_cjk_shape_purity(self) -> None: "'V' with no family; 1.4.0 read last 'V'"), Case("a_lone_shape_numeral_with_no_word_to_spare_is_the_family", "Malcolm X", {"given": "Malcolm", "family": "X"}, - ambiguities=("suffix-or-name",), classification="fix(#718)", + ambiguities=("suffix-or-name",), notes="the shape-only spelling of the row above: 'x' is in " "no vocabulary and reaches the fork by its shape"), Case("a_lone_numeral_in_a_one_case_name_is_counted", "JOHN V", {"given": "JOHN", "family": "V"}, - ambiguities=("suffix-or-name",), classification="fix(#718)", + ambiguities=("suffix-or-name",), notes="a name written in one case says nothing about the " "letter, so the count decides as it does for 'John V'"), Case("a_title_is_no_word_to_spare_for_a_lone_numeral", "Dr Smith V", {"title": "Dr", "given": "Smith", "family": "V"}, - ambiguities=("suffix-or-name",), classification="fix(#718)", + ambiguities=("suffix-or-name",), notes="S10's title clause (the trailing roman numeral); " "1.4.0 read first 'Smith', last 'V' too"), Case("a_multi_letter_numeral_needs_no_word_to_spare", "John III", {"given": "John", "suffix": "III"}, ambiguities=("given-or-family",), - notes="the boundary of the count: a listed numeral of two " - "letters is not initial-shaped and reads as any suffix " - "word, so the lone 'John' is what is reported (O5)"), + classification="fix(suffix-routing)", + notes="the boundary of the count: a listed numeral of two or " + "more letters is not initial-shaped and reads as any suffix " + "word, so the lone 'John' is what is reported (O5). " + "1.4.0 read last 'III', as it read 'Johnson PhD'"), Case("a_lowercase_numeral_in_a_mixed_case_name_reports_nothing", "John Smith vi", {"given": "John", "family": "Smith", "suffix": "vi"}, - classification="fix(#718)", notes="S10 (the trailing roman numeral): nobody writes an " "initial in lower case beside capitals, so the writing " "settles the fork. It " @@ -7486,6 +7489,72 @@ def _check_cjk_shape_purity(self) -> None: notes="the empty-head shape of the row above, which #718's " "first form read family 'von V'; 1.4.0 read last " "'von V'"), + Case("a_shape_numeral_with_no_word_to_spare_is_still_counted", + "John VI", {"given": "John", "suffix": "VI"}, + ambiguities=("suffix-or-name", "given-or-family"), + classification="fix(suffix-routing)", + notes="the count's other boundary: a numeral of two letters " + "read by its shape alone takes no count and still " + "reports (S10), so 'John VI' keeps the generation where " + "'John V' gives it up. 1.4.0 read last 'VI'"), + Case("a_lone_numeral_counts_in_units_under_family_first", + "John V", {"given": "V", "family": "John"}, + policy=Policy(name_order=FAMILY_FIRST), + ambiguities=("suffix-or-name",), classification="fix(#718)", + notes="the count is the same under a family-first order: " + "'John' is the only other word, so the letter is a name " + "word, and the order lays it out as the given name"), + Case("a_lone_numeral_the_chain_takes_reports", + "Freiherr von Richthofen V", + {"title": "Freiherr", "family": "von Richthofen V"}, + ambiguities=("particle-or-given", "suffix-or-name"), + notes="behind a title the chain is one name word, so the V " + "has none to spare and the chain takes it (P2); the " + "declined numeral is still the read's pick, reported " + "from the role it ends in. 2.2.0 to 2.4.2 read suffix " + "'V'; 1.4.0 read last 'von Richthofen V'"), + Case("a_lone_numeral_the_chain_takes_already_an_initial", + "FREIHERR VON RICHTHOFEN I", + {"title": "FREIHERR", "family": "VON RICHTHOFEN I"}, + ambiguities=("conjunction-or-initial", "particle-or-given"), + notes="the row above in one case with the connective 'I', " + "which classify has already reported as an initial: the " + "declined numeral reports nothing new (S10)"), + Case("a_lowercase_shape_numeral_in_a_mixed_case_name_is_settled", + "John x", {"given": "John", "suffix": "x"}, + ambiguities=("given-or-family",), classification="fix(#718)", + notes="lower case beside capitals settles the fork with no " + "count (S10): the generation, unreported, though no " + "word is spared -- the one shape that path reaches, " + "'v' and 'i' being listed suffix words. 1.4.0 read last " + "'x'"), + Case("a_lowercase_shape_numeral_in_a_one_case_name_is_counted", + "john x", {"given": "john", "family": "x"}, + ambiguities=("suffix-or-name",), + notes="the row above written wholly in lower case: the case " + "says nothing, so the count decides and reports"), + Case("a_lowercase_numeral_after_a_maiden_clause_reports_nothing", + "Jane Doe nee Smith vi", + {"given": "Jane", "family": "Doe", "suffix": "vi", + "maiden": "Smith"}, + classification="fix(#274)", + notes="the clause's take reports the numeral it gives up, and " + "asks the case as assign does: lower case in a mixed-" + "case name reports nothing (S10, #718's review). 1.4.0 " + "had no maiden markers"), + Case("a_lowercase_shape_numeral_after_a_maiden_clause_reports_nothing", + "Jane Doe nee Smith x", + {"given": "Jane", "family": "Doe", "suffix": "x", + "maiden": "Smith"}, + classification="fix(#274)", + notes="the unlisted spelling of the row above"), + Case("a_capital_numeral_after_a_maiden_clause_still_reports", + "Jane Doe nee Smith VI", + {"given": "Jane", "family": "Doe", "suffix": "VI", + "maiden": "Smith"}, + ambiguities=("suffix-or-name",), classification="fix(#274)", + notes="the contrast that makes the silence above a decision " + "about case"), Case("a_lowercase_numeral_in_a_one_case_name_still_reports", "john smith i", {"given": "john", "family": "smith", "suffix": "i"}, @@ -8761,7 +8830,6 @@ def _check_cjk_shape_purity(self) -> None: Case("comma_path_given_name_title_behind_a_listed_numeral", "John III, Sir", {"title": "Sir", "family": "John", "suffix": "III"}, - classification="fix(#296)", notes="the title row above with a GIVEN-NAME title. O5 is " "silenced all the same: what the report is about is a " "field NOTHING decided, and this one a title decided. " diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index b01d3abb..1cc3e0ab 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -185,12 +185,14 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # order; 1171 -> 1174 on 2026-10-17: the エリザベス二世 row (#707, # one given-or-family claim under each order); 1174 -> 1172 on # 2026-10-17: #718's re-pinned and added numeral rows (the lone - # numeral rows lose their given-or-family claims; 'John III' and - # 'Malcolm X' rows bring some back); 1172 -> 1191 on 2026-10-18: + # numeral rows lose their given-or-family claims; the 'John III' + # row brings some back); 1172 -> 1191 on 2026-10-18: # #718's declined-numeral report, which names the field the letter - # was read into, on those rows under each order. - assert claims == 1191, ( - f"the field sweep checks {claims} claims, recorded as 1191 on " + # was read into, on those rows under each order; 1191 -> 1210 the + # same day: the PR review's rows (the chain-taken, family-first, + # lowercase and maiden-clause numerals). + assert claims == 1210, ( + f"the field sweep checks {claims} claims, recorded as 1210 on " f"2026-10-18") diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index 0629cb61..6573cd29 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -73,6 +73,9 @@ #: to the unsubtracted default pushes every parenthesis-maiden row #: into this set and turns nothing red. _CORE_ONLY_IDS = frozenset({ + # #718: S10's count under a declared family-first order, which v1 + # cannot express + "a_lone_numeral_counts_in_units_under_family_first", # #701: H6's peel under a declared family-first order, which v1 # cannot express "a_title_before_a_family_comma_reads_the_same_family_first", diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 04414943..904c4500 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -511,6 +511,7 @@ "john doe q.c." "john e smith" "john i" +"john nee smith i" "john smith EdD" "john smith MEng" "john smith bsc" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index b6ea1593..8a5b5bab 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -3380,6 +3380,7 @@ {"name": "john e smith", "given": "john", "middle": "e", "family": "smith", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "j. e. s."} {"name": "john e. smith", "given": "john", "middle": "e.", "family": "smith", "initials": "j. e. s."} {"name": "john i", "given": "john", "family": "i", "ambiguities": ["CONJUNCTION_OR_INITIAL"], "initials": "j. i."} +{"name": "john nee smith i", "family": "john", "suffix": "i", "maiden": "smith", "initials": "j."} {"name": "john smith", "given": "john", "family": "smith", "initials": "j. s."} {"name": "john smith EdD", "given": "john", "family": "smith", "suffix": "EdD", "initials": "j. s."} {"name": "john smith MA", "given": "john", "family": "smith", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "j. s."} From 4ac5a06601531392d02ca76e405d8cb37020fef4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 18 Oct 2026 13:03:10 -0700 Subject: [PATCH 7/7] Simplify the numeral fork: decide the case once, in the walk (#718) /simplify pass over PR #719, behavior-neutral (snapshot unchanged): - a numeral its lower case settles is taken by the walk like any suffix word and recorded as no call, so assign and the maiden take stop re-asking numeral_settled_by_case -- the drift #718's review found in the clause can no longer recur - the one-letter test asks is_single_letter_numeral rather than a hand `len(text) > 1` - reported_as_initial moves into _assign, its one caller, and _pieces drops the two ambiguity imports it needed - flatter control flow in assign's numeral pick; one duplicated comment sentence and a long case note trimmed; the given-slot bound's comment says only the numeral half is uncounted Co-Authored-By: Claude Opus 5.5 --- nameparser/_pipeline/_assign.py | 39 +++++++++++++++++++------------- nameparser/_pipeline/_group.py | 13 ++++------- nameparser/_pipeline/_pieces.py | 40 ++++++++++++++------------------- nameparser/_pipeline/_vocab.py | 7 +++--- tests/v2/cases.py | 9 +++----- 5 files changed, 51 insertions(+), 57 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 3c63dbc0..f4a885b6 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -89,15 +89,13 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - effective_script, is_suffix_lenient, numeral_settled_by_case, - resolve_script_set, + effective_script, is_suffix_lenient, resolve_script_set, ) from nameparser._pipeline._pieces import ( anchor_in_reach, credential_at_the_given_slot, given_slot_anchors, _NOT_A_RUN_START, has_name_content, is_lone_never_given_particle, Peel, TailRead, is_suffix_piece, is_title_piece, leading_titles, - particle_tail, listed_lean, peel_walk, reported_as_initial, - starts_a_credential_run, + particle_tail, listed_lean, peel_walk, starts_a_credential_run, tail_reading, trailing_titles, ) from nameparser._pipeline._state import ( @@ -308,6 +306,21 @@ def _placed(read: TailRead, rest: list[int], peel.absorbed) +# rules/S10: "a letter already reported as read as an initial reports +# nothing new" -- asked only by the numeral fork's declined pick +# (#718). The maiden clause's report on the numeral it keeps cannot +# meet the case: classify reads no connective inside a clause +def _reported_as_initial(ambiguities: Sequence[PendingAmbiguity], + index: int) -> bool: + """Whether classify's connective-or-initial fork already reported + the token at `index`, reading it as an initial. Declining the + numeral reads the letter as that same name word, so a second + report would name one reading twice. A rare shape, so the scan + costs ordinary names nothing.""" + return any(a.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL + and index in a.indices for a in ambiguities) + + def _assign_main(seg_idx: int, state: ParseState, tokens: list[WorkToken], ambiguities: list[PendingAmbiguity], @@ -408,10 +421,7 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], Role.TITLE) for piece in peeled.absorbed: ambiguities.append(_absorbed(piece, tokens)) - # rules/S10: "a numeral written in lower case in a name of mixed - # case is the generation and reports nothing" - if peeled.numeral is not None and not numeral_settled_by_case( - tokens[peeled.numeral[0]].text, state.one_case): + if peeled.numeral is not None: # a trailing single letter is a name part unless it happens # to be a roman numeral -- and V/X/I are ordinary middle # initials, so taking it as a suffix is a call, not a fact @@ -540,13 +550,12 @@ def _assign_main(seg_idx: int, state: ParseState, # declined pick (#718), which carries neither credential tag # (the acronym fork's picks carry one) and is never a suffix if token.tags.isdisjoint(_AMBIGUOUS_CREDENTIAL_TAGS): - if reported_as_initial(ambiguities, piece[0]): - continue - ambiguities.append(PendingAmbiguity( - AmbiguityKind.SUFFIX_OR_NAME, - f"{token.text!r} is a roman numeral, but the name has " - f"no other word to spare for it; read as ", piece, - field_tail=" rather than a generational suffix")) + if not _reported_as_initial(ambiguities, piece[0]): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{token.text!r} is a roman numeral, but the name " + f"has no other word to spare for it; read as ", piece, + field_tail=" rather than a generational suffix")) continue declined = ("a name part" if token.role is Role.SUFFIX else "a post-nominal") diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index b2920a8e..b71b8aae 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -81,7 +81,7 @@ import nameparser._pipeline._comma as _comma from nameparser._pipeline._vocab import D, PH from nameparser._pipeline._vocab import ( - comma_follows_marker, delimiter_cores, numeral_settled_by_case, + comma_follows_marker, delimiter_cores, ) from nameparser._types import AmbiguityKind, Role @@ -332,14 +332,9 @@ def _maiden_take(pieces: Sequence[Sequence[int]], # -- the take consumed the run, so the forks the reading called # on it are reported here, where assign will never see them consumed = {i for j in range(end, len(pieces)) for i in pieces[j]} - # rules/S10: "a numeral written in lower case in a name of mixed - # case is the generation and reports nothing" -- here as at - # assign's numeral report (#718's review) - numeral = (() if vpeel.numeral is None - or numeral_settled_by_case( - tokens[vpeel.numeral[0]].text, one_case) - else (vpeel.numeral,)) - for piece in (*vpeel.picks, *vpeel.absorbed, *numeral): + for piece in (*vpeel.picks, *vpeel.absorbed, + *(() if vpeel.numeral is None + else (vpeel.numeral,))): if piece[0] in consumed: ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 1389f79f..e772fec0 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -59,7 +59,7 @@ from nameparser._pipeline._state import ( AMBIGUOUS_ACRONYM_TAG, NAME_ROLES, SHAPE_ACRONYM_TAG, SUFFIX_OR_UNREAD, - AmbiguityKind, PendingAmbiguity, WorkToken, + WorkToken, ) from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, _ROMAN, Lean, ambiguous_lean, in_initialless_script, @@ -918,7 +918,9 @@ class Peel(NamedTuple): pieces and rest[names:] the suffixes. The other two are pieces -- token-index tuples, as PendingAmbiguity wants them -- and each is one token long: `numeral` is the piece the roman-numeral fork - took (None when it did not fire; always the walk's last piece), + took as a call, which is reported (None when it did not fire, and + for a numeral its lower case settled, which is taken unrecorded; + always the walk's last piece), `picks` the bare ambiguous acronyms the peel had to resolve, in peel order, either way (the last may sit at rest[names - 1]), and a one-letter numeral the fork declined for want of words to spare @@ -1158,8 +1160,15 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], and is_trailing_numeral_suffix( text := tokens[piece[0]].text, tokens[pieces[rest[k - 2]][0]].text)): - if (len(text) > 1 or not count_numeral or k - 2 >= second - or numeral_settled_by_case(text, one_case)): + # rules/S10: "a numeral written in lower case in a name of + # mixed case is the generation and reports nothing" -- + # decided here once: taken as any suffix word is, and not + # recorded as the fork's call, so no reporter meets it + if numeral_settled_by_case(text, one_case): + k -= 1 + continue + if (not is_single_letter_numeral(text) or not count_numeral + or k - 2 >= second): numeral = tuple(piece) k -= 1 continue @@ -1363,22 +1372,6 @@ def particle_tail(seg: Sequence[Sequence[int]], return lo, hi -# rules/S10: "a letter already reported as read as an initial reports -# nothing new" -- asked by assign's report of the numeral fork's -# declined pick (#718). The maiden clause's report on the numeral it -# keeps cannot meet the case: classify reads no connective inside a -# clause, so a clause word never carries that report -def reported_as_initial(ambiguities: Iterable[PendingAmbiguity], - index: int) -> bool: - """Whether classify's connective-or-initial fork already reported - the token at `index`, which it reads as an initial when it reaches - the trailing peel as a lone piece. Declining the numeral reads the - letter as that same name word, so a second report would name one - reading twice. Asked only of a declined numeral pick, a rare - shape, so the scan costs ordinary names nothing.""" - return any(a.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL - and index in a.indices for a in ambiguities) - def has_name_content(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: @@ -1960,9 +1953,10 @@ def trailing_run_start(start: int, pieces: Sequence[Sequence[int]], comma (#623), where `trailing_start` sees only the suffixes and `trailing_start_past_titles` leaves the titles inside the bound.""" # rules/S4: "the given part's trailing slot after a family comma is - # read without it (S5)" -- the count, which assign's walk there - # never asks of the numeral either, so the chain this bounds stops - # where that walk will read the suffix (#718's review) + # read without it (S5)" -- the numeral half of the count, which + # assign's walk there never asks either, so the chain this bounds + # stops where that walk will read the suffix (#718's review); the + # acronym half is still counted here, as before #718 rest, titled, peel = tail_reading(peel_walk(start, ptags), pieces, ptags, tokens, one_case, count_numeral=False) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 9fffa568..52514325 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -255,8 +255,7 @@ def is_initial_shaped(text: str) -> bool: # rules/S10: "A roman numeral ending a name written without a comma # reads as the generation" -- which the initial veto would otherwise # take: V and I are suffix vocabulary, X a numeral by shape alone, and -# all three are bare capitals, which inside a name are middle initials. Whether a one-letter -# numeral has the words to spare is the caller's count (`peel_trailing`). +# all three are bare capitals, which inside a name are middle initials. def is_trailing_numeral_suffix(text: str, preceding: str) -> bool: """assign's roman-numeral fork, shared with group's bound-given reserve (#401): a FINAL single-token piece that is a roman numeral @@ -279,8 +278,8 @@ def is_trailing_numeral_suffix(text: str, preceding: str) -> bool: # rules/S10: "a numeral written in lower case in a name of mixed case is # the generation and reports nothing" -- the one writing that settles # the fork, since nobody writes an initial in lower case beside -# capitals. Asked by the walk, which then needs no words to spare, and -# by assign, which then reports nothing, so the two cannot drift. +# capitals. Asked by the walk alone, which takes such a numeral as any +# suffix word and records no call for a reporter to report. def numeral_settled_by_case(text: str, one_case: bool | None) -> bool: """Whether a numeral the trailing fork reads is lower case in a name written in more than one case. `one_case` is diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 2d3bf16f..7a7328ef 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -8834,12 +8834,9 @@ def _check_cjk_shape_purity(self) -> None: "silenced all the same: what the report is about is a " "field NOTHING decided, and this one a title decided. " "The word lands in `family`, AGAINST H1's given-name " - "title clause, as it did before #718 ('John Jr., Sir' " - "and 'John PhD, Sir' too). Until #718 this row was " - "'comma_path_given_name_title_decides_the_lone_word' on " - "'John V, Sir', the one spelling that read H1's given " - "name, and #718 made that 'V' a name word (the next " - "row), so nothing pins H1's reading behind a comma now"), + "title clause ('John Jr., Sir' and 'John PhD, Sir' " + "too, #720). 'John V, Sir' read H1's given name until " + "#718; nothing pins that reading behind a comma now"), Case("a_lone_numeral_before_a_given_name_title_comma", "John V, Sir", {"title": "Sir", "given": "John", "family": "V"},