From f3ea3b77837c4e379ed4301f42df96bb8684022c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 03:33:48 -0700 Subject: [PATCH 1/4] Ask one predicate for every numeral rule (#752, phase 2f) Every rule that weighs a roman numeral now asks _vocab.reads_as_generation, which reads the vocabulary's own numerals inside it: the trailing peel (anywhere in the run for two or more letters), S8's run start, C12's evidence, C1's listing form, the given part's walk after a family comma, the maiden take and tail_reading. - John Smith VI MD reads suffix 'VI MD' as III MD does (#784) - John Smith V, M.J. and VI, M.J. keep the numeral as suffix (#774) - Smith, John PhD I, Jr. reads middle 'I' again (#819) - Georg VI. and the other GND regnal rows read the suffix - a Title-case numeral in a mixed-case name leans to the name - John Smith V PhD keeps family 'V' (#721), measured on tse-donors Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../decisions/C1-suffix-comma-decision.md | 5 +- ...x-word-ending-the-name-before-the-comma.md | 2 +- .../C5-paired-initials-after-the-comma.md | 2 +- ...aracter-suffix-word-inside-a-comma-part.md | 9 +- .../decisions/S10-trailing-roman-numeral.md | 3 + .../decisions/S8-credentials-trailing-run.md | 4 +- docs/design/decisions/vocabulary-model.md | 9 +- docs/design/mechanisms.md | 2 +- docs/design/rules/C1-suffix-comma-decision.md | 8 +- ...x-word-ending-the-name-before-the-comma.md | 2 +- .../C5-paired-initials-after-the-comma.md | 2 +- ...aracter-suffix-word-inside-a-comma-part.md | 15 +- .../rules/S10-trailing-roman-numeral.md | 51 ++++- .../rules/S8-credentials-trailing-run.md | 4 +- docs/design/vocabulary-model-rfc.md | 2 +- docs/release_log.rst | 5 + nameparser/_pipeline/_assign.py | 141 ++++++++++-- nameparser/_pipeline/_comma.py | 5 +- nameparser/_pipeline/_group.py | 8 +- nameparser/_pipeline/_pieces.py | 202 ++++++++++++------ nameparser/_pipeline/_vocab.py | 108 +++++++--- tests/v2/cases.py | 87 +++++++- tests/v2/rules_budget.json | 12 +- tests/v2/test_cases.py | 9 +- tests/v2/test_properties.py | 4 +- tools/differential/corpus_rules.jsonl | 26 +++ tools/differential/gnd_agreement.json | 33 --- tools/differential/snapshot.jsonl | 35 ++- 29 files changed, 593 insertions(+), 204 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 350fa3ae..e8e24ec6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -278,7 +278,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Adding a flag-gated post-parse transform** (reorder/adjust) — add a `Constants` boolean (default `False`), implement a `handle_*()` method, and call it in `post_process()` after `handle_firstnames()` and before `handle_capitalization()`, gated on the flag. Default-off keeps existing parses byte-for-byte unchanged. Two shipped examples: `patronymic_name_order` gates both `handle_east_slavic_patronymic_name_order()` (#85) and `handle_turkic_patronymic_name_order()` (#185) — one flag driving two independent handlers, added in the same `post_process()` slot; `middle_name_as_last` gates `handle_middle_name_as_last()` (#133), which folds `middle_list` into `last_list`. -**Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. After the change, `tests/v2/test_snapshot.py` lists every corpus name it moved, grouped by changed field -- read them, and re-record with `uv run python tools/differential/snapshot.py --record` once every move is intended, so the PR diff of `snapshot.jsonl` shows the reviewer the same list. `uv run python tools/differential/cldr_agreement.py` reports how the tree's agreement with CLDR's stated name fields moved; `tests/v2/test_cldr_agreement.py` fails on any move, and `tools/differential/cldr_agreement.json` doubles as a measured list of names the parser and CLDR read differently. The same command and test score the GND corpus's real names (`gnd_fields.jsonl`, record `gnd_agreement.json`) by the same rules, and `--record` rewrites every record, printing per source what it accepts. A locale-pack change reads the second view the same command prints, per pack (`cldr_agreement_locales.json`, #672, and `wikidata_native_agreement_locales.json` for real Han and kana names); its `JA` rows run only with the `ja` extra installed. A change to Han, Hangul or kana order or to Korean segmentation reads `wikidata_native_agreement.json` too: the only real names in those scripts the corpora hold. It holds no honorific, so a change to the honorific peel moves nothing there. A change to comma or credential handling also reads the local corpus of hand-typed names (`snapshot.py --local ctgov-officials`, recorded first against the base checkout with `--tree` (a base read once from the same fetch is copied rather than parsed again; `--sample N` reads a fixed sample while iterating); tools/differential/README.md's "Local corpora"): the one population of names nobody chose for a test or a catalog, so it shows how often each shape is typed, and the one no CI run sees. A change to title handling reads `ukec-individuals` the same way: UK donors' names in spoken order and mixed case, in 8,219 of whose 15,214 `parse()` read a title on 2026-10-22. A change to particles or generational words reads `snapshot.py --local tse-donors` the same way, Brazilian names in spoken order dense with both. A change to initials, to a title word outside English, or to how a script's combining marks are read reads `snapshot.py --local nepal-candidates` the same way: Nepal's local election candidates, each a romanized name (surnames written as initials, `B.K.`, `K.C.`; honorific titles such as `Shekh` in front, and `Md.`, which its period makes a title by shape) beside the same name in Devanagari, written with vowel signs and the virama, combining marks the letter-shape tests do not count as letters (decisions/nfd-shapes), and with the joiners ZWJ and ZWNJ. It is the only large population of South Asian names the corpora hold, and its corpus reads both scripts. A change to non-Latin scripts or to Arabic-script names reads `snapshot.py --local jrc-names` the same way, and `local_corpora.py groups jrc-names`: the spellings of one person across scripts and transliterations, harvested from news text, with 17,074 Arabic-script spellings where the committed corpora hold 315 distinct names with an Arabic letter, 96 of them CLDR's fictional renderings (2026-10-24), and under a licence that keeps even a reading of one of its names in the cache. A change to which field a word lands in also reads `local_agreement.py psc-individuals`, recorded the same way: real names with the four boxes their filers typed them into (UK companies' people with significant control), scored field by field and by nationality. It is the only real fielded population outside the committed fields files, and no CI run sees it either. +**Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. After the change, `tests/v2/test_snapshot.py` lists every corpus name it moved, grouped by changed field -- read them, and re-record with `uv run python tools/differential/snapshot.py --record` once every move is intended, so the PR diff of `snapshot.jsonl` shows the reviewer the same list. `uv run python tools/differential/cldr_agreement.py` reports how the tree's agreement with CLDR's stated name fields moved; `tests/v2/test_cldr_agreement.py` fails on any move, and `tools/differential/cldr_agreement.json` doubles as a measured list of names the parser and CLDR read differently. The same command and test score the GND corpus's real names (`gnd_fields.jsonl`, record `gnd_agreement.json`) by the same rules, and `--record` rewrites every record, printing per source what it accepts. A locale-pack change reads the second view the same command prints, per pack (`cldr_agreement_locales.json`, #672, and `wikidata_native_agreement_locales.json` for real Han and kana names); its `JA` rows run only with the `ja` extra installed. A change to Han, Hangul or kana order or to Korean segmentation reads `wikidata_native_agreement.json` too: the only real names in those scripts the corpora hold. It holds no honorific, so a change to the honorific peel moves nothing there. A change to comma or credential handling also reads the local corpus of hand-typed names (`snapshot.py --local ctgov-officials`, recorded first against the base checkout with `--tree` (a base read once from the same fetch is copied rather than parsed again; `--sample N` reads a fixed sample while iterating; the recorded `snapshot.jsonl` and psc's `agreement.jsonl` are one file per source in the shared cache, so a session measuring in another worktree overwrites your baseline: record it again right before each check, as #752's phase 2f found when ctgov reported a sibling branch's 1,334 joined-credential readings as its own moves); tools/differential/README.md's "Local corpora"): the one population of names nobody chose for a test or a catalog, so it shows how often each shape is typed, and the one no CI run sees. A change to title handling reads `ukec-individuals` the same way: UK donors' names in spoken order and mixed case, in 8,219 of whose 15,214 `parse()` read a title on 2026-10-22. A change to particles or generational words reads `snapshot.py --local tse-donors` the same way, Brazilian names in spoken order dense with both. A change to initials, to a title word outside English, or to how a script's combining marks are read reads `snapshot.py --local nepal-candidates` the same way: Nepal's local election candidates, each a romanized name (surnames written as initials, `B.K.`, `K.C.`; honorific titles such as `Shekh` in front, and `Md.`, which its period makes a title by shape) beside the same name in Devanagari, written with vowel signs and the virama, combining marks the letter-shape tests do not count as letters (decisions/nfd-shapes), and with the joiners ZWJ and ZWNJ. It is the only large population of South Asian names the corpora hold, and its corpus reads both scripts. A change to non-Latin scripts or to Arabic-script names reads `snapshot.py --local jrc-names` the same way, and `local_corpora.py groups jrc-names`: the spellings of one person across scripts and transliterations, harvested from news text, with 17,074 Arabic-script spellings where the committed corpora hold 315 distinct names with an Arabic letter, 96 of them CLDR's fictional renderings (2026-10-24), and under a licence that keeps even a reading of one of its names in the cache. A change to which field a word lands in also reads `local_agreement.py psc-individuals`, recorded the same way: real names with the four boxes their filers typed them into (UK companies' people with significant control), scored field by field and by nationality. It is the only real fielded population outside the committed fields files, and no CI run sees it either. ## Gotchas diff --git a/docs/design/decisions/C1-suffix-comma-decision.md b/docs/design/decisions/C1-suffix-comma-decision.md index a4851ffa..4994b292 100644 --- a/docs/design/decisions/C1-suffix-comma-decision.md +++ b/docs/design/decisions/C1-suffix-comma-decision.md @@ -10,11 +10,11 @@ - The CJK stage asks the comma decision itself rather than a copy of it — 2026-10-06 #613, the CJK radar tier. - An empty part before the comma is read by the decision, not left to the listing walk — 2026-10-06 #613, the PR review. - A bound caps word that carries the name's case contrast keeps its capitals' marks, so case repair leaves it as written (rules/R4) — 2026-10-06 #613, the PR review. -- A generation ending the name before the comma meets paired initials after it this way: dotted, they stay the given name and the generation the suffix (C5); undotted as two capitals, they are the credential (C12); a roman numeral S10 reads does not yet stay the suffix, tracked by #774 — 2026-10-19 #761. +- A generation ending the name before the comma meets paired initials after it this way: dotted, they stay the given name and the generation the suffix (C5); undotted as two capitals, they are the credential (C12); a roman numeral S10 reads stays the suffix too — 2026-10-19 #761; 2026-10-25 #774. Not standing: 2026-10-04 #611's "the rethinks are filed, not done" — #613 rethought C1 (folding P6 in) and #614 S2, both shipped in 2.4.0, which overtook its "neither for 2.4". -Open: #774 — a roman numeral S10 reads by shape is lost into the family before dotted paired initials (`John Smith V, M.J.`). +Open: none in this file. ## Log @@ -34,3 +34,4 @@ Open: #774 — a roman numeral S10 reads by shape is lost into the family before SUPERSEDES: decisions/C4's 2026-10-01 #562 bullet's mechanism and its accepted reports (its pair report and the settled-run test); decisions/C13's 2026-10-04 #603 bullet's "segment decides" wording; and decisions/C3's #575 bullet's account of WHY segment builds its unit facts from vocabulary -- `_comma._whole_name` counts over classify's tags, and `name_word_count`, which reads the vocabulary, is asked by decide and by script_segment, the agreement sweep `test_surname_unit_tags_agree_with_classify` keeping the vocabulary view equal to the tags. - 2026-10-18 #728 — SPLIT. rules/C1 held what are now C1 and C3–C11, and this file their history; each entry moved to the file of the rule it decided, and the #429, #436/#437 and #511 entries, which decide R1's entry pass, to decisions/R1. The entries kept here moved nothing but the pointers the split broke and the measurement paragraphs, each cut to one line naming its population, counts, baseline and PR. The #611 entry stays here, an open question on #728. - 2026-10-19 (Derek), #761 — A GENERATION BEFORE THE COMMA AGAINST PAIRED INITIALS AFTER IT: DOTTED, THE INITIALS ARE THE GIVEN NAME AND THE GENERATION STAYS THE SUFFIX; UNDOTTED, THEY ARE THE CREDENTIAL. rules/C12 (a suffix word ending the name before the comma makes the comma a suffix comma) and rules/C5 (paired initials after the comma are the given name) both claim `John Smith Jr., M.J.`, and no rule text said which wins; the C section's move to the skeleton format, which gives a hub Precedence bullets, asked. Measured on 732a2716: `John Smith Jr., M.J.` reads given 'M.J.', family 'John Smith', suffix 'Jr.', and `John Smith Jr., MJ` reads given 'John', family 'Smith', suffix 'Jr., MJ', each reporting `suffix-or-name`. Decided: that reading is the rule, for every generation C12 reads, written in rules/C1's Precedence with both as examples and in C5's and C12's Precedence where they meet. It is the reading C12's Accepted clause (`Lloret Palol II, JA`) and its `Smith Jones Jr., J.A.` boundary already gave behind a two-word surname. The review found a roman numeral S10 reads by shape breaking it today: `John Smith V, M.J.` gives given 'M.J.', family 'John Smith V' and no suffix, while `John Smith V, MJ` reads suffix 'V, MJ'. So does every numeral S10 reads (`John Smith VI, M.J.`, `I`, `IX`, `X` alike). Derek ruled that a bug, so C1's Precedence carries the intended reading as two `deviates:` examples, V for the one-letter count and VI for the shape-only path, tracked by #774. The same pass collected into C1's Precedence the order among C4, C5 and C6 that their own texts already state (C4 counts name words first and asks written case second, C5 is the exception to that count, C6 gates when an unlisted all-caps word joins the class), deciding nothing new there; and the strict switch left C1 for rules/C14 (decisions/C14). +- 2026-10-25 #774 — A NUMERAL ENDING THE FAMILY PART IS READ AS THE COMMA-LESS NAME READS IT. The family part's walk took a word into the suffix by the suffix-piece test alone, so a listed generation stayed the suffix (`John Smith III, M.J.` → suffix 'III') while `V`, which the initial veto refuses, and `VI`, which no list holds, went into the family (`John Smith V, M.J.` → family 'John Smith V'), the split rules/C1's #761 Precedence example tracked as a `deviates:` row. The walk now asks the numeral's predicate (decisions/vocabulary-model's 2026-10-25 2f entry) of a single-word piece shaped as a roman numeral with nothing but suffix pieces behind it: the generation where it reads as one, a one-letter numeral needing two name pieces of the part in front of it as S10's count asks, and reported where a shape reads it, as the comma-less name reports. `John Smith V, M.J.` and `John Smith VI, M.J.` keep the numeral as the suffix, `Smith VI, John` reads suffix 'VI', and `Smith V, John` keeps family 'Smith V' (one word in front), silent, the comma having named the family. The two `deviates:` markers come off. diff --git a/docs/design/decisions/C12-suffix-word-ending-the-name-before-the-comma.md b/docs/design/decisions/C12-suffix-word-ending-the-name-before-the-comma.md index 3933ece1..e3f48738 100644 --- a/docs/design/decisions/C12-suffix-word-ending-the-name-before-the-comma.md +++ b/docs/design/decisions/C12-suffix-word-ending-the-name-before-the-comma.md @@ -10,7 +10,7 @@ Elsewhere: paired initials after a generation, dotted the given name and undotted the credential (`John Smith Jr., M.J.` → given 'M.J.'; `John Smith Jr., MJ` → suffix 'Jr., MJ') — decisions/C1's 2026-10-19 (Derek), #761 entry; `mbchb`, whose addition #656 declined, is listed (`Smith, MBChB` → suffix 'MBChB') — decisions/suffix-acronym-collisions' 2026-10-21 (#797) entry. -Open: #774 — a roman numeral S10 reads is lost into the family before dotted paired initials (`John Smith V, M.J.`), decisions/C1's Open line. +Open: none in this file. ## Log diff --git a/docs/design/decisions/C5-paired-initials-after-the-comma.md b/docs/design/decisions/C5-paired-initials-after-the-comma.md index 5903a145..2aef0b04 100644 --- a/docs/design/decisions/C5-paired-initials-after-the-comma.md +++ b/docs/design/decisions/C5-paired-initials-after-the-comma.md @@ -6,7 +6,7 @@ Elsewhere: paired initials are the exception to the ambiguous class's count, read as the given name after the comma (`García Márquez, G.J.` → given 'G.J.', family 'García Márquez'), and what in front of them makes them the credential — decisions/S9's 2026-09-30 (Derek), #563 entry; behind a one-word particle surname (`De La Cruz, M.J. K.L.` → given 'M.J.', middle 'K.L.', the second pair ending the given part as decisions/S3's 2026-10-24 #864 entry reads it) — decisions/C3's 2026-10-01 (Derek), #575 entry; a generation ending the name before the comma keeps dotted paired initials the given name and makes two undotted capitals the credential (`John Smith Jr., M.J.` → given 'M.J.', suffix 'Jr.'; `John Smith Jr., MJ` → suffix 'Jr., MJ') — decisions/C1's 2026-10-19 (Derek), #761 entry. -Open: #774 — a roman numeral S10 reads is lost into the family before dotted paired initials (`John Smith V, M.J.` → family 'John Smith V'), decisions/C1's Open line. +Open: none in this file. ## Log diff --git a/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md b/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md index 94aa3594..e50df12e 100644 --- a/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md +++ b/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md @@ -5,11 +5,13 @@ - Inside a comma part a one-character suffix word is read by what stands before it: behind another suffix word it continues the run, with a period or without (`Smith, PSM I` → suffix 'PSM I'; `Smith, PSM I.` → suffix 'PSM I.'), and behind a name word an ASCII period makes it a middle initial while the bare letter, with no further comma after it, is the suffix (`Smith, John V.` → middle 'V.'; `Smith, John V` → suffix 'V', v1 parity) — 2026-08-25 #430. - The class is one-character suffix vocabulary, not a roman numeral (`Smith, John 2.` → middle '2.'), and a run's first word that is title vocabulary too stays in the run (`Smith, RA I` → suffix 'RA I'; `Smith, Jr. I` → suffix 'Jr. I') — 2026-08-25 #430. - The no-comma numeral predicate is not asked in the given part, so a numeral behind an initial stays the suffix (`Chang, Andy C I` → middle 'C', suffix 'I') — 2026-08-25 #430. +- A lone letter ending the given part ahead of a further comma is the middle initial whatever stands before it, a credential run in front stopping short of it (`Smith, John PhD I, Jr.` → middle 'I', suffix 'PhD, Jr.'), at the cost of splitting a certification level from its credential (`Smith, John PSM I, Jr.`) — 2026-10-25 #819. +- The given part's walk asks the numeral's predicate, which reads a one-letter numeral there by the vocabulary, so an unlisted letter stays the middle initial (`SMITH, JOHN X`), and a numeral of two or more letters as the generation (`Smith, John VI` → suffix 'VI') — 2026-10-25 #819. - The given part's walk is strict-blind and the fork reports nothing: `Smith, John V` reads suffix 'V' under `lenient_comma_suffixes=False` as under the default, with no `suffix-or-name`, where `John Smith V` reports — 2026-08-25 #430. -Elsewhere: the strict switch, which vetoes the numeral in the comma's decision and in a part that is nothing but suffix words — decisions/C14's 2026-10-19 (Derek), #761 entry; a credential starting a run to the end of its part, which reads `Smith, John PSM I I.` as suffix 'PSM I I.' where #430 had accepted middle 'I I.' — decisions/S8's 2026-10-04 (Derek), #602 entry; the same run, unrecorded in that entry, reads `Smith, John PhD I, Jr.` and `Smith, John PhD I., Jr.` as suffix 'PhD I, Jr.' and 'PhD I., Jr.' (middle 'I' and 'I.' before #602), which rules/C9's further-comma clause and its Accepted bullet still give as the middle initial; a bare letter behind a name word with a suffix comma after it stays a middle initial (`Smith, John V, Jr.` → middle 'V', suffix 'Jr.') — decisions/v1-xfail-triage's 2026-09-01 (Derek's triage) entry; an ideographic or fullwidth full stop is no abbreviation mark there (`Smith, John V。` → suffix 'V。') — decisions/cjk-full-stops' 2026-09-10 #322/#323 bundle. +Elsewhere: the strict switch, which vetoes the numeral in the comma's decision and in a part that is nothing but suffix words — decisions/C14's 2026-10-19 (Derek), #761 entry; a credential starting a run to the end of its part, which reads `Smith, John PSM I I.` as suffix 'PSM I I.' where #430 had accepted middle 'I I.' — decisions/S8's 2026-10-04 (Derek), #602 entry; a bare letter behind a name word with a suffix comma after it stays a middle initial (`Smith, John V, Jr.` → middle 'V', suffix 'Jr.') — decisions/v1-xfail-triage's 2026-09-01 (Derek's triage) entry; an ideographic or fullwidth full stop is no abbreviation mark there (`Smith, John V。` → suffix 'V。') — decisions/cjk-full-stops' 2026-09-10 #322/#323 bundle. -Open: [#819](https://github.com/derek73/python-nameparser/issues/819) (whether rules/C9's further-comma clause or the parser is right for a letter behind a credential since #602, `Smith, John PhD I, Jr.`). +Open: none in this file. ## Log @@ -22,3 +24,6 @@ Open: [#819](https://github.com/derek73/python-nameparser/issues/819) (whether r The walk is also strict-blind, pre-existing and unchanged: it calls is_suffix_lenient unconditionally, so `Smith, John V` reads suffix 'V' under `lenient_comma_suffixes=False` as under the default. The reading consults the knob and the walk does not, which is now conspicuous where it was merely unexamined. Not this change's to fix. Not done, and recorded rather than left implicit: neither half emits an ambiguity, though the no-comma `John Smith V` reports `suffix-or-name` for the same fork. The comma path has never reported it (decisions/C8's #428 review entry records PARTICLE_OR_GIVEN suppressed there for a related reason), and adding an emitter needs a trigger, a stage-ownership entry and a case row — more than this fix, and #432 raises it as a secondary observation rather than a requirement. - 2026-10-18 #728 — SPLIT FROM decisions/C1 with the rule. The entries above were written under decisions/C1 while this rule was part of rules/C1, and moved here unchanged but for the pointers the move broke, the measurement paragraphs, each cut to one line naming its population, counts, baseline and PR, and the superseded passages, each cut to its decision and what replaced it; where one says rules/C1 or C1, it means the rule as it then stood, which held this one. +- 2026-10-25 #819 — A LONE LETTER ENDING THE GIVEN PART AHEAD OF A FURTHER COMMA IS THE MIDDLE INITIAL, AND THE WALK ASKS THE NUMERAL'S PREDICATE. Derek agreed reading (A) of #819 on 2026-10-23 with the numeral predicate (decisions/vocabulary-model's 2026-10-23 #752 entry, item 8): the further comma has named the suffix, so a letter ending the given part in front of it has no generation left to be, and #602's credential run (decisions/S8) stops short of it. `Smith, John PhD I, Jr.` and `Smith, John PhD I., Jr.` read middle 'I' and 'I.' again, as before #602, and the run's report on them goes with the reading; `Smith, John PhD I` (no further comma) keeps suffix 'PhD I'. The clause reads any lone letter a script with capitals writes as an initial, bare or with its period, numeral or not (`Smith, John PSM X, Jr.` → middle 'X'), since what it asks is the slot, not the word. + WHAT IT COSTS: `Smith, John PSM I, Jr.`, a certification level written ahead of a generation, splits from its credential (rules/C9's Accepted bullet). The spike that proposed this measured no corpus moving; on this change the committed snapshot holds no such name either, and the local corpora were read with it (decisions/vocabulary-model's 2026-10-25 2f entry). + THE WALK'S NUMERAL QUESTIONS are the predicate's (decisions/vocabulary-model, mechanisms.md#ONE-PREDICATE-PER-QUESTION): the last-piece test asks it of a bare one-letter numeral, and in this part, where neither the count nor an initial in front is asked, it answers by the vocabulary, so `Smith, John V` keeps suffix 'V' and `SMITH, JOHN X` middle 'X' (FEC writes a middle initial `X` there, the constraint the predicate's design recorded); a one-letter numeral behind a suffix word in a part of nothing but suffix words is asked the same way, its answer unchanged. A numeral of two or more letters, which this rule never governed, is read in this part by S10 since the same change (`Smith, John VI` → suffix 'VI', reported). diff --git a/docs/design/decisions/S10-trailing-roman-numeral.md b/docs/design/decisions/S10-trailing-roman-numeral.md index 7a527c67..b018726e 100644 --- a/docs/design/decisions/S10-trailing-roman-numeral.md +++ b/docs/design/decisions/S10-trailing-roman-numeral.md @@ -6,6 +6,7 @@ - A lowercase numeral in a mixed-case name is the generation and reports nothing (`John Smith vi`); a name written in one case is counted as its capitals are — 2026-10-17 #718; 2026-10-18 #721. - The declined numeral reports `suffix-or-name` as the taken one does, assign alone reporting it — 2026-10-18 #718; 2026-10-18 #718's PR review. - The count is not asked in the given part after a family comma: `Smith, Freiherr von Berg V` keeps suffix 'V' — 2026-10-18 #718 review. +- Every numeral rule asks one predicate, which reads a numeral of two or more letters as the generation wherever the trailing run holds it (`John Smith VI MD` → suffix 'VI MD'), written with its period too (`Georg VI.` → suffix 'VI.'), and leans a Title-case one in a mixed-case name to the name, reported (`Nguyen Van Vi` → family 'Van Vi'); in the given part after a family comma a numeral of two or more letters is the generation (`Smith, John VI`) and a one-letter one only where listed (`SMITH, JOHN X` → middle 'X') — 2026-10-25 #752. - A one-letter numeral with a credential behind it is a name word and reports (`John Smith V PhD` → family 'V'); requiring all three name fields, or never reading the comma-less letter as the generation, was declined (`John W. Ingram V` keeps suffix 'V') — 2026-10-18 #721's PR review. - A marked letter with no given, middle or family word after it in its part loses its connective report, withdrawn once over final roles, in assemble — 2026-10-18 #722's /simplify pass. - A bare letter in a one-case name is the initial its capital is (rules/T4), and the one-letter numerals are an instance of that rule — 2026-10-18 #723. @@ -37,3 +38,5 @@ Open: none in this file. Open: "a name written wholly in one case says nothing about any letter in it" is applied to the roman-numeral letters only. In a one-case name a bare capital is tagged an initial and a bare lowercase letter is not, unless it is `v`, `i` or `x` (this PR) or a marked connective (P8), so the lowercase and all-caps spellings of a one-case name read differently wherever a reader honors the tag — before #721 too: M4 makes a lone name word before a marker the family but not an initial (`j nee smith` family 'j', `J NEE SMITH` given 'J'), O5 is silenced by one (a lone `j` reports given-or-family, `J` does not), and the connective join defers to one (`jose j y lopez` family 'j y lopez', `JOSE J Y LOPEZ` middle 'J Y', family 'LOPEZ'). Tagging every one-case single letter as its capital is tagged is a behavior change beyond this PR: #723. - 2026-10-18, #723 — THE OPEN QUESTION ABOVE IS ANSWERED, FOR EVERY LETTER. A bare letter in a one-case name is the initial its capital is (rules/T4), and the one-letter numerals are an instance of that rule rather than a case of their own: S10's own one-case sentences are unchanged, and classify's numeral test is gone into T4's. History: decisions/T4. - 2026-10-19 (Derek), #761 — THE RARITY PREMISE LEAVES THE RATIONALE, AND THE SUFFIX-COMMA PRECEDENCE NAMES C12. The Because no longer says the generation a one-letter numeral would name is rare: the one-letter reading rests on the words-to-spare count (#722 above), not on rarity. The Precedence bullet for the part before a suffix comma had said it reads as the same name written without the comma (C1); that was false for a one-case part before a capitalized credential (`john j v, PhD` reads family 'j', suffix 'v, PhD', where `john j v` reads family 'v'; 2.5.0 the same). That reading is C12 deciding the comma and written case being asked of the whole name, credential included, so the part belongs to a name of mixed case (T4); the bullet now says so and carries both shapes as examples. No reading moved. +- 2026-10-25 #752 — EVERY NUMERAL RULE ASKS ONE PREDICATE, AND THE ONE-LETTER NUMERAL BEFORE A CREDENTIAL STAYS A NAME. Derek agreed on 2026-10-23 that every rule weighing a numeral asks one predicate, answering for `i` to `x` as this rule's fork does (decisions/vocabulary-model's 2026-10-23 #752 entry, item 8); the sites and the measurements are that file's 2026-10-25 #752, phase 2f entry. For this rule: the fork reads a numeral of two or more letters anywhere in the trailing run, not only as its last word, so `John Smith VI MD` reads suffix 'VI MD' as the listed `III MD` did (#784); a numeral of two or more letters keeps its reading written with a period, as `III.` did (`Georg VI.`, the German ordinal, and GND's two other regnal rows read suffix where they read family); a Title-case numeral in a mixed-case name leans to the name and reports, the third reading agreed (`Nguyen Van Vi`, 1.4.0's fields; PSC's three Vietnamese `Vi` surnames fixed, one `Viii` written as a generation broken). A numeral the vocabulary lists in two or more letters is not weighed: neither an initial in front nor Title case moves it (`John J. III`, `John Smith Iii` keep the suffix). + KEPT: the #721 reading of a one-letter numeral before a credential (`John Smith V PhD` → family 'V', reported). Read with the count instead, as the predicate reads one ending the name, it moved 6 tse-donors names, all wrong (a lone letter before `JR` abbreviating a surname), and the committed rows `John Smith V PhD` and its maiden twins; so the peel reads a one-letter numeral only as the walk's last piece, as before. diff --git a/docs/design/decisions/S8-credentials-trailing-run.md b/docs/design/decisions/S8-credentials-trailing-run.md index 8746fab6..ab994572 100644 --- a/docs/design/decisions/S8-credentials-trailing-run.md +++ b/docs/design/decisions/S8-credentials-trailing-run.md @@ -5,6 +5,7 @@ Examples name each field as the default order reads it; under the family-first orders the given and family names trade places. - An unambiguous suffix word of two or more letters, in any script, other than the split `Ph. D.` in a main segment (below), standing after two name words with no comma or before a suffix comma, or after the given word in the part after a family comma, starts a run to the end of its part: every later word reads as a suffix except a title word, which reads as a title, and an absorbed name word reports `suffix-or-name` (`John Smith PhD Jones` → suffix 'PhD Jones'; `Eric H. Holder Jr. Attorney General` → suffix 'Jr.', title 'Attorney General') — 2026-10-04 #602; 2026-10-14 #679. +- A roman numeral of two or more letters that reads as the generation starts the run as a listed one does (`John Smith VI Jones` → suffix 'VI Jones', as `John Smith III Jones`), its shape alone reported — 2026-10-25 #784. - What never starts the run: a title word (`Mary Jane King Smith` keeps family 'Smith'), a member of the ambiguous class, a single letter, a connective (`Josep Carod i Rovira` keeps family 'Carod i Rovira'), and a word that is also particle or bound given-name vocabulary (`Mohamed Ali Abd Allah` → family 'Allah'); a title/suffix dual does start it — 2026-10-04 #602. - An honorific or generational word starts the run as a credential does, whatever its script (`राम शर्मा साहब मोहन` → suffix 'साहब मोहन'); the one-character honorifics stay out by the two-letter minimum — 2026-10-14 #679. - A lone particle counts toward the two name words only where the credential stands right behind it (`John von PhD Jones` → family 'von', suffix 'PhD Jones', under the family-first orders given 'von'); a particle run counts once, and a particle the run took is not asked (`Freiherr von vd PhD Jones` keeps middle 'PhD') — 2026-10-10 #627. @@ -15,7 +16,7 @@ Not standing: the split of 2026-10-17 #682, carried out. Elsewhere: the split `Ph. D.` in a main segment (no comma, or before a suffix comma) starts no run, the trailing peel's walk never holding the merged piece (`John Smith Ph. D. Jones` → family 'Jones', suffix 'Ph. D.', unreported; `John Smith Ph. D. Jones, Jr.` → suffix 'Ph. D., Jr.') — decisions/C13's 2026-10-04 (Derek), #603 entry, and the #602 entry's own REVERSED paragraph; a credential opening the part after the comma, and the split `Ph. D.` starting the run as `PhD` does (`Smith, John Ph. D. Jones` → suffix 'Ph. D. Jones') — decisions/C13's 2026-10-04 (Derek), #603 entry; the run taking a lone letter behind a credential ahead of a further comma (`Smith, John PhD I, Jr.` → suffix 'PhD I, Jr.'), which the #602 entry does not record — decisions/C9's Elsewhere line. -Open: [#785](https://github.com/derek73/python-nameparser/issues/785) (whether capitals in a mixed-case name start the run as periods do); [#819](https://github.com/derek73/python-nameparser/issues/819) (whether a letter behind a credential ahead of a further comma stays a middle initial). +Open: [#785](https://github.com/derek73/python-nameparser/issues/785) (whether capitals in a mixed-case name start the run as periods do). ## Log @@ -31,3 +32,4 @@ Open: [#785](https://github.com/derek73/python-nameparser/issues/785) (whether c - 2026-10-14 (Derek), #679 — ANY UNAMBIGUOUS SUFFIX WORD OF TWO OR MORE LETTERS STARTS THE RUN, WHATEVER ITS SCRIPT; THE HONORIFIC EXCEPTION IS RETRACTED. #602 excluded "the NON-LATIN HONORIFIC words, an honorific followed by a name word being a different question from a credential followed by one", and drew that line by script, the vocabulary having no honorific marker. #661 then added Cyrillic and Devanagari GENERATIONAL words the script line refused too, splitting Serbian by the script it is written in (#generational-words). Measured 2026-10-14 over every non-Latin suffix word (51) in six shapes under three orders (918 parses), the exception protected no correct reading: an honorific followed by a name word read as a MIDDLE name, in silence (`राम शर्मा साहब मोहन` gave middle 'शर्मा साहब', family 'मोहन'; `משה כהן ז"ל דוד` middle 'כהן ז"ל'), and dropping it reads the honorific as the suffix it is with the word behind it absorbed and reported, as `John Smith PhD Jones` reads. 360 of the 576 honorific parses move and 285 of the 342 generational ones, the same 120 and 95 under each order; the one-character honorifics (씨 様 氏 殿 군 님 양, and जी, whose vowel sign is no letter) never move, S2's two-letter minimum keeping them out already. No realistic name with a name word after an honorific was found, and no corpus name outside rules.md's own examples moves; of the three S2 examples that had pinned the refusal, two now pin the run, and the mixed-script `Taro Tanaka さん Jones` gives way to `राम शर्मा साहब मोहन`. Recompute: the words are master's `Lexicon.default().suffix_words` (after #681) whose letters are not all LATIN by Unicode name; each goes between same-script name words in the shapes `{a} {b} {w} {c}`, `{b}, {a} {w} {c}`, `{a} {b}, {w} {c}`, `{a} {b} {w} {c} {c}`, `{a} {w} {c}` and `{b} {a} {w} {c}`, under each of the three orders, keyed by the order's NAME (the first count keyed by its first role and lost an order, two orders both beginning with FAMILY); compare `as_dict()` and the ambiguity kinds with `_pieces.starts_a_credential_run`'s script test in and out. Generational words are not a class of their own here: a suffix word is a suffix word, which is what drops `_pieces._all_latin` (#681, the same day) and the need for any honorific or generational marker as #679's roster adds non-Latin words. - 2026-10-17 #682 — SPLIT FROM decisions/S2 with the rule. The entries above were written under decisions/S2 while this rule was part of rules/S2, and moved here unchanged but for pointers the move broke; where one says rules/S2 it means the rule as it then stood, which held this one. - 2026-10-20 (Derek), #761 — AN AMBIGUOUS ACRONYM WRITTEN WITH ITS PERIODS STARTS THE RUN. The skeleton migration (PR #783) found S8 ("a member of the ambiguous class" never starts the run) and S3 (an ambiguous acronym written with its periods "counts unambiguously") both claiming `John Smith M.A. Jones`, with no text naming the winner; the parser reads suffix `M.A. Jones`. Derek: "writing periods makes it unambiguously not a name". S3 wins, written as a Precedence bullet in rules/S8; no behavior moves. The bullet is scoped to the periods: the capitals alone do not start the run (`John Smith MA Jones` reads middle `Smith MA`, family `Jones`), a question tracked in [#785](https://github.com/derek73/python-nameparser/issues/785). +- 2026-10-25 #784 — A NUMERAL THAT READS AS THE GENERATION STARTS THE RUN, WHATEVER ITS SPELLING. The run start asked the suffix vocabulary, so a listed numeral started it and a numeral S10 reads by its shape did not: `John Smith III Jones` → suffix 'III Jones', `John Smith VI Jones` → family 'Jones'. It now asks the numeral's predicate (decisions/vocabulary-model's 2026-10-25 2f entry) for a numeral of two or more letters, with the word in front and the name's case, so a Title-case `Vi` in a mixed-case name, which leans to the name, and a numeral behind an initial start nothing. A single letter still never starts the run. The numeral starting the run reports `suffix-or-name` where its shape alone reads it, as S10's numeral does anywhere it is weighed. The given part after a family comma starts its run the same way (`Smith, John VI Jones`). Separately, #819's reading of a lone letter ahead of a further comma stops the run short of that letter (decisions/C9's 2026-10-25 #819 entry). diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 88f344bb..b1bddae3 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -15,6 +15,7 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - Name readings are hand-declared, and only on a word some language file claims as vocabulary or refuses, so the vocabulary carries no surname list (the segmentation lists, Korean and the ZH pack's, are outside the per-word model) — 2026-10-15/16; 2026-10-19. - Bearer counts are structured provenance (source and count), never used to compute a reading — 2026-10-15/16. - Phase 2 is to put vocabulary, declared name and shape readings in one pool, derive `may_be_given` and `may_be_name` from declared name readings (a count-free one carrying a `note`, `ld` left ungated, `may_be_initial` kept), and decide every clash by one gate of four steps, floor, form, structure and count, asked at the sites that decide today; the reading not taken is to surface as `Ambiguity.alternative` in 2.x and the reading taken as `Token.reading` in 3.0 — 2026-10-23 #752. +- Every numeral rule asks one predicate, `_vocab.reads_as_generation`, the vocabulary's own numerals read inside it; a numeral of two or more letters is the generation wherever the trailing run holds it, while a one-letter one before a credential stays the name word decisions/S10's #721 entry made it — 2026-10-25 #752, phase 2f. - Phase 2's title batches go in order of lone-head breakage in PSC, `buddha`, `siddha` and `gyani` in the default gate and `guru` left to a `ne` pack — 2026-10-24 #752. - `Dean Smith` reads given `Dean`. The contested-head gate is to cover the words outside `titles` (`dean`, `justice`, `major`), and a title that is also a given name gets a declared given reading one word at a time, in measured batches — 2026-10-15/16; 2026-10-23 #752. - Order and shape are separate: a language supplies only a default order, and the shapes (where titles, suffixes, particles and commas sit) belong to each order and live in the core; vocabulary readings are to settle a clash and name readings only to keep the gate on — 2026-10-15/16. @@ -184,7 +185,13 @@ Open: #752 (phase 2, derived ambiguity; its sub-phases 2a to 2m, each its own PR - **2026-10-25 #752 — one predicate per question (neutral).** Three questions that #752's later changes widen had their answers copied between sites, and each now has a shared predicate in `_pipeline/_vocab.py` for the widening to land in (mechanisms.md#ONE-PREDICATE-PER-QUESTION). Two are whole: every site asks them. The numeral's is not yet: the bullet below names the sites found still recognizing a numeral on their own. No reading moves. - **`split_pair(first, second)`**, the split credential (`Ph.` `D.`, decisions/phd-merge): group's merge, `_comma._pieces` and `is_wholly_suffix` each held their own copy of the two-pattern test. The test runs per adjacent pair, so each caller asks `SPLIT_PAIR_OPENS`, the predicate's own first-word pattern, C-level before the call, and a name holding no `Ph.` pays no frame. The split-credential change widens both together (item 6 of the 2026-10-23 #752 entry above). - - **`reads_as_generation(text, preceding, one_case)`**, the numeral: `is_trailing_numeral_suffix` renamed, the predicate every numeral rule is to ask (item 8 of the 2026-10-23 #752 entry above, #784, #774, #819). Today it is the fork's SHAPE half alone. rules/C12's evidence asks a broader question beside its call: whether the word ending the part before the comma is any suffix word, by the suffix-word tag (`John Smith Jr., AD`, `John Smith Esq., AD`), or reads as a generation by the predicate (`John Smith VI, AD`); the tag stays at the call site because `Jr.` and `Esq.` are no numerals. The listed numerals belong in the predicate too, or each numeral rule tests the vocabulary itself again, the hand copy #656's first C12 draft was (it asked the suffix-word tag and missed `VI`). The shape test already takes the listed `ii` to `iv`, so reading the vocabulary inside it adds only the one-letter `i` and `v`, and that moves readings, which this PR may not: answered yes on a grid of 3,927 readings (17 heads, 11 numerals, 7 tails, three orders; the second review, 2026-10-25 on 30dc7aac), `i` and `v` moved 375 readings at the peel's fork, 348 of them in their fields (`John J. V` from middle 'J.', family 'V' to family 'J.', suffix 'V', with a new report), and 144 reports at `reads_as_a_reported_numeral` with no field moving (`John J. V MD` gains a `suffix-or-name`), the ones those sites meet carrying the `initial` tag (`I`, `V`, and their one-case spellings; a lower-case `v` in a name of mixed case carries none, and the peel takes it as a suffix word). Adding `ii` to `iv` alone moved nothing. That half is left to the change that has every numeral rule ask the predicate. Until it lands, these sites recognize a numeral on their own, each keeping the answer it had (found 2026-10-25 by two reviews and a grep of `_pipeline/` for the numeral pattern, the suffix-word tag and the suffix predicates; a list found by search, not a proof that no other site exists): the predicate's own missing `i` and `v`, above; rules/S8's run start asks the vocabulary alone (`John Smith III MD` → suffix 'III MD', `John Smith VI MD` → family 'VI', #784); rules/C1's listing form asks the suffix-word tag (`John Smith III, M.J.` keeps suffix 'III' where `John Smith VI, M.J.` and `John Smith V, M.J.` read family 'John Smith VI' and 'John Smith V', #774); the peel takes `ii` to `iv` through `is_suffix_piece` before its fork asks the predicate; rules/C9's walk over the given part after a family comma asks `is_suffix_lenient`, the vocabulary alone (`Smith, John V` → suffix 'V', `Smith, John VI` and `Smith, John X` → middle, where `John Smith VI` reads suffix 'VI'); and after a suffix word in that part a listed one-letter numeral is taken by its tags (`_pieces._numeral_behind_the_initial_veto`), so `Smith, John PhD V` reads suffix 'PhD V' with no report where `Smith, John PhD VI` reports `suffix-or-name`. + - **`reads_as_generation(text, preceding, one_case)`**, the numeral: `is_trailing_numeral_suffix` renamed, the predicate every numeral rule is to ask (item 8 of the 2026-10-23 #752 entry above, #784, #774, #819). Today it is the fork's SHAPE half alone. rules/C12's evidence asks a broader question beside its call: whether the word ending the part before the comma is any suffix word, by the suffix-word tag (`John Smith Jr., AD`, `John Smith Esq., AD`), or reads as a generation by the predicate (`John Smith VI, AD`); the tag stays at the call site because `Jr.` and `Esq.` are no numerals. The listed numerals belong in the predicate too, or each numeral rule tests the vocabulary itself again, the hand copy #656's first C12 draft was (it asked the suffix-word tag and missed `VI`). The shape test already takes the listed `ii` to `iv`, so reading the vocabulary inside it adds only the one-letter `i` and `v`, and that moves readings, which this PR may not: answered yes on a grid of 3,927 readings (17 heads, 11 numerals, 7 tails, three orders; the second review, 2026-10-25 on 30dc7aac), `i` and `v` moved 375 readings at the peel's fork, 348 of them in their fields (`John J. V` from middle 'J.', family 'V' to family 'J.', suffix 'V', with a new report), and 144 reports at `reads_as_a_reported_numeral` with no field moving (`John J. V MD` gains a `suffix-or-name`), the ones those sites meet carrying the `initial` tag (`I`, `V`, and their one-case spellings; a lower-case `v` in a name of mixed case carries none, and the peel takes it as a suffix word). Adding `ii` to `iv` alone moved nothing. That half is left to the change that has every numeral rule ask the predicate. Until it lands (it did: the 2026-10-25 #752, phase 2f entry below, which asks the predicate at every site listed here), these sites recognize a numeral on their own, each keeping the answer it had (found 2026-10-25 by two reviews and a grep of `_pipeline/` for the numeral pattern, the suffix-word tag and the suffix predicates; a list found by search, not a proof that no other site exists): the predicate's own missing `i` and `v`, above; rules/S8's run start asks the vocabulary alone (`John Smith III MD` → suffix 'III MD', `John Smith VI MD` → family 'VI', #784); rules/C1's listing form asks the suffix-word tag (`John Smith III, M.J.` keeps suffix 'III' where `John Smith VI, M.J.` and `John Smith V, M.J.` read family 'John Smith VI' and 'John Smith V', #774); the peel takes `ii` to `iv` through `is_suffix_piece` before its fork asks the predicate; rules/C9's walk over the given part after a family comma asks `is_suffix_lenient`, the vocabulary alone (`Smith, John V` → suffix 'V', `Smith, John VI` and `Smith, John X` → middle, where `John Smith VI` reads suffix 'VI'); and after a suffix word in that part a listed one-letter numeral is taken by its tags (`_pieces._numeral_behind_the_initial_veto`), so `Smith, John PhD V` reads suffix 'PhD V' with no report where `Smith, John PhD VI` reports `suffix-or-name`. - **`is_cased_paired_initials(text)`**, rules/S3's paired initials in a script with capitals: classify asked the script half inline (#872, decisions/S3). This is the lift the 2026-10-24 #872 entry above names, for classify and the split merge to share. One overlap is left to the split-credential PR to state: item 6 refuses a split credential whose first chunk is a single letter, so for a split `A. B.` that length test answers before this predicate is asked. rules/C5's comma reading keeps the bare `is_paired_initials`, since after a comma a caseless pair reads as a name too (`John Smith, พ.บ.` → given 'พ.บ.'), and moving C5 onto this predicate would change that reading. - **Measured neutral** against the branch, first on 632685c8 and again on 2dd0414b once phase 2a had merged: a grid of 20,364 texts (22 heads covering no comma, a whole name, one word and nothing before a comma, a nickname-only head, a maiden clause, a family comma with a title-particle head and a quoted clause before the pair; 33 spellings including `MA`, `M.D.`, `X.Y.Z.`, `XYZ`, `Ph. D.` and its case variants, `V`, `VI`, `III`, `XI`, `M.J.`, `K.C.` and `พ.บ.`; each text also in lower and upper case), under all three orders, read identically (fields, ambiguity kinds and details, token roles and tags, `initials()`, `capitalized()`), and the same harness reported 45,871, 18, 3,974 and 2,986 differing readings with `split_pair`, C12's call to the numeral predicate, the cased half and the peel's call to it each knocked out. The old and new predicates gave the same answer on 19,769 direct calls. The committed snapshot, every CLDR, GND, Wikidata and conventions agreement record, psc-individuals' agreement record and the ctgov-officials, fec-individuals and tse-donors local snapshots (each recorded from the base with `--tree`) all match, on both bases. psc-individuals' record was itself stale against 632685c8 (3 regressions, 5 progress, 5 changed before this change), so it was re-recorded from each base first. - **Frames**, counted with `sys.setprofile` on both trees: `_CALL_BASELINE`'s reference name and `Smith, John` cost what they did (305 and 158). A word matching `SPLIT_PAIR_OPENS` (`Ph`, `ph.`) with a word after it pays one frame at each site that tests it, whether or not a `D.` follows (`John Smith Ph. D.` 233 → 234, `Smith, John Ph. D.` 239 → 241; the review counted `ph ph ph ph ph ph` 282 → 286), so the cost grows only with the number of such words. C12's evidence keeps its tag test ahead of the call, so it costs what it did (`John Smith Jr., AD` 372). That is a rare shape that no guard counts. +- **2026-10-25 #752, phase 2f — EVERY NUMERAL RULE ASKS ONE PREDICATE.** Carries out item 8 of the 2026-10-23 #752 entry above (#784, #774, #819), and the list of sites in the 2026-10-25 one-predicate entry. `_vocab.reads_as_generation(text, tags, preceding, one_case)` answers for every spelling from `i` to `x` as S10's trailing fork does: the shape on the word without its trailing periods; a single letter written with its period an initial (rules/T5); a numeral the vocabulary lists in two or more letters (`ii` to `iv`, read from the token's `vocab:suffix` tag, not copied at a call site) the generation whatever stands around it; for the rest, an initial in front a name word, and a Title-case numeral in a name of mixed case declined, which `numeral_leans_to_the_name` tells apart so the decline is reported. `preceding` is None where a slot asks no initial in front, the given part after a family comma, where the predicate reads a one-letter numeral by the vocabulary. The count for one letter stays the caller's, which alone knows where the name's words are. + - **The sites.** The trailing peel asks it of a numeral anywhere in the run, behind the suffix test, so `John Smith VI MD` reads suffix 'VI MD' as `III MD` does (#784); rules/S8's run start asks it of a numeral of two or more letters (`John Smith VI Jones` → suffix 'VI Jones'); rules/C12's evidence asks it with the token's tags, its broader suffix-word test still beside the call; C1's listing form asks it of a numeral ending the family part, with S10's count over that part's pieces (`John Smith V, M.J.` and `John Smith VI, M.J.` keep the suffix, #774); the given part's walk asks it of a numeral of two or more letters (`Smith, John VI` → suffix 'VI') and of the bare one-letter numeral its last-piece test reads, and a one-letter numeral behind a suffix word in a part of suffix words asks it too; the maiden take and `tail_reading` reach it through the peel, `trailing_candidates` with no case (its superset); the report on a numeral a post-nominal follows asks it, with the Title-case decline. A numeral of two or more letters read by its shape alone reports `suffix-or-name` wherever it is taken, the run start and the given part included. + - **The three readings agreed on 2026-10-23.** #819: a lone letter ending the given part ahead of a further comma is the middle initial, the credential run in front stopping short of it (decisions/C9's 2026-10-25 #819 entry; `Smith, John PSM I, Jr.` the accepted cost). FEC's constraint: in that part a one-letter numeral is the generation only where listed, so `SMITH, JOHN X` keeps middle 'X'. A Title-case numeral in a mixed-case name leans to the name (`Nguyen Van Vi` → family 'Van Vi', reported); a numeral the vocabulary lists in two or more letters is not leaned (`John Smith Iii` keeps suffix 'Iii'), being unambiguous vocabulary as an unambiguous credential is. + - **One reading kept rather than widened: a one-letter numeral before a credential stays a name word** (decisions/S10's #721 entry, `John Smith V PhD` → family 'V', reported). Reading it with the count instead moved 6 tse-donors names, every one wrong: a lone `V` or `I` behind a surname or a particle before `JR` or `JUNIOR` (shape `Aa A Aa de V JUNIOR` read family 'de', suffix 'V JUNIOR'), where the letter abbreviates a surname; and 3 committed names (`John Smith V PhD`, `Jane Doe nee Smith V PhD`, `Jane Doe nee Smith V MA`). A split credential behind such a numeral (`John Smith V Ph. D.`) is left to the split-credential sub-phase (item 6), which must keep its suffix 'V Ph. D.' by its own rule. + - **Measured** (2026-10-25, against master 2af8da8e, each local baseline re-recorded from that checkout with `--tree` immediately before its check, since the shared cache's `snapshot.jsonl` is overwritten by any session measuring in parallel): the committed snapshot moves 6 names, every one intended: the three GND regnal rows (`Georg VI.`, `Konstantin VII.`, `Bonifatius VIII.`, family → suffix), `Smith VI, AD` (family 'Smith VI' → 'Smith', suffix 'VI'), and #774's `John Smith V, M.J.` and `John Smith VI, M.J.`. GND agreement goes from 618 to 621 of 801 on family and 779 to 782 on suffix, three progress rows each, and every other agreement record matches. ctgov-officials moves 1 reading (a job title ending in a level before a comma, `Aa Aa I, A` → suffix 'I', reported); fec-individuals 22, each a numeral ending the given part or the family part (`A, A VI` → suffix 'VI'; `A VI, A` → suffix 'VI'; `A A V, A` → suffix 'V'), about 14 read as generations and about 8 a middle name or a fragment read as one, all reported; tse-donors none. psc-individuals (1,039,171 records scored): 3 fixed, all Vietnamese names ending in the surname `Vi` that the Title-case lean keeps a name; 1 broken, a Title-case `Viii` written as the bearer's generation, which the lean reads as a name (the lean's cost, rules/S10); 1 changed, a record whose title box the scorer sets aside. Spike figures above (listing `vi` to `x`: 152 FEC moves) are the 2026-10-23 entry's and were not re-taken. + - **Recompute:** `uv run python tools/differential/snapshot.py` on the parent of this change's merge with its code, and for each local corpus `snapshot.py --local --record --tree ` then `snapshot.py --local ` from this tree, back to back; `uv run python tools/differential/local_agreement.py psc-individuals` the same way, `--list N` printing the moved records locally. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 41d9d0e0..a2634027 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -160,7 +160,7 @@ This catalog converts discovery into a one-time cost — it does not remove disc **How it works.** A hand-written mirror agrees with its original only until one of them moves, and the drift is invisible in both directions: each site keeps passing its own tests while they disagree about an input neither covers. Every instance below was found as a defect before it was found as a pattern. The destination follows the LAYER, not the topic: a predicate over token text goes to `_vocab`, one over pieces and tags to `_pieces`, both leaves the stages sit on. The piece layer got its own module only in #439; until then those predicates collected in `_group`, the one place both stages could reach, so the sharing between group and assign predates the module (decisions/parse-cost measured #439 as a pure relocation). The cost is a second evaluation of the same predicate, measured for #429 at 1.2–2.2% of a family-comma parse and 0% of every other — the right price there over plumbing a state field the two sites would not otherwise share. Folding two copies into one, take from each caller only what genuinely differs between them, and ask whether a parameter one caller had been filling with a guard is a question the other caller must answer too (#617, decisions/P3). The census of which stage calls which predicate, and how each instance was found, was kept here until #759 and is in git history (`git show fe85bb3d:docs/design/mechanisms.md`); the decisions entries carry each instance's argument. -**Lives in.** nameparser/_pipeline/_vocab.py over token text: `is_wholly_suffix` (#319, decisions/W2), `reads_as_generation` (#401/#421 as `is_trailing_numeral_suffix`, decisions/P5; since #752 the predicate every numeral rule is to ask, today its shape half alone: the listed one-letter `i` and `v`, S8's run start, C1's listing form, the peel's vocabulary path, C9's family-comma walk and the one-letter numeral behind a suffix word there still recognize a numeral on their own until #784, #774 and #819, decisions/vocabulary-model), `split_pair` (#752, the split credential `Ph.` `D.` that group's merge, `_comma._pieces` and `is_wholly_suffix` each tested with their own copy, each asking `SPLIT_PAIR_OPENS` C-level first since the test runs per adjacent pair, decisions/phd-merge), `is_cased_paired_initials` (#752, rules/S3's paired initials in a script with capitals, which classify asked inline; C5 asks the bare `is_paired_initials`, decisions/S3), `maiden_marker_run` (#434, decisions/M2), `delimiter_cores` (#436/#437, decisions/R1), `in_initialless_script` (#322/#323, decisions/cjk-full-stops), `caps_shape_candidate` (#516, decisions/S9). nameparser/_pipeline/_pieces.py over pieces and tags: `is_suffix_piece`, `leading_titles` and `is_leading_title` (#424), `is_title_piece`, `peel_walk`, `trailing_start`, `tail_reading` (the peel-and-chain fixed point that answers where the name pieces end, decisions/H5), `segment_suffix_reading` (#430, decisions/R1), `particle_tail` and `trailing_candidates` (#610, decisions/M2), `join_connectives` (#617, decisions/P3). nameparser/_pipeline/_post_rules.py: `suffix_entries` (#511, decisions/R1), the one instance in a stage rather than a leaf, being a pass over a whole ParseState. tests/v2/test_layering.py holds each module's contract, and a piece predicate growing a dependency on a STAGE shows up there as a widened entry. +**Lives in.** nameparser/_pipeline/_vocab.py over token text: `is_wholly_suffix` (#319, decisions/W2), `reads_as_generation` (#401/#421 as `is_trailing_numeral_suffix`, decisions/P5; since #752 the predicate every numeral rule asks, the vocabulary's own numerals read inside it, decisions/vocabulary-model), `split_pair` (#752, the split credential `Ph.` `D.` that group's merge, `_comma._pieces` and `is_wholly_suffix` each tested with their own copy, each asking `SPLIT_PAIR_OPENS` C-level first since the test runs per adjacent pair, decisions/phd-merge), `is_cased_paired_initials` (#752, rules/S3's paired initials in a script with capitals, which classify asked inline; C5 asks the bare `is_paired_initials`, decisions/S3), `maiden_marker_run` (#434, decisions/M2), `delimiter_cores` (#436/#437, decisions/R1), `in_initialless_script` (#322/#323, decisions/cjk-full-stops), `caps_shape_candidate` (#516, decisions/S9). nameparser/_pipeline/_pieces.py over pieces and tags: `is_suffix_piece`, `leading_titles` and `is_leading_title` (#424), `is_title_piece`, `peel_walk`, `trailing_start`, `tail_reading` (the peel-and-chain fixed point that answers where the name pieces end, decisions/H5), `segment_suffix_reading` (#430, decisions/R1), `particle_tail` and `trailing_candidates` (#610, decisions/M2), `join_connectives` (#617, decisions/P3). nameparser/_pipeline/_post_rules.py: `suffix_entries` (#511, decisions/R1), the one instance in a stage rather than a leaf, being a pass over a whole ParseState. tests/v2/test_layering.py holds each module's contract, and a piece predicate growing a dependency on a STAGE shows up there as a widened entry. **Reach for it when.** You are about to write a condition that mirrors, matches or "does what X does" — or you find a comment saying one does. Grep for the other site's predicate and call it instead. diff --git a/docs/design/rules/C1-suffix-comma-decision.md b/docs/design/rules/C1-suffix-comma-decision.md index 0fe42238..0bbca320 100644 --- a/docs/design/rules/C1-suffix-comma-decision.md +++ b/docs/design/rules/C1-suffix-comma-decision.md @@ -31,12 +31,14 @@ - **The count against written case** ("Smith, MA", "Smith Jr., Ma"): C4 counts the name words before the comma first and asks written case (S6) second, only where the count has left the word a name. - **Paired initials against the count** ("García Márquez, G.J."): C5 is the stated exception to C4's count, since two words before the comma may be one surname written in two. - **An unlisted all-caps word** ("John Smith, XYZ"): C6 decides whether it joins C4's class at all, only where the name carries a case contrast, and once it has joined, C4's count reads it. -- **A generation ending the name before the comma against paired initials after it** ("John Smith Jr., M.J."): written dotted, the initials keep C5's reading and are the given name, the generation staying the suffix; written undotted as two capitals, they are C12's credential, read as the credential they look like. +- **A generation ending the name before the comma against paired initials after it** ("John Smith Jr., M.J."): written dotted, the initials keep C5's reading and are the given name, the generation staying the suffix; written undotted as two capitals, they are C12's credential, read as the credential they look like. A numeral ending the family part is read as the comma-less name reads it (S10), its words to spare counted in that part, so the roman numerals S10 reads by their shape stay the suffix as the listed ones do. ``` "John Smith Jr., M.J." → given="M.J." "John Smith Jr., MJ" → suffix="Jr., MJ" - "John Smith V, M.J." → suffix="V" deviates: #774 (today: suffix="") - "John Smith VI, M.J." → suffix="VI" deviates: #774 (today: suffix="") + "John Smith V, M.J." → suffix="V" + "John Smith VI, M.J." → suffix="VI" + "Smith VI, John" → suffix="VI" + "Smith V, John" → family="Smith V" ``` **Accepted:** diff --git a/docs/design/rules/C12-suffix-word-ending-the-name-before-the-comma.md b/docs/design/rules/C12-suffix-word-ending-the-name-before-the-comma.md index 0d557de1..785d9aee 100644 --- a/docs/design/rules/C12-suffix-word-ending-the-name-before-the-comma.md +++ b/docs/design/rules/C12-suffix-word-ending-the-name-before-the-comma.md @@ -51,7 +51,7 @@ **Precedence:** -- **Paired initials after a generation** ("John Smith Jr., M.J."): written dotted, they keep C5's reading and are the given name, the generation staying the suffix; undotted as two capitals, they are this rule's credential (C1, where #774 tracks the roman numerals S10 reads, which do not yet stay the suffix). +- **Paired initials after a generation** ("John Smith Jr., M.J."): written dotted, they keep C5's reading and are the given name, the generation staying the suffix; undotted as two capitals, they are this rule's credential (C1, where the roman numerals S10 reads stay the suffix as the listed ones do). **Accepted:** diff --git a/docs/design/rules/C5-paired-initials-after-the-comma.md b/docs/design/rules/C5-paired-initials-after-the-comma.md index d308ecb8..62d14079 100644 --- a/docs/design/rules/C5-paired-initials-after-the-comma.md +++ b/docs/design/rules/C5-paired-initials-after-the-comma.md @@ -38,7 +38,7 @@ **Precedence:** -- **A generation ending the name before the comma** ("John Smith Jr., M.J."): dotted, the paired initials keep this rule's reading and are the given name, the generation staying the suffix; undotted as two capitals, they are the credential C12 reads (C1, where #774 tracks the roman numerals S10 reads, which do not yet stay the suffix). +- **A generation ending the name before the comma** ("John Smith Jr., M.J."): dotted, the paired initials keep this rule's reading and are the given name, the generation staying the suffix; undotted as two capitals, they are the credential C12 reads (C1, where a roman numeral S10 reads stays the suffix as a listed generation does). **Accepted:** diff --git a/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md b/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md index a887cb1d..01779a51 100644 --- a/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md +++ b/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md @@ -4,9 +4,15 @@ **Then:** it is read by what stands before it: behind another suffix it is describing that suffix and continues the run, written with a period or without, an initial being no shape anyone writes there, and behind a name word it is the generation only when written bare, a period marking it the abbreviation of a name and so a middle initial (T5). -- **Both branches then ask that the generation slot still be open**: where a further comma has already named the suffix (C2), a single letter ending the GIVEN part has no generation left to be, and stays a middle initial — with a period or without, and whether a name word or another suffix word stands before it. +- **Both branches then ask that the generation slot still be open**: where a further comma has already named the suffix (C2), a single letter ending the GIVEN part has no generation left to be, and stays a middle initial — with a period or without, and whether a name word or another suffix word stands before it. So a lone letter ending the given part ahead of a further comma is the middle initial, whatever stands before it, a credential run in front of it stopping short of it. + ``` + "Smith, John V, Jr." → middle="V" + "Smith, John PhD I, Jr." → middle="I" + "Smith, John PhD I., Jr." → middle="I." + "Smith, John PhD I, Jr." → suffix="PhD, Jr." + ``` - **Only the given part is touched**: a part after the comma that is nothing but suffix words is the credential run, and a letter in it continues that run up to the further comma. -- **Longer suffix words are not in question either way.** +- **Longer suffix words are not in question either way**, a numeral of two or more letters being read as S10 reads one in this part. **Because:** one letter is the shape a middle initial is written in, so only what stands before it says whether it is a generation. @@ -24,7 +30,10 @@ **Accepted:** -- The further-comma qualifier carries no example line of its own. It discriminates PAIRS and spans both branches, so exemplifying it means a with-comma partner for each — every one of them a promise about behavior that has not moved since v1. Its executable witness is instead the pair already standing in the v1-style bank, tests/test_suffixes.py's test_roman_numeral_i_after_single_initial_lastname_comma_format and test_roman_numeral_i_with_explicit_suffix_comma_stays_a_middle_initial. Read the rule's examples with the qualifier in hand: `Smith, John V` reads the suffix and `Smith, John PhD I.` continues the run, while adding a suffix comma after either turns that same letter into the middle initial. +- A letter that is a level of the credential in front of it, written ahead of a further comma, is read as the middle initial, the further comma having named the suffix: the certification level is split from its credential. No corpus holds the shape (#819, measured 2026-10-25). + ``` + "Smith, John PSM I, Jr." → middle="I" + ``` --- **Signal:** neighbor: what stands before a one-character suffix word inside a comma part diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index 9e95f83d..c8861659 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -1,26 +1,45 @@ # S10 — the trailing roman numeral -**When:** a name written without a comma ends in a roman numeral, as in "John Smith III". +**When:** a roman numeral stands in a name's trailing run — ending the name, or with other suffix words behind it — as in "John Smith III" or "John Smith VI MD". **Then:** the numeral reads as the generation: the numerals the vocabulary lists, and those up to X it does not, by their shape alone. +- **Wherever it stands in the run, in two or more letters.** A numeral of two or more letters with a credential or another suffix word behind it reads as one ending the name does, so every such spelling joins the run as the listed ones always have. + ``` + "John Smith VI MD" → suffix="VI MD" + "John Smith III MD" → suffix="III MD" + "Jane Doe nee Smith VI PhD" → suffix="VI PhD" + ``` +- **A numeral of two or more letters keeps its reading written with a period**, as the German ordinal writes one, the listed numerals reading so too. + ``` + "Georg VI." → suffix="VI." + "Konstantin VII." → suffix="VII." + ``` - **What reports.** A numeral of one letter reports whichever way the words-to-spare count reads it: either reading is reported, as either reading of an ambiguous acronym is. A numeral read by its shape alone reports too; a numeral the vocabulary lists in two or more letters is not in question and reports nothing. ``` "John Smith V" → ambiguities=("suffix-or-name",) "John V" → ambiguities=("suffix-or-name",) "John Smith VI" → ambiguities=("suffix-or-name",) + "John Smith VI MD" → ambiguities=("suffix-or-name",) "John Smith III" → ambiguities=() ``` -- **Written case settles the fork in one way only:** a numeral written in lower case in a name of mixed case is the generation and reports nothing, being no initial there (T4). +- **Written case settles the fork one way:** a numeral written in lower case in a name of mixed case is the generation and reports nothing, being no initial there (T4). ``` "John Smith vi" → ambiguities=() ``` +- **And leans it the other:** a Title-case numeral in a name of mixed case leans to the name, as a Title-case member of the ambiguous class does (S6), and reports, the fork having weighed it. + ``` + "Nguyen Van Vi" → family="Van Vi" + "Nguyen Van Vi" → ambiguities=("suffix-or-name",) + "John Smith Vi" → family="Vi" + ``` - **A name written wholly in one case says nothing about any letter in it** (T4). ``` "JOHN V" → family="V" "john v" → family="v" "john smith v" → ambiguities=("suffix-or-name",) "john smith i" → ambiguities=("suffix-or-name",) + "JOHN SMITH VI" → suffix="VI" ``` - **A marked letter (P8) that is such a numeral reports this way and not as a connective.** ``` @@ -29,7 +48,7 @@ "john i dr." → ambiguities=("suffix-or-name",) ``` -**Because:** a roman numeral ending a name is its generation. But one letter is also how an initial is written. A name of two words has no word to spare for it. +**Because:** a roman numeral ending a name is its generation, and the post-nominals written after the generation do not change what it is. But one letter is also how an initial is written. A name of two words has no word to spare for it. ``` "John Smith V" → suffix="V" @@ -49,9 +68,11 @@ "Dr Smith V" → family="V" "John V, Dr." → family="V" ``` -- **An initial in front makes it a name word too.** When the word in front of the numeral is written as an initial, the numeral is a name word; in a name written wholly in one case a bare letter in front is written as one (T4). +- **An initial in front makes it a name word too.** When the word in front of the numeral is written as an initial, the numeral is a name word, unless the vocabulary lists it in two or more letters; in a name written wholly in one case a bare letter in front is written as one (T4). ``` "John J. V" → family="V" · boundary + "John J. VI" → family="VI" · boundary + "John J. III" → suffix="III" "john j v" → family="v" ``` - **A single letter written with a period is an initial** (T5). @@ -61,6 +82,7 @@ - **A numeral past X is a name word.** ``` "John Smith XI" → family="XI" · boundary + "John Smith XI MD" → family="XI" · boundary ``` **Precedence:** @@ -70,11 +92,20 @@ "john smith v, PhD" → ambiguities=() "john j v, PhD" → family="j" ``` -- **In the given part after a family comma** ("Smith, John V"): the given part is not counted, the comma having settled the count there (S5). +- **Before a family comma** ("John Smith V, M.J."): a numeral ending the family part is read by this rule, its words to spare counted in that part (C1). +- **In the given part after a family comma** ("Smith, John VI"): the given part is not counted, the comma having settled the count there (S5), and no initial in front is asked. A numeral of two or more letters reads as the generation, reporting where its shape alone reads it; a one-letter numeral is the generation only where the vocabulary lists it and it ends the part (C9), so an unlisted one stays the middle initial. + ``` + "Smith, John VI" → suffix="VI" + "Smith, John VI MD" → suffix="VI MD" + "Smith, John V" → suffix="V" + "Smith, John X" → middle="X" + "Smith, John V MD" → middle="V" + "Smith, John Vi" → middle="Vi" + ``` **Accepted:** -- A numeral with another suffix word behind it is not the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name, and the reading is reported, as either reading of the fork is. +- A numeral of one letter with another suffix word behind it is not the last word of the name and is read as any word there is, so a one-letter numeral before a credential is the family name, and the reading is reported, as either reading of the fork is. A lone letter there abbreviates a surname as often as it numbers a generation. ``` "John Smith V PhD" → family="V" "John Smith V PhD" → ambiguities=("suffix-or-name",) @@ -85,6 +116,14 @@ ``` "John Smith V Ph. D." → suffix="V Ph. D." ``` +- A generation written in Title case in a mixed-case name reads as a name word, the lean that keeps a surname such as 'Vi' a name taking it too. + ``` + "John Smith Viii" → family="Viii" + ``` +- A numeral the vocabulary lists in two or more letters is not weighed: an initial in front does not make it a name word, nor does Title case lean it to the name, the vocabulary settling it as it settles any unambiguous suffix word. No one writes 'III' as an initial or bears 'Ii' as a name. + ``` + "John Smith Iii" → suffix="Iii" + ``` --- **Signal:** shape diff --git a/docs/design/rules/S8-credentials-trailing-run.md b/docs/design/rules/S8-credentials-trailing-run.md index a3c31211..a3f89634 100644 --- a/docs/design/rules/S8-credentials-trailing-run.md +++ b/docs/design/rules/S8-credentials-trailing-run.md @@ -4,8 +4,10 @@ **Then:** a credential after the name core starts a run to the end of its part (#602): every later word of that part reads as a suffix, except a title word, which reads as a title. -- **Where the run starts.** An unambiguous suffix word of two or more letters, in any script, standing after two name words where no comma divides the name or before a suffix comma, or after the given word in the part after a family comma, makes every later word of that part a suffix. +- **Where the run starts.** An unambiguous suffix word of two or more letters, in any script, standing after two name words where no comma divides the name or before a suffix comma, or after the given word in the part after a family comma, makes every later word of that part a suffix. So does a roman numeral of two or more letters that reads as the generation (S10), whether the vocabulary lists it or its shape alone reads it. ``` + "John Smith VI Jones" → suffix="VI Jones" + "Smith, John VI Jones" → suffix="VI Jones" "Smith, John PhD Jones" → suffix="PhD Jones" "John Smith PhD Prof. Ma" → suffix="PhD Ma" "Eric H. Holder Jr. Attorney General" → title="Attorney General" diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index f510365f..cee93521 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -108,7 +108,7 @@ A clash not settled at step 2 is reported at the decision site (mechanisms.md#AM A reading whose writing offers no cue is one the gate could only guess, so it is `default = false`: recorded in the TOML, compiled to nothing by default, turned on by a pack in phase 3. Spanish `H.`, German regnal `I.` and Czech `st.` are the first. -**The numerals ask one predicate.** `i` to `v` are vocabulary and `vi` to `x` are shape, and today some rules ask only the vocabulary, so `John Smith III MD` and `John Smith VI MD` read differently (#784, #774). One predicate answers for every spelling the way the trailing numeral rule does (shape, the count for one letter, an initial in front, the period, the case), and every rule that asks either recognizer asks it. Listing `vi` to `x` instead was measured and declined: a middle initial `X` would read as a generation. +**The numerals ask one predicate.** `i` to `v` are vocabulary and `vi` to `x` are shape, and until 2f some rules asked only the vocabulary, so `John Smith III MD` and `John Smith VI MD` read differently (#784, #774). One predicate answers for every spelling the way the trailing numeral rule does (shape, the count for one letter, an initial in front, the period, the case), and every rule that asks either recognizer asks it. Listing `vi` to `x` instead was measured and declined: a middle initial `X` would read as a generation. **`fils` and the agnomens show what kind-specific cues buy.** `fils` is gated by its case and the count, so `Alexandre Dumas fils` is to read suffix `fils` while `Marie Fils` keeps her family name, reported. The Brazilian agnomens are to be gated by the count alone: case would say nothing, and the plain generation reading breaks real surnames (`Ana Neto`) and given names. diff --git a/docs/release_log.rst b/docs/release_log.rst index 2aad3e20..3e1eb3d6 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -4,6 +4,11 @@ Release Log nameparser 2.7 is under development. + **Behavior Changes** + + - **Fix roman numerals VI to X reading as names where III reads as a suffix.** ``John Smith VI MD`` gives suffix ``VI MD`` and ``Smith, John VI`` suffix ``VI``, as ``III`` does, where 1.4.0 through 2.6.0 gave last ``VI``, suffix ``MD`` and middle ``VI``. ``John Smith V, M.J.`` keeps suffix ``V`` and ``Georg VI.`` gives suffix ``VI.``, where 1.4.0 through 2.6.0 gave last ``John Smith V`` and ``VI.``. A one-letter numeral before a credential is still a name (``John Smith V PhD`` keeps last ``V``), and a Title-case ``Vi`` in a mixed-case name reads as a name (``Nguyen Van Vi`` gives last ``Van Vi``, where 2.2.0 through 2.6.0 gave suffix ``Vi``). :issue:`784`, :issue:`774` + - **Fix a lone letter before a further comma reading as part of a credential.** ``Smith, John PhD I, Jr.`` gives middle ``I``, suffix ``PhD, Jr.``, as 1.4.0 through 2.3.0 did, where 2.4.0 through 2.6.0 gave suffix ``PhD I, Jr.``. :issue:`819` + * 2.6.0 - October 25, 2026 nameparser 2.6 ships the default vocabulary as per-word data files, adds an opt-in reading for names with two family names, and fixes words read as a credential or a title that are part of the name. Highlights: diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 91fb4cdc..19d79240 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -89,8 +89,10 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - _ROMAN, effective_script, is_suffix_lenient, - reads_as_a_reported_numeral, resolve_script_set, + _ROMAN, effective_script, is_initial_shaped, is_listed_numeral, + is_suffix_lenient, + numeral_leans_to_the_name, numeral_settled_by_case, + reads_as_a_reported_numeral, reads_as_generation, resolve_script_set, ) from nameparser._pipeline._pieces import ( anchor_in_reach, credential_at_the_given_slot, given_slot_anchors, @@ -303,7 +305,7 @@ def _placed(read: TailRead, rest: list[int], names.extend(suffixes) # built, not `_replace`d: that is two frames on every read return names, tuple(titled), Peel( - count, peel.numeral, peel.picks, peel.anchors, tuple(run_titles), + count, peel.numerals, peel.picks, peel.anchors, tuple(run_titles), peel.absorbed) @@ -407,16 +409,20 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], Role.TITLE) for piece in peeled.absorbed: ambiguities.append(_absorbed(piece, tokens)) - if peeled.numeral is not None: - # a trailing single letter is a name part unless it happens - # to be a roman numeral -- and V/X/I are ordinary middle - # initials, so taking it as a suffix is a call, not a fact + for numeral in peeled.numerals: + # a single letter is a name part unless it happens to be a + # roman numeral -- and V/X/I are ordinary middle initials, so + # taking it as a suffix is a call, not a fact; a numeral read + # by its shape alone is in no wordlist, and may be a name word + text = tokens[numeral[0]].text ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, - f"{tokens[peeled.numeral[0]].text!r} is a roman numeral, so " - f"it reads as a generational suffix; any other single " - f"letter there would be a middle initial", - peeled.numeral)) + f"{text!r} is a roman numeral, so it reads as a " + f"generational suffix; " + + ("any other single letter there would be a middle initial" + if len(text.rstrip(".")) == 1 + else "it may be a name word"), + numeral)) name_pieces, suffix_pieces = rest[:peeled.names], rest[peeled.names:] if peeled.run_titles: suffix_pieces = [q for q in suffix_pieces @@ -537,11 +543,12 @@ def _assign_main(seg_idx: int, state: ParseState, # `_ROMAN` first, C-level, so an ordinary name word pays no frame if (len(name_pieces) >= 2 and (suffix_pieces or titled_tail) and len(last := pieces[name_pieces[-1]]) == 1 - and _ROMAN.match(word := tokens[last[0]].text) is not None + and _ROMAN.match((word := tokens[last[0]].text).rstrip(".")) + is not None and tuple(last) not in peeled.picks and reads_as_a_reported_numeral( - word, tokens[pieces[name_pieces[-2]][0]].text, - state.one_case)): + word, tokens[last[0]].tags, + tokens[pieces[name_pieces[-2]][0]].text, state.one_case)): ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, f"{word!r} is a roman numeral, but a post-nominal or title " @@ -596,11 +603,32 @@ def _assign_main(seg_idx: int, state: ParseState, return order +def _family_part_numeral(word: WorkToken, preceding: str, names: int, + one_case: bool | None) -> Role | None: + """The role of a roman numeral ending the family part before a + family comma, behind `names` pieces of that part, as the comma-less + name reads one ending it (rules/S10), or None where the numeral's + predicate reads no numeral there (an initial in front, a letter with + its period): the suffix where it reads as the generation, a + one-letter numeral needing two name words in front of it, and the + family name where that count fails or its Title case leans to the + name.""" + if reads_as_generation(word.text, word.tags, preceding, one_case): + if (names < 2 and len(word.text) == 1 + and not numeral_settled_by_case(word.text, one_case)): + return Role.FAMILY + return Role.SUFFIX + if numeral_leans_to_the_name(word.text, word.tags, preceding, one_case): + return Role.FAMILY + return None + + def _reads_as_a_trailing_suffix(piece: Sequence[int], prev_piece: Sequence[int], prev_ptags: Set[str], tokens: Sequence[WorkToken], - lexicon: Lexicon) -> bool: + lexicon: Lexicon, + one_case: bool | None) -> bool: """The lenient tail test for a trailing one-token piece after a family comma, and #432's carve-out from it. @@ -626,10 +654,16 @@ def _reads_as_a_trailing_suffix(piece: Sequence[int], # rules/C9: "behind a name word it is the generation only when # written bare, a period marking it the abbreviation of a name and # so a middle initial" - text = tokens[piece[0]].text + tok = tokens[piece[0]] + text = tok.text if text.endswith(".") and not is_suffix_piece( prev_piece, prev_ptags, tokens): return False + # the numeral's predicate, which in this slot reads a bare + # one-letter numeral by the vocabulary ('Smith, John X' keeps + # middle 'X'); `_ROMAN` first, C-level + if len(text) == 1 and _ROMAN.match(text) is not None: + return reads_as_generation(text, tok.tags, None, one_case) return is_suffix_lenient(text, lexicon) @@ -1115,6 +1149,16 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: """ if is_suffix_piece(pieces[m], ptags[m], tokens): return True + # rules/S10: "A numeral of two or more letters reads as + # the generation" in the given part, which asks no + # initial in front -- 'Smith, John VI' as 'Smith, John + # III' (#752); `_ROMAN` first, C-level + if (len(pieces[m]) == 1 + and len(stem := (tok := tokens[pieces[m][0]]).text + .rstrip(".")) >= 2 + and _ROMAN.match(stem) is not None): + return reads_as_generation(tok.text, tok.tags, None, + state.one_case) # rules/S5: "a word that ENDS that part reads as the credential # unless its WRITING says otherwise" -- the given part's # trailing slot (#531). @@ -1209,7 +1253,7 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: and len(pieces[m]) == 1 and _reads_as_a_trailing_suffix( pieces[m], pieces[prev], ptags[prev], - tokens, state.lexicon)) + tokens, state.lexicon, state.one_case)) # rules/C1: "anything else after the comma means the # listing form" -- and this walk reads it. A segment of no @@ -1302,10 +1346,13 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: tags = tokens[pieces[m][0]].tags if (m not in titled_idx and ("suffix" in ptags[m] - or ("vocab:suffix" in tags + or (("vocab:suffix" in tags + or _ROMAN.match(tokens[pieces[m][0]].text + .rstrip(".")) is not None) and tags.isdisjoint(_NOT_A_RUN_START))) and starts_a_credential_run(pieces[m], ptags[m], - tokens)): + tokens, None, + state.one_case)): sticky_from = m break # rules/P6: "a particle ending the name attaches to that @@ -1337,6 +1384,34 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: continue suffix_here = (reads_as_a_suffix(m, titled_idx) if titled_idx else m not in walkable) + # rules/S10: "A numeral read by its shape alone reports + # too" -- one of two or more letters the walk reads as + # the generation; `_ROMAN` first, C-level + if (suffix_here and len(pieces[m]) == 1 + and len(stem := (tok := tokens[pieces[m][0]]).text + .rstrip(".")) >= 2 + and _ROMAN.match(stem) is not None + and not is_listed_numeral(tok.text, tok.tags) + and not numeral_settled_by_case(tok.text, + state.one_case)): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{tok.text!r} is a roman numeral, so it reads as " + f"a generational suffix; it may be a name word", + (pieces[m][0],))) + # rules/C9: "a lone letter ending the given part ahead of + # a further comma is the middle initial, whatever stands + # before it" -- the further comma has named the suffix, + # so the credential run in front stops short of it + # (#819); `len` first, C-level + if (m > sticky_from and len(state.segments) > 2 + and len(pieces[m]) == 1 + and len(tokens[pieces[m][0]].text.rstrip(".")) == 1 + and m == previous_kept(len(pieces), titled_idx) + and is_initial_shaped( + tokens[pieces[m][0]].text.upper())): + _set_roles(tokens, pieces[m], Role.MIDDLE) + continue if m >= sticky_from and not k6 <= m < end6: # inside the run: a title word reads as a title, # every other word as a suffix, and a word the walk @@ -1512,6 +1587,34 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: piece = fam_pieces[k] if k >= floor: _set_roles(tokens, piece, Role.SUFFIX) + # rules/C1: "a numeral ending the family part is read as the + # comma-less name reads it" -- the numeral's predicate, a + # one-letter numeral with two name words in front of it + # (rules/S10), so 'John Smith V, M.J.' keeps suffix 'V' as + # 'John Smith III, M.J.' keeps 'III' (#774). `_ROMAN` + # first, C-level, so a family word pays no frame + elif (k > lead and len(piece) == 1 + and _ROMAN.match( + (word := tokens[piece[0]]).text.rstrip(".")) + is not None + and all(is_suffix_piece(fam_pieces[j], fam_tags[j], + tokens) + for j in range(k + 1, len(fam_pieces))) + and (numeral := _family_part_numeral( + word, tokens[fam_pieces[k - 1][0]].text, + k - lead, state.one_case)) is not None): + _set_roles(tokens, piece, numeral) + # reported where taken: a name word here is the family + # name the comma has just named, which nobody misreads + if (numeral is Role.SUFFIX + and not is_listed_numeral(word.text, word.tags) + and not numeral_settled_by_case(word.text, + state.one_case)): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{word.text!r} ending the family part is a roman " + f"numeral, so it reads as a generational suffix; " + f"it may be a name word", tuple(piece))) elif (k > lead and is_suffix_piece(piece, fam_tags[k], tokens) and not (k + 1 < len(fam_pieces) and "particle" in tokens[piece[0]].tags diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index 681a9eaa..081ba98f 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -238,7 +238,8 @@ def _behind_a_generation(state: ParseState, seg0: Sequence[int], # any suffix word ('Jr.', 'Esq.') counts, by its tag if ("vocab:suffix-word" not in tokens[seg0[-1]].tags and not reads_as_generation( - last, tokens[seg0[-2]].text, state.one_case)): + last, tokens[seg0[-1]].tags, tokens[seg0[-2]].text, + state.one_case)): return None # the generation set aside, the name's own words in front of it are # a whole name: 'Smith VI' is one word and a numeral, as 'Smith III' @@ -373,7 +374,7 @@ def decide(state: ParseState) -> ParseState: # the helper) generation = bool(seg0) and ( "vocab:suffix-word" in tokens[seg0[-1]].tags - or ROMAN.match(tokens[seg0[-1]].text) is not None) + or ROMAN.match(tokens[seg0[-1]].text.rstrip(".")) is not None) behind_generation = listing = False contrast: bool | None = None if not maybe: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 4d5a553a..82096f11 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -333,8 +333,7 @@ def _maiden_take(pieces: Sequence[Sequence[int]], # on it are reported here, where assign will never see them consumed = {i for j in range(end, len(pieces)) for i in pieces[j]} for piece in (*vpeel.picks, *vpeel.absorbed, - *(() if vpeel.numeral is None - else (vpeel.numeral,))): + *vpeel.numerals): if piece[0] in consumed: ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, @@ -362,7 +361,8 @@ def _maiden_take(pieces: Sequence[Sequence[int]], f"a generational suffix", tuple(last))) elif (released and len(last) == 1 and end - 2 >= lo and reads_as_a_reported_numeral( - word.text, tokens[pieces[end - 2][0]].text, one_case)): + word.text, word.tags, tokens[pieces[end - 2][0]].text, + one_case)): # rules/S10: "a one-letter numeral before a credential is the # family name, and the reading is reported" -- the clause's # twin of assign's report (#721's review): the clause keeps the @@ -1041,7 +1041,7 @@ def chain(tail: int) -> None: # built, not `_replace`d: two frames a call peel = read.peel read = TailRead(read.tail, read.titles, read.run_titles, Peel( - peel.names, peel.numeral, + peel.names, peel.numerals, tuple([p for p in peel.picks if p not in took]), peel.anchors, peel.run_titles, peel.absorbed)) # rules/P5: "a recognized bound given-name word joins the diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index aa0f49da..8c2b427c 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -63,8 +63,8 @@ ) from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, _ROMAN, Lean, ambiguous_lean, in_initialless_script, - is_single_letter_numeral, numeral_settled_by_case, - reads_as_generation, tag_marker_runs, + is_listed_numeral, is_single_letter_numeral, numeral_leans_to_the_name, + numeral_settled_by_case, reads_as_generation, tag_marker_runs, ) @@ -467,7 +467,9 @@ def is_suffix_piece(piece: Sequence[int], ptags: Set[str], # starts #602's credential run, one predicate for the trailing peel and # the given-part walk (mechanisms.md#ONE-PREDICATE-PER-QUESTION) def starts_a_credential_run(piece: Sequence[int], ptags: Set[str], - tokens: Sequence[WorkToken]) -> bool: + tokens: Sequence[WorkToken], + preceding: str | None = None, + one_case: bool | None = None) -> bool: """Whether one piece starts #602's run: an unambiguous suffix word of two or more letters, in any script -- a credential acronym, a generational word or an honorific -- that no other vocabulary @@ -492,17 +494,30 @@ def starts_a_credential_run(piece: Sequence[int], ptags: Set[str], asks only of the pieces `peel_walk` keeps, and the walk drops the merged piece, so 'John Smith Ph. D. Jones' still reads family 'Jones'. + + A roman numeral of two or more letters starts it where + `_vocab.reads_as_generation` reads it as the generation, with + `preceding` and `one_case` that predicate's (#784): 'John Smith VI + Jones' as 'John Smith III Jones'. """ if "suffix" in ptags: # the Ph./D. pair alone: a connective join keeps the tag on the # wider piece ('Ph. D. and Mary'), which starts nothing, as # 'PhD and Mary' does not return len(piece) == 2 - if len(piece) != 1 or not is_suffix_piece(piece, ptags, tokens): + if len(piece) != 1: return False tok = tokens[piece[0]] if not tok.tags.isdisjoint(_NOT_A_RUN_START): return False + # rules/S8: "a roman numeral of two or more letters that reads as + # the generation" -- asked of the numeral's predicate, which the + # vocabulary's own numerals answer too + stem = tok.text.rstrip(".") + if len(stem) >= 2 and _ROMAN.match(stem) is not None: + return reads_as_generation(tok.text, tok.tags, preceding, one_case) + if not is_suffix_piece(piece, ptags, tokens): + return False # C-level, no generator frame per character letters = "".join(filter(str.isalpha, tok.text.lower())) return len(letters) >= 2 @@ -551,8 +566,14 @@ def _numeral_behind_the_initial_veto(piece: Sequence[int], """ if len(piece) != 1: return False - tags = tokens[piece[0]].tags - return "vocab:suffix" in tags and "initial" in tags + tok = tokens[piece[0]] + if "vocab:suffix" not in tok.tags or "initial" not in tok.tags: + return False + # a one-letter numeral is the numeral's predicate's to answer, which + # in a comma part reads it by the vocabulary, as here (#752) + if _ROMAN.match(stem := tok.text.rstrip(".")) is not None and stem: + return reads_as_generation(stem, tok.tags, None, None) + return True def _anchors(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: @@ -925,10 +946,12 @@ class Peel(NamedTuple): of positions in the caller's `rest`: rest[:names] are the name pieces and rest[names:] the suffixes. The other two are pieces -- token-index tuples, as PendingAmbiguity wants them -- and each is - one token long: `numeral` is the piece the roman-numeral fork - took as a call, which is reported (None when it did not fire, and - for a numeral its lower case settled, which is taken unrecorded; - always the walk's last piece), + one token long: `numerals` are the pieces the roman-numeral fork + took as a call, each reported, in peel order (none for a numeral + its lower case settled or the vocabulary lists in two or more + letters, which is taken unrecorded; a one-letter one is always the + walk's last piece, one of two or more letters anywhere in its + trailing run, #752), `picks` the bare ambiguous acronyms the peel had to resolve, in peel order, either way (the last may sit at rest[names - 1]), and a one-letter numeral the fork declined for want of words to spare @@ -940,7 +963,7 @@ class Peel(NamedTuple): `tail_reading` returns carries None.""" names: int - numeral: tuple[int, ...] | None + numerals: tuple[tuple[int, ...], ...] picks: tuple[tuple[int, ...], ...] anchors: list[bool] | None #: #602 (`credential_run`): the pieces in rest[names:] that are @@ -997,7 +1020,7 @@ def trailing_start(start: int, pieces: Sequence[Sequence[int]], rest = peel_walk(start, ptags) names = run_start(rest, peel_trailing(rest, pieces, ptags, tokens, one_case).names, - pieces, ptags, tokens) + pieces, ptags, tokens, one_case=one_case) return rest[names] if names < len(rest) else len(pieces) @@ -1123,8 +1146,9 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], `start` on were already peeled, with `anchors` the pass an earlier walk built (or None). Only `tail_reading` passes them, and its docstring states when that answers what a fresh walk would. The - returned picks are the resumed walk's own; the numeral is never - read, `start` standing short of the walk's last piece. + returned picks and numerals are the resumed walk's own; a + one-letter numeral is never read, `start` standing short of the + walk's last piece. `second` is where the second name unit opens in `rest` (`second_unit`), the caller's to find once; positions are units @@ -1141,9 +1165,11 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], `count_numeral` False takes a one-letter numeral without asking for words to spare: the given part after a family comma is read without the count, and its chain bound must not ask it (#718). + Every numeral is weighed by `_vocab.reads_as_generation`, the one + predicate every numeral rule asks (#752). """ picks: list[tuple[int, ...]] = [] - numeral: tuple[int, ...] | None = None + numerals: list[tuple[int, ...]] = [] # where the second name unit opens in `rest`, found once a member # asks; positions are units where `units` is None k = len(rest) if start is None else start @@ -1156,32 +1182,48 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], if is_suffix_piece(piece, ptags[rest[k - 1]], tokens): k -= 1 continue - # a final single token that is a roman numeral, after a piece - # that is not initial-shaped; the predicate's docstring carries - # the is_initial_shaped reasoning (#320). rules/S10: "A numeral - # of one letter needs words to spare, as a bare ambiguous - # acronym does" -- the acronym fork's own count, - # below, so 'John V' keeps family 'V' (#718). Declined, it is a - # pick, since "either reading is reported", and no branch below - # can take a numeral - if (k == len(rest) and k >= 2 and len(piece) == 1 - and reads_as_generation( - text := tokens[piece[0]].text, - tokens[pieces[rest[k - 2]][0]].text, one_case)): - # rules/S10: "a numeral written in lower case in a name of - # mixed case is the generation and reports nothing" -- - # decided here once: taken as any suffix word is, and not - # recorded as the fork's call, so no reporter meets it - if numeral_settled_by_case(text, one_case): - k -= 1 - continue - if (not is_single_letter_numeral(text) or not count_numeral - or k - 2 >= second): - numeral = tuple(piece) - k -= 1 - continue - picks.append(tuple(piece)) - break + # rules/S10: "the numeral reads as the generation" -- every + # numeral the suffix test above refused, the listed ones it took + # being the predicate's yes too. One of two or more letters is + # read anywhere in the trailing run ('John Smith VI MD' as 'John + # Smith III MD', #784); one of one letter only where it ends the + # walk, since with a credential behind it it is the name word + # S10's Accepted clause keeps (#721). `_ROMAN` on the stem first, + # C-level, so a name word pays no frame. A one-letter numeral + # "needs words to spare, as a bare ambiguous acronym does" -- + # the acronym fork's own count, below, so 'John V' keeps family + # 'V' (#718). Declined, it is a pick, since "either reading is + # reported", and no branch below can take a numeral + if (k >= 2 and len(piece) == 1 + and _ROMAN.match(text := tokens[piece[0]].text.rstrip(".")) + is not None and text + and (k == len(rest) or len(text) > 1)): + tok = tokens[piece[0]] + if reads_as_generation(tok.text, tok.tags, + tokens[pieces[rest[k - 2]][0]].text, + one_case): + # rules/S10: "a numeral written in lower case in a name + # of mixed case is the generation and reports nothing" + # -- decided here once: taken as any suffix word is, and + # not recorded as the fork's call, so no reporter meets + # it + if numeral_settled_by_case(tok.text, one_case): + k -= 1 + continue + if (len(text) > 1 or not count_numeral + or k - 2 >= second): + numerals.append(tuple(piece)) + k -= 1 + continue + picks.append(tuple(piece)) + break + # rules/S10: "a Title-case numeral in a name of mixed case + # leans to the name" -- weighed and declined, so reported + if numeral_leans_to_the_name( + tok.text, tok.tags, + tokens[pieces[rest[k - 2]][0]].text, one_case): + picks.append(tuple(piece)) + break # A bare ambiguous acronym ("MA", not "M.A.") is a credential # only when peeling it still leaves a given AND a family name. # With two pieces, "one of them is a credential" is the less @@ -1257,7 +1299,7 @@ def peel_trailing(rest: Sequence[int], pieces: Sequence[Sequence[int]], k -= 1 continue break - return Peel(k, numeral, tuple(picks), anchors) + return Peel(k, tuple(numerals), tuple(picks), anchors) # rules/S8: "A credential after the name core starts a run to the @@ -1268,7 +1310,8 @@ def run_start(rest: Sequence[int], names: int, pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], tokens: Sequence[WorkToken], - units: Sequence[int] | None = None) -> int: + units: Sequence[int] | None = None, + one_case: bool | None = None) -> int: """Where #602's run starts in `rest[:names]`: the position of the first credential that starts one (`starts_a_credential_run`) with two name pieces in front of it, or `names` where none does -- with @@ -1299,9 +1342,14 @@ def run_start(rest: Sequence[int], names: int, for p in range(names): q = rest[p] tags = tokens[pieces[q][0]].tags - if (core + lone >= 2 and "vocab:suffix" in tags + if (core + lone >= 2 + and ("vocab:suffix" in tags + or _ROMAN.match(tokens[pieces[q][0]].text.rstrip(".")) + is not None) and tags.isdisjoint(_NOT_A_RUN_START) - and starts_a_credential_run(pieces[q], ptags[q], tokens)): + and starts_a_credential_run( + pieces[q], ptags[q], tokens, + tokens[pieces[rest[p - 1]][0]].text, one_case)): return p # a lone particle is not yet a name: 'de Mesnil' is one surname # -- unless it opens a chain run, which is then the surname @@ -1392,7 +1440,8 @@ def has_name_content(piece: Sequence[int], def credential_run(rest: Sequence[int], peel: Peel, p: int, pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], - tokens: Sequence[WorkToken]) -> Peel: + tokens: Sequence[WorkToken], + one_case: bool | None = None) -> Peel: """`peel` with its name count cut back to `p`, where #602's run starts (`run_start`, which the caller asks first so that a name with no run pays one frame, not two). @@ -1406,6 +1455,15 @@ def credential_run(rest: Sequence[int], peel: Peel, p: int, a run, leaves both lists without the title test's frame.""" run_titles: list[int] = [] absorbed: list[tuple[int, ...]] = [] + # a numeral starting the run is read as the generation, which a + # shape alone makes a call, reported as the peel's are (rules/S10) + numerals = peel.numerals + start = pieces[rest[p]] + if (len(start) == 1 and _ROMAN.match( + (tok := tokens[start[0]]).text.rstrip(".")) is not None + and not is_listed_numeral(tok.text, tok.tags) + and not numeral_settled_by_case(tok.text, one_case)): + numerals = (tuple(start), *numerals) for q in rest[p + 1:peel.names]: if is_suffix_piece(pieces[q], ptags[q], tokens): continue @@ -1415,7 +1473,7 @@ def credential_run(rest: Sequence[int], peel: Peel, p: int, piece = tuple(pieces[q]) if piece not in peel.picks and has_name_content(piece, tokens): absorbed.append(piece) - return Peel(p, peel.numeral, peel.picks, peel.anchors, + return Peel(p, numerals, peel.picks, peel.anchors, tuple(run_titles), tuple(absorbed)) @@ -1534,12 +1592,14 @@ def trailing_candidates(lo: int, pieces: Sequence[Sequence[int]], Back from the end over every piece the peel or the chain admits on its own terms -- a suffix piece, a member of the ambiguous class, an initial-shaped suffix word, a period-marked title word -- and - over a roman numeral by the numeral fork's SHAPE test (which asks - no vocabulary: 'VI' is in no list), where nothing stands behind it - but such title words and group-flagged suffix pieces: the chain - takes the titles first and the walk never holds the flagged pieces - ('Ph. D.'), so either way the numeral is the walk's last piece and - the fork reads it. Then back to the first + over a roman numeral the numeral's predicate admits, asked with no + case so it stays a superset: one of two or more letters wherever it + stands, the peel reading it anywhere in the run (#752), and one of + one letter where nothing stands behind it but such title words and + group-flagged suffix pieces: the chain takes the titles first and + the walk never holds the flagged pieces ('Ph. D.'), so either way + that numeral is the walk's last piece and the fork reads it. Then + back to the first credential that starts #602's run, whose words the run takes whatever they are. A BARE title word is no candidate: H5's chain does not take one, and in the view it would only inflate the count @@ -1558,9 +1618,11 @@ def trailing_candidates(lo: int, pieces: Sequence[Sequence[int]], or is_suffix_piece(piece, ptags[k], tokens) or (len(piece) == 1 and ( not tokens[piece[0]].tags.isdisjoint(_TRAILING_WORD_TAGS) - or (titles_behind and reads_as_generation( - tokens[piece[0]].text, - tokens[pieces[k - 1][0]].text, None))))): + or ((titles_behind + or len(tokens[piece[0]].text.rstrip(".")) > 1) + and reads_as_generation( + tokens[piece[0]].text, tokens[piece[0]].tags, + tokens[pieces[k - 1][0]].text, None))))): break # a group-flagged suffix piece is outside the peel's walk # altogether (`peel_walk` drops it), so it is behind the @@ -1571,8 +1633,12 @@ def trailing_candidates(lo: int, pieces: Sequence[Sequence[int]], # name word pays no frame return next((j for j in range(lo + 1, c) if ("suffix" in ptags[j] - or "vocab:suffix" in tokens[pieces[j][0]].tags) - and starts_a_credential_run(pieces[j], ptags[j], tokens)), + or "vocab:suffix" in tokens[pieces[j][0]].tags + or _ROMAN.match(tokens[pieces[j][0]].text.rstrip(".")) + is not None) + and starts_a_credential_run( + pieces[j], ptags[j], tokens, + tokens[pieces[j - 1][0]].text)), c) @@ -1678,9 +1744,10 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], kept = trailing_titles(rest, pieces, ptags, tokens, end=peeled.names, salutations=salutations) if kept == peeled.names: - p = run_start(rest, peeled.names, pieces, ptags, tokens, units) + p = run_start(rest, peeled.names, pieces, ptags, tokens, units, + one_case) return rest, (), (peeled if p == peeled.names else credential_run( - rest, peeled, p, pieces, ptags, tokens)) + rest, peeled, p, pieces, ptags, tokens, one_case)) # `rest[:hi]` stands as written; `behind` holds the peeled runs # the splices left after it, and `titled` the chained ones, each # back to front -- a pass's run goes in FRONT of what the pass @@ -1690,7 +1757,7 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], behind_count = 0 titled: list[list[int]] = [] picks = list(peeled.picks) - numeral = peeled.numeral + numerals = list(peeled.numerals) while kept < peeled.names: names = peeled.names titled.append(rest[kept:names]) @@ -1714,6 +1781,7 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], units=units, second=second, count_numeral=count_numeral) picks.extend(peeled.picks) + numerals.extend(peeled.numerals) else: rest = rest[:hi] + [j for run in reversed(behind) for j in run] if units is not None and second >= hi: @@ -1724,16 +1792,17 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], units=units, second=second, count_numeral=count_numeral) picks = list(peeled.picks) - numeral = peeled.numeral + numerals = list(peeled.numerals) kept = trailing_titles(rest, pieces, ptags, tokens, end=peeled.names, salutations=salutations) if behind: rest = rest[:hi] + [j for run in reversed(behind) for j in run] - final = Peel(peeled.names, numeral, tuple(picks), None) - p = run_start(rest, final.names, pieces, ptags, tokens, units) + final = Peel(peeled.names, tuple(numerals), tuple(picks), None) + p = run_start(rest, final.names, pieces, ptags, tokens, units, + one_case) return (rest, tuple(j for run in reversed(titled) for j in run), final if p == final.names else credential_run( - rest, final, p, pieces, ptags, tokens)) + rest, final, p, pieces, ptags, tokens, one_case)) class TailRead(NamedTuple): @@ -1899,7 +1968,8 @@ def _weighed(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: return False tok = tokens[piece[0]] return (not tok.tags.isdisjoint(_WEIGHED_TAGS) - or _ROMAN.match(tok.text) is not None + or _ROMAN.match(stem := tok.text.rstrip(".")) is not None + and stem != "" or "vocab:title" in tok.tags and _PERIOD_ABBREV.match(tok.text) is not None) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 78a61e75..e74332ec 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -287,32 +287,37 @@ def is_initial_shaped(text: str) -> bool: # rules/S10's Except bullet "A numeral of one letter needs words to # spare", is the callers'. decisions/S10 (#718, #721) # mechanisms.md#ONE-PREDICATE-PER-QUESTION ("exactly one predicate answers it and every other site calls that one") -# -- the numeral's predicate, which every numeral rule is to ask; -# today its shape half, and the sites still recognizing a numeral on -# their own are named in the decisions entry (decisions/vocabulary- -# model, #752) -def reads_as_generation(text: str, preceding: str, +# -- the numeral's predicate, which every numeral rule asks +# (decisions/vocabulary-model, #752; #784, #774, #819) +def reads_as_generation(text: str, tags: Set[str], preceding: str | None, one_case: bool | None) -> bool: - """Whether a word ending a name reads as the generation: the - trailing walk's roman-numeral fork, a FINAL single-token piece - that is a roman numeral reading as the suffix when the piece - before it does not look like part of an initial run -- for a - one-letter numeral a necessary condition only, the caller's words-to-spare count deciding - (rules/S10, #718). `preceding` is that piece's first token; - the callers establish that `text` is last and that a name piece - precedes it. - - It answers by SHAPE alone. That covers the listed 'ii' to 'iv', - which the shape test takes as it takes 'VI'; of the listed - numerals only the one-letter 'i' and 'v' would be new here, and - the vocabulary half is the change that has every numeral rule ask - this predicate (#784, #774, #819). It cannot read them yet without - moving readings: answering yes for 'i' and 'v' moves the peel's - fork ('John J. V' family 'V' → suffix 'V') and the report on a - numeral before a credential ('John J. V MD' gains one), since the - ones those sites meet carry the `initial` tag. Other sites still recognize a - numeral without this predicate, each keeping the answer it had - before #752; decisions/vocabulary-model lists those found. + """Whether a word reads as the generation: S10's roman-numeral + fork, asked by every rule that weighs a numeral (#752). For a + one-letter numeral a necessary condition only: the words-to-spare + count is the caller's, which alone knows where the name's words + are (rules/S10, #718). + + A numeral is I to X by shape, written with its periods or without + ('VI.', the German ordinal), save that a single letter written + with its period is an initial (rules/T5). The vocabulary's own + numerals of two or more letters ('ii' to 'iv', `tags` carrying + `vocab:suffix`) are the generation whatever stands around them, + the vocabulary settling them; the rest ('vi' to 'x', and the + one-letter 'i' and 'v') are weighed: + + - an initial in front makes the numeral a name word: `preceding` + is the first token of the piece in front, read for layout, and + None where the slot never asks (the given part after a family + comma, where 'Chang, Andy C I' reads suffix 'I', v1's reading); + - a Title-case numeral in a name of mixed case leans to the name, + as a Title-case member of the ambiguous class does + (`numeral_leans_to_the_name`). + + In the given part after a family comma (`preceding` None) a + one-letter numeral is the generation only where the vocabulary + lists it, rules/C9's "one-character suffix word": no count is + asked there (S5), and an unlisted letter is the middle initial + ('SMITH, JOHN X'). is_initial_shaped, not is_initial: this asks whether the preceding piece looks like part of an initial run, which is a question @@ -325,11 +330,47 @@ def reads_as_generation(text: str, preceding: str, is initial-shaped as 'J' is (rules/S10, #721's review: 'john j v' reads as 'JOHN J V'), while 'é', 'ж' or an uncased word, whose capitals are not, stay as they were. None (the maiden take's - superset view passes it) gets the shape test alone, which admits + superset view passes it) asks neither case question, which admits more and so stays a superset.""" - return (_ROMAN.match(text) is not None - and not is_initial_shaped(preceding) - and not (one_case and is_initial_shaped(preceding.upper()))) + stem = text.rstrip(".") + if not stem or _ROMAN.match(stem) is None: + return False + if len(stem) == 1: + # rules/T5: "A one-letter word written with its period is an + # initial" + if len(text) > 1: + return False + if preceding is None: + return "vocab:suffix" in tags + elif "vocab:suffix" in tags: + return True + elif one_case is False and stem.istitle(): + return False + return (preceding is None + or not (is_initial_shaped(preceding) + or one_case and is_initial_shaped(preceding.upper()))) + + +# rules/S10: "a Title-case numeral in a name of mixed case leans to the +# name" -- the fork weighed it and the writing declined it, so it is +# reported as a declined member of the ambiguous class is (#752) +def numeral_leans_to_the_name(text: str, tags: Set[str], + preceding: str | None, + one_case: bool | None) -> bool: + """Whether `reads_as_generation` declined a numeral for its Title + case alone: two or more letters, unlisted, in a name of mixed case, + and no initial in front (which makes it a name word unweighed).""" + return (one_case is False and text.rstrip(".").istitle() + and len(text.rstrip(".")) >= 2 and "vocab:suffix" not in tags + and reads_as_generation(text, tags, preceding, None)) + + +def is_listed_numeral(text: str, tags: Set[str]) -> bool: + """A numeral the vocabulary lists in two or more letters ('III'): + not in question, so its reading is never reported (rules/S10).""" + stem = text.rstrip(".") + return (len(stem) >= 2 and "vocab:suffix" in tags + and _ROMAN.match(stem) is not None) # rules/S10: "a numeral written in lower case in a name of mixed case is @@ -349,11 +390,14 @@ def numeral_settled_by_case(text: str, one_case: bool | None) -> bool: # reports on a numeral a post-nominal follows ask (assign's, and the # maiden take's for a numeral its clause keeps): the fork would read # this numeral, and its writing does not settle it (#721) -def reads_as_a_reported_numeral(text: str, preceding: str, +def reads_as_a_reported_numeral(text: str, tags: Set[str], + preceding: str, one_case: bool | None) -> bool: """Whether a numeral the walk left a name word, with `preceding` - in front of it, is one the fork would weigh and so reports.""" - return (reads_as_generation(text, preceding, one_case) + in front of it, is one the fork would weigh and so reports: one it + reads as the generation, or declines for its Title case.""" + return ((reads_as_generation(text, tags, preceding, one_case) + or numeral_leans_to_the_name(text, tags, preceding, one_case)) and not numeral_settled_by_case(text, one_case)) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 8ee5889b..f58080c9 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -4513,10 +4513,14 @@ def _check_cjk_shape_purity(self) -> None: classification="fix(#701)"), Case("a_title_before_a_family_comma_peels_ahead_of_a_shape_numeral", "Dr. Smith VI, John", - {"title": "Dr.", "given": "John", "family": "Smith VI"}, - classification="fix(#701)", - notes="VI is read by shape alone, and the part before a " - "family comma keeps it as it does without the title"), + {"title": "Dr.", "given": "John", "family": "Smith", + "suffix": "VI"}, + ambiguities=("suffix-or-name",), classification="fix(#774)", + notes="VI is read by shape alone, and since #752 the part " + "before a family comma reads a numeral ending it as the " + "comma-less name does (C1), as it reads 'III'; reported, " + "being in no wordlist. #701 to 2.6.0 kept family " + "'Smith VI'"), Case("a_title_before_a_family_comma_stops_at_a_period_marked_dual", "Dr. Freiherr. von Berg, Ed", {"title": "Dr.", "given": "Ed", "family": "Freiherr. von Berg"}, @@ -6316,7 +6320,7 @@ def _check_cjk_shape_purity(self) -> None: ambiguities=("suffix-or-name",), notes="#424's own fork, pinned beside the acronym one " "because the two now share a single peel: the numeral " - "half answers off Peel.numeral exactly as it did, and " + "half answers off Peel.numerals exactly as it did, and " "this row is what says the shared call did not move " "it. Unchanged since #424", shape=1), @@ -8552,6 +8556,79 @@ def _check_cjk_shape_purity(self) -> None: "about case: written wholly in lower case the letter " "could be an initial, and the fork reports (Derek, " "#718)"), + Case("a_shape_numeral_before_a_credential_joins_the_run", + "John Smith VI MD", + {"given": "John", "family": "Smith", "suffix": "VI MD"}, + ambiguities=("suffix-or-name",), classification="fix(#784)", + notes="S10: a numeral of two or more letters is read anywhere " + "in the trailing run, so 'VI' joins it in front of a " + "credential as the listed 'III' does, and reports, being " + "read by its shape alone. 1.4.0 through 2.6.0 read family " + "'VI', suffix 'MD'"), + Case("a_shape_numeral_starts_the_credential_run", + "John Smith VI Jones", + {"given": "John", "family": "Smith", "suffix": "VI Jones"}, + ambiguities=("suffix-or-name", "suffix-or-name"), + classification="fix(#784)", + notes="S8: a numeral of two or more letters that reads as the " + "generation starts the run as 'III' does; both the " + "numeral and the absorbed name word report"), + Case("a_numeral_behind_an_initial_stays_a_name_before_a_credential", + "John J. VI MD", + {"given": "John", "middle": "J.", "family": "VI", "suffix": "MD"}, + notes="S10's Except: an initial in front makes an unlisted " + "numeral a name word, before a credential as at the end; " + "no fork is weighed, so nothing reports"), + Case("a_shape_numeral_before_dotted_initials_stays_the_suffix", + "John Smith V, M.J.", + {"given": "M.J.", "family": "John Smith", "suffix": "V"}, + ambiguities=("suffix-or-name", "suffix-or-name"), + classification="fix(#774)", + notes="C1: a numeral ending the family part is read as the " + "comma-less name reads it, the two name words in front " + "sparing the one-letter one, so 'V' stays the suffix as " + "'Jr.' and 'III' do. 1.4.0 through 2.6.0 read family " + "'John Smith V'"), + Case("a_lone_letter_ahead_of_a_further_comma_is_the_initial", + "Smith, John PhD I, Jr.", + {"given": "John", "middle": "I", "family": "Smith", + "suffix": "PhD, Jr."}, + notes="C9 (#819): the further comma has named the suffix, so " + "the credential run stops short of the letter, which is " + "the middle initial, as 1.4.0 through 2.3.0 read it; " + "2.4.0 through 2.6.0 read suffix 'PhD I, Jr.'"), + Case("a_title_case_numeral_leans_to_the_name", + "Nguyen Van Vi", + {"given": "Nguyen", "family": "Van Vi"}, + ambiguities=("suffix-or-name",), + notes="S10: a Title-case numeral in a name of mixed case leans " + "to the name, as a Title-case member of the ambiguous " + "class does, and reports. 1.4.0 read the same fields; " + "2.2.0 through 2.6.0 read suffix 'Vi'"), + Case("a_numeral_written_with_its_period_is_the_generation", + "Georg VI.", + {"given": "Georg", "suffix": "VI."}, + ambiguities=("suffix-or-name", "given-or-family"), + classification="fix(#752)", + notes="S10: a numeral of two or more letters keeps its reading " + "written with a period, the German ordinal, as 'III.' " + "does. 1.4.0 through 2.6.0 read family 'VI.'"), + Case("a_shape_numeral_ending_the_given_part_is_the_generation", + "Smith, John VI", + {"given": "John", "family": "Smith", "suffix": "VI"}, + ambiguities=("suffix-or-name",), classification="fix(#784)", + notes="S10's Precedence: in the given part after a family comma " + "a numeral of two or more letters reads as the " + "generation, as 'III' does, and reports by its shape. " + "1.4.0 through 2.6.0 read middle 'VI'"), + Case("an_unlisted_letter_ending_the_given_part_is_the_initial", + "SMITH, JOHN X", + {"given": "JOHN", "middle": "X", "family": "SMITH"}, + notes="S10's Precedence and C9: in the given part after a " + "family comma a one-letter numeral is the generation only " + "where the vocabulary lists it, so 'X' stays the middle " + "initial it nearly always is in donor lists (the " + "predicate's FEC constraint), where 'V' reads the suffix"), Case("initial_not_suffix", "John V. Smith", {"given": "John", "middle": "V.", "family": "Smith"}, notes="shape 1's Middle slot filled by an initial-shaped " diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index 204e8262..6267b3e7 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -1,18 +1,18 @@ { "A1": {"words": 144, "then_words": 120, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 1, "accepted_words": 32}, "A2": {"words": 60, "then_words": 31, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, - "C1": {"words": 416, "then_words": 196, "because_sentences": 1, "except": 0, "precedence": 5, "accepted": 1, "accepted_words": 116}, + "C1": {"words": 456, "then_words": 196, "because_sentences": 1, "except": 0, "precedence": 5, "accepted": 1, "accepted_words": 116}, "C2": {"words": 492, "then_words": 399, "because_sentences": 1, "except": 2, "precedence": 0, "accepted": 1, "accepted_words": 81}, "C3": {"words": 260, "then_words": 228, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 1, "accepted_words": 173}, "C4": {"words": 498, "then_words": 371, "because_sentences": 1, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, - "C5": {"words": 320, "then_words": 79, "because_sentences": 1, "except": 2, "precedence": 1, "accepted": 1, "accepted_words": 54}, + "C5": {"words": 319, "then_words": 79, "because_sentences": 1, "except": 2, "precedence": 1, "accepted": 1, "accepted_words": 54}, "C6": {"words": 207, "then_words": 162, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "C7": {"words": 119, "then_words": 68, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "C8": {"words": 176, "then_words": 110, "because_sentences": 1, "except": 0, "precedence": 1, "accepted": 1, "accepted_words": 36}, - "C9": {"words": 241, "then_words": 162, "because_sentences": 1, "except": 0, "precedence": 1, "accepted": 1, "accepted_words": 96}, + "C9": {"words": 289, "then_words": 210, "because_sentences": 1, "except": 0, "precedence": 1, "accepted": 1, "accepted_words": 48}, "C10": {"words": 377, "then_words": 225, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 5, "accepted_words": 311}, "C11": {"words": 372, "then_words": 247, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 0, "accepted_words": 0}, - "C12": {"words": 284, "then_words": 107, "because_sentences": 2, "except": 1, "precedence": 1, "accepted": 2, "accepted_words": 59}, + "C12": {"words": 283, "then_words": 107, "because_sentences": 2, "except": 1, "precedence": 1, "accepted": 2, "accepted_words": 59}, "C13": {"words": 259, "then_words": 74, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, "C14": {"words": 231, "then_words": 111, "because_sentences": 3, "except": 1, "precedence": 0, "accepted": 0, "accepted_words": 0}, "D1": {"words": 73, "then_words": 26, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, @@ -59,9 +59,9 @@ "S5": {"words": 632, "then_words": 317, "because_sentences": 1, "except": 2, "precedence": 1, "accepted": 0, "accepted_words": 0}, "S6": {"words": 337, "then_words": 121, "because_sentences": 2, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, "S7": {"words": 570, "then_words": 388, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 1, "accepted_words": 147}, - "S8": {"words": 354, "then_words": 240, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, + "S8": {"words": 381, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, "S9": {"words": 430, "then_words": 229, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, - "S10": {"words": 401, "then_words": 140, "because_sentences": 3, "except": 4, "precedence": 2, "accepted": 2, "accepted_words": 73}, + "S10": {"words": 624, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 4, "accepted_words": 176}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 14d584ae..35af0fbc 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -234,9 +234,12 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # the same day: #857's sa rows, after the word left every set (the # one reporting row, 'Smith, S.A.', claims under the orders it # reaches); 1406 -> 1415 the same day: the review's comma rows for - # sa and the listed `ma` twins, three of them reporting. - assert claims == 1415, ( - f"the field sweep checks {claims} claims, recorded as 1415 on " + # sa and the listed `ma` twins, three of them reporting; 1415 -> + # 1445 the same day: #752's numeral rows (phase 2f) and the + # 'Dr. Smith VI, John' row, which reports since, each claim under + # the three orders. + assert claims == 1445, ( + f"the field sweep checks {claims} claims, recorded as 1445 on " f"2026-10-25") diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 0a8dbae0..e1ac63ed 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -3517,8 +3517,8 @@ def test_the_one_case_grid_sees_the_letter_in_front( from nameparser._pipeline import _vocab monkeypatch.setattr( "nameparser._pipeline._pieces.reads_as_generation", - lambda text, preceding, one_case=None: - _vocab.reads_as_generation(text, preceding, None)) + lambda text, tags, preceding, one_case=None: + _vocab.reads_as_generation(text, tags, preceding, None)) assert (len(_one_case_disagreements()) == _ONE_CASE_LETTER_IN_FRONT_UNGUARDED) diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index d0f9a937..6267001d 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -176,6 +176,7 @@ "García Márquez, MJ JK" "García Márquez, MJ PhD" "García Márquez, Ms G.J." +"Georg VI." "Graf Lambsdorff, Otto" "Greve Anna" "Greve Smith, Anna" @@ -206,6 +207,7 @@ "JOHN SMITH MA" "JOHN SMITH NEE X X" "JOHN SMITH PH.D." +"JOHN SMITH VI" "JOHN SMITH, FNP-BC" "JOHN SMITH, VD MA" "JOHN V" @@ -248,6 +250,7 @@ "Jane Doe nee Smith Prof. MA" "Jane Doe nee Smith V PhD" "Jane Doe nee Smith V Prof." +"Jane Doe nee Smith VI PhD" "Jane Doe nee Smith VI Prof." "Jane Doe nee Smith X.Y.Z." "Jane Doe née Jones Smith V" @@ -287,7 +290,9 @@ "John III, Sir" "John Ingram V" "John J VI, AD" +"John J. III" "John J. V" +"John J. VI" "John Ma" "John PhD Smith" "John Prof. MA" @@ -303,6 +308,8 @@ "John Smith Esq." "John Smith II, AD" "John Smith III" +"John Smith III MD" +"John Smith Iii" "John Smith J.u.n.i.o.r." "John Smith Jr." "John Smith Jr. Prof." @@ -341,10 +348,15 @@ "John Smith V, M.J." "John Smith V." "John Smith VI" +"John Smith VI Jones" +"John Smith VI MD" "John Smith VI, AD" "John Smith VI, M.J." +"John Smith Vi" +"John Smith Viii" "John Smith X.Y.Z." "John Smith XI" +"John Smith XI MD" "John Smith XYZ" "John Smith Xyz." "John Smith vi" @@ -453,6 +465,7 @@ "Juan Ó. Pérez" "Kim김민준씨, MA" "King Smith, John" +"Konstantin VII." "LT.GOV. JOHN DOE" "Lloret Palol II, JA" "Lloret Palol II, Josep Antoni" @@ -490,6 +503,7 @@ "Ménil de" "Nguyen Thi Van" "Nguyen Van Lac" +"Nguyen Van Vi" "Nguyen, Thi Van" "Nguyen, Van" "Nguyen, Van Le" @@ -558,7 +572,9 @@ "Smith Prof." "Smith Sir." "Smith V, AD" +"Smith V, John" "Smith VI, AD" +"Smith VI, John" "Smith and King" "Smith née Jones" "Smith née Jones PhD" @@ -573,13 +589,23 @@ "Smith, Jan van Prof." "Smith, Jane Mrs" "Smith, John" +"Smith, John PSM I, Jr." +"Smith, John PhD I, Jr." "Smith, John PhD I." +"Smith, John PhD I., Jr." "Smith, John PhD Jones" "Smith, John PhD de Jr." "Smith, John Prof." "Smith, John V" +"Smith, John V MD" +"Smith, John V, Jr." "Smith, John V." "Smith, John V;" +"Smith, John VI" +"Smith, John VI Jones" +"Smith, John VI MD" +"Smith, John Vi" +"Smith, John X" "Smith, John van der Berg Prof." "Smith, John, MD - née Jones Smith" "Smith, John, PhD - and MD" diff --git a/tools/differential/gnd_agreement.json b/tools/differential/gnd_agreement.json index 2ddbec55..927ee05d 100644 --- a/tools/differential/gnd_agreement.json +++ b/tools/differential/gnd_agreement.json @@ -736,17 +736,6 @@ }, "text": "Abu-'l-Fidāʾ Ismāʿīl Ibn-ʿAlī" }, - "118513257/regnal/givenFirst": { - "family": { - "expected": "", - "got": "VIII." - }, - "suffix": { - "expected": "VIII.", - "got": "" - }, - "text": "Bonifatius VIII." - }, "118528947/regnal/givenFirst": { "family": { "expected": "", @@ -835,17 +824,6 @@ }, "text": "Meister E. S." }, - "118690477/regnal/givenFirst": { - "family": { - "expected": "", - "got": "VI." - }, - "suffix": { - "expected": "VI.", - "got": "" - }, - "text": "Georg VI." - }, "118716158/direct-ir-af/givenFirst": { "family": { "expected": "", @@ -1000,17 +978,6 @@ }, "text": "Manuel Marulanda Vélez" }, - "119022397/regnal/givenFirst": { - "family": { - "expected": "", - "got": "VII." - }, - "suffix": { - "expected": "VII.", - "got": "" - }, - "text": "Konstantin VII." - }, "119075067/compound-latam/givenFirst": { "family": { "expected": "López Portillo y Pacheco", diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index afe7ab01..fe24a3b0 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -390,7 +390,7 @@ {"name": "Boluwatife", "given": "Boluwatife", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "B."} {"name": "Boluwatife", "order": "FAMILY_FIRST", "family": "Boluwatife", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "B."} {"name": "Bonafini, Hebe de", "given": "Hebe", "family": "de Bonafini", "initials": "H. B."} -{"name": "Bonifatius VIII.", "given": "Bonifatius", "family": "VIII.", "initials": "B. V.", "capitalized_force": "Bonifatius Viii."} +{"name": "Bonifatius VIII.", "given": "Bonifatius", "suffix": "VIII.", "ambiguities": ["GIVEN_OR_FAMILY", "SUFFIX_OR_NAME"], "initials": "B."} {"name": "Bonneville, Benjamin Louis Eulalie de", "given": "Benjamin", "middle": "Louis Eulalie", "family": "de Bonneville", "initials": "B. L. E. B."} {"name": "Bonvi", "given": "Bonvi", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "B."} {"name": "Borcke, Karl August Ferdinand von", "given": "Karl", "middle": "August Ferdinand", "family": "von Borcke", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "K. A. F. B."} @@ -1054,7 +1054,7 @@ {"name": "Gemmingen, Pleikard Dietrich von", "given": "Pleikard", "middle": "Dietrich", "family": "von Gemmingen", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "P. D. G."} {"name": "General of Minnesota", "title": "General of Minnesota"} {"name": "Geng, Biao", "given": "Biao", "family": "Geng", "initials": "B. G."} -{"name": "Georg VI.", "given": "Georg", "family": "VI.", "initials": "G. V.", "capitalized_force": "Georg Vi."} +{"name": "Georg VI.", "given": "Georg", "suffix": "VI.", "ambiguities": ["GIVEN_OR_FAMILY", "SUFFIX_OR_NAME"], "initials": "G."} {"name": "Georg von Hülsen-Haeseler", "given": "Georg", "family": "von Hülsen-Haeseler", "initials": "G. H."} {"name": "George D. Cummins", "given": "George", "middle": "D.", "family": "Cummins", "initials": "G. D. C."} {"name": "George Edward Stanhope Molyneux Herbert of Carnarvon", "given": "George", "middle": "Edward Stanhope Molyneux", "family": "Herbert of Carnarvon", "initials": "G. E. S. M. H. C."} @@ -1337,6 +1337,7 @@ {"name": "JOHN SMITH MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} {"name": "JOHN SMITH NEE X X", "given": "JOHN", "family": "SMITH", "maiden": "X X", "initials": "J. S.", "capitalized": "John Smith (X X)"} {"name": "JOHN SMITH PH.D.", "given": "JOHN", "family": "SMITH", "suffix": "PH.D.", "initials": "J. S.", "capitalized": "John Smith Ph.D."} +{"name": "JOHN SMITH VI", "given": "JOHN", "family": "SMITH", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith VI"} {"name": "JOHN SMITH, FNP-BC", "given": "JOHN", "family": "SMITH", "suffix": "FNP-BC", "initials": "J. S.", "capitalized": "John Smith FNP-BC"} {"name": "JOHN SMITH, MA", "given": "JOHN", "family": "SMITH", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith MA"} {"name": "JOHN SMITH, VD MA", "given": "JOHN", "family": "SMITH", "suffix": "VD MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized": "John Smith VD MA"} @@ -1432,6 +1433,7 @@ {"name": "Jane Doe nee Smith V MA", "given": "Jane", "family": "Doe", "suffix": "MA", "maiden": "Smith V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee Smith V PhD", "given": "Jane", "family": "Doe", "suffix": "PhD", "maiden": "Smith V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee Smith V Prof.", "title": "Prof.", "given": "Jane", "family": "Doe", "suffix": "V", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} +{"name": "Jane Doe nee Smith VI PhD", "given": "Jane", "family": "Doe", "suffix": "VI PhD", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee Smith VI Prof.", "title": "Prof.", "given": "Jane", "family": "Doe", "suffix": "VI", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee Smith X.Y.Z.", "given": "Jane", "family": "Doe", "suffix": "X.Y.Z.", "maiden": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. D."} {"name": "Jane Doe nee Smith XYZ", "given": "Jane", "family": "Doe", "maiden": "Smith XYZ", "initials": "J. D.", "capitalized_force": "Jane Doe (Smith Xyz)"} @@ -1572,7 +1574,9 @@ {"name": "John III, Sir", "title": "Sir", "given": "John", "suffix": "III", "initials": "J."} {"name": "John Ingram V", "given": "John", "family": "Ingram", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. I."} {"name": "John J VI, AD", "given": "AD", "family": "John J VI", "initials": "A. J. J. V.", "capitalized_force": "Ad John J Vi"} +{"name": "John J. III", "given": "John", "family": "J.", "suffix": "III", "initials": "J. J."} {"name": "John J. V", "given": "John", "middle": "J.", "family": "V", "initials": "J. J. V."} +{"name": "John J. VI", "given": "John", "middle": "J.", "family": "VI", "initials": "J. J. V.", "capitalized_force": "John J. Vi"} {"name": "John Jack Andrew Kennedy", "given": "John", "middle": "Jack Andrew", "family": "Kennedy", "initials": "J. J. A. K."} {"name": "John Jones (Google Docs)", "given": "John", "family": "Jones", "nickname": "Google Docs", "initials": "J. J."} {"name": "John Jones (Google Docs), Jr. (Unknown)", "given": "John", "family": "Jones", "suffix": "Jr.", "nickname": "Google Docs Unknown", "initials": "J. J."} @@ -1605,6 +1609,8 @@ {"name": "John Smith I", "given": "John", "family": "Smith", "suffix": "I", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith II, AD", "given": "John", "family": "Smith", "suffix": "II, AD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith III", "given": "John", "family": "Smith", "suffix": "III", "initials": "J. S."} +{"name": "John Smith III MD", "given": "John", "family": "Smith", "suffix": "III MD", "initials": "J. S."} +{"name": "John Smith Iii", "given": "John", "family": "Smith", "suffix": "Iii", "initials": "J. S.", "capitalized_force": "John Smith III"} {"name": "John Smith J.u.n.i.o.r.", "given": "John", "family": "Smith", "suffix": "J.u.n.i.o.r.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S.", "capitalized_force": "John Smith J.U.N.I.O.R."} {"name": "John Smith Jr", "given": "John", "family": "Smith", "suffix": "Jr", "initials": "J. S."} {"name": "John Smith Jr.", "given": "John", "family": "Smith", "suffix": "Jr.", "initials": "J. S."} @@ -1651,13 +1657,18 @@ {"name": "John Smith V Ph. D.", "given": "John", "family": "Smith", "suffix": "V Ph. D.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith V PhD", "given": "John", "middle": "Smith", "family": "V", "suffix": "PhD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. V."} {"name": "John Smith V, AD", "given": "John", "family": "Smith", "suffix": "V, AD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John Smith V, M.J.", "given": "M.J.", "family": "John Smith V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. J. S. V."} +{"name": "John Smith V, M.J.", "given": "M.J.", "family": "John Smith", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. J. S."} {"name": "John Smith V.", "given": "John", "middle": "Smith", "family": "V.", "initials": "J. S. V."} {"name": "John Smith VI", "given": "John", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith VI Jones", "given": "John", "family": "Smith", "suffix": "VI Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "John Smith VI MD", "given": "John", "family": "Smith", "suffix": "VI MD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith VI, AD", "given": "John", "family": "Smith", "suffix": "VI, AD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} -{"name": "John Smith VI, M.J.", "given": "M.J.", "family": "John Smith VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. J. S. V.", "capitalized_force": "M.J. John Smith Vi"} +{"name": "John Smith VI, M.J.", "given": "M.J.", "family": "John Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. J. S."} +{"name": "John Smith Vi", "given": "John", "middle": "Smith", "family": "Vi", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. V."} +{"name": "John Smith Viii", "given": "John", "middle": "Smith", "family": "Viii", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S. V."} {"name": "John Smith X.Y.Z.", "given": "John", "family": "Smith", "suffix": "X.Y.Z.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "John Smith XI", "given": "John", "middle": "Smith", "family": "XI", "initials": "J. S. X.", "capitalized_force": "John Smith Xi"} +{"name": "John Smith XI MD", "given": "John", "middle": "Smith", "family": "XI", "suffix": "MD", "initials": "J. S. X.", "capitalized_force": "John Smith Xi MD"} {"name": "John Smith XYZ", "given": "John", "middle": "Smith", "family": "XYZ", "initials": "J. S. X.", "capitalized_force": "John Smith Xyz"} {"name": "John Smith Xyz.", "given": "John", "middle": "Smith", "family": "Xyz.", "initials": "J. S. X."} {"name": "John Smith nee Jones R.A.I.", "given": "John", "family": "Smith", "suffix": "R.A.I.", "maiden": "Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} @@ -2021,7 +2032,7 @@ {"name": "Kolko, Gabriel", "given": "Gabriel", "family": "Kolko", "initials": "G. K."} {"name": "Kong, Yingda", "given": "Yingda", "family": "Kong", "initials": "Y. K."} {"name": "Konráð Gíslason", "given": "Konráð", "family": "Gíslason", "initials": "K. G."} -{"name": "Konstantin VII.", "given": "Konstantin", "family": "VII.", "initials": "K. V.", "capitalized_force": "Konstantin Vii."} +{"name": "Konstantin VII.", "given": "Konstantin", "suffix": "VII.", "ambiguities": ["GIVEN_OR_FAMILY", "SUFFIX_OR_NAME"], "initials": "K."} {"name": "Kotryna Miliūtė", "given": "Kotryna", "family": "Miliūtė", "initials": "K. M."} {"name": "Kovač Ana Marija Vera", "order": "FAMILY_FIRST", "given": "Ana", "middle": "Marija Vera", "family": "Kovač", "initials": "A. M. V. K."} {"name": "Kovač, Ana Marija Vera", "given": "Ana", "middle": "Marija Vera", "family": "Kovač", "initials": "A. M. V. K."} @@ -2499,6 +2510,7 @@ {"name": "Ngugi wa Thiong'o", "given": "Ngugi", "middle": "wa", "family": "Thiong'o", "initials": "N. w. T.", "capitalized_force": "Ngugi Wa Thiong'O"} {"name": "Nguyen Thi Van", "given": "Nguyen", "middle": "Thi", "family": "Van", "initials": "N. T. V."} {"name": "Nguyen Van Lac", "given": "Nguyen", "family": "Van Lac", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "N. L.", "capitalized_force": "Nguyen van Lac"} +{"name": "Nguyen Van Vi", "given": "Nguyen", "family": "Van Vi", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "N. V.", "capitalized_force": "Nguyen van Vi"} {"name": "Nguyen, Thi Van", "given": "Thi", "family": "Van Nguyen", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "T. N.", "capitalized_force": "Thi van Nguyen"} {"name": "Nguyen, Van", "given": "Van", "family": "Nguyen", "initials": "V. N."} {"name": "Nguyen, Van Le", "given": "Van", "middle": "Le", "family": "Nguyen", "initials": "V. L. N."} @@ -2959,7 +2971,9 @@ {"name": "Smith Prof.", "title": "Prof.", "family": "Smith", "initials": "S."} {"name": "Smith Sir.", "title": "Sir.", "given": "Smith", "initials": "S."} {"name": "Smith V, AD", "given": "AD", "family": "Smith V", "initials": "A. S. V.", "capitalized_force": "Ad Smith V"} -{"name": "Smith VI, AD", "given": "AD", "family": "Smith VI", "initials": "A. S. V.", "capitalized_force": "Ad Smith Vi"} +{"name": "Smith V, John", "given": "John", "family": "Smith V", "initials": "J. S. V."} +{"name": "Smith VI, AD", "given": "AD", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "A. S.", "capitalized_force": "Ad Smith VI"} +{"name": "Smith VI, John", "given": "John", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith and King", "given": "Smith and King", "ambiguities": ["TITLE_OR_NAME"], "initials": "S. K."} {"name": "Smith born", "given": "Smith", "family": "born", "initials": "S. b.", "capitalized_force": "Smith Born"} {"name": "Smith nee Jones, Jane MA", "given": "Jane", "family": "Smith", "suffix": "MA", "maiden": "Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} @@ -2994,14 +3008,23 @@ {"name": "Smith, John I", "given": "John", "family": "Smith", "suffix": "I", "initials": "J. S."} {"name": "Smith, John Jr.", "given": "John", "family": "Smith", "suffix": "Jr.", "initials": "J. S."} {"name": "Smith, John MEng", "given": "John", "middle": "MEng", "family": "Smith", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. M. S.", "capitalized_force": "John Meng Smith"} +{"name": "Smith, John PSM I, Jr.", "given": "John", "middle": "I", "family": "Smith", "suffix": "PSM, Jr.", "initials": "J. I. S."} +{"name": "Smith, John PhD I, Jr.", "given": "John", "middle": "I", "family": "Smith", "suffix": "PhD, Jr.", "initials": "J. I. S."} {"name": "Smith, John PhD I.", "given": "John", "family": "Smith", "suffix": "PhD I.", "initials": "J. S."} +{"name": "Smith, John PhD I., Jr.", "given": "John", "middle": "I.", "family": "Smith", "suffix": "PhD, Jr.", "initials": "J. I. S."} {"name": "Smith, John PhD Jones", "given": "John", "family": "Smith", "suffix": "PhD Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John PhD de Jr.", "given": "John", "family": "de Smith", "suffix": "PhD, Jr.", "initials": "J. S."} {"name": "Smith, John Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "Smith, John V", "given": "John", "family": "Smith", "suffix": "V", "initials": "J. S."} +{"name": "Smith, John V MD", "given": "John", "middle": "V", "family": "Smith", "suffix": "MD", "initials": "J. V. S."} {"name": "Smith, John V, Jr.", "given": "John", "middle": "V", "family": "Smith", "suffix": "Jr.", "initials": "J. V. S."} {"name": "Smith, John V.", "given": "John", "middle": "V.", "family": "Smith", "initials": "J. V. S."} {"name": "Smith, John V;", "given": "John", "middle": "V;", "family": "Smith", "initials": "J. V. S."} +{"name": "Smith, John VI", "given": "John", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "Smith, John VI Jones", "given": "John", "family": "Smith", "suffix": "VI Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "Smith, John VI MD", "given": "John", "family": "Smith", "suffix": "VI MD", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "Smith, John Vi", "given": "John", "middle": "Vi", "family": "Smith", "initials": "J. V. S."} +{"name": "Smith, John X", "given": "John", "middle": "X", "family": "Smith", "initials": "J. X. S."} {"name": "Smith, John e, III, Jr", "given": "John", "middle": "e", "family": "Smith", "suffix": "III, Jr", "initials": "J. e. S."} {"name": "Smith, John van der Berg Prof.", "title": "Prof.", "given": "John", "middle": "van der Berg", "family": "Smith", "initials": "J. B. S."} {"name": "Smith, John, Extra, Jr.", "given": "John", "family": "Smith", "suffix": "Extra, Jr.", "ambiguities": ["COMMA_STRUCTURE"], "initials": "J. S."} From e20c0687264c7799e99f4d5bee1202cd89064e1c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 03:42:52 -0700 Subject: [PATCH 2/4] Address the 2f reviews: report wording, C3's count, stale docs (#752) - a numeral declined for its Title case says so, and one a credential run in front takes reports as an absorbed name word - the family part counts its words as C3 does, so 'Vega y Lopez V, Mary' keeps suffix 'V' as the comma-less name does - S10 states that comma sites report only a generation read by shape; #819's clause is scoped to a letter written as an initial (T5) - stale docstrings, Standing lines and the S8 Elsewhere pointer Co-Authored-By: Claude Opus 5.5 --- .../decisions/C1-suffix-comma-decision.md | 2 +- ...aracter-suffix-word-inside-a-comma-part.md | 4 +- .../decisions/S8-credentials-trailing-run.md | 2 +- docs/design/decisions/vocabulary-model.md | 4 +- docs/design/mechanisms.md | 2 +- docs/design/rules/C1-suffix-comma-decision.md | 4 +- ...aracter-suffix-word-inside-a-comma-part.md | 3 +- .../rules/S10-trailing-roman-numeral.md | 13 +++++- nameparser/_pipeline/_assign.py | 46 +++++++++++++++---- nameparser/_pipeline/_group.py | 11 +++-- nameparser/_pipeline/_pieces.py | 20 ++++---- tests/v2/rules_budget.json | 6 +-- tools/differential/corpus_rules.jsonl | 3 ++ tools/differential/snapshot.jsonl | 3 ++ 14 files changed, 83 insertions(+), 40 deletions(-) diff --git a/docs/design/decisions/C1-suffix-comma-decision.md b/docs/design/decisions/C1-suffix-comma-decision.md index 4994b292..f4ba7643 100644 --- a/docs/design/decisions/C1-suffix-comma-decision.md +++ b/docs/design/decisions/C1-suffix-comma-decision.md @@ -34,4 +34,4 @@ Open: none in this file. SUPERSEDES: decisions/C4's 2026-10-01 #562 bullet's mechanism and its accepted reports (its pair report and the settled-run test); decisions/C13's 2026-10-04 #603 bullet's "segment decides" wording; and decisions/C3's #575 bullet's account of WHY segment builds its unit facts from vocabulary -- `_comma._whole_name` counts over classify's tags, and `name_word_count`, which reads the vocabulary, is asked by decide and by script_segment, the agreement sweep `test_surname_unit_tags_agree_with_classify` keeping the vocabulary view equal to the tags. - 2026-10-18 #728 — SPLIT. rules/C1 held what are now C1 and C3–C11, and this file their history; each entry moved to the file of the rule it decided, and the #429, #436/#437 and #511 entries, which decide R1's entry pass, to decisions/R1. The entries kept here moved nothing but the pointers the split broke and the measurement paragraphs, each cut to one line naming its population, counts, baseline and PR. The #611 entry stays here, an open question on #728. - 2026-10-19 (Derek), #761 — A GENERATION BEFORE THE COMMA AGAINST PAIRED INITIALS AFTER IT: DOTTED, THE INITIALS ARE THE GIVEN NAME AND THE GENERATION STAYS THE SUFFIX; UNDOTTED, THEY ARE THE CREDENTIAL. rules/C12 (a suffix word ending the name before the comma makes the comma a suffix comma) and rules/C5 (paired initials after the comma are the given name) both claim `John Smith Jr., M.J.`, and no rule text said which wins; the C section's move to the skeleton format, which gives a hub Precedence bullets, asked. Measured on 732a2716: `John Smith Jr., M.J.` reads given 'M.J.', family 'John Smith', suffix 'Jr.', and `John Smith Jr., MJ` reads given 'John', family 'Smith', suffix 'Jr., MJ', each reporting `suffix-or-name`. Decided: that reading is the rule, for every generation C12 reads, written in rules/C1's Precedence with both as examples and in C5's and C12's Precedence where they meet. It is the reading C12's Accepted clause (`Lloret Palol II, JA`) and its `Smith Jones Jr., J.A.` boundary already gave behind a two-word surname. The review found a roman numeral S10 reads by shape breaking it today: `John Smith V, M.J.` gives given 'M.J.', family 'John Smith V' and no suffix, while `John Smith V, MJ` reads suffix 'V, MJ'. So does every numeral S10 reads (`John Smith VI, M.J.`, `I`, `IX`, `X` alike). Derek ruled that a bug, so C1's Precedence carries the intended reading as two `deviates:` examples, V for the one-letter count and VI for the shape-only path, tracked by #774. The same pass collected into C1's Precedence the order among C4, C5 and C6 that their own texts already state (C4 counts name words first and asks written case second, C5 is the exception to that count, C6 gates when an unlisted all-caps word joins the class), deciding nothing new there; and the strict switch left C1 for rules/C14 (decisions/C14). -- 2026-10-25 #774 — A NUMERAL ENDING THE FAMILY PART IS READ AS THE COMMA-LESS NAME READS IT. The family part's walk took a word into the suffix by the suffix-piece test alone, so a listed generation stayed the suffix (`John Smith III, M.J.` → suffix 'III') while `V`, which the initial veto refuses, and `VI`, which no list holds, went into the family (`John Smith V, M.J.` → family 'John Smith V'), the split rules/C1's #761 Precedence example tracked as a `deviates:` row. The walk now asks the numeral's predicate (decisions/vocabulary-model's 2026-10-25 2f entry) of a single-word piece shaped as a roman numeral with nothing but suffix pieces behind it: the generation where it reads as one, a one-letter numeral needing two name pieces of the part in front of it as S10's count asks, and reported where a shape reads it, as the comma-less name reports. `John Smith V, M.J.` and `John Smith VI, M.J.` keep the numeral as the suffix, `Smith VI, John` reads suffix 'VI', and `Smith V, John` keeps family 'Smith V' (one word in front), silent, the comma having named the family. The two `deviates:` markers come off. +- 2026-10-25 #774 — A NUMERAL ENDING THE FAMILY PART IS READ AS THE COMMA-LESS NAME READS IT. The family part's walk took a word into the suffix by the suffix-piece test alone, so a listed generation stayed the suffix (`John Smith III, M.J.` → suffix 'III') while `V`, which the initial veto refuses, and `VI`, which no list holds, went into the family (`John Smith V, M.J.` → family 'John Smith V'), the split rules/C1's #761 Precedence example tracked as a `deviates:` row. The walk now asks the numeral's predicate (decisions/vocabulary-model's 2026-10-25 2f entry) of a single-word piece shaped as a roman numeral with nothing but suffix pieces behind it: the generation where it reads as one, a one-letter numeral needing two name words of the part in front of it as S10's count asks, counted as C3 counts them (a connective join is its words, a particle run and its word one: `Vega y Lopez V, Mary` → suffix 'V', the PR review's find), and reported where a shape reads it, as the comma-less name reports. `John Smith V, M.J.` and `John Smith VI, M.J.` keep the numeral as the suffix, `Smith VI, John` reads suffix 'VI', and `Smith V, John` keeps family 'Smith V' (one word in front), silent, the comma having named the family. The two `deviates:` markers come off. diff --git a/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md b/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md index e50df12e..cfbc3347 100644 --- a/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md +++ b/docs/design/decisions/C9-one-character-suffix-word-inside-a-comma-part.md @@ -4,7 +4,7 @@ - Inside a comma part a one-character suffix word is read by what stands before it: behind another suffix word it continues the run, with a period or without (`Smith, PSM I` → suffix 'PSM I'; `Smith, PSM I.` → suffix 'PSM I.'), and behind a name word an ASCII period makes it a middle initial while the bare letter, with no further comma after it, is the suffix (`Smith, John V.` → middle 'V.'; `Smith, John V` → suffix 'V', v1 parity) — 2026-08-25 #430. - The class is one-character suffix vocabulary, not a roman numeral (`Smith, John 2.` → middle '2.'), and a run's first word that is title vocabulary too stays in the run (`Smith, RA I` → suffix 'RA I'; `Smith, Jr. I` → suffix 'Jr. I') — 2026-08-25 #430. -- The no-comma numeral predicate is not asked in the given part, so a numeral behind an initial stays the suffix (`Chang, Andy C I` → middle 'C', suffix 'I') — 2026-08-25 #430. +- The given part asks no initial in front of a numeral, the no-comma reading of one, so a numeral behind an initial stays the suffix (`Chang, Andy C I` → middle 'C', suffix 'I') — 2026-08-25 #430. - A lone letter ending the given part ahead of a further comma is the middle initial whatever stands before it, a credential run in front stopping short of it (`Smith, John PhD I, Jr.` → middle 'I', suffix 'PhD, Jr.'), at the cost of splitting a certification level from its credential (`Smith, John PSM I, Jr.`) — 2026-10-25 #819. - The given part's walk asks the numeral's predicate, which reads a one-letter numeral there by the vocabulary, so an unlisted letter stays the middle initial (`SMITH, JOHN X`), and a numeral of two or more letters as the generation (`Smith, John VI` → suffix 'VI') — 2026-10-25 #819. - The given part's walk is strict-blind and the fork reports nothing: `Smith, John V` reads suffix 'V' under `lenient_comma_suffixes=False` as under the default, with no `suffix-or-name`, where `John Smith V` reports — 2026-08-25 #430. @@ -24,6 +24,6 @@ Open: none in this file. The walk is also strict-blind, pre-existing and unchanged: it calls is_suffix_lenient unconditionally, so `Smith, John V` reads suffix 'V' under `lenient_comma_suffixes=False` as under the default. The reading consults the knob and the walk does not, which is now conspicuous where it was merely unexamined. Not this change's to fix. Not done, and recorded rather than left implicit: neither half emits an ambiguity, though the no-comma `John Smith V` reports `suffix-or-name` for the same fork. The comma path has never reported it (decisions/C8's #428 review entry records PARTICLE_OR_GIVEN suppressed there for a related reason), and adding an emitter needs a trigger, a stage-ownership entry and a case row — more than this fix, and #432 raises it as a secondary observation rather than a requirement. - 2026-10-18 #728 — SPLIT FROM decisions/C1 with the rule. The entries above were written under decisions/C1 while this rule was part of rules/C1, and moved here unchanged but for the pointers the move broke, the measurement paragraphs, each cut to one line naming its population, counts, baseline and PR, and the superseded passages, each cut to its decision and what replaced it; where one says rules/C1 or C1, it means the rule as it then stood, which held this one. -- 2026-10-25 #819 — A LONE LETTER ENDING THE GIVEN PART AHEAD OF A FURTHER COMMA IS THE MIDDLE INITIAL, AND THE WALK ASKS THE NUMERAL'S PREDICATE. Derek agreed reading (A) of #819 on 2026-10-23 with the numeral predicate (decisions/vocabulary-model's 2026-10-23 #752 entry, item 8): the further comma has named the suffix, so a letter ending the given part in front of it has no generation left to be, and #602's credential run (decisions/S8) stops short of it. `Smith, John PhD I, Jr.` and `Smith, John PhD I., Jr.` read middle 'I' and 'I.' again, as before #602, and the run's report on them goes with the reading; `Smith, John PhD I` (no further comma) keeps suffix 'PhD I'. The clause reads any lone letter a script with capitals writes as an initial, bare or with its period, numeral or not (`Smith, John PSM X, Jr.` → middle 'X'), since what it asks is the slot, not the word. +- 2026-10-25 #819 — A LONE LETTER ENDING THE GIVEN PART AHEAD OF A FURTHER COMMA IS THE MIDDLE INITIAL, AND THE WALK ASKS THE NUMERAL'S PREDICATE. Derek agreed reading (A) of #819 on 2026-10-23 with the numeral predicate (decisions/vocabulary-model's 2026-10-23 #752 entry, item 8): the further comma has named the suffix, so a letter ending the given part in front of it has no generation left to be, and #602's credential run (decisions/S8) stops short of it. `Smith, John PhD I, Jr.` and `Smith, John PhD I., Jr.` read middle 'I' and 'I.' again, as before #602, and the run's report on them goes with the reading; `Smith, John PhD I` (no further comma) keeps suffix 'PhD I'. The clause reads a letter written as an initial (rules/T5: a bare capital A to Z, or a letter with its period), numeral or not (`Smith, John PSM X, Jr.` → middle 'X'), since what it asks is the slot, not the word; a bare accented or non-Latin capital is no initial there and stays in the run (`Smith, John PhD É, Jr.` → suffix 'PhD É, Jr.'). WHAT IT COSTS: `Smith, John PSM I, Jr.`, a certification level written ahead of a generation, splits from its credential (rules/C9's Accepted bullet). The spike that proposed this measured no corpus moving; on this change the committed snapshot holds no such name either, and the local corpora were read with it (decisions/vocabulary-model's 2026-10-25 2f entry). THE WALK'S NUMERAL QUESTIONS are the predicate's (decisions/vocabulary-model, mechanisms.md#ONE-PREDICATE-PER-QUESTION): the last-piece test asks it of a bare one-letter numeral, and in this part, where neither the count nor an initial in front is asked, it answers by the vocabulary, so `Smith, John V` keeps suffix 'V' and `SMITH, JOHN X` middle 'X' (FEC writes a middle initial `X` there, the constraint the predicate's design recorded); a one-letter numeral behind a suffix word in a part of nothing but suffix words is asked the same way, its answer unchanged. A numeral of two or more letters, which this rule never governed, is read in this part by S10 since the same change (`Smith, John VI` → suffix 'VI', reported). diff --git a/docs/design/decisions/S8-credentials-trailing-run.md b/docs/design/decisions/S8-credentials-trailing-run.md index ab994572..0bfe24ab 100644 --- a/docs/design/decisions/S8-credentials-trailing-run.md +++ b/docs/design/decisions/S8-credentials-trailing-run.md @@ -14,7 +14,7 @@ Examples name each field as the default order reads it; under the family-first o Not standing: the split of 2026-10-17 #682, carried out. -Elsewhere: the split `Ph. D.` in a main segment (no comma, or before a suffix comma) starts no run, the trailing peel's walk never holding the merged piece (`John Smith Ph. D. Jones` → family 'Jones', suffix 'Ph. D.', unreported; `John Smith Ph. D. Jones, Jr.` → suffix 'Ph. D., Jr.') — decisions/C13's 2026-10-04 (Derek), #603 entry, and the #602 entry's own REVERSED paragraph; a credential opening the part after the comma, and the split `Ph. D.` starting the run as `PhD` does (`Smith, John Ph. D. Jones` → suffix 'Ph. D. Jones') — decisions/C13's 2026-10-04 (Derek), #603 entry; the run taking a lone letter behind a credential ahead of a further comma (`Smith, John PhD I, Jr.` → suffix 'PhD I, Jr.'), which the #602 entry does not record — decisions/C9's Elsewhere line. +Elsewhere: the split `Ph. D.` in a main segment (no comma, or before a suffix comma) starts no run, the trailing peel's walk never holding the merged piece (`John Smith Ph. D. Jones` → family 'Jones', suffix 'Ph. D.', unreported; `John Smith Ph. D. Jones, Jr.` → suffix 'Ph. D., Jr.') — decisions/C13's 2026-10-04 (Derek), #603 entry, and the #602 entry's own REVERSED paragraph; a credential opening the part after the comma, and the split `Ph. D.` starting the run as `PhD` does (`Smith, John Ph. D. Jones` → suffix 'Ph. D. Jones') — decisions/C13's 2026-10-04 (Derek), #603 entry; the run stopping short of a lone letter ending the given part ahead of a further comma (`Smith, John PhD I, Jr.` → middle 'I', suffix 'PhD, Jr.') — decisions/C9's 2026-10-25 #819 entry. Open: [#785](https://github.com/derek73/python-nameparser/issues/785) (whether capitals in a mixed-case name start the run as periods do). diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index b1bddae3..0419966d 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -143,7 +143,7 @@ Open: #752 (phase 2, derived ambiguity; its sub-phases 2a to 2m, each its own PR 5. **Joined credentials land: a word joined by `-` or `/` whose every chunk is listed and none ambiguous reads as a credential, and an unlisted word of capitals so joined, with a chunk of two or more letters, joins the ambiguous class after a comma, gated by C6's case contrast.** The stricter chunk test is because a hyphen also joins names (`Smith-Jones`). Together they moved 1,782 distinct ctgov readings (1,254 and 759, overlapping on 231), nothing in PSC, the committed corpora or the agreement records; FEC is one case, so the contrast never admits a word there. 6. **A split credential merges only where its first chunk has two or more letters, and the merged piece is a member of the ambiguous class, decided by C4's count and S4's words to spare; it starts S8's run (#827), landing after the numeral predicate answers the numeral before a credential.** FEC decided the opening letter: it writes a middle initial before a trailing salutation (`SMITH, JOHN A. MS.`), and `A. MS.` folds to the listed `ams` (merging any pair ending its part, but for two single letters, moved 5,122 FEC names, `A. MS.` 2,246 of them). A two-letter opening chunk needs the count: the 30 FEC merges it made all stood behind a one-word family (`SAM E.` → `same`), against 351 of 352 ctgov merges behind two or more name words. Accepted cost: `M. D.` and `M D` stay unmerged. Left to the split-credential PR: rules/S8 refuses a member of the ambiguous class as a run start, so how a merged piece can be both is that PR's to state (today's `Ph. D.` merge is unambiguous, and S8's #827 row covers it). 7. **A word taken into the credential's run by position alone (`John Smith PhD Jones`) gets no new ambiguity kind.** It is not a clash between two readings; the surfaced reading's source tells it from a word with a declared name reading (item 13). This answers #785's fourth question; its first three (case drops the report and starts the run, the one-case row keeps its report) land through the pool, since retagging the word was measured to erase its other readings (`Doe, John DO` became family `DO Doe` once `do` stopped being an ambiguous acronym, its particle reading meeting P6). - 8. **Every numeral rule asks one predicate, answering for `i` to `x` the way S10's trailing fork does (shape, the count for one letter, an initial before it, the period, the case), with three readings stated:** a lone letter ending the given part ahead of a further comma is the initial (#819; `Smith, John PSM I, Jr.` the accepted cost), a lone letter in a given part after a family comma stays the initial, and a Title-case numeral in a name of mixed case leans to the name. It answers #784 (`John Smith VI MD` reads family `VI` where `III` starts the run) and #774. + 8. (narrowed by the 2026-10-25 #752, phase 2f entry: in the given part after a family comma a one-letter numeral the vocabulary lists stays the suffix, and a one-letter numeral before a credential stays a name word) **Every numeral rule asks one predicate, answering for `i` to `x` the way S10's trailing fork does (shape, the count for one letter, an initial before it, the period, the case), with three readings stated:** a lone letter ending the given part ahead of a further comma is the initial (#819; `Smith, John PSM I, Jr.` the accepted cost), a lone letter in a given part after a family comma stays the initial, and a Title-case numeral in a name of mixed case leans to the name. It answers #784 (`John Smith VI MD` reads family `VI` where `III` starts the run) and #774. 9. **Generation readings are gated by their kind's cue, and a reading with none is `default = false`, compiled to nothing by default and turned on by a pack in phase 3.** `fils` is gated by case and the count. The Brazilian agnomens `Filho`, `Filha`, `Neto`, `Neta`, `Netto`, `Sobrinho` and `Sobrinha` become gated generation readings in `pt.toml` (`pt-BR` where only Brazil writes one), with the count as their only cue: they are written in the case of the name around them. In tse-donors, 405 filed/registry pairs differ only by a trailing agnomen; master gives both spellings the same last family word in 1, and a gate emulated as an ambiguous acronym, so carrying S6's Title-case lean beside the count, in all 405, keeping every PSC surname (`Ana Neto`) and FEC given name (`SMITH, NETO`) the plain reading broke, by the count and the lean. The count-only gate the design chooses was not measured: it is to be measured before 2h lands. They are not part of the surname: a surname is what a family shares, and an agnomen marks the bearer against a relative of the same name, as `Júnior` does. Spanish `H.` (*hijo*), German regnal `I.` and Czech `st.` are `default = false`. When 2h lands it replaces decisions/generational-words' Standing line that the agnomens stay family names. 10. **Decision 10 is unparked at the head of the name only: a modifier is a title only beside a head title there, never in a comma part,** after `resident` and `rt` move to `occupation` and `pro` to `academic`. At the head it fixed 177 PSC records and broke 7; in a comma part it reverses names whose part after the comma is the person's role (247 of 367 ctgov comma-part moves, the part being exactly `Resident` 120 times and `resident` 50, counted as the family taking master's given name). 11. **A contested title's phrase partner is a modifier or a style before it, `of` after it, or a head title after it that the word `attaches` to;** a salutation is never one, and nor is a title before the word that is neither modifier nor style (#853). `attaches` is a property of a title reading naming the title kinds it qualifies, `chief` attaching to `office` keeping `Chief Justice`. @@ -185,7 +185,7 @@ Open: #752 (phase 2, derived ambiguity; its sub-phases 2a to 2m, each its own PR - **2026-10-25 #752 — one predicate per question (neutral).** Three questions that #752's later changes widen had their answers copied between sites, and each now has a shared predicate in `_pipeline/_vocab.py` for the widening to land in (mechanisms.md#ONE-PREDICATE-PER-QUESTION). Two are whole: every site asks them. The numeral's is not yet: the bullet below names the sites found still recognizing a numeral on their own. No reading moves. - **`split_pair(first, second)`**, the split credential (`Ph.` `D.`, decisions/phd-merge): group's merge, `_comma._pieces` and `is_wholly_suffix` each held their own copy of the two-pattern test. The test runs per adjacent pair, so each caller asks `SPLIT_PAIR_OPENS`, the predicate's own first-word pattern, C-level before the call, and a name holding no `Ph.` pays no frame. The split-credential change widens both together (item 6 of the 2026-10-23 #752 entry above). - - **`reads_as_generation(text, preceding, one_case)`**, the numeral: `is_trailing_numeral_suffix` renamed, the predicate every numeral rule is to ask (item 8 of the 2026-10-23 #752 entry above, #784, #774, #819). Today it is the fork's SHAPE half alone. rules/C12's evidence asks a broader question beside its call: whether the word ending the part before the comma is any suffix word, by the suffix-word tag (`John Smith Jr., AD`, `John Smith Esq., AD`), or reads as a generation by the predicate (`John Smith VI, AD`); the tag stays at the call site because `Jr.` and `Esq.` are no numerals. The listed numerals belong in the predicate too, or each numeral rule tests the vocabulary itself again, the hand copy #656's first C12 draft was (it asked the suffix-word tag and missed `VI`). The shape test already takes the listed `ii` to `iv`, so reading the vocabulary inside it adds only the one-letter `i` and `v`, and that moves readings, which this PR may not: answered yes on a grid of 3,927 readings (17 heads, 11 numerals, 7 tails, three orders; the second review, 2026-10-25 on 30dc7aac), `i` and `v` moved 375 readings at the peel's fork, 348 of them in their fields (`John J. V` from middle 'J.', family 'V' to family 'J.', suffix 'V', with a new report), and 144 reports at `reads_as_a_reported_numeral` with no field moving (`John J. V MD` gains a `suffix-or-name`), the ones those sites meet carrying the `initial` tag (`I`, `V`, and their one-case spellings; a lower-case `v` in a name of mixed case carries none, and the peel takes it as a suffix word). Adding `ii` to `iv` alone moved nothing. That half is left to the change that has every numeral rule ask the predicate. Until it lands (it did: the 2026-10-25 #752, phase 2f entry below, which asks the predicate at every site listed here), these sites recognize a numeral on their own, each keeping the answer it had (found 2026-10-25 by two reviews and a grep of `_pipeline/` for the numeral pattern, the suffix-word tag and the suffix predicates; a list found by search, not a proof that no other site exists): the predicate's own missing `i` and `v`, above; rules/S8's run start asks the vocabulary alone (`John Smith III MD` → suffix 'III MD', `John Smith VI MD` → family 'VI', #784); rules/C1's listing form asks the suffix-word tag (`John Smith III, M.J.` keeps suffix 'III' where `John Smith VI, M.J.` and `John Smith V, M.J.` read family 'John Smith VI' and 'John Smith V', #774); the peel takes `ii` to `iv` through `is_suffix_piece` before its fork asks the predicate; rules/C9's walk over the given part after a family comma asks `is_suffix_lenient`, the vocabulary alone (`Smith, John V` → suffix 'V', `Smith, John VI` and `Smith, John X` → middle, where `John Smith VI` reads suffix 'VI'); and after a suffix word in that part a listed one-letter numeral is taken by its tags (`_pieces._numeral_behind_the_initial_veto`), so `Smith, John PhD V` reads suffix 'PhD V' with no report where `Smith, John PhD VI` reports `suffix-or-name`. + - **`reads_as_generation(text, preceding, one_case)`**, the numeral: `is_trailing_numeral_suffix` renamed, the predicate every numeral rule is to ask (item 8 of the 2026-10-23 #752 entry above, #784, #774, #819). Today it is the fork's SHAPE half alone. rules/C12's evidence asks a broader question beside its call: whether the word ending the part before the comma is any suffix word, by the suffix-word tag (`John Smith Jr., AD`, `John Smith Esq., AD`), or reads as a generation by the predicate (`John Smith VI, AD`); the tag stays at the call site because `Jr.` and `Esq.` are no numerals. The listed numerals belong in the predicate too, or each numeral rule tests the vocabulary itself again, the hand copy #656's first C12 draft was (it asked the suffix-word tag and missed `VI`). The shape test already takes the listed `ii` to `iv`, so reading the vocabulary inside it adds only the one-letter `i` and `v`, and that moves readings, which this PR may not: answered yes on a grid of 3,927 readings (17 heads, 11 numerals, 7 tails, three orders; the second review, 2026-10-25 on 30dc7aac), `i` and `v` moved 375 readings at the peel's fork, 348 of them in their fields (`John J. V` from middle 'J.', family 'V' to family 'J.', suffix 'V', with a new report), and 144 reports at `reads_as_a_reported_numeral` with no field moving (`John J. V MD` gains a `suffix-or-name`), the ones those sites meet carrying the `initial` tag (`I`, `V`, and their one-case spellings; a lower-case `v` in a name of mixed case carries none, and the peel takes it as a suffix word). Adding `ii` to `iv` alone moved nothing. That half is left to the change that has every numeral rule ask the predicate. Until it lands (it did: the 2026-10-25 #752, phase 2f entry below, which asks the predicate at every site listed here, the peel taking `ii` to `iv` by the suffix test, which the predicate answers alike), these sites recognize a numeral on their own, each keeping the answer it had (found 2026-10-25 by two reviews and a grep of `_pipeline/` for the numeral pattern, the suffix-word tag and the suffix predicates; a list found by search, not a proof that no other site exists): the predicate's own missing `i` and `v`, above; rules/S8's run start asks the vocabulary alone (`John Smith III MD` → suffix 'III MD', `John Smith VI MD` → family 'VI', #784); rules/C1's listing form asks the suffix-word tag (`John Smith III, M.J.` keeps suffix 'III' where `John Smith VI, M.J.` and `John Smith V, M.J.` read family 'John Smith VI' and 'John Smith V', #774); the peel takes `ii` to `iv` through `is_suffix_piece` before its fork asks the predicate; rules/C9's walk over the given part after a family comma asks `is_suffix_lenient`, the vocabulary alone (`Smith, John V` → suffix 'V', `Smith, John VI` and `Smith, John X` → middle, where `John Smith VI` reads suffix 'VI'); and after a suffix word in that part a listed one-letter numeral is taken by its tags (`_pieces._numeral_behind_the_initial_veto`), so `Smith, John PhD V` reads suffix 'PhD V' with no report where `Smith, John PhD VI` reports `suffix-or-name`. - **`is_cased_paired_initials(text)`**, rules/S3's paired initials in a script with capitals: classify asked the script half inline (#872, decisions/S3). This is the lift the 2026-10-24 #872 entry above names, for classify and the split merge to share. One overlap is left to the split-credential PR to state: item 6 refuses a split credential whose first chunk is a single letter, so for a split `A. B.` that length test answers before this predicate is asked. rules/C5's comma reading keeps the bare `is_paired_initials`, since after a comma a caseless pair reads as a name too (`John Smith, พ.บ.` → given 'พ.บ.'), and moving C5 onto this predicate would change that reading. - **Measured neutral** against the branch, first on 632685c8 and again on 2dd0414b once phase 2a had merged: a grid of 20,364 texts (22 heads covering no comma, a whole name, one word and nothing before a comma, a nickname-only head, a maiden clause, a family comma with a title-particle head and a quoted clause before the pair; 33 spellings including `MA`, `M.D.`, `X.Y.Z.`, `XYZ`, `Ph. D.` and its case variants, `V`, `VI`, `III`, `XI`, `M.J.`, `K.C.` and `พ.บ.`; each text also in lower and upper case), under all three orders, read identically (fields, ambiguity kinds and details, token roles and tags, `initials()`, `capitalized()`), and the same harness reported 45,871, 18, 3,974 and 2,986 differing readings with `split_pair`, C12's call to the numeral predicate, the cased half and the peel's call to it each knocked out. The old and new predicates gave the same answer on 19,769 direct calls. The committed snapshot, every CLDR, GND, Wikidata and conventions agreement record, psc-individuals' agreement record and the ctgov-officials, fec-individuals and tse-donors local snapshots (each recorded from the base with `--tree`) all match, on both bases. psc-individuals' record was itself stale against 632685c8 (3 regressions, 5 progress, 5 changed before this change), so it was re-recorded from each base first. - **Frames**, counted with `sys.setprofile` on both trees: `_CALL_BASELINE`'s reference name and `Smith, John` cost what they did (305 and 158). A word matching `SPLIT_PAIR_OPENS` (`Ph`, `ph.`) with a word after it pays one frame at each site that tests it, whether or not a `D.` follows (`John Smith Ph. D.` 233 → 234, `Smith, John Ph. D.` 239 → 241; the review counted `ph ph ph ph ph ph` 282 → 286), so the cost grows only with the number of such words. C12's evidence keeps its tag test ahead of the call, so it costs what it did (`John Smith Jr., AD` 372). That is a rare shape that no guard counts. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index a2634027..371fb91e 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -160,7 +160,7 @@ This catalog converts discovery into a one-time cost — it does not remove disc **How it works.** A hand-written mirror agrees with its original only until one of them moves, and the drift is invisible in both directions: each site keeps passing its own tests while they disagree about an input neither covers. Every instance below was found as a defect before it was found as a pattern. The destination follows the LAYER, not the topic: a predicate over token text goes to `_vocab`, one over pieces and tags to `_pieces`, both leaves the stages sit on. The piece layer got its own module only in #439; until then those predicates collected in `_group`, the one place both stages could reach, so the sharing between group and assign predates the module (decisions/parse-cost measured #439 as a pure relocation). The cost is a second evaluation of the same predicate, measured for #429 at 1.2–2.2% of a family-comma parse and 0% of every other — the right price there over plumbing a state field the two sites would not otherwise share. Folding two copies into one, take from each caller only what genuinely differs between them, and ask whether a parameter one caller had been filling with a guard is a question the other caller must answer too (#617, decisions/P3). The census of which stage calls which predicate, and how each instance was found, was kept here until #759 and is in git history (`git show fe85bb3d:docs/design/mechanisms.md`); the decisions entries carry each instance's argument. -**Lives in.** nameparser/_pipeline/_vocab.py over token text: `is_wholly_suffix` (#319, decisions/W2), `reads_as_generation` (#401/#421 as `is_trailing_numeral_suffix`, decisions/P5; since #752 the predicate every numeral rule asks, the vocabulary's own numerals read inside it, decisions/vocabulary-model), `split_pair` (#752, the split credential `Ph.` `D.` that group's merge, `_comma._pieces` and `is_wholly_suffix` each tested with their own copy, each asking `SPLIT_PAIR_OPENS` C-level first since the test runs per adjacent pair, decisions/phd-merge), `is_cased_paired_initials` (#752, rules/S3's paired initials in a script with capitals, which classify asked inline; C5 asks the bare `is_paired_initials`, decisions/S3), `maiden_marker_run` (#434, decisions/M2), `delimiter_cores` (#436/#437, decisions/R1), `in_initialless_script` (#322/#323, decisions/cjk-full-stops), `caps_shape_candidate` (#516, decisions/S9). nameparser/_pipeline/_pieces.py over pieces and tags: `is_suffix_piece`, `leading_titles` and `is_leading_title` (#424), `is_title_piece`, `peel_walk`, `trailing_start`, `tail_reading` (the peel-and-chain fixed point that answers where the name pieces end, decisions/H5), `segment_suffix_reading` (#430, decisions/R1), `particle_tail` and `trailing_candidates` (#610, decisions/M2), `join_connectives` (#617, decisions/P3). nameparser/_pipeline/_post_rules.py: `suffix_entries` (#511, decisions/R1), the one instance in a stage rather than a leaf, being a pass over a whole ParseState. tests/v2/test_layering.py holds each module's contract, and a piece predicate growing a dependency on a STAGE shows up there as a widened entry. +**Lives in.** nameparser/_pipeline/_vocab.py over token text: `is_wholly_suffix` (#319, decisions/W2), `reads_as_generation` (#401/#421 as `is_trailing_numeral_suffix`, decisions/P5; since #752 the predicate every rule weighing a numeral as the generation asks, the vocabulary's own numerals read inside it; C4's run refuses a one-letter numeral by its shape alone, `is_single_letter_numeral`, a question about the shape of an initial and not a generation, decisions/vocabulary-model), `split_pair` (#752, the split credential `Ph.` `D.` that group's merge, `_comma._pieces` and `is_wholly_suffix` each tested with their own copy, each asking `SPLIT_PAIR_OPENS` C-level first since the test runs per adjacent pair, decisions/phd-merge), `is_cased_paired_initials` (#752, rules/S3's paired initials in a script with capitals, which classify asked inline; C5 asks the bare `is_paired_initials`, decisions/S3), `maiden_marker_run` (#434, decisions/M2), `delimiter_cores` (#436/#437, decisions/R1), `in_initialless_script` (#322/#323, decisions/cjk-full-stops), `caps_shape_candidate` (#516, decisions/S9). nameparser/_pipeline/_pieces.py over pieces and tags: `is_suffix_piece`, `leading_titles` and `is_leading_title` (#424), `is_title_piece`, `peel_walk`, `trailing_start`, `tail_reading` (the peel-and-chain fixed point that answers where the name pieces end, decisions/H5), `segment_suffix_reading` (#430, decisions/R1), `particle_tail` and `trailing_candidates` (#610, decisions/M2), `join_connectives` (#617, decisions/P3). nameparser/_pipeline/_post_rules.py: `suffix_entries` (#511, decisions/R1), the one instance in a stage rather than a leaf, being a pass over a whole ParseState. tests/v2/test_layering.py holds each module's contract, and a piece predicate growing a dependency on a STAGE shows up there as a widened entry. **Reach for it when.** You are about to write a condition that mirrors, matches or "does what X does" — or you find a comment saying one does. Grep for the other site's predicate and call it instead. diff --git a/docs/design/rules/C1-suffix-comma-decision.md b/docs/design/rules/C1-suffix-comma-decision.md index 0bbca320..5be50472 100644 --- a/docs/design/rules/C1-suffix-comma-decision.md +++ b/docs/design/rules/C1-suffix-comma-decision.md @@ -31,14 +31,12 @@ - **The count against written case** ("Smith, MA", "Smith Jr., Ma"): C4 counts the name words before the comma first and asks written case (S6) second, only where the count has left the word a name. - **Paired initials against the count** ("García Márquez, G.J."): C5 is the stated exception to C4's count, since two words before the comma may be one surname written in two. - **An unlisted all-caps word** ("John Smith, XYZ"): C6 decides whether it joins C4's class at all, only where the name carries a case contrast, and once it has joined, C4's count reads it. -- **A generation ending the name before the comma against paired initials after it** ("John Smith Jr., M.J."): written dotted, the initials keep C5's reading and are the given name, the generation staying the suffix; written undotted as two capitals, they are C12's credential, read as the credential they look like. A numeral ending the family part is read as the comma-less name reads it (S10), its words to spare counted in that part, so the roman numerals S10 reads by their shape stay the suffix as the listed ones do. +- **A generation ending the name before the comma against paired initials after it** ("John Smith Jr., M.J."): written dotted, the initials keep C5's reading and are the given name, the generation staying the suffix; written undotted as two capitals, they are C12's credential, read as the credential they look like. A numeral ending the family part is read as the comma-less name reads it (S10), so the roman numerals S10 reads by their shape stay the suffix as the listed ones do. ``` "John Smith Jr., M.J." → given="M.J." "John Smith Jr., MJ" → suffix="Jr., MJ" "John Smith V, M.J." → suffix="V" "John Smith VI, M.J." → suffix="VI" - "Smith VI, John" → suffix="VI" - "Smith V, John" → family="Smith V" ``` **Accepted:** diff --git a/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md b/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md index 01779a51..1f7af5c2 100644 --- a/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md +++ b/docs/design/rules/C9-one-character-suffix-word-inside-a-comma-part.md @@ -4,12 +4,13 @@ **Then:** it is read by what stands before it: behind another suffix it is describing that suffix and continues the run, written with a period or without, an initial being no shape anyone writes there, and behind a name word it is the generation only when written bare, a period marking it the abbreviation of a name and so a middle initial (T5). -- **Both branches then ask that the generation slot still be open**: where a further comma has already named the suffix (C2), a single letter ending the GIVEN part has no generation left to be, and stays a middle initial — with a period or without, and whether a name word or another suffix word stands before it. So a lone letter ending the given part ahead of a further comma is the middle initial, whatever stands before it, a credential run in front of it stopping short of it. +- **Both branches then ask that the generation slot still be open**: where a further comma has already named the suffix (C2), a single letter ending the GIVEN part has no generation left to be, and stays a middle initial — with a period or without, and whether a name word or another suffix word stands before it. So a lone letter ending the given part ahead of a further comma is the middle initial, whatever stands before it, a credential run in front of it stopping short of it, where the letter is written as an initial (T5). ``` "Smith, John V, Jr." → middle="V" "Smith, John PhD I, Jr." → middle="I" "Smith, John PhD I., Jr." → middle="I." "Smith, John PhD I, Jr." → suffix="PhD, Jr." + "Smith, John PhD É, Jr." → suffix="PhD É, Jr." · boundary ``` - **Only the given part is touched**: a part after the comma that is nothing but suffix words is the credential run, and a letter in it continues that run up to the further comma. - **Longer suffix words are not in question either way**, a numeral of two or more letters being read as S10 reads one in this part. diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index c8861659..cbd5fe80 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -92,8 +92,16 @@ "john smith v, PhD" → ambiguities=() "john j v, PhD" → family="j" ``` -- **Before a family comma** ("John Smith V, M.J."): a numeral ending the family part is read by this rule, its words to spare counted in that part (C1). -- **In the given part after a family comma** ("Smith, John VI"): the given part is not counted, the comma having settled the count there (S5), and no initial in front is asked. A numeral of two or more letters reads as the generation, reporting where its shape alone reads it; a one-letter numeral is the generation only where the vocabulary lists it and it ends the part (C9), so an unlisted one stays the middle initial. +- **Before a family comma** ("John Smith V, M.J."): a numeral ending the family part is read by this rule, its words to spare counted in that part (C1). Only a generation its shape alone reads reports there; a numeral the comma leaves in the family name is silent, the comma having named the family. + ``` + "Smith VI, John" → suffix="VI" + "Smith VI, John" → ambiguities=("suffix-or-name",) + "Smith V, John" → family="Smith V" + "Smith V, John" → ambiguities=() + "Vega y Lopez V, Mary" → suffix="V" + "Smith Vi, John" → ambiguities=() + ``` +- **In the given part after a family comma** ("Smith, John VI"): the given part is not counted, the comma having settled the count there (S5), and no initial in front is asked. A numeral of two or more letters reads as the generation, reporting where its shape alone reads it, save a Title-case one in a mixed-case name, which stays a name word in silence; a one-letter numeral is the generation only where the vocabulary lists it and it ends the part (C9), so an unlisted one stays the middle initial. ``` "Smith, John VI" → suffix="VI" "Smith, John VI MD" → suffix="VI MD" @@ -101,6 +109,7 @@ "Smith, John X" → middle="X" "Smith, John V MD" → middle="V" "Smith, John Vi" → middle="Vi" + "Smith, John Vi" → ambiguities=() ``` **Accepted:** diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 19d79240..f92d05e6 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -60,7 +60,9 @@ and, since #613, at P6's attachment after a family comma where the run holds an ambiguous particle ("Beethoven, Ludwig van"). Emits SUFFIX_OR_NAME at these sites: the trailing roman numeral, taken -or (since #718) declined for want of words to spare, each +or declined (since #718 for want of words to spare, since #752 for its +Title case), a numeral ending the family part before a family comma, +one of two or more letters ending a given part's walk, each ambiguous acronym the trailing peel had to resolve, the bare-suffix carve-out where an input that is nothing but post-nominal vocabulary gets its first word made into the name (H4's suffix half, #491), @@ -564,10 +566,20 @@ def _assign_main(seg_idx: int, state: ParseState, # declined pick (#718), which carries neither credential tag # (the acronym fork's picks carry one) and is never a suffix if token.tags.isdisjoint(_AMBIGUOUS_CREDENTIAL_TAGS): + # a pick a credential run in front then took is the run's + # absorbed name word (rules/S8); otherwise it is declined + # for want of words to spare (one letter) or for its Title + # case (two or more, rules/S10's lean) + if token.role is Role.SUFFIX: + ambiguities.append(_absorbed(piece, tokens)) + continue + why = ("is a roman numeral, but the name has no other word " + "to spare for it" if len(token.text.rstrip(".")) == 1 + else "is a roman numeral written in Title case, which " + "leans to the name") ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, - f"{token.text!r} is a roman numeral, but the name has " - f"no other word to spare for it; read as ", piece, + f"{token.text!r} {why}; read as ", piece, field_tail=" rather than a generational suffix")) continue declined = ("a name part" if token.role is Role.SUFFIX @@ -603,10 +615,23 @@ def _assign_main(seg_idx: int, state: ParseState, return order +def _family_words(pieces: Sequence[Sequence[int]], + tokens: Sequence[WorkToken]) -> int: + """The name words of the family part in front of a numeral, counted + as rules/C3 counts them: a particle run and the word it attaches to + are one word, a connective join counts as its words ('Vega y Lopez' + is two). Asked only of a numeral piece, so a family part pays no + frame for it.""" + return sum(max(1, sum("conjunction" not in tokens[i].tags + and "particle" not in tokens[i].tags + for i in piece)) + for piece in pieces) + + def _family_part_numeral(word: WorkToken, preceding: str, names: int, one_case: bool | None) -> Role | None: """The role of a roman numeral ending the family part before a - family comma, behind `names` pieces of that part, as the comma-less + family comma, behind `names` name words of that part, as the comma-less name reads one ending it (rules/S10), or None where the numeral's predicate reads no numeral there (an initial in front, a letter with its period): the suffix where it reads as the generation, a @@ -643,11 +668,11 @@ def _reads_as_a_trailing_suffix(piece: Sequence[int], is_suffix_piece said no, and a lenient single token it refuses is one carrying the `initial` tag, so the shape is already implied. - NOT reads_as_generation, though it answers the period half: - it also refuses a numeral behind an initial-shaped piece, which is - a no-comma rule and the opposite of this path's v1 parity -- - 'Chang, Andy C I' is first Andy, middle C, suffix I, and asking - that predicate here made the numeral a middle. The #401/#421 entry + reads_as_generation is asked with no word in front (`preceding` + None): its initial-in-front refusal is a no-comma rule and the + opposite of this path's v1 parity -- 'Chang, Andy C I' is first + Andy, middle C, suffix I, and asking it with the word in front + made the numeral a middle. The #401/#421 entry under decisions/P5 records the same fork declining to transfer to this walk under LENIENT. """ @@ -1602,7 +1627,8 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: for j in range(k + 1, len(fam_pieces))) and (numeral := _family_part_numeral( word, tokens[fam_pieces[k - 1][0]].text, - k - lead, state.one_case)) is not None): + _family_words(fam_pieces[lead:k], tokens), + state.one_case)) is not None): _set_roles(tokens, piece, numeral) # reported where taken: a name word here is the family # name the comma has just named, which nobody misreads diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 82096f11..6cfb22a7 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -352,13 +352,16 @@ def _maiden_take(pieces: Sequence[Sequence[int]], f"than reading it as one", tuple(last))) elif tuple(last) in picked: # rules/S10: "either reading is reported" -- a numeral the read - # declined for want of words to spare, which the clause keeps + # declined, for want of words to spare or for its Title case, + # which the clause keeps + why = ("but the name has no other word to spare for it" + if len(word.text.rstrip(".")) == 1 + else "but written in Title case it leans to the name") ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_NAME, f"{word.text!r} ending the maiden name is a roman numeral, " - f"but the name has no other word to spare for it; the " - f"maiden marker's clause keeps it rather than reading it as " - f"a generational suffix", tuple(last))) + f"{why}; the maiden marker's clause keeps it rather than " + f"reading it as a generational suffix", tuple(last))) elif (released and len(last) == 1 and end - 2 >= lo and reads_as_a_reported_numeral( word.text, word.tags, tokens[pieces[end - 2][0]].text, diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 8c2b427c..d6fa3e76 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -566,14 +566,11 @@ def _numeral_behind_the_initial_veto(piece: Sequence[int], """ if len(piece) != 1: return False - tok = tokens[piece[0]] - if "vocab:suffix" not in tok.tags or "initial" not in tok.tags: - return False - # a one-letter numeral is the numeral's predicate's to answer, which - # in a comma part reads it by the vocabulary, as here (#752) - if _ROMAN.match(stem := tok.text.rstrip(".")) is not None and stem: - return reads_as_generation(stem, tok.tags, None, None) - return True + # A one-letter numeral here is the numeral's predicate's yes by + # construction: in a comma part it reads one by the vocabulary, and + # the tag below is that vocabulary (#752), so it is not asked again. + tags = tokens[piece[0]].tags + return "vocab:suffix" in tags and "initial" in tags def _anchors(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: @@ -954,8 +951,10 @@ class Peel(NamedTuple): trailing run, #752), `picks` the bare ambiguous acronyms the peel had to resolve, in peel order, either way (the last may sit at rest[names - 1]), and - a one-letter numeral the fork declined for want of words to spare - (#718), always the walk's last piece and a name. + a numeral the fork declined, a one-letter one for want of words to + spare (#718), always the walk's last piece, or one of two or more + letters for its Title case (#752), anywhere in the trailing run, + and a name unless a credential run in front of it takes it. `anchors` is `peel_trailing`'s working state for `tail_reading` alone -- the walk's `credential_anchors` pass over `rest` where a member needed one, handed to the next pass of the fixed point @@ -1347,6 +1346,7 @@ def run_start(rest: Sequence[int], names: int, or _ROMAN.match(tokens[pieces[q][0]].text.rstrip(".")) is not None) and tags.isdisjoint(_NOT_A_RUN_START) + # p >= 2 here: two name words stand in front and starts_a_credential_run( pieces[q], ptags[q], tokens, tokens[pieces[rest[p - 1]][0]].text, one_case)): diff --git a/tests/v2/rules_budget.json b/tests/v2/rules_budget.json index 6267b3e7..3eb51eb6 100644 --- a/tests/v2/rules_budget.json +++ b/tests/v2/rules_budget.json @@ -1,7 +1,7 @@ { "A1": {"words": 144, "then_words": 120, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 1, "accepted_words": 32}, "A2": {"words": 60, "then_words": 31, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, - "C1": {"words": 456, "then_words": 196, "because_sentences": 1, "except": 0, "precedence": 5, "accepted": 1, "accepted_words": 116}, + "C1": {"words": 448, "then_words": 196, "because_sentences": 1, "except": 0, "precedence": 5, "accepted": 1, "accepted_words": 116}, "C2": {"words": 492, "then_words": 399, "because_sentences": 1, "except": 2, "precedence": 0, "accepted": 1, "accepted_words": 81}, "C3": {"words": 260, "then_words": 228, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 1, "accepted_words": 173}, "C4": {"words": 498, "then_words": 371, "because_sentences": 1, "except": 0, "precedence": 2, "accepted": 0, "accepted_words": 0}, @@ -9,7 +9,7 @@ "C6": {"words": 207, "then_words": 162, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "C7": {"words": 119, "then_words": 68, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "C8": {"words": 176, "then_words": 110, "because_sentences": 1, "except": 0, "precedence": 1, "accepted": 1, "accepted_words": 36}, - "C9": {"words": 289, "then_words": 210, "because_sentences": 1, "except": 0, "precedence": 1, "accepted": 1, "accepted_words": 48}, + "C9": {"words": 298, "then_words": 219, "because_sentences": 1, "except": 0, "precedence": 1, "accepted": 1, "accepted_words": 48}, "C10": {"words": 377, "then_words": 225, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 5, "accepted_words": 311}, "C11": {"words": 372, "then_words": 247, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 0, "accepted_words": 0}, "C12": {"words": 283, "then_words": 107, "because_sentences": 2, "except": 1, "precedence": 1, "accepted": 2, "accepted_words": 59}, @@ -61,7 +61,7 @@ "S7": {"words": 570, "then_words": 388, "because_sentences": 1, "except": 1, "precedence": 2, "accepted": 1, "accepted_words": 147}, "S8": {"words": 381, "then_words": 267, "because_sentences": 1, "except": 1, "precedence": 1, "accepted": 0, "accepted_words": 0}, "S9": {"words": 430, "then_words": 229, "because_sentences": 2, "except": 0, "precedence": 3, "accepted": 0, "accepted_words": 0}, - "S10": {"words": 624, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 4, "accepted_words": 176}, + "S10": {"words": 665, "then_words": 246, "because_sentences": 3, "except": 4, "precedence": 3, "accepted": 4, "accepted_words": 176}, "T1": {"words": 93, "then_words": 18, "because_sentences": 1, "except": 1, "precedence": 0, "accepted": 3, "accepted_words": 83}, "T2": {"words": 43, "then_words": 14, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, "T3": {"words": 56, "then_words": 23, "because_sentences": 1, "except": 0, "precedence": 0, "accepted": 0, "accepted_words": 0}, diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 6267001d..8e1bb4a7 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -575,6 +575,7 @@ "Smith V, John" "Smith VI, AD" "Smith VI, John" +"Smith Vi, John" "Smith and King" "Smith née Jones" "Smith née Jones PhD" @@ -595,6 +596,7 @@ "Smith, John PhD I., Jr." "Smith, John PhD Jones" "Smith, John PhD de Jr." +"Smith, John PhD É, Jr." "Smith, John Prof." "Smith, John V" "Smith, John V MD" @@ -652,6 +654,7 @@ "VI, AD" "Van Buren, Ed" "Van Johnson" +"Vega y Lopez V, Mary" "Vega, Juan de la" "Vincent van Gogh van Beethoven" "Wang M.Eng." diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index fe24a3b0..b5006376 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -2974,6 +2974,7 @@ {"name": "Smith V, John", "given": "John", "family": "Smith V", "initials": "J. S. V."} {"name": "Smith VI, AD", "given": "AD", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "A. S.", "capitalized_force": "Ad Smith VI"} {"name": "Smith VI, John", "given": "John", "family": "Smith", "suffix": "VI", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} +{"name": "Smith Vi, John", "given": "John", "family": "Smith Vi", "initials": "J. S. V."} {"name": "Smith and King", "given": "Smith and King", "ambiguities": ["TITLE_OR_NAME"], "initials": "S. K."} {"name": "Smith born", "given": "Smith", "family": "born", "initials": "S. b.", "capitalized_force": "Smith Born"} {"name": "Smith nee Jones, Jane MA", "given": "Jane", "family": "Smith", "suffix": "MA", "maiden": "Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} @@ -3014,6 +3015,7 @@ {"name": "Smith, John PhD I., Jr.", "given": "John", "middle": "I.", "family": "Smith", "suffix": "PhD, Jr.", "initials": "J. I. S."} {"name": "Smith, John PhD Jones", "given": "John", "family": "Smith", "suffix": "PhD Jones", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John PhD de Jr.", "given": "John", "family": "de Smith", "suffix": "PhD, Jr.", "initials": "J. S."} +{"name": "Smith, John PhD É, Jr.", "given": "John", "family": "Smith", "suffix": "PhD É, Jr.", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "J. S."} {"name": "Smith, John Prof.", "title": "Prof.", "given": "John", "family": "Smith", "initials": "J. S."} {"name": "Smith, John V", "given": "John", "family": "Smith", "suffix": "V", "initials": "J. S."} {"name": "Smith, John V MD", "given": "John", "middle": "V", "family": "Smith", "suffix": "MD", "initials": "J. V. S."} @@ -3280,6 +3282,7 @@ {"name": "Varano, Costanza da", "given": "Costanza", "family": "da Varano", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "C. V."} {"name": "Vatson Con Hamiş", "order": "FAMILY_FIRST", "given": "Con", "middle": "Hamiş", "family": "Vatson", "initials": "C. H. V."} {"name": "Vega Smith", "given": "Vega", "family": "Smith", "initials": "V. S."} +{"name": "Vega y Lopez V, Mary", "given": "Mary", "family": "Vega y Lopez", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. V. L."} {"name": "Vega, Juan de la", "given": "Juan", "family": "de la Vega", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "J. V."} {"name": "Velasquez y Garcia, Dr. Juan", "title": "Dr.", "given": "Juan", "family": "Velasquez y Garcia", "initials": "J. V. G."} {"name": "Velasquez y Garcia, Dr. Juan III", "title": "Dr.", "given": "Juan", "family": "Velasquez y Garcia", "suffix": "III", "initials": "J. V. G."} From d71e7d717596155c122f2b6a1f0b151d6c1d35fd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 03:48:39 -0700 Subject: [PATCH 3/4] Count the family part by pieces again, recording the connective split (#752) The second review of the fix commit found that counting a connective join's words, which fixed 'Vega y Lopez V, Mary', broke the joins both spellings make ('Smith and Jones V, Mary'). The piece count stands, and decisions/C1 records the remaining split with case rows for both. Co-Authored-By: Claude Opus 5.5 --- .../decisions/C1-suffix-comma-decision.md | 3 ++- .../design/rules/S10-trailing-roman-numeral.md | 1 - nameparser/_pipeline/_assign.py | 18 ++---------------- tests/v2/cases.py | 17 +++++++++++++++++ tools/differential/corpus_rules.jsonl | 1 - tools/differential/snapshot.jsonl | 1 - 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/design/decisions/C1-suffix-comma-decision.md b/docs/design/decisions/C1-suffix-comma-decision.md index f4ba7643..19fb4064 100644 --- a/docs/design/decisions/C1-suffix-comma-decision.md +++ b/docs/design/decisions/C1-suffix-comma-decision.md @@ -34,4 +34,5 @@ Open: none in this file. SUPERSEDES: decisions/C4's 2026-10-01 #562 bullet's mechanism and its accepted reports (its pair report and the settled-run test); decisions/C13's 2026-10-04 #603 bullet's "segment decides" wording; and decisions/C3's #575 bullet's account of WHY segment builds its unit facts from vocabulary -- `_comma._whole_name` counts over classify's tags, and `name_word_count`, which reads the vocabulary, is asked by decide and by script_segment, the agreement sweep `test_surname_unit_tags_agree_with_classify` keeping the vocabulary view equal to the tags. - 2026-10-18 #728 — SPLIT. rules/C1 held what are now C1 and C3–C11, and this file their history; each entry moved to the file of the rule it decided, and the #429, #436/#437 and #511 entries, which decide R1's entry pass, to decisions/R1. The entries kept here moved nothing but the pointers the split broke and the measurement paragraphs, each cut to one line naming its population, counts, baseline and PR. The #611 entry stays here, an open question on #728. - 2026-10-19 (Derek), #761 — A GENERATION BEFORE THE COMMA AGAINST PAIRED INITIALS AFTER IT: DOTTED, THE INITIALS ARE THE GIVEN NAME AND THE GENERATION STAYS THE SUFFIX; UNDOTTED, THEY ARE THE CREDENTIAL. rules/C12 (a suffix word ending the name before the comma makes the comma a suffix comma) and rules/C5 (paired initials after the comma are the given name) both claim `John Smith Jr., M.J.`, and no rule text said which wins; the C section's move to the skeleton format, which gives a hub Precedence bullets, asked. Measured on 732a2716: `John Smith Jr., M.J.` reads given 'M.J.', family 'John Smith', suffix 'Jr.', and `John Smith Jr., MJ` reads given 'John', family 'Smith', suffix 'Jr., MJ', each reporting `suffix-or-name`. Decided: that reading is the rule, for every generation C12 reads, written in rules/C1's Precedence with both as examples and in C5's and C12's Precedence where they meet. It is the reading C12's Accepted clause (`Lloret Palol II, JA`) and its `Smith Jones Jr., J.A.` boundary already gave behind a two-word surname. The review found a roman numeral S10 reads by shape breaking it today: `John Smith V, M.J.` gives given 'M.J.', family 'John Smith V' and no suffix, while `John Smith V, MJ` reads suffix 'V, MJ'. So does every numeral S10 reads (`John Smith VI, M.J.`, `I`, `IX`, `X` alike). Derek ruled that a bug, so C1's Precedence carries the intended reading as two `deviates:` examples, V for the one-letter count and VI for the shape-only path, tracked by #774. The same pass collected into C1's Precedence the order among C4, C5 and C6 that their own texts already state (C4 counts name words first and asks written case second, C5 is the exception to that count, C6 gates when an unlisted all-caps word joins the class), deciding nothing new there; and the strict switch left C1 for rules/C14 (decisions/C14). -- 2026-10-25 #774 — A NUMERAL ENDING THE FAMILY PART IS READ AS THE COMMA-LESS NAME READS IT. The family part's walk took a word into the suffix by the suffix-piece test alone, so a listed generation stayed the suffix (`John Smith III, M.J.` → suffix 'III') while `V`, which the initial veto refuses, and `VI`, which no list holds, went into the family (`John Smith V, M.J.` → family 'John Smith V'), the split rules/C1's #761 Precedence example tracked as a `deviates:` row. The walk now asks the numeral's predicate (decisions/vocabulary-model's 2026-10-25 2f entry) of a single-word piece shaped as a roman numeral with nothing but suffix pieces behind it: the generation where it reads as one, a one-letter numeral needing two name words of the part in front of it as S10's count asks, counted as C3 counts them (a connective join is its words, a particle run and its word one: `Vega y Lopez V, Mary` → suffix 'V', the PR review's find), and reported where a shape reads it, as the comma-less name reports. `John Smith V, M.J.` and `John Smith VI, M.J.` keep the numeral as the suffix, `Smith VI, John` reads suffix 'VI', and `Smith V, John` keeps family 'Smith V' (one word in front), silent, the comma having named the family. The two `deviates:` markers come off. +- 2026-10-25 #774 — A NUMERAL ENDING THE FAMILY PART IS READ AS THE COMMA-LESS NAME READS IT. The family part's walk took a word into the suffix by the suffix-piece test alone, so a listed generation stayed the suffix (`John Smith III, M.J.` → suffix 'III') while `V`, which the initial veto refuses, and `VI`, which no list holds, went into the family (`John Smith V, M.J.` → family 'John Smith V'), the split rules/C1's #761 Precedence example tracked as a `deviates:` row. The walk now asks the numeral's predicate (decisions/vocabulary-model's 2026-10-25 2f entry) of a single-word piece shaped as a roman numeral with nothing but suffix pieces behind it: the generation where it reads as one, a one-letter numeral needing two pieces of the part in front of it, as S10's count asks of the comma-less name's pieces, and reported where a shape reads it, as the comma-less name reports. `John Smith V, M.J.` and `John Smith VI, M.J.` keep the numeral as the suffix, `Smith VI, John` reads suffix 'VI', and `Smith V, John` keeps family 'Smith V' (one word in front), silent, the comma having named the family. The two `deviates:` markers come off. + ACCEPTED, found by the PR's reviews: a part holding a connective surname reads its one-letter numeral as its comma-less twin does only where the two join the connective alike. The part before the comma joins `Vega y Lopez` into one piece where the comma-less `Vega y Lopez V` leaves it unjoined, so `Vega y Lopez V, Mary` keeps family 'Vega y Lopez V' where the twin reads suffix 'V'. Counting a join's words instead fixed that row and broke the longer joins, which both spellings join (`Smith and Jones V, Mary`, `Ortega y Gasset y Lopez V, Mary`, family 'V' comma-less and suffix 'V' with the count), so the piece count stands. Also as on master: `St Smith V, Mary` (a title-particle is no title run before a family comma, rules/H6) and `Jane nee Smith VI, Mary` (the clause keeps the numeral) differ from their twins. diff --git a/docs/design/rules/S10-trailing-roman-numeral.md b/docs/design/rules/S10-trailing-roman-numeral.md index cbd5fe80..fed6c9fe 100644 --- a/docs/design/rules/S10-trailing-roman-numeral.md +++ b/docs/design/rules/S10-trailing-roman-numeral.md @@ -98,7 +98,6 @@ "Smith VI, John" → ambiguities=("suffix-or-name",) "Smith V, John" → family="Smith V" "Smith V, John" → ambiguities=() - "Vega y Lopez V, Mary" → suffix="V" "Smith Vi, John" → ambiguities=() ``` - **In the given part after a family comma** ("Smith, John VI"): the given part is not counted, the comma having settled the count there (S5), and no initial in front is asked. A numeral of two or more letters reads as the generation, reporting where its shape alone reads it, save a Title-case one in a mixed-case name, which stays a name word in silence; a one-letter numeral is the generation only where the vocabulary lists it and it ends the part (C9), so an unlisted one stays the middle initial. diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index f92d05e6..cc4d987b 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -615,23 +615,10 @@ def _assign_main(seg_idx: int, state: ParseState, return order -def _family_words(pieces: Sequence[Sequence[int]], - tokens: Sequence[WorkToken]) -> int: - """The name words of the family part in front of a numeral, counted - as rules/C3 counts them: a particle run and the word it attaches to - are one word, a connective join counts as its words ('Vega y Lopez' - is two). Asked only of a numeral piece, so a family part pays no - frame for it.""" - return sum(max(1, sum("conjunction" not in tokens[i].tags - and "particle" not in tokens[i].tags - for i in piece)) - for piece in pieces) - - def _family_part_numeral(word: WorkToken, preceding: str, names: int, one_case: bool | None) -> Role | None: """The role of a roman numeral ending the family part before a - family comma, behind `names` name words of that part, as the comma-less + family comma, behind `names` pieces of that part, as the comma-less name reads one ending it (rules/S10), or None where the numeral's predicate reads no numeral there (an initial in front, a letter with its period): the suffix where it reads as the generation, a @@ -1627,8 +1614,7 @@ def reads_as_a_suffix(m: int, titled: frozenset[int]) -> bool: for j in range(k + 1, len(fam_pieces))) and (numeral := _family_part_numeral( word, tokens[fam_pieces[k - 1][0]].text, - _family_words(fam_pieces[lead:k], tokens), - state.one_case)) is not None): + k - lead, state.one_case)) is not None): _set_roles(tokens, piece, numeral) # reported where taken: a name word here is the family # name the comma has just named, which nobody misreads diff --git a/tests/v2/cases.py b/tests/v2/cases.py index f58080c9..ff4d90e4 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -8589,6 +8589,23 @@ def _check_cjk_shape_purity(self) -> None: "sparing the one-letter one, so 'V' stays the suffix as " "'Jr.' and 'III' do. 1.4.0 through 2.6.0 read family " "'John Smith V'"), + Case("a_one_letter_numeral_behind_a_connective_surname_stays_family", + "Vega y Lopez V, Mary", + {"given": "Mary", "family": "Vega y Lopez V"}, + notes="C1's count of the family part goes by pieces, and the " + "part joins 'Vega y Lopez' into one, so the 'V' has no " + "word to spare and stays the family name, silent. The " + "comma-less twin leaves the connective unjoined and reads " + "suffix 'V' (decisions/C1's 2026-10-25 #774 entry, " + "ACCEPTED); counting the join's words was declined for " + "the row below"), + Case("a_one_letter_numeral_behind_a_joined_surname_matches_its_twin", + "Smith and Jones V, Mary", + {"given": "Mary", "family": "Smith and Jones V"}, + notes="both spellings join 'Smith and Jones', so the 'V' has " + "no word to spare with the comma or without it; counting " + "a join's words (the first fix for the row above) read " + "suffix 'V' here and broke the match"), Case("a_lone_letter_ahead_of_a_further_comma_is_the_initial", "Smith, John PhD I, Jr.", {"given": "John", "middle": "I", "family": "Smith", diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 8e1bb4a7..5988d1e6 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -654,7 +654,6 @@ "VI, AD" "Van Buren, Ed" "Van Johnson" -"Vega y Lopez V, Mary" "Vega, Juan de la" "Vincent van Gogh van Beethoven" "Wang M.Eng." diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index b5006376..f3ad7eec 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -3282,7 +3282,6 @@ {"name": "Varano, Costanza da", "given": "Costanza", "family": "da Varano", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "C. V."} {"name": "Vatson Con Hamiş", "order": "FAMILY_FIRST", "given": "Con", "middle": "Hamiş", "family": "Vatson", "initials": "C. H. V."} {"name": "Vega Smith", "given": "Vega", "family": "Smith", "initials": "V. S."} -{"name": "Vega y Lopez V, Mary", "given": "Mary", "family": "Vega y Lopez", "suffix": "V", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "M. V. L."} {"name": "Vega, Juan de la", "given": "Juan", "family": "de la Vega", "ambiguities": ["PARTICLE_OR_GIVEN"], "initials": "J. V."} {"name": "Velasquez y Garcia, Dr. Juan", "title": "Dr.", "given": "Juan", "family": "Velasquez y Garcia", "initials": "J. V. G."} {"name": "Velasquez y Garcia, Dr. Juan III", "title": "Dr.", "given": "Juan", "family": "Velasquez y Garcia", "suffix": "III", "initials": "J. V. G."} From 1114dd8417ed34268b351278c15364738ba4fdc8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 25 Oct 2026 10:54:04 -0700 Subject: [PATCH 4/4] Pin how a declined numeral is reported (#752) Covers the branch where a credential run in front takes a numeral the peel declined, which no test reached (Codecov's patch check). Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_parser.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 2851194e..798116ab 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -761,6 +761,28 @@ def test_the_numeral_fork_fires_on_the_last_piece_only() -> None: assert (n.family, n.suffix) == (family, "Jr") +@pytest.mark.parametrize("text, fragment", [ + # rules/S10's Title-case lean, worded for what declined the numeral + ("John Smith Vi", "written in Title case, which leans to the name"), + ("Jane Doe nee Smith Vi", "written in Title case it leans to the name"), + # one-letter, declined for want of words to spare, as #718 words it + ("John V", "the name has no other word to spare"), + # a declined numeral a credential run in front then takes is the + # run's absorbed name word (rules/S8), not a numeral read as a name + ("John Smith III Vi", "follows a credential"), + ("John Smith PhD Vi", "follows a credential"), +]) +def test_a_declined_numeral_is_reported_for_what_declined_it( + text: str, fragment: str) -> None: + # #752's review: the Title-case pick first took #718's "no other + # word to spare" wording, and one a run took read "as a suffix + # rather than a generational suffix" + details = [a.detail for a in parse(text).ambiguities] + assert any(fragment in d for d in details), (text, details) + assert not any("rather than a generational suffix" in d + and "suffix rather" in d for d in details), details + + @pytest.mark.parametrize("bound", ["abd", "abu"]) @pytest.mark.parametrize("numeral", ["I", "X"]) def test_every_bound_word_spares_the_family_before_every_numeral(