diff --git a/AGENTS.md b/AGENTS.md index f390d6935..40192bffc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,8 @@ uv run ruff check uv run python -m nameparser "Dr. Juan Q. Xavier de la Vega III" # Rebuild the compiled vocabulary after editing nameparser/vocab/*.toml -# (--check exits 1 if nameparser/config/_compiled.py is stale), and look +# (--check exits 1 if nameparser/config/_compiled.py or the reading +# index nameparser/config/_index.py is stale), and look # words up by text, kind, issue or exclusion uv run python tools/vocab/build.py uv run python tools/vocab/query.py --word 殿 @@ -204,7 +205,7 @@ Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely o ### Vocabulary (`nameparser/vocab/`) and the configuration layer (`nameparser/config/`) -**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` compiles it into the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line, and runs the build checks that replaced the modules' import-time asserts (its docstring numbers them). Each `config/` module re-exports its constants from there (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, naming the refresh command, and gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. +**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and phase 3's choice of languages compile alike; the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed at runtime by configuring a `Lexicon`, never by editing the constant, and the shipped default is changed by editing the TOML and rebuilding. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the hazard the freeze closes is still live for it: `CAPITALIZATION_EXCEPTIONS['dclinpsy'] = 'DClinPsy'` after the first parse misses the cached `Lexicon.default()`. Same advice — configure the object (`dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. @@ -225,8 +226,8 @@ The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2.0 and then 3.0 deleted. These conventions apply to all of it. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser`, `nameparser.locales` and the vocabulary data modules under `nameparser.config`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; `__main__` imports only the public package; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. +- **Module layout**: every module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_reading_index.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser`, `nameparser.locales` and the vocabulary data modules under `nameparser.config`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only, e.g. `config/maiden_markers.py`); `_reading_index` imports nothing internal at module level and the index, `config/_index.py`, at call time, since `tools/vocab/build.py` loads it by path; `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; `__main__` imports only the public package; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). **When the message hands the reader code to paste, that code has to survive a type checker, and then RUN** — nameparser ships `py.typed`, and a placeholder can type-check and still raise: the 2.4 release sweep's first fix to the `capitalization_exceptions` hint offered `('key', 'Value')`, which passes mypy and fails #459's mask check, so `test_add_capitalization_exceptions_raises_pointing_at_replace` now executes the offered code. #337's segmenterless warning offered `Policy(segment_scripts=())`, an `arg-type` error, because these fields are annotated with what they STORE rather than everything the constructor accepts. Prefer the `frozenset()` / `()` spellings in messages and docstrings, and pin the offered spelling in a test — the warning tests matched on `ja_segmenter` and never checked the actionable half of the message. **A warning emitted in `Parser.__post_init__` needs `parser_for` to re-emit it from its own frame** (the `catch_warnings(record=True)` block at its return): `__post_init__`'s `stacklevel` is sized for direct `Parser(...)` construction, and through `parser_for`'s extra frame the default one-line rendering attributes the warning to the library's own `return Parser(...)` — the exact call the message tells the user to change becomes invisible. No single stacklevel serves both entry points; a new construction warning gets the re-emission for free, but a new CONSTRUCTION SITE for `Parser` inside this package needs its own re-emission or its callers get library-attributed warnings (#337 review). @@ -252,7 +253,7 @@ The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2 **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). -**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `AFTER_GIVEN_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `tools/vocab/build.py`'s `BEHAVIORAL` table, a branch in `compile_pool` adding the words whose reading carries it, a constant in its `CONSTANTS` roster and a key of the same name in its `Compiled` TypedDict (an import-time assert holds the two to one list), and the `NAME = _compiled.NAME` assignment in the config module. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A marker that records a clash with a NAME is the other shape: derived from the word's name reading beside the clashing one (`SUFFIX_ACRONYMS_AMBIGUOUS`, `SUFFIX_WORDS_AMBIGUOUS`, and the complement `NON_GIVEN_NAME_PARTICLES` works from), not written as a property, so it too holds by construction (decisions/vocabulary-model, 2026-10-25). A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). A caller's `Lexicon.add()` to a sub-set does **not** add the word to its base, so a caller adding a brand-new word adds it to both. Two traps a gating set met (#752 phase 2i). Where a caller listing a word in the BASE set has always meant "read it as this everywhere" (the 1.4 recipe `c.titles.add("dean")`), the gated words compile OUT of the base rather than into both, or the gate overrides the caller's listing: `TITLES_AMBIGUOUS` is not a subset of `TITLES` for that reason, and a word in both reads as a title. And `_VOCAB_FIELDS` is read whole by two rosters in `_pipeline/_vocab.py`, the capitals shape's "unlisted" (`in_any_wordlist`) and the comma contrast's "claimed as non-name" (`_NON_NAME_FIELDS`), so a new field joins both by default; a field whose words may be names leaves both, which the grid found as `John Smith, DEAN` reversed where C6 had read a credential; #789's `given_name_abbreviations` met it again (`John Smith, JAS` and `Jas SMITH, XYZ` reversed), found by review because its grid held only the dotted spellings the set was for -- a new field's grid holds the spellings it must NOT move too. +**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `AFTER_GIVEN_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `nameparser/_reading_index.py`'s `BEHAVIORAL` table, a branch in its `compile_readings` adding the words whose reading carries it and a key in its `Compiled` TypedDict, a constant of the same name in `tools/vocab/build.py`'s `CONSTANTS` roster (an import-time assert there holds the two to one list), and the `NAME = _compiled.NAME` assignment in the config module. A property the compiler reads is kept in the reading index by `indexed()`, which reads `BEHAVIORAL`, so a new one reaches a choice of languages too. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A marker that records a clash with a NAME is the other shape: derived from the word's name reading beside the clashing one (`SUFFIX_ACRONYMS_AMBIGUOUS`, `SUFFIX_WORDS_AMBIGUOUS`, and the complement `NON_GIVEN_NAME_PARTICLES` works from), not written as a property, so it too holds by construction (decisions/vocabulary-model, 2026-10-25). A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). A caller's `Lexicon.add()` to a sub-set does **not** add the word to its base, so a caller adding a brand-new word adds it to both. Two traps a gating set met (#752 phase 2i). Where a caller listing a word in the BASE set has always meant "read it as this everywhere" (the 1.4 recipe `c.titles.add("dean")`), the gated words compile OUT of the base rather than into both, or the gate overrides the caller's listing: `TITLES_AMBIGUOUS` is not a subset of `TITLES` for that reason, and a word in both reads as a title. And `_VOCAB_FIELDS` is read whole by two rosters in `_pipeline/_vocab.py`, the capitals shape's "unlisted" (`in_any_wordlist`) and the comma contrast's "claimed as non-name" (`_NON_NAME_FIELDS`), so a new field joins both by default; a field whose words may be names leaves both, which the grid found as `John Smith, DEAN` reversed where C6 had read a credential; #789's `given_name_abbreviations` met it again (`John Smith, JAS` and `Jas SMITH, XYZ` reversed), found by review because its grid held only the dotted spellings the set was for -- a new field's grid holds the spellings it must NOT move too. **Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. After the change, `tests/v2/test_snapshot.py` lists every corpus name it moved, grouped by changed field -- read them, and re-record with `uv run python tools/differential/snapshot.py --record` once every move is intended, so the PR diff of `snapshot.jsonl` shows the reviewer the same list. `uv run python tools/differential/cldr_agreement.py` reports how the tree's agreement with CLDR's stated name fields moved; `tests/v2/test_cldr_agreement.py` fails on any move, and `tools/differential/cldr_agreement.json` doubles as a measured list of names the parser and CLDR read differently. The same command and test score the GND corpus's real names (`gnd_fields.jsonl`, record `gnd_agreement.json`) by the same rules, and `--record` rewrites every record, printing per source what it accepts. A locale-pack change reads the second view the same command prints, per pack (`cldr_agreement_locales.json`, #672, and `wikidata_native_agreement_locales.json` for real Han and kana names); its `JA` rows run only with the `ja` extra installed. A change to Han, Hangul or kana order or to Korean segmentation reads `wikidata_native_agreement.json` too: the only real names in those scripts the corpora hold. It holds no honorific, so a change to the honorific peel moves nothing there. A change to comma or credential handling also reads the local corpus of hand-typed names (`snapshot.py --local ctgov-officials --base `, which reads the base from a baseline kept for its package's files and never the shared snapshot beside the corpus (a base read once from the same fetch is copied rather than parsed again; `--sample N` reads a fixed sample while iterating; the shared snapshot beside the corpus is one file per source for every worktree, and #752's phase 2f found ctgov reporting a sibling branch's 1,334 joined-credential readings as its own moves against it); tools/differential/README.md's "Local corpora"): the one population of names nobody chose for a test or a catalog, so it shows how often each shape is typed, and the one no CI run sees. A change to title handling reads `ukec-individuals` the same way: UK donors' names in spoken order and mixed case, in 8,219 of whose 15,214 `parse()` read a title on 2026-10-22. A change to particles or generational words reads `snapshot.py --local tse-donors` the same way, Brazilian names in spoken order dense with both. A change to initials, to a title word outside English, or to how a script's combining marks are read reads `snapshot.py --local nepal-candidates` the same way: Nepal's local election candidates, each a romanized name (surnames written as initials, `B.K.`, `K.C.`; honorific titles such as `Guru` in front, `Shekh`, a title there until #752's phase 2j made it one only inside a title phrase, and `Md.`, which its period makes a title by shape) beside the same name in Devanagari, written with vowel signs and the virama, combining marks the letter-shape tests do not count as letters (decisions/nfd-shapes), and with the joiners ZWJ and ZWNJ. It is the only large population of South Asian names the corpora hold, and its corpus reads both scripts. A change to non-Latin scripts or to Arabic-script names reads `snapshot.py --local jrc-names` the same way, and `local_corpora.py groups jrc-names`: the spellings of one person across scripts and transliterations, harvested from news text, with 17,074 Arabic-script spellings where the committed corpora hold 315 distinct names with an Arabic letter, 96 of them CLDR's fictional renderings (2026-10-24), and under a licence that keeps even a reading of one of its names in the cache. A change to which field a word lands in also reads `local_agreement.py psc-individuals --base ` the same way: real names with the four boxes their filers typed them into (UK companies' people with significant control), scored field by field and by nationality; and `local_agreement.py orcid-researchers --base `, ORCID holders' own two boxes (given names, family name) from every country, scored by the country they work or live in -- a population each person split for themselves, academic but worldwide, the one to read for a change a language pack makes; and `local_agreement.py patentsview-inventors --base `, US patents' inventors in the two boxes their filings give, split by a professional filer, with how many people filed under each name in each country, the one to read for how common a name's word is among a country's US-patent inventors. These are the only real fielded populations outside the committed fields files, and no CI run sees them either. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 1b982c7ae..1e05104cd 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -10,7 +10,8 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - The default loads every language and keeps working for names of any language. Choosing a language is to narrow it in phase 3 (#753): a pack is to settle only the clashes where its language has a vocabulary reading, a strict pack to drop other languages' name-level readings too, and a pack's preferred reading to be able to outrank one a shape rule derives — 2026-10-15/16. - `e`, `y` and `i` stay in the default; choosing a language is to settle their one-case fork, which no shipped pack does yet — 2026-10-15/16. - A patronymic field is to come in 3.0, as a view that names its words without emptying `family`, and the particle kinds already draw the `clan`/`patronymic` line for it: a patronymic is followed by a parent's name, a clan prefix by a family name the lineage shares — 2026-10-15/16; 2026-10-19 #749; 2026-10-28 #754 (Derek). -- A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index loaded for every caller carries the languages to run time; strict waits for 3i, and third-party language files are deferred — 2026-10-27 #753 (Derek). +- A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index, loaded on its first lookup, carries the languages to run time; strict waits for 3i, and third-party language files are deferred — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3a (Derek). +- A name reading written `default = false` derives no marker in the default and counts for no check guarding the default's compile; a particle whose word's only `given` readings are off is refused until a pack needs one — 2026-10-29 #753, phase 3a. - A caller configures vocabulary by reading, `Lexicon` deriving the sets and gates the build derives, and the compiled sets become internal in 3.0 — 2026-10-28 #754, the `Lexicon` surface (Derek). - A `Locale` may carry CLDR-shaped render patterns, display and sorting, a later addition whose shape is agreed; how a call reaches one is open — 2026-10-28 #754, render patterns. - 3.0's output model keeps the seven roles and adds every new field as a view, read from `Token.reading`, the reading each token took, or for `surname2` from rules/O6's tag: `generation` and `credentials` partitioning `suffix`, `patronymic`, and `surname2` where rules/O6 read two family names; `middle` and `surnames` keep their meaning, a compound given name being a rule a caller or pack turns on; the default `str()` renders a name in the order it was read in with its written separators in a normalized form, through `render(spec: str | None = None)`, `None` being that default; a language file's `[rules]` compile into its `Locale`'s policy patch — 2026-10-28 #754 (Derek). @@ -272,3 +273,12 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 10. **What stays out of the output model.** Gender (decision 9) waits, `Token.reading` able to carry it later as one more addition. Names of given names only are a rule's question, not a field's: ORCID's empty family box holds 68,006 of 1,831,533 records, 15.9% to 18.7% of those from India, Pakistan, Bangladesh and Indonesia, and the parse fills `family` in 51,367 of the 51,415 with two or more given-box words. #694's two-word surname after a given name (`Tariq Abdul Aziz`) is a parse rule for the default, outside phase 4. Phase 1's four reserved names said to be phase 4's (`separators`, `bare_only`, `leading_only`, `trailing_only`) have no sub-phase yet. - **2026-10-28 #754, render patterns — A `Locale` MAY CARRY CLDR-SHAPED RENDER PATTERNS (A GIVEN-FIRST AND A SURNAME-FIRST DISPLAY PATTERN AND A SORTING PATTERN), A SHAPE AGREED NOW FOR A FEATURE THAT COMES LATER AS AN ADDITION.** The RFC's Phase 4 section, "Render patterns a pack may carry", holds the shape: the pack field, a table of Phase 4's views against CLDR's person-name fields, and the gap it shows, CLDR's `surname-prefix` and `surname-core` belonging to the first surname where `family_particles` and `family_base` read the whole family, which the per-surname units close. CLDR is a starting point, not the authority: its `pt` sorting pattern files under the first surname and leaves `surname2` out, which is not ABNT's last-surname entry (decisions/R2). Proposed there, not decided: a casefolded key method beside `comparison_key()`, collation left to the caller; and, open for a decision, whether a pattern is reached through a convention recorded on the name or through the parser. - **2026-10-28 #754, the `Lexicon` surface (Derek) — A CALLER CONFIGURES VOCABULARY BY READING, AND THE COMPILED SETS BECOME INTERNAL.** Derek decided in chat on 2026-10-28, choosing it over keeping a public field per compiled set and over keeping 2.0.0's eleven while hiding the eleven added since. A caller states what a word may be in the language files' terms (a `given` reading on `guru`, no credential reading on `dc`) and `Lexicon` derives the sets and gates as the build does. The reasons: the set mechanics are hard to keep straight even for the maintainer, a reading is what a caller can know about a word, and it is the form a pack contribution takes. Today the same configuration needs rules/H9's set algebra, `Lexicon.default().remove(titles={'guru'}).add(titles_ambiguous={'guru'})` (`Guru Prasad Sharma` → given `Guru`) and `remove(suffix_acronyms={'dc'})` (`Ram Bahadur DC` → family `DC`), while `add(suffix_acronyms_ambiguous={'dc'})` leaves suffix `DC` (2026-10-28, run on this branch's base). A caller's own removal may settle a clash: decision 15 binds the default and the packs, not a caller's data. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, eleven since 2.0.0's eleven; the RFC's Phase 4 "Left open" holds the weighing. A break of the frozen `Lexicon`, recorded in decisions/3-0-reevaluations. Open: the method names, and whether a caller's own language file can join the pool (phase 3's item 8 defers third-party files). The 2.x warning the freeze asks for was waived on 2026-10-29, 2.8 being the last 2.x feature release (decisions/3-0-reevaluations, THE LEXICON WARNING). + +- **2026-10-29 #753, phase 3a — THE READING INDEX AND THE SHARED COMPILER LAND, AND A NAME READING MAY BE WRITTEN `default = false`; NO PARSE MOVES.** Carries out items 3 and 5 of the 2026-10-27 #753 entry, and 3a of its item 11. + - **What landed.** tools/vocab/build.py writes a second generated module, nameparser/config/_index.py, beside _compiled.py: for each word, one row per reading in every file, on by default or not, `(tag, kind, properties)`, the properties those the compiler reads (each category's behavioral ones, and `same_as` on a `given` reading); counts, notes, exclusions and the stated `may_be_given` stay out, being checked at build time and compiled to nothing. The compiler moved into nameparser/_reading_index.py (`compile_readings`, with `CATEGORY`, `BEHAVIORAL` and `GATED_WORDS`), which the build loads by path as it loads _lexicon.py; the build keeps the checks and compiles _compiled.py from the index rows, so the module the parser reads is the index compiled. tests/v2/test_reading_index.py holds the no-choice invariant: `compile_readings(index())`, rendered as the build renders it, is _compiled.py byte for byte. The run-time reader is `readings_of(text)`, every `IndexReading` (text, tag, kind, props) of a word, for `Reading.languages`, which 4a (#942) adds but leaves unfilled and a follow-up after 3a and 4a both merge fills; `IndexReading.kind` is the vocabulary's kind (`credential`, `preposition`), and mapping it to a `ReadingKind` is that follow-up's. + - **A name reading written `default = false` derives no marker in the default** (item 3, the half before a pack turns it on): the word's other readings compile as they would without it. So it counts for none of the checks guarding the default's compile: an agnomen on by default needs a name reading on by default (check 11, else it reads ungated), and a bound given name that is also a particle, which needs a `given` reading on by default (else Lexicon raises at import), is refused by check 11's particle half below before check 9 reads it. A name reading off beside a suffix word kind no gate reads is still refused, gating nothing when on either. + - **Left open: a particle whose word's only `given` readings are off is refused.** Check 11 asks such a particle to state `may_be_given = false` exactly where its word has no `given` reading. The default compiles it never-given, a pack turning the given reading on compiles it possibly given, and which of the two the stated property describes is a choice no 3.0 pack needs yet (`nl`'s prepositions wait with strict). The build refuses the shape, naming both ways out, rather than pick a side; the first pack that needs it decides, with its measurement. + - **The index is written grouped by word, and an `IndexReading` is built only for a word looked up.** On a flat draft of this change, one row per line, grouping the rows at run time took about 0.56 ms; building every row's object takes about 1.4 ms (`index()`, which only a compile over the whole pool calls); each against the import's own cost, so the build writes one line per word and the lookup table is a single `dict()` call. A change to a word's readings is still a one-line diff. + - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Today, and until the follow-up that fills it after 3a and 4a (#942) both merge, no parse reads the index, so no caller pays the import, parsing or not, where loading it with the default lexicon would cost every parsing caller the import for nothing. Once the follow-up fills `Reading.languages`, the first parse will load it, so every parsing caller will pay it once, as item 5 measured, and only a caller who imports nameparser without parsing is spared. Wherever the follow-up calls `readings_of`, the import moves with the call. + - **Measured** 2026-10-29 on `9210f221` plus this change, Python 3.11.17, medians of fresh processes with bytecode cached as an installed wheel has it: importing the index takes 0.56 to 0.68 ms across three runs on a shared machine, about 3% of `import nameparser`'s 18 to 24 ms in the same runs, above the 0.45 ms item 5 measured on a flat draft of 2,140 rows; the first lookup after it, 0.07 ms; compiling the whole index, 2.1 ms; uncached, the import takes about 13 to 18 ms (13 after `import nameparser` in this session, 17 to 19 in review's runs, a figure no wheel install pays). The index holds 2,157 readings over 1,798 words, 106 of them in more than one file, and 3 readings written `default = false`; it is about 95 KB of source. Recompute: `uv run python tools/vocab/build.py`, then with `PYTHONDONTWRITEBYTECODE` unset, time `import nameparser.config._index` after `import nameparser` in fresh processes (the first run writes the bytecode), and count with `from nameparser.config._index import READINGS`. + - Not touched: `Lexicon` and every other pickled type gain no field, so no pickle layout moves, and 3a forces no public name; the configure-by-reading methods stay open. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index fc214e6af..dbca571b1 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -230,9 +230,9 @@ This catalog converts discovery into a one-time cost — it does not remove disc **Contract statement.** Compile the source into a committed, generated module that the runtime imports, and test that the committed module is byte-identical to a fresh compile, failing with the command that refreshes it; run the compiler's checks over the shipped source in the same test, with warnings as errors, and give each check a mutation it must refuse. -**How it works.** The parser imports nameparser/config/_compiled.py, frozenset literals one entry per line so a one-word change is a one-line diff; tools/vocab/build.py writes it from nameparser/vocab/*.toml; tests/v2/test_vocab.py compiles in memory and compares. The source can grow descriptive data (notes, counts, expansions) that compiles to nothing and costs the runtime nothing. Cost: a data change touches two files and needs the tool run, which the staleness test turns from a silent drift into a CI failure carrying the fix. +**How it works.** The parser imports nameparser/config/_compiled.py, frozenset literals one entry per line so a one-word change is a one-line diff; tools/vocab/build.py writes it from nameparser/vocab/*.toml, and beside it a second committed artifact, the reading index nameparser/config/_index.py, both from the same in-memory rows, which the shared compiler in nameparser/_reading_index.py compiles into the first (#753, phase 3a); tests/v2/test_vocab.py and tests/v2/test_reading_index.py compile in memory and compare each. The source can grow descriptive data (notes, counts, expansions) that compiles to nothing and costs the runtime nothing. Cost: a data change touches three files (the source and the two generated modules) and needs the tool run, which the staleness test turns from a silent drift into a CI failure carrying the fix. -**Lives in.** tools/vocab/build.py, tests/v2/test_vocab.py (decisions/vocabulary-model). +**Lives in.** tools/vocab/build.py, nameparser/_reading_index.py, tests/v2/test_vocab.py, tests/v2/test_reading_index.py (decisions/vocabulary-model). **Reach for it when.** A data file the runtime would otherwise parse, or a hand-kept Python copy of data that has a better home — and not for a measured expectation, which is RECORDED-ROSTERS. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index 0115eae3c..d87602013 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -26,7 +26,7 @@ Four phases, in a fixed order: |---|---|---|---| | 1 | The per-word TOML format, compiled to today's sets | no | shipped in 2.6 (#737, #744); #749–#751 gave the words in `und.toml` their languages, leaving two particles of unrecorded origin | | 2 | Ambiguity derived from clashing readings; the ambiguity gate extended to every reading kind | yes, the first phase that does | #752, design agreed 2026-10-23; 2a, the two name-clash markers derived, landed without moving a parse (decisions/vocabulary-model, 2026-10-25) | -| 3 | Language packs that prefer or restrict their language | yes, for callers who choose one | #753, for 3.0, needs 1 and 2; design agreed 2026-10-28, 3g's connective open | +| 3 | Language packs that prefer or restrict their language | yes, for callers who choose one | #753, for 3.0, needs 1 and 2; design agreed 2026-10-28, 3g's connective open; 3a, the reading index and the shared compiler, landed without moving a parse (decisions/vocabulary-model, 2026-10-29) | | 4 | Order and shape; the rules a pack can choose; the output model | yes | #754, 3.0 work; its output model agreed 2026-10-28, ahead of or alongside phase 3's mechanism | Two commitments hold through all four. **The default loads every language and keeps working for names of any language, as today**; choosing a language narrows it (decision 5 in decisions/vocabulary-model). And **languages own vocabulary and defaults, the core owns structure**: a pack configures rules the core implements, and never ships code (mechanisms.md#LOCALE-PACKS-PURE-DATA). @@ -164,7 +164,7 @@ Each sub-phase is one PR, measured on its own base: the committed snapshot read ## Phase 3: packs that prefer or restrict -Status: **agreed** (proposed 2026-10-27, #753; Derek accepted every item on 2026-10-28, six and then the other six). The record is decisions/vocabulary-model's 2026-10-27 #753 entry; this section describes its items. One design inside the work order is still open: 3g's connective (below), which accepting the work order did not decide. Phase 3 is 3.0 work (re-planned 2026-10-28): 2.8 is the last 2.x feature release, and 3.0 ships phases 3 and 4 with the facade's removal. Packs stay opt-in, so a caller who chooses no language parses as the default does, and nothing in this phase moves the default's parses. One standing text is amended with it, in 3b's PR, which builds the selection: mechanisms.md#LOCALE-PACKS-PURE-DATA's contract that a pack is add-only and declares its deviations in its docstring (a pack adds, and selects among the default's readings by language, its deviations derived for a vocabulary pack). decisions/3-0-reevaluations' line that removing base vocabulary through a pack is part of 3.0's pack design is carried out rather than moved. +Status: **agreed** (proposed 2026-10-27, #753; Derek accepted every item on 2026-10-28, six and then the other six); **3a landed 2026-10-29**, the reading index, the compiler moved into the package and `default = false` on name readings (decisions/vocabulary-model's 2026-10-29 entry, which records what 3a decided and measured, and the index's lazy load, which Derek chose on 2026-10-29 over loading it with `Lexicon.default()`; "Where the provenance lives at runtime" below describes it). The record is decisions/vocabulary-model's 2026-10-27 #753 entry; this section describes its items. One design inside the work order is still open: 3g's connective (below), which accepting the work order did not decide. Phase 3 is 3.0 work (re-planned 2026-10-28): 2.8 is the last 2.x feature release, and 3.0 ships phases 3 and 4 with the facade's removal. Packs stay opt-in, so a caller who chooses no language parses as the default does, and nothing in this phase moves the default's parses. One standing text is amended with it, in 3b's PR, which builds the selection: mechanisms.md#LOCALE-PACKS-PURE-DATA's contract that a pack is add-only and declares its deviations in its docstring (a pack adds, and selects among the default's readings by language, its deviations derived for a vocabulary pack). decisions/3-0-reevaluations' line that removing base vocabulary through a pack is part of 3.0's pack design is carried out rather than moved. **Phase 4's shape questions came first.** The output model decides what a pack can express: a patronymic field, a second surname, credentials apart from generation suffixes (#326), and what `middle` means (#704). A pack reading of a kind with no field to land in is a reading nothing can surface, so those questions were to be settled before or with phase 3's mechanism, by the same shape-before-value rule, and the output model was agreed on 2026-10-28 (decisions/vocabulary-model's 2026-10-28 #754 entry): the seven roles stay, and `generation`, `credentials`, `patronymic` and `surname2` are views read from `Token.reading` or rules/O6's tag. Of the work order below, 3a (the reading index and the shared compiler) and 3c (the pack-shaped exclusions converted) are independent of the output model: the index carries whatever kinds the files hold, and 3c moves data between two encodings that compile alike. 3b's selection semantics are independent too, and its API's dependency, what `Locale` and `Policy` look like once the facade is gone, is answered (below). The packs depend on the output model in part, each on the phase 4 sub-phase that builds its view: `cs`'s generations and `pt-BR`'s agnomens on 4b's `generation` (3f and 3h), `es` and `pt` on 4d's `surname2` and 4g's compound given name (3g); `ne`'s readings do not. @@ -219,9 +219,9 @@ The 2 broken are pairs whose romanization opens with a listed title, which reads ### Where the provenance lives at runtime -The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded for every caller, with `Lexicon.default()`, because phase 4's `Reading.languages` (its sub-phase 4a) is filled from it on every parse. Loading it on first access to `reading.languages` instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0. +The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Today no parse makes one, so no caller loads it. Phase 4's `Reading.languages` will be filled from it by a follow-up once 3a and 4a (#942, which adds the field unfilled) have both merged; from then on the first parse will load it, every caller who parses will pay it once, and a caller who only imports nameparser will not. The import moves with the lookup, wherever the follow-up makes it. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. -Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 words, 106 of them in more than one file. An index of their behavioral properties is about 93 KB of generated source. Imported with its bytecode cached, as an installed wheel has it, it takes about 0.45 ms (median of nine fresh processes, 2026-10-28 on `c8357e74`, Python 3.11.17), against about 20 ms for `import nameparser` cached the same way; with no cache it takes about 11 ms, the figure this section gave on 2026-10-27, measured in a shell that sets `PYTHONDONTWRITEBYTECODE`. That load is paid once per process by every caller. Compiling the pool takes about 9 ms, paid once per choice and only by a caller who makes one. Reading the TOML instead costs about 39 ms. Recompute: the build's `read_dir`, `load` and `compile_pool`, timed; for the import, write the index as a module, `py_compile` it, and time its import in fresh processes beside a cached `import nameparser`. +Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 words, 106 of them in more than one file. An index of their behavioral properties is about 93 KB of generated source. Imported with its bytecode cached, as an installed wheel has it, it takes about 0.45 ms (median of nine fresh processes, 2026-10-28 on `c8357e74`, Python 3.11.17), against about 20 ms for `import nameparser` cached the same way; with no cache it takes about 11 ms, the figure this section gave on 2026-10-27, measured in a shell that sets `PYTHONDONTWRITEBYTECODE`. That load is paid once per process, at the first lookup. Compiling the pool takes about 9 ms, paid once per choice and only by a caller who makes one. Reading the TOML instead costs about 39 ms. Recompute: the build's `read_dir`, `load` and `compile_pool`, timed; for the import, write the index as a module, `py_compile` it, and time its import in fresh processes beside a cached `import nameparser`. Two cheaper-looking forms do not work. Per-language deltas compiled at build time do not compose: under prefer `{pt, en}`, `do` keeps both its particle and its credential, but Portuguese's delta removes the credential and English's removes the particle. And precompiling every combination of languages is combinatorial. @@ -400,7 +400,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar A shape question, decided 2026-10-28: -- **Whether `Lexicon` keeps a public field for each compiled set**, or answers word, kind and property queries from phase 3's reading index and keeps its sets internal. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, 11 of them added since 2.0.0's 11 (`surnames`, `honorific_tails`, `conjunctions_ambiguous`, `after_given_titles`, `suffix_words_ambiguous`, `agnomens`, `given_name_abbreviations`, `titles_ambiguous`, `title_modifiers`, `title_styles`, `title_phrases`), each a new parse behavior's set, and a rule keyed on a title kind or on one language's reading of a kind (phase 3's design, "a property a language's own reading should take only under its pack") would add more. Keeping the fields: each is frozen API, shown in the bounded repr (`Lexicon(default + titles: +1)`), and pickled, where `Lexicon`'s own `__setstate__`, unlike the `ADDED_IN` guard the other types share, refuses any pickle whose fields differ, so every new field breaks every older `Lexicon` pickle, and with it every pickled `Parser` and locale pack, until that guard learns the marker. Turning them internal: a break that has to land in 3.0, where a query added beside the fields is an addition that can come later; a caller who configures vocabulary by the field today (`Lexicon.add(titles=...)`, `dataclasses.replace`) needs another way in, for words that carry no kind and no language; and phase 3's reading index, which a `Lexicon` answering queries would read, is already loaded for every caller with `Lexicon.default()` (about 0.45 ms with its bytecode cached, by phase 3's timing above), so answering from it adds no load. The parser need not pay either way: a frozenset test per word costs no frame and an internal cache keeps that path, while a query method called per word would add one per token, the cost the frame budget counts as scaling with the input. **Derek decided on 2026-10-28 that a caller configures vocabulary by reading and the compiled sets become internal** (decisions/vocabulary-model's 2026-10-28 #754 entry on the `Lexicon` surface). A caller states what a word may be in the language files' terms, a `given` reading on `guru`, no credential reading on `dc`, and `Lexicon` derives the sets and gates the build derives, so making `Guru` a possible given name needs no knowledge of rules/H9's sets, where today it is `remove(titles={'guru'}).add(titles_ambiguous={'guru'})`, and `add(given_name_titles=...)` or `add(suffix_acronyms_ambiguous=...)`, the plausible wrong guesses, do something else. A reading is also what a caller can know about a word and what a pack contribution states. Removing a reading from a caller's own lexicon may settle a clash; decision 15 binds the default and the packs, not a caller's choice about their own data. It breaks the frozen `Lexicon` (decisions/3-0-reevaluations); the method names and whether a caller's own language file can join the pool later (phase 3 defers it) are open, and the 2.x warning the freeze asks for was waived on 2026-10-29, no 2.x feature release being left to carry it (decisions/3-0-reevaluations, THE LEXICON WARNING). +- **Whether `Lexicon` keeps a public field for each compiled set**, or answers word, kind and property queries from phase 3's reading index and keeps its sets internal. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, 11 of them added since 2.0.0's 11 (`surnames`, `honorific_tails`, `conjunctions_ambiguous`, `after_given_titles`, `suffix_words_ambiguous`, `agnomens`, `given_name_abbreviations`, `titles_ambiguous`, `title_modifiers`, `title_styles`, `title_phrases`), each a new parse behavior's set, and a rule keyed on a title kind or on one language's reading of a kind (phase 3's design, "a property a language's own reading should take only under its pack") would add more. Keeping the fields: each is frozen API, shown in the bounded repr (`Lexicon(default + titles: +1)`), and pickled, where `Lexicon`'s own `__setstate__`, unlike the `ADDED_IN` guard the other types share, refuses any pickle whose fields differ, so every new field breaks every older `Lexicon` pickle, and with it every pickled `Parser` and locale pack, until that guard learns the marker. Turning them internal: a break that has to land in 3.0, where a query added beside the fields is an addition that can come later; a caller who configures vocabulary by the field today (`Lexicon.add(titles=...)`, `dataclasses.replace`) needs another way in, for words that carry no kind and no language; and phase 3's reading index, which a `Lexicon` answering queries would read, is loaded on its first lookup (decided lazy on 2026-10-29 rather than loaded with `Lexicon.default()`; about 0.6 ms with its bytecode cached, by phase 3's timing above). No parse makes one today; once a follow-up to 4a fills `Reading.languages`, every parse will, so answering queries from it will add no load beyond that first lookup. The parser need not pay either way: a frozenset test per word costs no frame and an internal cache keeps that path, while a query method called per word would add one per token, the cost the frame budget counts as scaling with the input. **Derek decided on 2026-10-28 that a caller configures vocabulary by reading and the compiled sets become internal** (decisions/vocabulary-model's 2026-10-28 #754 entry on the `Lexicon` surface). A caller states what a word may be in the language files' terms, a `given` reading on `guru`, no credential reading on `dc`, and `Lexicon` derives the sets and gates the build derives, so making `Guru` a possible given name needs no knowledge of rules/H9's sets, where today it is `remove(titles={'guru'}).add(titles_ambiguous={'guru'})`, and `add(given_name_titles=...)` or `add(suffix_acronyms_ambiguous=...)`, the plausible wrong guesses, do something else. A reading is also what a caller can know about a word and what a pack contribution states. Removing a reading from a caller's own lexicon may settle a clash; decision 15 binds the default and the packs, not a caller's choice about their own data. It breaks the frozen `Lexicon` (decisions/3-0-reevaluations); the method names and whether a caller's own language file can join the pool later (phase 3 defers it) are open, and the 2.x warning the freeze asks for was waived on 2026-10-29, no 2.x feature release being left to carry it (decisions/3-0-reevaluations, THE LEXICON WARNING). Value questions inside the shapes above: diff --git a/nameparser/_reading_index.py b/nameparser/_reading_index.py new file mode 100644 index 000000000..a37114ab8 --- /dev/null +++ b/nameparser/_reading_index.py @@ -0,0 +1,410 @@ +"""The reading index, and the compiler that folds readings into sets. + +The default vocabulary is written per word in nameparser/vocab/*.toml, +each word listing its READINGS (what it can be read as, keyed by kind) +in the file of the language whose names it occurs in +(decisions/vocabulary-model). tools/vocab/build.py checks the files and +writes two modules from them, each generated and committed: + +* nameparser/config/_index.py, the READING INDEX: for each word, one + row per reading, ``(tag, kind, properties)``, the tag being its file's + BCP 47 tag and the properties the ones the compiler reads. It is what + a set has forgotten, which file each word came from. +* nameparser/config/_compiled.py, the sets the parser reads, which is + :func:`compile_readings` run over the index. + +This module is the compiler both the build and, from phase 3b, a choice +of languages run, so there is one implementation of what a reading +compiles to; tests/v2/test_reading_index.py holds that compiling the +index reproduces _compiled.py exactly. It is also the one way to read +the index at run time (:func:`readings_of`, :func:`index`). + +It imports nothing internal at module level, and the index only inside +:func:`index`: the build loads this file by path, as it loads +_lexicon.py, so that a generated module left broken cannot stop the +tool that rewrites it. +""" +from __future__ import annotations + +import functools +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any, TypedDict + +#: kind -> category. The category decides which compiled field a reading +#: feeds. For title, suffix, particle and conjunction, the category's own +#: name is also a kind (`title = {}`: a title not yet classified), so +#: classifying a word later moves no set. bound_given's one kind shares +#: its category's name; the relation (birth_marker) and name (given, +#: family) categories have no generic kind. Kind +#: names are unique across categories; a fine kind arrives with its +#: category's classification, in a core PR, as one row here. +CATEGORY: dict[str, str] = { + "title": "title", + "salutation": "title", + "style": "title", + "academic": "title", + "religious": "title", + "nobility": "title", + "military": "title", + "office": "title", + "occupation": "title", + "kinship": "title", + "modifier": "title", + "suffix": "suffix", + "generation": "suffix", + "agnomen": "suffix", + "credential": "suffix", + "honorific": "suffix", + "professional_honorific": "suffix", + "status": "suffix", + "courtesy": "suffix", + "particle": "particle", + "preposition": "particle", + "article": "particle", + "patronymic": "particle", + "teknonymic": "particle", + "clan": "particle", + "rank": "particle", + "saint": "particle", + "alias": "particle", + "conjunction": "conjunction", + "bound_given": "bound_given", + "birth_marker": "relation", + "given": "name", + "family": "name", +} + +#: Behavioral properties per category: each compiles into a field, and +#: its allowed values are exactly what today's pipeline does. A name +#: reading's one is `default` (phase 3a): its other keys are evidence. +BEHAVIORAL: dict[str, frozenset[str]] = { + "title": frozenset({"addresses", "after_given", "attaches", "cased", + "default"}), + "suffix": frozenset({"acronym", "attached", "cased", "default"}), + "particle": frozenset({"cased", "default"}), + "conjunction": frozenset({"may_be_initial", "cased", "default"}), + "bound_given": frozenset({"cased", "default"}), + "relation": frozenset({"cased", "default"}), + "name": frozenset({"default"}), +} + +#: The suffix word kinds a name reading gates (rules/S11): the third +#: derived marker reads these, and a name reading beside another word +#: kind marks nothing, no gate reading that clash yet. +GATED_WORDS = frozenset({"generation", "agnomen"}) + + +def indexed(kind: str) -> frozenset[str]: + """The properties of a `kind` reading the index keeps: its + category's behavioral ones, and on a `given` reading `same_as`, + which compiles the word into GIVEN_NAME_ABBREVIATIONS (#789). + Everything else a reading carries is evidence, compiled to + nothing.""" + keep = BEHAVIORAL[CATEGORY[kind]] + return keep | {"same_as"} if kind == "given" else keep + + +class ReadingProperties: + """The unwritten values of the properties whose default is not + false, defined once for every reading type that has a `kind` and + `props`: the build's Reading, which holds a reading as its file + wrote it, and :class:`IndexReading`, which holds it as the index + does.""" + + __slots__ = () + text: str + kind: str + props: Mapping[str, Any] + + @property + def category(self) -> str: + return CATEGORY[self.kind] + + @property + def acronym(self) -> bool: + """Matched with its periods removed: unwritten, true only for a + credential.""" + return bool(self.props.get("acronym", self.kind == "credential")) + + @property + def on_by_default(self) -> bool: + """Compiled into the default vocabulary: unwritten, true. A + reading written `default = false` is recorded and checked as a + reading but compiles to nothing, a name reading included (3a), + for a pack to turn on in phase 3 (decisions/vocabulary-model).""" + return self.props.get("default", True) is not False + + def flag(self, prop: str) -> bool: + """A `may_be_*` flag or `after_given`: unwritten, false.""" + return bool(self.props.get(prop, False)) + + @property + def addresses(self) -> object: + """Which name a title addresses: unwritten, the family name. An + object, not a str, because the build reads it before its check + 3 has refused a value that is not one of the allowed strings.""" + return self.props.get("addresses", "family") + + +@dataclass(frozen=True, slots=True) +class IndexReading(ReadingProperties): + """One row of the reading index. + + ``text`` is the word, normalized as the vocabulary stores it; + ``tag`` the BCP 47 tag of the file holding the reading (``"en"``, + ``"pt-BR"``, ``"mul"``, ``"und"``); ``kind`` its vocabulary kind, + a key of :data:`CATEGORY` (``"credential"``, ``"preposition"``, + ``"given"``), finer than :class:`nameparser.ReadingKind`; + ``props`` the properties :func:`indexed` keeps, as written, read + through the accessors above for their unwritten values. A word in + several files has a row per file, and a reading written + ``default = false`` has its row too, with + ``on_by_default`` false. + + A lookup result, not a value to keep: its ``props`` is a read-only + view, so an IndexReading neither hashes nor pickles. What a parse + would keep of it (``Reading.languages``, filled by a follow-up to + phase 4a) is its tag.""" + + text: str + tag: str + kind: str + props: Mapping[str, Any] + + +#: A row as nameparser/config/_index.py writes it, under its word: +#: (tag, kind, properties), the properties as sorted (key, value) pairs, +#: a list value as a sorted tuple and the `attached` table as its side, +#: so the whole index is one constant. +Row = tuple[str, str, tuple[tuple[str, Any], ...]] +#: The index as written: (text, rows) per word, sorted by text, its +#: rows by tag and then kind. +Index = tuple[tuple[str, tuple[Row, ...]], ...] + + +def to_row(r: IndexReading) -> Row: + """`r` as the index writes it under its word.""" + pairs = [] + for key in sorted(r.props): + value = r.props[key] + if key == "attached": + value = value["side"] if isinstance(value, Mapping) else value + elif isinstance(value, (list, tuple, set, frozenset)): + value = tuple(sorted(value)) + pairs.append((key, value)) + return (r.tag, r.kind, tuple(pairs)) + + +def from_row(text: str, row: Row) -> IndexReading: + tag, kind, pairs = row + return IndexReading(text, tag, kind, MappingProxyType(dict(pairs))) + + +@functools.cache +def _by_text() -> Mapping[str, tuple[Row, ...]]: + # the index is written grouped by word, so this is one dict() call + # in C, and an IndexReading is built only for a word looked up: + # building all of them costs more than importing the index + # (decisions/vocabulary-model, 2026-10-29 3a) + from nameparser.config._index import READINGS + return MappingProxyType(dict(READINGS)) + + +@functools.cache +def index() -> tuple[IndexReading, ...]: + """Every row of the shipped reading index, in its order (by text, + then tag, then kind), for a compile over the whole pool. Built on + first call and kept: the index describes the shipped files, which + do not change in a process.""" + return tuple(from_row(text, row) + for text, rows in _by_text().items() for row in rows) + + +def readings_of(text: str) -> tuple[IndexReading, ...]: + """Every reading the shipped files hold for `text`, on by default + or not, one per file and kind, in index order; empty for a word no + file holds. + + `text` is looked up as the vocabulary stores it (lowercase, NFC, + edge periods removed; _lexicon._normalize), and a phrase whole + (``"z domu"``). A reading the parse took from a caller's own + vocabulary, from a shape or from position has no row here, which is + what ``Reading.languages`` is to read as empty once a follow-up to + phase 4a fills it from here; no parse calls this yet.""" + return tuple(from_row(text, row) for row in _by_text().get(text, ())) + + +class CompileError(ValueError): + """The readings cannot compile: a fact about the data, which the + build reports as a vocabulary error (exit 2) where any other + exception from the compiler is an internal one (exit 3).""" + + +class Compiled(TypedDict): + """What :func:`compile_readings` returns: one key per constant, in + the order nameparser/config/_compiled.py defines them, typed as it + declares them.""" + GIVEN_NAME_TITLES: frozenset[str] + TITLES: frozenset[str] + AFTER_GIVEN_TITLES: frozenset[str] + TITLES_AMBIGUOUS: frozenset[str] + TITLE_MODIFIERS: frozenset[str] + TITLE_STYLES: frozenset[str] + TITLE_PHRASES: frozenset[str] + NON_GIVEN_NAME_PARTICLES: frozenset[str] + PARTICLES: frozenset[str] + CONJUNCTIONS: frozenset[str] + CONJUNCTIONS_AMBIGUOUS: frozenset[str] + BOUND_GIVEN_NAMES: frozenset[str] + GIVEN_NAME_ABBREVIATIONS: frozenset[str] + MAIDEN_MARKERS: frozenset[str] + SUFFIX_WORDS: frozenset[str] + GLUED_HONORIFICS: frozenset[str] + SUFFIX_WORDS_AMBIGUOUS: frozenset[str] + AGNOMENS: frozenset[str] + SUFFIX_ACRONYMS_AMBIGUOUS: frozenset[str] + SUFFIX_ACRONYMS: frozenset[str] + CAPITALIZATION_EXCEPTIONS: dict[str, str] + + +def compile_readings(readings: Iterable[ReadingProperties]) -> Compiled: + """Fold the readings on by default into the default vocabulary's + constants. The readings are taken as already checked (the build's + checks run before it calls this); each needs ``text``, ``kind`` + and ``props``. + + Raises :class:`CompileError` on a post-nominal that is both an + acronym and a word, a backstop the build's checks 4 and 5 always + reach first. Any other exception is a bug in this function.""" + texts = [(r, r.text) for r in readings if r.on_by_default] + # the derived markers (the build's check 11): a clash with a name is + # a word's name reading beside the reading it clashes with, and a + # name reading written `default = false` marks nothing in the + # default, as it compiles to nothing (3a) + named = {t for r, t in texts if r.category == "name"} + given = {t for r, t in texts if r.kind == "given"} + titles: set[str] = set() + given_titles: set[str] = set() + after_given: set[str] = set() + title_kinds: dict[str, set[str]] = {} + attaches: dict[str, set[str]] = {} + particles: set[str] = set() + never_given: set[str] = set() + conjunctions: set[str] = set() + initial_conjunctions: set[str] = set() + bound: set[str] = set() + abbreviations: set[str] = set() + markers: set[str] = set() + words: set[str] = set() + glued: set[str] = set() + ambiguous_words: set[str] = set() + agnomens: set[str] = set() + ambiguous: set[str] = set() + acronyms: set[str] = set() + cased: dict[str, str] = {} + for r, t in texts: + p = r.props + if "cased" in p: + cased[t] = p["cased"] + if r.category == "title": + titles.add(t) + if r.addresses == "given": + given_titles.add(t) + if r.flag("after_given"): + after_given.add(t) + title_kinds.setdefault(t, set()).add(r.kind) + attaches.setdefault(t, set()).update(p.get("attaches", ())) + elif r.category == "suffix": + if r.acronym: + acronyms.add(t) + if t in named: + ambiguous.add(t) + else: + words.add(t) + if "attached" in p: + glued.add(t) + if t in named and r.kind in GATED_WORDS: + ambiguous_words.add(t) + if r.kind == "agnomen": + agnomens.add(t) + elif r.category == "particle": + particles.add(t) + if t not in given: + never_given.add(t) + elif r.category == "conjunction": + conjunctions.add(t) + if r.flag("may_be_initial"): + initial_conjunctions.add(t) + elif r.category == "bound_given": + bound.add(t) + elif r.category == "relation": + markers.add(t) + elif r.kind == "given" and "same_as" in p: + abbreviations.add(t) + contested = titles & given - particles + out = Compiled( + GIVEN_NAME_TITLES=frozenset(given_titles), + # a title that may be a given name compiles into TITLES_AMBIGUOUS + # INSTEAD: a word in TITLES is a title wherever it stands, which + # is what a caller listing one there has always asked for + TITLES=frozenset(titles - contested), + AFTER_GIVEN_TITLES=frozenset(after_given), + # the fourth derived marker (check 11): a title that may be a + # given name, a title only inside a title phrase (rules/H9), and + # so not in TITLES above. A + # title that is also a particle ('st', 'freiherr') is left to the + # particle rules, which weigh its given reading at the front + # already (rules/P4) + TITLES_AMBIGUOUS=frozenset(contested), + # a word every title reading of which is a modifier: a title only + # with a head title behind it in its run (decision 10, rules/H9) + TITLE_MODIFIERS=frozenset( + t for t, ks in title_kinds.items() if ks == {"modifier"}), + TITLE_STYLES=frozenset( + t for t, ks in title_kinds.items() if "style" in ks), + # `attaches` resolved to the two-word phrases it licenses, the + # word before and each title word of a kind it names + TITLE_PHRASES=frozenset( + f"{t} {h}" for t, kinds in attaches.items() if kinds + for h, hk in title_kinds.items() + if h != t and (hk & kinds or h in kinds)), + NON_GIVEN_NAME_PARTICLES=frozenset(never_given), + PARTICLES=frozenset(particles), + CONJUNCTIONS=frozenset(conjunctions), + CONJUNCTIONS_AMBIGUOUS=frozenset(initial_conjunctions), + BOUND_GIVEN_NAMES=frozenset(bound), + # a given name written short (`chas`, Filipino `ma`): opening the + # given part period-marked, it is the given name, which outranks + # rules/H2's shape (#789) + GIVEN_NAME_ABBREVIATIONS=frozenset(abbreviations), + MAIDEN_MARKERS=frozenset(markers), + SUFFIX_WORDS=frozenset(words), + GLUED_HONORIFICS=frozenset(glued), + SUFFIX_WORDS_AMBIGUOUS=frozenset(ambiguous_words), + AGNOMENS=frozenset(agnomens), + SUFFIX_ACRONYMS_AMBIGUOUS=frozenset(ambiguous), + SUFFIX_ACRONYMS=frozenset(acronyms), + CAPITALIZATION_EXCEPTIONS=dict(sorted(cased.items())), + ) + # One form per post-nominal: the word test strips only edge periods + # and the acronym test strips all of them, so a word in both is + # matched twice by two rules, and which one fired is unreadable from + # outside. The sharper stake is the ambiguous set's: suffix_as_written + # ORs the two branches, so an ambiguous acronym that were also a word + # would be claimed through the word and bypass the period gate the + # marking exists to impose. The single overlap was 'esq', dropped + # 2026-09-08 (decisions/suffix-acronym-collisions). Lexicon guards + # only that ambiguous half (its gate-bypass raise), so a caller may + # still put an unambiguous word in both. In the TOML the overlap is + # unreachable -- the build's check 4 refuses two suffix kinds on one + # word, check 5 one kind with `acronym` differing between files, and + # check 2 a word twice in one file -- so this is a backstop kept + # because it costs one set intersection, not a check any input + # reaches today. + both = out["SUFFIX_ACRONYMS"] & out["SUFFIX_WORDS"] + if both: + raise CompileError(f"{sorted(both)}: a post-nominal is an acronym or a " + f"word, never both (check `acronym` in every file)") + return out diff --git a/nameparser/config/_index.py b/nameparser/config/_index.py new file mode 100644 index 000000000..9b022b4bb --- /dev/null +++ b/nameparser/config/_index.py @@ -0,0 +1,1814 @@ +"""The reading index. GENERATED from nameparser/vocab/*.toml by +tools/vocab/build.py -- do not edit; edit the TOML and run + + uv run python tools/vocab/build.py + +One line per word in the vocabulary files, holding a row per reading, +on by default or not: (tag, kind, properties), the tag being the file's +BCP 47 tag and the properties the ones the compiler reads, as sorted +(key, value) pairs. +Read it through nameparser._reading_index (index, readings_of), which +documents the row; _compiled.py is this index compiled. +""" + + +READINGS: tuple[tuple[str, tuple[tuple[str, str, tuple[tuple[str, object], ...]], ...]], ...] = ( + ('&', (('mul', 'conjunction', ()),)), + ("'t", (('nl', 'article', ()),)), + ('10th', (('en', 'modifier', ()),)), + ('1lt', (('en', 'military', ()),)), + ('1sgt', (('en', 'military', ()),)), + ('1st', (('en', 'modifier', ()),)), + ('1stlt', (('en', 'military', ()),)), + ('1stsgt', (('en', 'military', ()),)), + ('2', (('mul', 'generation', ()),)), + ('2lt', (('en', 'military', ()),)), + ('2nd', (('en', 'modifier', ()),)), + ('2ndlt', (('en', 'military', ()),)), + ('3rd', (('en', 'modifier', ()),)), + ('4th', (('en', 'modifier', ()),)), + ('5th', (('en', 'modifier', ()),)), + ('6th', (('en', 'modifier', ()),)), + ('7th', (('en', 'modifier', ()),)), + ('8-vsb', (('en', 'credential', ()),)), + ('8th', (('en', 'modifier', ()),)), + ('9th', (('en', 'modifier', ()),)), + ('a1c', (('en', 'military', ()),)), + ('aan', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('aas', (('en', 'credential', ()),)), + ('ab', (('en', 'military', ()),)), + ('aba', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('abbess', (('en', 'religious', ()),)), + ('abbot', (('en', 'religious', ()),)), + ('abc', (('en', 'credential', ()),)), + ('abd', (('ar', 'bound_given', ()), ('en', 'credential', ()))), + ('abdal', (('ar', 'bound_given', ()),)), + ('abdel', (('ar', 'bound_given', ()),)), + ('abdul', (('ar', 'bound_given', ()),)), + ('abolitionist', (('en', 'occupation', ()),)), + ('abou', (('ar', 'bound_given', ()),)), + ('abpp', (('en', 'credential', ()),)), + ('abr', (('en', 'credential', ()),)), + ('abu', (('ar', 'teknonymic', ()), ('ar', 'bound_given', ()), ('ar', 'given', ()))), + ('aca', (('en', 'credential', ()), ('en', 'family', ()))), + ('academic', (('en', 'occupation', ()),)), + ('acas', (('en', 'credential', ()),)), + ('ace', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('acha', (('en', 'credential', ()), ('en', 'family', ()))), + ('acolyte', (('en', 'religious', ()),)), + ('acp', (('en', 'credential', ()),)), + ('activist', (('en', 'occupation', ()),)), + ('actor', (('en', 'occupation', ()),)), + ('actress', (('en', 'occupation', ()),)), + ('ad-', (('ar', 'article', ()),)), + ('adept', (('en', 'religious', ()),)), + ('adh-', (('ar', 'article', ()),)), + ('adjutant', (('en', 'military', ()),)), + ('adm', (('en', 'military', ()),)), + ('administrator', (('en', 'office', ()),)), + ('admiral', (('en', 'military', ()),)), + ('advertising', (('en', 'modifier', ()),)), + ('adviser', (('en', 'occupation', ()),)), + ('advocate', (('en', 'academic', ()),)), + ('ae', (('en', 'credential', ()),)), + ('aem', (('en', 'credential', ()),)), + ('aen', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('af', (('da', 'preposition', ()), ('nb', 'preposition', ()), ('sv', 'preposition', ()))), + ('afasma', (('en', 'credential', ()),)), + ('afc', (('en', 'credential', ()),)), + ('afm', (('en', 'credential', ()),)), + ('agacnp-bc', (('en', 'credential', ()),)), + ('agsf', (('en', 'credential', ()),)), + ('aia', (('en', 'credential', ()),)), + ('aicp', (('en', 'credential', ()),)), + ('air', (('en', 'modifier', ()),)), + ('akhoond', (('fa', 'religious', ()),)), + ('al', (('ar', 'article', ()), ('ar', 'given', ()))), + ('al-', (('ar', 'article', ()),)), + ('ala', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('alc', (('en', 'credential', ()),)), + ('alderman', (('en', 'office', ()),)), + ('almoner', (('en', 'office', ()),)), + ('alp', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('am', (('en', 'credential', ()),)), + ('ambassador', (('en', 'office', ()),)), + ('amd', (('en', 'credential', ()),)), + ('ame', (('en', 'credential', ()),)), + ('amieee', (('en', 'credential', ()),)), + ('amn', (('en', 'military', ()),)), + ('ams', (('en', 'credential', ()), ('en', 'family', ()))), + ('an-', (('ar', 'article', ()),)), + ('analytics', (('en', 'modifier', ()),)), + ('anarchist', (('en', 'occupation', ()),)), + ('and', (('en', 'conjunction', ()),)), + ('animator', (('en', 'occupation', ()),)), + ('anthropologist', (('en', 'occupation', ()),)), + ('aphr', (('en', 'credential', ()),)), + ('apn', (('en', 'credential', ()),)), + ('appellate', (('en', 'modifier', ()),)), + ('apprentice', (('en', 'occupation', ()),)), + ('apr', (('en', 'credential', ()),)), + ('aprn', (('en', 'credential', ()),)), + ('aprn-cnp', (('en', 'credential', ()),)), + ('apss', (('en', 'credential', ()),)), + ('aqp', (('en', 'credential', ()),)), + ('ar-', (('ar', 'article', ()),)), + ('arbitrator', (('en', 'office', ()),)), + ('archbishop', (('en', 'religious', ()),)), + ('archdeacon', (('en', 'religious', ()),)), + ('archdruid', (('en', 'religious', ()),)), + ('archduchess', (('en', 'nobility', ()),)), + ('archduke', (('en', 'nobility', ()),)), + ('archeologist', (('en', 'occupation', ()),)), + ('architect', (('en', 'occupation', ()),)), + ('arhat', (('en', 'religious', ()),)), + ('arm', (('en', 'credential', ()), ('en', 'family', ()))), + ('army', (('en', 'modifier', ()),)), + ('arranger', (('en', 'occupation', ()),)), + ('arrc', (('en', 'credential', ()),)), + ('as-', (('ar', 'article', ()),)), + ('asa', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('asc', (('en', 'credential', ()),)), + ('ash-', (('ar', 'article', ()),)), + ('asid', (('en', 'credential', ()),)), + ('asla', (('en', 'credential', ()),)), + ('asp', (('en', 'credential', ()), ('en', 'family', ()))), + ('assistant', (('en', 'modifier', ()),)), + ('assoc', (('en', 'modifier', ()),)), + ('associate', (('en', 'modifier', ()),)), + ('asst', (('en', 'modifier', ()),)), + ('astronomer', (('en', 'occupation', ()),)), + ('at-', (('ar', 'article', ()),)), + ('atc', (('en', 'credential', ()),)), + ('ath-', (('ar', 'article', ()),)), + ('attache', (('en', 'office', ()),)), + ('attaché', (('en', 'office', ()), ('fr', 'office', ()))), + ('attorney', (('en', 'academic', ()),)), + ('auf', (('de', 'preposition', ()),)), + ('aunt', (('en', 'kinship', (('addresses', 'given'),)),)), + ('auntie', (('en', 'kinship', (('addresses', 'given'),)),)), + ('author', (('en', 'occupation', ()),)), + ('av', (('nb', 'preposition', ()), ('sv', 'preposition', ()))), + ('award-winning', (('en', 'modifier', ()),)), + ('awb', (('en', 'credential', ()),)), + ('ayatollah', (('fa', 'religious', ()),)), + ('az-', (('ar', 'article', ()),)), + ('aš-', (('ar', 'article', ()),)), + ('aḍ-', (('ar', 'article', ()),)), + ('aḏ-', (('ar', 'article', ()),)), + ('aṣ-', (('ar', 'article', ()),)), + ('aṭ-', (('ar', 'article', ()),)), + ('aṯ-', (('ar', 'article', ()),)), + ('aẓ-', (('ar', 'article', ()),)), + ('ba', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('baba', (('bn', 'religious', (('addresses', 'given'),)), ('hi', 'religious', (('addresses', 'given'),)))), + ('bailiff', (('en', 'office', ()),)), + ('ballet', (('en', 'modifier', ()),)), + ('bandleader', (('en', 'occupation', ()),)), + ('banker', (('en', 'occupation', ()),)), + ('banner', (('en', 'modifier', ()),)), + ('bapak', (('id', 'salutation', (('after_given', True),)),)), + ('bar', (('he', 'patronymic', ()), ('he', 'given', ()))), + ('bard', (('en', 'occupation', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('baron', (('de', 'nobility', ()), ('en', 'nobility', ()), ('fr', 'nobility', ()), ('nl', 'nobility', ()))), + ('baroness', (('en', 'nobility', ()),)), + ('barrister', (('en', 'occupation', ()),)), + ('baseball', (('en', 'modifier', ()),)), + ('bat', (('he', 'patronymic', ()), ('he', 'given', ()))), + ('batoni', (('ka', 'salutation', (('after_given', True),)),)), + ('bay', (('tr', 'family', ()),)), + ('bca', (('en', 'credential', ()),)), + ('bcl', (('en', 'credential', ()),)), + ('bcss', (('en', 'credential', ()),)), + ('bds', (('en', 'credential', ()),)), + ('bearer', (('en', 'modifier', ()),)), + ('behavioral', (('en', 'modifier', ()),)), + ('bem', (('en', 'credential', ()), ('en', 'family', ()))), + ('bench', (('en', 'modifier', ()),)), + ('benj', (('en', 'given', (('same_as', 'benjamin'),)),)), + ('bg', (('en', 'military', ()),)), + ('bgen', (('en', 'military', ()),)), + ('biblical', (('en', 'modifier', ()),)), + ('bibliographer', (('en', 'occupation', ()),)), + ('bin', (('ar', 'patronymic', ()), ('ar', 'given', ()), ('ms', 'patronymic', ()))), + ('binibini', (('fil', 'salutation', (('after_given', True),)),)), + ('bint', (('ar', 'patronymic', ()),)), + ('binte', (('ms', 'patronymic', ()), ('ms', 'given', ()))), + ('binti', (('ms', 'patronymic', ()), ('ms', 'given', ()))), + ('biochemist', (('en', 'occupation', ()),)), + ('biographer', (('en', 'occupation', ()),)), + ('biologist', (('en', 'occupation', ()),)), + ('bishop', (('en', 'religious', ()),)), + ('blessed', (('en', 'religious', ()),)), + ('blogger', (('en', 'occupation', ()),)), + ('bls-i', (('en', 'credential', ()),)), + ('blues', (('en', 'modifier', ()),)), + ('bn', (('en', 'credential', ()),)), + ('bodhisattva', (('en', 'religious', ()),)), + ('bon', (('und', 'given', ()), ('und', 'family', ()))), + ('bookseller', (('en', 'occupation', ()),)), + ('botanist', (('en', 'occupation', ()),)), + ('bp', (('en', 'religious', ()),)), + ('bpe', (('en', 'credential', ()),)), + ('bpi', (('en', 'credential', ()),)), + ('bpt', (('en', 'credential', ()),)), + ('brigadier', (('en', 'military', ()),)), + ('briggen', (('en', 'military', ()),)), + ('british', (('en', 'modifier', ()),)), + ('broadcaster', (('en', 'occupation', ()),)), + ('brother', (('en', 'religious', (('addresses', 'given'),)),)), + ('bs', (('en', 'credential', ()),)), + ('bsc', (('en', 'credential', (('cased', 'BSc'),)),)), + ('bt', (('en', 'credential', (('cased', 'Bt'),)),)), + ('btcs', (('en', 'credential', ()),)), + ('bts', (('en', 'credential', ()),)), + ('buddha', (('en', 'religious', ()), ('en', 'given', ()))), + ('burgess', (('en', 'office', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('burlesque', (('en', 'modifier', ()),)), + ('business', (('en', 'modifier', ()),)), + ('businessman', (('en', 'occupation', ()),)), + ('businesswoman', (('en', 'occupation', ()),)), + ('bwana', (('sw', 'salutation', (('after_given', True),)), ('sw', 'family', ()))), + ('cacts', (('en', 'credential', ()),)), + ('cae', (('en', 'credential', ()),)), + ('caha', (('en', 'credential', ()), ('en', 'family', ()))), + ('caia', (('en', 'credential', ()),)), + ('cams', (('en', 'credential', ()),)), + ('canon', (('en', 'religious', ()),)), + ('cap', (('en', 'credential', ()),)), + ('capa', (('en', 'credential', ()), ('en', 'family', ()))), + ('capm', (('en', 'credential', ()),)), + ('capp', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('caps', (('en', 'credential', ()),)), + ('capt', (('en', 'military', ()),)), + ('captain', (('en', 'military', ()),)), + ('cardinal', (('en', 'religious', ()),)), + ('caro', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('cartographer', (('en', 'occupation', ()),)), + ('cartoonist', (('en', 'occupation', ()),)), + ('cas', (('en', 'credential', ()),)), + ('casp', (('en', 'credential', ()),)), + ('catholicos', (('en', 'religious', ()),)), + ('cb', (('en', 'credential', ()),)), + ('cbe', (('en', 'credential', ()),)), + ('cbm', (('en', 'credential', ()),)), + ('cbne', (('en', 'credential', ()),)), + ('cbnt', (('en', 'credential', ()),)), + ('cbp', (('en', 'credential', ()),)), + ('cbrte', (('en', 'credential', ()),)), + ('cbs', (('en', 'credential', ()),)), + ('cbsp', (('en', 'credential', ()),)), + ('cbt', (('en', 'credential', ()),)), + ('cbte', (('en', 'credential', ()),)), + ('cbv', (('en', 'credential', ()),)), + ('cca', (('en', 'credential', ()),)), + ('ccc', (('en', 'credential', ()),)), + ('ccc-a', (('en', 'credential', ()),)), + ('ccc-slp', (('en', 'credential', ()),)), + ('ccc/slp', (('en', 'credential', ()),)), + ('ccca', (('en', 'credential', ()),)), + ('cccm', (('en', 'credential', ()),)), + ('cce', (('en', 'credential', ()),)), + ('cchp', (('en', 'credential', ()),)), + ('ccie', (('en', 'credential', ()),)), + ('ccim', (('en', 'credential', ()),)), + ('cciso', (('en', 'credential', ()),)), + ('ccm', (('en', 'credential', ()),)), + ('ccmsgt', (('en', 'military', ()),)), + ('ccmt', (('en', 'credential', ()),)), + ('ccna', (('en', 'credential', ()),)), + ('ccnp', (('en', 'credential', ()),)), + ('ccp', (('en', 'credential', ()),)), + ('ccp-c', (('en', 'credential', ()),)), + ('ccpr', (('en', 'credential', ()),)), + ('ccs', (('en', 'credential', ()),)), + ('ccufc', (('en', 'credential', ()),)), + ('cd', (('en', 'credential', ()),)), + ('cdal', (('en', 'credential', ()),)), + ('cdfm', (('en', 'credential', ()),)), + ('cdmp', (('en', 'credential', ()),)), + ('cdr', (('en', 'military', ()),)), + ('cds', (('en', 'credential', ()),)), + ('cdt', (('en', 'credential', ()),)), + ('cea', (('en', 'credential', ()), ('en', 'family', ()))), + ('ceas', (('en', 'credential', ()),)), + ('cebs', (('en', 'credential', ()),)), + ('ceds', (('en', 'credential', ()),)), + ('ceh', (('en', 'credential', ()),)), + ('cela', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('celebrity', (('en', 'occupation', ()),)), + ('cem', (('en', 'credential', ()),)), + ('ceo', (('en', 'office', ()),)), + ('cep', (('en', 'credential', ()),)), + ('cera', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('cet', (('en', 'credential', ()),)), + ('cf-slp', (('en', 'credential', ()),)), + ('cfa', (('en', 'credential', ()),)), + ('cfc', (('en', 'credential', ()),)), + ('cfcc', (('en', 'credential', ()),)), + ('cfce', (('en', 'credential', ()),)), + ('cfcm', (('en', 'credential', ()),)), + ('cfe', (('en', 'credential', ()),)), + ('cfeds', (('en', 'credential', ()),)), + ('cfi', (('en', 'credential', ()),)), + ('cfm', (('en', 'credential', ()),)), + ('cfo', (('en', 'office', ()),)), + ('cfp', (('en', 'credential', ()),)), + ('cfps', (('en', 'credential', ()),)), + ('cfr', (('en', 'credential', ()),)), + ('cfre', (('en', 'credential', ()),)), + ('cga', (('en', 'credential', ()),)), + ('cgap', (('en', 'credential', ()),)), + ('cgb', (('en', 'credential', ()),)), + ('cgc', (('en', 'credential', ()),)), + ('cgfm', (('en', 'credential', ()),)), + ('cgfo', (('en', 'credential', ()),)), + ('cgm', (('en', 'credential', ()),)), + ('cgma', (('en', 'credential', ()),)), + ('cgp', (('en', 'credential', ()),)), + ('cgr', (('en', 'credential', ()),)), + ('cgsp', (('en', 'credential', ()),)), + ('ch', (('en', 'credential', ()),)), + ('ch.ma', (('it', 'style', (('cased', 'Ch.ma'),)),)), + ('ch.mo', (('it', 'style', (('cased', 'Ch.mo'),)),)), + ('chair', (('en', 'office', ()),)), + ('chairs', (('en', 'office', ()),)), + ('chancellor', (('en', 'office', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('chaplain', (('en', 'religious', ()),)), + ('charge', (('en', 'office', ()),)), + ('chargé', (('en', 'office', ()), ('fr', 'office', ()))), + ('chas', (('en', 'given', (('same_as', 'charles'),)),)), + ('chba', (('en', 'credential', ()),)), + ('chdm', (('en', 'credential', ()),)), + ('che', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('chef', (('en', 'occupation', ()),)), + ('cheikh', (('ar', 'style', (('addresses', 'given'),)), ('fr', 'style', (('addresses', 'given'),)), ('fr', 'given', ()), ('fr', 'family', ()))), + ('chemist', (('en', 'occupation', ()),)), + ('ches', (('en', 'credential', ()),)), + ('chfc', (('en', 'credential', (('cased', 'ChFC'),)),)), + ('chi', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('chief', (('en', 'nobility', (('attaches', ('military', 'office')),)),)), + ('chieftain', (('en', 'nobility', ()),)), + ('chmc', (('en', 'credential', ()),)), + ('chmm', (('en', 'credential', ()),)), + ('choreographer', (('en', 'occupation', ()),)), + ('chp', (('en', 'credential', ()),)), + ('chpa', (('en', 'credential', ()),)), + ('chpe', (('en', 'credential', ()),)), + ('chpln', (('en', 'credential', ()),)), + ('chpse', (('en', 'credential', ()),)), + ('chrm', (('en', 'credential', ()),)), + ('chsc', (('en', 'credential', ()),)), + ('chse', (('en', 'credential', ()),)), + ('chse-a', (('en', 'credential', ()),)), + ('chsos', (('en', 'credential', ()),)), + ('chss', (('en', 'credential', ()),)), + ('cht', (('en', 'credential', ()),)), + ('cia', (('en', 'credential', ()),)), + ('cic', (('en', 'credential', ()),)), + ('cie', (('en', 'credential', ()),)), + ('cig', (('en', 'credential', ()),)), + ('cip', (('en', 'credential', ()),)), + ('cipm', (('en', 'credential', ()),)), + ('cips', (('en', 'credential', ()),)), + ('ciro', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('cisa', (('en', 'credential', ()),)), + ('cism', (('en', 'credential', ()),)), + ('cissp', (('en', 'credential', ()),)), + ('civil', (('en', 'modifier', ()),)), + ('cla', (('en', 'credential', ()),)), + ('classical', (('en', 'modifier', ()),)), + ('clergyman', (('en', 'occupation', ()),)), + ('clerk', (('en', 'office', ()),)), + ('cllr', (('en', 'office', ()),)), + ('clsd', (('en', 'credential', ()),)), + ('cltd', (('en', 'credential', ()),)), + ('clu', (('en', 'credential', ()),)), + ('cm', (('en', 'credential', ()),)), + ('cma', (('en', 'credential', ()),)), + ('cmas', (('en', 'credential', ()),)), + ('cmc', (('en', 'credential', ()),)), + ('cmfo', (('en', 'credential', ()),)), + ('cmg', (('en', 'credential', ()),)), + ('cmp', (('en', 'credential', ()),)), + ('cms', (('en', 'credential', ()),)), + ('cmsaf', (('en', 'military', ()),)), + ('cmsgt', (('en', 'military', ()),)), + ('cmsp', (('en', 'credential', ()),)), + ('cmt', (('en', 'credential', ()),)), + ('cna', (('en', 'credential', ()),)), + ('cnm', (('en', 'credential', ()),)), + ('cnp', (('en', 'credential', ()),)), + ('co-chair', (('en', 'office', ()),)), + ('co-chairs', (('en', 'office', ()),)), + ('co-founder', (('en', 'occupation', ()),)), + ('coach', (('en', 'occupation', ()),)), + ('col', (('en', 'military', ()),)), + ('collector', (('en', 'occupation', ()),)), + ('colonel', (('en', 'military', ()),)), + ('comedian', (('en', 'occupation', ()),)), + ('comedienne', (('en', 'occupation', ()),)), + ('comic', (('en', 'occupation', ()),)), + ('commander', (('en', 'military', ()),)), + ('commander-in-chief', (('en', 'military', ()),)), + ('commissioner', (('en', 'office', ()),)), + ('commodore', (('en', 'military', ()),)), + ('composer', (('en', 'occupation', ()),)), + ('compositeur', (('fr', 'occupation', ()),)), + ('comptroller', (('en', 'office', ()),)), + ('computer', (('en', 'modifier', ()),)), + ('comtesse', (('fr', 'nobility', ()),)), + ('conductor', (('en', 'occupation', ()),)), + ('consort', (('en', 'modifier', (('addresses', 'given'),)),)), + ('consultant', (('en', 'occupation', ()),)), + ('controller', (('en', 'office', ()),)), + ('corporal', (('en', 'military', ()),)), + ('corporate', (('en', 'modifier', ()),)), + ('correspondent', (('en', 'occupation', ()),)), + ('cota/l', (('en', 'credential', ()),)), + ('councillor', (('en', 'office', ()),)), + ('counsel', (('en', 'office', ()),)), + ('counselor', (('en', 'occupation', ()),)), + ('count', (('en', 'nobility', ()),)), + ('countess', (('en', 'nobility', ()),)), + ('courtier', (('en', 'office', ()),)), + ('cp', (('en', 'credential', ()),)), + ('cp-c', (('en', 'credential', ()),)), + ('cpa', (('en', 'credential', ()),)), + ('cpacc', (('en', 'credential', ()),)), + ('cpbe', (('en', 'credential', ()),)), + ('cpcm', (('en', 'credential', ()),)), + ('cpcu', (('en', 'credential', ()),)), + ('cpe', (('en', 'credential', ()),)), + ('cpfa', (('en', 'credential', ()),)), + ('cpfo', (('en', 'credential', ()),)), + ('cpg', (('en', 'credential', ()),)), + ('cph', (('en', 'credential', ()),)), + ('cpht', (('en', 'credential', (('cased', 'CPhT'),)),)), + ('cpim', (('en', 'credential', ()),)), + ('cpl', (('en', 'military', ()), ('en', 'credential', ()))), + ('cplp', (('en', 'credential', ()),)), + ('cpm', (('en', 'credential', ()),)), + ('cpnp-pc', (('en', 'credential', ()),)), + ('cpo', (('en', 'military', ()), ('en', 'credential', ()))), + ('cpp', (('en', 'credential', ()),)), + ('cppm', (('en', 'credential', ()),)), + ('cprc', (('en', 'credential', ()),)), + ('cpre', (('en', 'credential', ()),)), + ('cprp', (('en', 'credential', ()),)), + ('cpsc', (('en', 'credential', ()),)), + ('cpsi', (('en', 'credential', ()),)), + ('cpss', (('en', 'credential', ()),)), + ('cpt', (('en', 'military', ()), ('en', 'credential', ()))), + ('cpwa', (('en', 'credential', ()),)), + ('crde', (('en', 'credential', ()),)), + ('credit', (('en', 'modifier', ()),)), + ('criminal', (('en', 'modifier', ()),)), + ('criminologist', (('en', 'occupation', ()),)), + ('crisc', (('en', 'credential', ()),)), + ('critic', (('en', 'occupation', ()),)), + ('crma', (('en', 'credential', ()),)), + ('crme', (('en', 'credential', ()),)), + ('crna', (('en', 'credential', ()),)), + ('cro', (('en', 'credential', ()),)), + ('crp', (('en', 'credential', ()),)), + ('crt', (('en', 'credential', ()),)), + ('crtt', (('en', 'credential', ()),)), + ('csa', (('en', 'credential', ()),)), + ('csbe', (('en', 'credential', ()),)), + ('csc', (('en', 'credential', ()),)), + ('cscp', (('en', 'credential', ()),)), + ('cscu', (('en', 'credential', ()),)), + ('csep', (('en', 'credential', ()),)), + ('csi', (('en', 'credential', ()), ('en', 'family', ()))), + ('csm', (('en', 'military', ()), ('en', 'credential', ()))), + ('csp', (('en', 'credential', ()),)), + ('cspo', (('en', 'credential', ()),)), + ('csre', (('en', 'credential', ()),)), + ('csrte', (('en', 'credential', ()),)), + ('csslp', (('en', 'credential', ()),)), + ('cssm', (('en', 'credential', ()),)), + ('cst', (('en', 'credential', ()),)), + ('cste', (('en', 'credential', ()),)), + ('ctbs', (('en', 'credential', ()),)), + ('ctfa', (('en', 'credential', ()),)), + ('cto', (('en', 'credential', ()),)), + ('ctp', (('en', 'credential', ()),)), + ('cts', (('en', 'credential', ()),)), + ('cua', (('en', 'credential', ()), ('en', 'family', ()))), + ('curator', (('en', 'occupation', ()),)), + ('cusp', (('en', 'credential', ()),)), + ('customs', (('en', 'modifier', ()),)), + ('cva', (('en', 'credential', ()),)), + ('cvo', (('en', 'credential', ()),)), + ('cvp', (('en', 'credential', ()),)), + ('cvrs', (('en', 'credential', ()),)), + ('cwap', (('en', 'credential', ()),)), + ('cwb', (('en', 'credential', ()),)), + ('cwdp', (('en', 'credential', ()),)), + ('cwep', (('en', 'credential', ()),)), + ('cwna', (('en', 'credential', ()),)), + ('cwne', (('en', 'credential', ()),)), + ('cwo-2', (('en', 'military', ()),)), + ('cwo-3', (('en', 'military', ()),)), + ('cwo-4', (('en', 'military', ()),)), + ('cwo-5', (('en', 'military', ()),)), + ('cwo2', (('en', 'military', ()),)), + ('cwo3', (('en', 'military', ()),)), + ('cwo4', (('en', 'military', ()),)), + ('cwo5', (('en', 'military', ()),)), + ('cwp', (('en', 'credential', ()),)), + ('cwsp', (('en', 'credential', ()),)), + ('cxa', (('en', 'credential', ()),)), + ('cyclist', (('en', 'occupation', ()),)), + ('cyds', (('en', 'credential', ()),)), + ('cysa', (('en', 'credential', ()),)), + ("d'", (('fr', 'preposition', ()), ('it', 'preposition', ()))), + ("d'affaires", (('en', 'office', ()), ('fr', 'office', ()))), + ('da', (('it', 'preposition', ()), ('it', 'given', ()), ('pt', 'preposition', ()))), + ('dabfm', (('en', 'credential', ()),)), + ('dabvlm', (('en', 'credential', ()),)), + ('dacvim', (('en', 'credential', ()),)), + ('dal', (('it', 'preposition', ()), ('it', 'given', ()))), + ('dame', (('en', 'nobility', (('addresses', 'given'),)),)), + ('dancer', (('en', 'occupation', ()),)), + ('das', (('pt', 'preposition', ()),)), + ('dbe', (('en', 'credential', ()),)), + ('dc', (('en', 'credential', ()),)), + ('dcb', (('en', 'credential', ()),)), + ('dcm', (('en', 'credential', ()),)), + ('dcmg', (('en', 'credential', ()),)), + ('dcn', (('en', 'religious', ()),)), + ('dcvo', (('en', 'credential', ()),)), + ('dd', (('en', 'credential', ()),)), + ('dds', (('en', 'credential', ()),)), + ('de', (('es', 'preposition', ()), ('fr', 'preposition', ()), ('it', 'preposition', ()), ('nl', 'article', ()), ('pt', 'preposition', ()))), + ("de'", (('it', 'preposition', ()),)), + ('deacon', (('en', 'religious', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('dean', (('en', 'office', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('ded', (('en', 'credential', ()),)), + ('degli', (('it', 'preposition', ()),)), + ('dei', (('it', 'preposition', ()),)), + ('del', (('es', 'preposition', ()), ('es', 'given', ()), ('it', 'preposition', ()))), + ('dela', (('fil', 'preposition', ()), ('fil', 'given', ()))), + ('delegate', (('en', 'office', ()),)), + ("dell'", (('it', 'preposition', ()),)), + ('della', (('it', 'preposition', ()), ('it', 'given', ()))), + ('delle', (('it', 'preposition', ()),)), + ('delli', (('it', 'preposition', ()),)), + ('dello', (('it', 'preposition', ()),)), + ('dell’', (('it', 'preposition', ()),)), + ('dem', (('de', 'article', ()),)), + ('den', (('de', 'article', ()), ('de', 'given', ()), ('nl', 'article', ()))), + ('dep', (('en', 'credential', ()),)), + ('deputy', (('en', 'office', (('attaches', ('office',)),)),)), + ('der', (('de', 'article', ()), ('nl', 'article', ()))), + ('designated', (('en', 'modifier', ()),)), + ('designer', (('en', 'occupation', ()),)), + ('detective', (('en', 'military', ()),)), + ('developer', (('en', 'occupation', ()),)), + ('de’', (('it', 'preposition', ()),)), + ('dfc', (('en', 'credential', ()),)), + ('dfm', (('en', 'credential', ()),)), + ('dhr', (('nl', 'salutation', (('after_given', True),)),)), + ('di', (('it', 'preposition', ()), ('it', 'given', ()))), + ('dipl.-ing', (('de', 'academic', ()),)), + ('diplac', (('en', 'credential', (('cased', 'DiplAc'),)),)), + ('diplom', (('en', 'credential', ()),)), + ('diplomat', (('en', 'occupation', ()),)), + ('dir', (('en', 'office', ()),)), + ('director', (('en', 'office', ()),)), + ('discovery', (('en', 'modifier', ()),)), + ('dissident', (('en', 'occupation', ()),)), + ('district', (('en', 'modifier', ()),)), + ('division', (('en', 'modifier', ()),)), + ('djur', (('en', 'credential', ()),)), + ('dm', (('en', 'credential', ()),)), + ('dma', (('en', 'credential', ()),)), + ('dmd', (('en', 'credential', ()),)), + ('dmin', (('en', 'credential', (('cased', 'DMin'),)),)), + ('dmsc', (('en', 'credential', (('cased', 'DMSc'),)),)), + ('dmsci', (('en', 'credential', (('cased', 'DMSci'),)),)), + ('dnp', (('en', 'credential', ()),)), + ('do', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()), ('pt', 'preposition', ()))), + ('doamna', (('ro', 'salutation', (('after_given', True),)),)), + ('doamnă', (('ro', 'salutation', (('after_given', True),)),)), + ('docent', (('mul', 'academic', ()),)), + ('docket', (('en', 'modifier', ()),)), + ('doctor', (('mul', 'academic', ()),)), + ('domnișoara', (('ro', 'salutation', (('after_given', True),)),)), + ('domnul', (('ro', 'salutation', (('after_given', True),)),)), + ('dos', (('pt', 'preposition', ()),)), + ('dott.ssa', (('it', 'academic', (('cased', 'Dott.ssa'),)),)), + ('doyen', (('en', 'style', ()),)), + ('dphil', (('en', 'credential', (('cased', 'DPhil'),)),)), + ('dpm', (('en', 'credential', ()),)), + ('dpt', (('en', 'credential', ()),)), + ('dpty', (('en', 'office', ()),)), + ('dr', (('mul', 'academic', ()),)), + ('dra', (('es', 'academic', ()), ('pt', 'academic', ()))), + ('dramatist', (('en', 'occupation', ()),)), + ('drb', (('en', 'credential', ()),)), + ('drmp', (('en', 'credential', ()),)), + ('drph', (('en', 'credential', (('cased', 'DrPH'),)),)), + ('druid', (('en', 'religious', ()),)), + ('drummer', (('en', 'occupation', ()),)), + ('dsc', (('en', 'credential', (('cased', 'DSc'),)),)), + ('dsm', (('en', 'credential', ()),)), + ('dso', (('en', 'credential', ()),)), + ('dss', (('en', 'credential', ()),)), + ('dtr', (('en', 'credential', ()),)), + ('du', (('fr', 'preposition', ()), ('fr', 'given', ()))), + ('duchesse', (('fr', 'nobility', ()),)), + ('duke', (('en', 'given', ()), ('en', 'family', ()))), + ('dutchess', (('en', 'nobility', ()),)), + ('dvep', (('en', 'credential', ()),)), + ('dvm', (('en', 'credential', ()),)), + ('dí', (('und', 'given', ()),)), + ('d’', (('fr', 'preposition', ()), ('it', 'preposition', ()))), + ('e', (('it', 'conjunction', (('may_be_initial', True),)), ('pt', 'conjunction', (('may_be_initial', True),)))), + ('ea', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('ecologist', (('en', 'occupation', ()),)), + ('economist', (('en', 'occupation', ()),)), + ('ed', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('ed-', (('ar', 'article', ()),)), + ('edd', (('en', 'credential', ()),)), + ('edh-', (('ar', 'article', ()),)), + ('editor', (('en', 'occupation', ()),)), + ('edler', (('de', 'nobility', ()),)), + ('edmi', (('en', 'modifier', ()),)), + ('edohen', (('bin', 'nobility', ()),)), + ('educator', (('en', 'occupation', ()),)), + ('edw', (('en', 'given', (('same_as', 'edward'),)),)), + ('effendi', (('tr', 'style', ()),)), + ('ei', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('eit', (('en', 'credential', ()),)), + ('ekegbian', (('bin', 'nobility', ()),)), + ('el-', (('ar', 'article', ()),)), + ('eldri', (('is', 'generation', ()),)), + ('elerunwon', (('yo', 'nobility', ()),)), + ('els', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('emd', (('en', 'credential', ()),)), + ('eminence', (('en', 'style', ()),)), + ('emperor', (('en', 'nobility', ()),)), + ('empress', (('en', 'nobility', ()),)), + ('emt-b', (('en', 'credential', ()),)), + ('emt-i/85', (('en', 'credential', ()),)), + ('emt-i/99', (('en', 'credential', ()),)), + ('emt-p', (('en', 'credential', ()),)), + ('en-', (('ar', 'article', ()),)), + ('encik', (('ms', 'salutation', (('after_given', True),)),)), + ('engineer', (('en', 'academic', ()),)), + ('english', (('en', 'modifier', ()),)), + ('enp', (('en', 'credential', ()),)), + ('ens', (('en', 'military', ()),)), + ('entertainer', (('en', 'occupation', ()),)), + ('entrepreneur', (('en', 'occupation', ()),)), + ('envoy', (('en', 'office', ()),)), + ('er-', (('ar', 'article', ()),)), + ('erd', (('en', 'credential', ()), ('en', 'family', ()))), + ('erzbischof', (('de', 'religious', ()),)), + ('es-', (('ar', 'article', ()),)), + ('esh-', (('ar', 'article', ()),)), + ('esq', (('en', 'courtesy', ()),)), + ('esquire', (('en', 'courtesy', ()),)), + ('essayist', (('en', 'occupation', ()),)), + ('et', (('fr', 'conjunction', ()),)), + ('et-', (('ar', 'article', ()),)), + ('eth-', (('ar', 'article', ()),)), + ('evangelist', (('en', 'religious', ()),)), + ('evp', (('en', 'credential', ()),)), + ('excellency', (('en', 'style', ()),)), + ('excellent', (('en', 'modifier', ()),)), + ('exec', (('en', 'modifier', ()),)), + ('executive', (('en', 'modifier', ()),)), + ('expert', (('en', 'occupation', ()),)), + ('ez-', (('ar', 'article', ()),)), + ('faafp', (('en', 'credential', ()),)), + ('faan', (('en', 'credential', ()),)), + ('faap', (('en', 'credential', ()),)), + ('fac-c', (('en', 'credential', ()),)), + ('facc', (('en', 'credential', ()),)), + ('facd', (('en', 'credential', ()),)), + ('facem', (('en', 'credential', ()),)), + ('facep', (('en', 'credential', ()),)), + ('facha', (('en', 'credential', ()),)), + ('facofp', (('en', 'credential', ()),)), + ('facog', (('en', 'credential', ()),)), + ('facp', (('en', 'credential', ()),)), + ('facph', (('en', 'credential', ()),)), + ('facs', (('en', 'credential', ()),)), + ('fadm', (('en', 'military', ()),)), + ('faia', (('en', 'credential', ()), ('en', 'family', ()))), + ('faicp', (('en', 'credential', ()),)), + ('fala', (('en', 'credential', ()),)), + ('familie', (('da', 'salutation', (('after_given', True),)), ('de', 'salutation', (('after_given', True),)), ('nb', 'salutation', (('after_given', True),)))), + ('family', (('en', 'modifier', ()),)), + ('fashp', (('en', 'credential', ()),)), + ('fasid', (('en', 'credential', ()),)), + ('fasla', (('en', 'credential', ()),)), + ('fasma', (('en', 'credential', ()),)), + ('faspen', (('en', 'credential', ()),)), + ('father', (('en', 'religious', (('addresses', 'given'),)),)), + ('fca', (('en', 'credential', ()),)), + ('fcas', (('en', 'credential', ()),)), + ('fcela', (('en', 'credential', ()),)), + ('fd', (('en', 'credential', ()),)), + ('fec', (('en', 'credential', ()),)), + ('federal', (('en', 'modifier', ()),)), + ('fh', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('fh-prof', (('de', 'academic', ()),)), + ('fhames', (('en', 'credential', ()),)), + ('fic', (('en', 'credential', ()), ('en', 'family', ()))), + ('ficf', (('en', 'credential', ()),)), + ('fieee', (('en', 'credential', ()),)), + ('field', (('en', 'modifier', ()),)), + ('filh', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('filha', (('pt', 'agnomen', ()), ('pt', 'family', ()))), + ('filho', (('pt', 'agnomen', ()), ('pt', 'family', ()))), + ('film', (('en', 'modifier', ()),)), + ('fils', (('fr', 'generation', ()), ('fr', 'family', ()))), + ('financial', (('en', 'modifier', ()),)), + ('first', (('en', 'modifier', ()),)), + ('flag', (('en', 'modifier', ()),)), + ('flying', (('en', 'modifier', ()),)), + ('fmp', (('en', 'credential', ()),)), + ('fmva', (('en', 'credential', ()),)), + ('fnp-bc', (('en', 'credential', ()),)), + ('fnp-c', (('en', 'credential', ()),)), + ('fnss', (('en', 'credential', ()),)), + ('fo', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('foreign', (('en', 'modifier', ()),)), + ('forester', (('en', 'occupation', ()),)), + ('founder', (('en', 'occupation', ()),)), + ('fp&a', (('en', 'credential', ()),)), + ('fp-c', (('en', 'credential', ()),)), + ('fpc', (('en', 'credential', ()),)), + ('fr', (('en', 'religious', ()),)), + ('frau', (('de', 'salutation', (('after_given', True),)), ('de', 'family', ()))), + ('freifrau', (('de', 'nobility', ()),)), + ('freiherr', (('de', 'nobility', ()), ('de', 'rank', ()), ('de', 'given', ()))), + ('freiherrin', (('de', 'rank', ()), ('de', 'given', ()))), + ('friar', (('en', 'religious', ()),)), + ('frk', (('da', 'salutation', (('after_given', True),)), ('nb', 'salutation', (('after_given', True),)), ('sv', 'salutation', (('after_given', True),)))), + ('frm', (('en', 'credential', ()),)), + ('fru', (('da', 'salutation', ()), ('nb', 'salutation', ()), ('sv', 'salutation', ()), ('sv', 'family', ()))), + ('fräulein', (('de', 'salutation', (('after_given', True),)),)), + ('fröken', (('sv', 'salutation', (('after_given', True),)),)), + ('frøken', (('da', 'salutation', (('after_given', True),)), ('nb', 'salutation', (('after_given', True),)))), + ('frú', (('is', 'salutation', ()),)), + ('fsa', (('en', 'credential', ()),)), + ('fsdp', (('en', 'credential', ()),)), + ('fws', (('en', 'credential', ()),)), + ('född', (('sv', 'birth_marker', ()),)), + ('fødd', (('nn', 'birth_marker', ()),)), + ('født', (('da', 'birth_marker', ()), ('nb', 'birth_marker', ()))), + ('fürst', (('de', 'nobility', ()),)), + ('fürsterzbischof', (('de', 'religious', ()),)), + ('gaf', (('en', 'military', ()),)), + ('gba', (('en', 'credential', ()),)), + ('gbe', (('en', 'credential', ()),)), + ('gcb', (('en', 'credential', ()),)), + ('gchs', (('en', 'credential', ()),)), + ('gcie', (('en', 'credential', ()),)), + ('gcmg', (('en', 'credential', ()),)), + ('gcsi', (('en', 'credential', ()),)), + ('gcvo', (('en', 'credential', ()),)), + ('geb', (('de', 'birth_marker', ()),)), + ('geboren', (('nl', 'birth_marker', ()),)), + ('geborene', (('de', 'birth_marker', ()),)), + ('gen', (('en', 'military', ()),)), + ('general', (('en', 'military', ()),)), + ('generalissimo', (('en', 'military', ()),)), + ('gent.le', (('it', 'style', (('cased', 'Gent.le'),)),)), + ('gent.ma', (('it', 'style', (('cased', 'Gent.ma'),)),)), + ('gent.mo', (('it', 'style', (('cased', 'Gent.mo'),)),)), + ('gentiluomo', (('it', 'nobility', ()),)), + ('geo', (('en', 'given', (('same_as', 'george'),)),)), + ('giani', (('pa', 'religious', ()),)), + ('ginang', (('fil', 'salutation', (('after_given', True),)),)), + ('ginoo', (('fil', 'salutation', (('after_given', True),)),)), + ('gisp', (('en', 'credential', ()),)), + ('git', (('en', 'credential', ()),)), + ('gm', (('en', 'credential', ()),)), + ('gmb', (('en', 'credential', ()),)), + ('gmr', (('en', 'credential', ()),)), + ('goodman', (('en', 'salutation', ()), ('en', 'family', ()))), + ('goodwife', (('en', 'salutation', (('after_given', True),)),)), + ('gospodin', (('bs', 'salutation', (('after_given', True),)), ('hr', 'salutation', (('after_given', True),)), ('sr', 'salutation', (('after_given', True),)))), + ('gospođa', (('bs', 'salutation', (('after_given', True),)), ('hr', 'salutation', (('after_given', True),)), ('sr', 'salutation', (('after_given', True),)))), + ('gospođica', (('bs', 'salutation', (('after_given', True),)), ('hr', 'salutation', (('after_given', True),)), ('sr', 'salutation', (('after_given', True),)))), + ('gov', (('en', 'office', ()),)), + ('governor', (('en', 'office', ()),)), + ('gphr', (('en', 'credential', ()),)), + ('graaf', (('nl', 'nobility', ()),)), + ('grace', (('en', 'style', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('graf', (('de', 'nobility', ()),)), + ('grafas', (('lt', 'nobility', ()),)), + ('grafi', (('ka', 'nobility', ()),)), + ('grand', (('en', 'modifier', ()),)), + ('greifi', (('is', 'nobility', ()),)), + ('greve', (('da', 'nobility', ()), ('nb', 'nobility', ()), ('sv', 'nobility', ()))), + ('gri', (('en', 'credential', ()),)), + ('group', (('en', 'modifier', ()),)), + ('großfürst', (('de', 'nobility', ()),)), + ('grp', (('en', 'credential', ()),)), + ('gräfin', (('de', 'nobility', ()),)), + ('gsmieee', (('en', 'credential', ()),)), + ('guitarist', (('en', 'occupation', ()),)), + ('guru', (('bn', 'religious', (('addresses', 'given'),)), ('hi', 'religious', (('addresses', 'given'),)), ('pa', 'religious', (('addresses', 'given'),)))), + ('gyani', (('pa', 'religious', ()), ('pa', 'given', ()))), + ('gysgt', (('en', 'military', ()),)), + ('h', (('es', 'generation', (('default', False),)),)), + ('hajji', (('ar', 'style', ()),)), + ('hccp', (('en', 'credential', ()),)), + ('headman', (('en', 'office', ()),)), + ('heer', (('nl', 'given', ()), ('nl', 'family', ()))), + ('heir', (('en', 'nobility', ()),)), + ('heiress', (('en', 'nobility', ()),)), + ('her', (('en', 'modifier', ()),)), + ('hereditary', (('en', 'modifier', ()),)), + ('heren', (('nl', 'family', ()),)), + ('herr', (('da', 'salutation', ()), ('de', 'salutation', ()), ('de', 'family', ()), ('nb', 'salutation', ()), ('sv', 'salutation', ()))), + ('herra', (('fi', 'salutation', ()), ('fi', 'family', ()), ('is', 'salutation', ()))), + ('herren', (('de', 'family', ()),)), + ('herrn', (('de', 'salutation', (('after_given', True),)),)), + ('herzog', (('de', 'nobility', ()),)), + ('het', (('nl', 'article', ()),)), + ('high', (('en', 'modifier', ()),)), + ('highness', (('en', 'style', ()),)), + ('his', (('en', 'modifier', ()),)), + ('historian', (('en', 'occupation', ()),)), + ('historicus', (('nl', 'occupation', ()),)), + ('historien', (('fr', 'occupation', ()),)), + ('holiness', (('en', 'style', ()),)), + ('hon', (('en', 'style', ()),)), + ('honorable', (('en', 'style', ()),)), + ('honourable', (('en', 'style', ()),)), + ('host', (('en', 'occupation', ()),)), + ('hr', (('da', 'salutation', ()), ('de', 'salutation', ()), ('nb', 'salutation', ()), ('sv', 'salutation', ()))), + ('hrabia', (('pl', 'nobility', ()),)), + ('hrs', (('en', 'credential', ()),)), + ('härra', (('et', 'salutation', ()), ('et', 'family', ()))), + ('i', (('ca', 'conjunction', (('may_be_initial', True),)), ('de', 'generation', (('default', False),)), ('mul', 'generation', ()), ('pl', 'conjunction', (('may_be_initial', True),)))), + ('iaccp', (('en', 'credential', ()),)), + ('iaee', (('en', 'credential', ()),)), + ('ibn', (('ar', 'patronymic', ()),)), + ('iccm-d', (('en', 'credential', ()),)), + ('iccm-f', (('en', 'credential', ()),)), + ('idsm', (('en', 'credential', ()),)), + ('ifgict', (('en', 'credential', ()),)), + ('ii', (('mul', 'generation', ()),)), + ('iii', (('mul', 'generation', ()),)), + ('ill.ma', (('it', 'style', (('cased', 'Ill.ma'),)),)), + ('ill.mo', (('it', 'style', (('cased', 'Ill.mo'),)),)), + ('illustrator', (('en', 'occupation', ()),)), + ('imam', (('ar', 'religious', ()),)), + ('industrialist', (('en', 'occupation', ()),)), + ('information', (('en', 'modifier', ()),)), + ('ing', (('mul', 'credential', (('cased', 'Ing'),)), ('mul', 'given', ()), ('mul', 'family', ()))), + ('instructor', (('en', 'occupation', ()),)), + ('intelligence', (('en', 'modifier', ()),)), + ('intendant', (('en', 'office', ()),)), + ('inventor', (('en', 'occupation', ()),)), + ('investigator', (('en', 'occupation', ()),)), + ('investor', (('en', 'occupation', ()),)), + ('iom', (('en', 'credential', ()),)), + ('ipep', (('en', 'credential', ()),)), + ('ipm', (('en', 'credential', ()),)), + ('iso', (('en', 'credential', ()),)), + ('issp-csp', (('en', 'credential', ()),)), + ('issp-sa', (('en', 'credential', ()),)), + ('itil', (('en', 'credential', ()),)), + ('iv', (('mul', 'generation', ()),)), + ('jas', (('en', 'given', (('same_as', 'james'),)),)), + ('jaunesnioji', (('lt', 'generation', ()),)), + ('jaunesnysis', (('lt', 'generation', ()),)), + ('jaunākais', (('lv', 'generation', ()),)), + ('jaunākā', (('lv', 'generation', ()),)), + ('jd', (('en', 'credential', ()), ('en', 'given', ()))), + ('jno', (('en', 'given', (('same_as', 'john'),)),)), + ('jnr', (('en', 'generation', ()),)), + ('jo', (('fil', 'given', (('same_as', 'josé'),)),)), + ('journalist', (('en', 'occupation', ()),)), + ('journeyman', (('en', 'occupation', ()),)), + ('jp', (('en', 'credential', ()),)), + ('jr', (('en', 'generation', ()),)), + ('judge', (('en', 'office', ()),)), + ('judicial', (('en', 'modifier', ()),)), + ('juffrouw', (('nl', 'salutation', (('after_given', True),)),)), + ('junior', (('en', 'generation', ()),)), + ('jurist', (('en', 'occupation', ()),)), + ('justice', (('en', 'office', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('júnior', (('pt', 'generation', ()),)), + ('kalbatoni', (('ka', 'salutation', (('after_given', True),)),)), + ('kbe', (('en', 'credential', ()),)), + ('kcb', (('en', 'credential', ()),)), + ('kchs/dchs', (('en', 'credential', ()),)), + ('kcie', (('en', 'credential', ()),)), + ('kcmg', (('en', 'credential', ()),)), + ('kcsi', (('en', 'credential', ()),)), + ('kcvo', (('en', 'credential', ()),)), + ('keyboardist', (('en', 'occupation', ()),)), + ('kg', (('en', 'credential', ()),)), + ('khs/dhs', (('en', 'credential', ()),)), + ('king', (('en', 'nobility', (('addresses', 'given'), ('attaches', ('consort',)))), ('en', 'given', ()), ('en', 'family', ()))), + ("king's", (('en', 'modifier', ()),)), + ('kingdom', (('en', 'modifier', ()),)), + ('knight', (('en', 'nobility', ()),)), + ('knowledge', (('en', 'modifier', ()),)), + ('knt', (('en', 'credential', (('cased', 'Knt'),)),)), + ('kp', (('en', 'credential', ()),)), + ('krahv', (('et', 'nobility', ()),)), + ('kreivi', (('fi', 'nobility', ()),)), + ('kt', (('en', 'credential', (('cased', 'Kt'),)),)), + ("l'", (('fr', 'article', ()), ('it', 'article', ()))), + ('la', (('es', 'article', ()), ('es', 'given', ()), ('fr', 'article', ()), ('it', 'article', ()))), + ('lac', (('en', 'credential', ()), ('en', 'family', ()))), + ('lady', (('en', 'nobility', (('attaches', ('justice',)),)),)), + ('lama', (('bo', 'religious', (('addresses', 'given'),)), ('mn', 'religious', (('addresses', 'given'),)))), + ('lamido', (('ff', 'nobility', ()),)), + ('las', (('es', 'article', ()),)), + ('law', (('en', 'modifier', ()),)), + ('lawyer', (('en', 'occupation', ()),)), + ('lcdr', (('en', 'military', ()),)), + ('lcmt', (('en', 'credential', ()),)), + ('lcpc', (('en', 'credential', ()),)), + ('lcpl', (('en', 'military', ()),)), + ('lcsw', (('en', 'credential', ()),)), + ('lcsw-c', (('en', 'credential', ()),)), + ('lcsw-r', (('en', 'credential', ()),)), + ('ld', (('en', 'credential', ()),)), + ('le', (('fr', 'article', ()), ('fr', 'given', ()))), + ('leader', (('en', 'office', ()),)), + ('lecturer', (('en', 'occupation', ()),)), + ('legal', (('en', 'modifier', ()),)), + ('lg', (('en', 'credential', ()),)), + ('librarian', (('en', 'occupation', ()),)), + ('lieutenant', (('en', 'military', ()),)), + ('linguist', (('en', 'occupation', ()),)), + ('lisw-s', (('en', 'credential', ()),)), + ('literary', (('en', 'modifier', ()),)), + ('litk', (('en', 'credential', ()),)), + ('litl', (('en', 'credential', ()),)), + ('litp', (('en', 'credential', ()),)), + ('llm', (('en', 'credential', ()),)), + ('lm', (('en', 'credential', ()),)), + ('lmsw', (('en', 'credential', ()),)), + ('lmt', (('en', 'credential', ()),)), + ('lord', (('en', 'nobility', (('attaches', ('chancellor', 'justice')),)),)), + ('los', (('es', 'article', ()),)), + ('lp', (('en', 'credential', ()),)), + ('lpa', (('en', 'credential', ()),)), + ('lpc', (('en', 'credential', ()),)), + ('lpn', (('en', 'credential', ()),)), + ('lpss', (('en', 'credential', ()),)), + ('lsi', (('en', 'credential', ()),)), + ('lsit', (('en', 'credential', ()),)), + ('lt', (('en', 'military', ()), ('en', 'credential', ()))), + ('ltc', (('en', 'military', ()),)), + ('ltcol', (('en', 'military', ()),)), + ('ltg', (('en', 'military', ()),)), + ('ltgen', (('en', 'military', ()),)), + ('ltjg', (('en', 'military', ()),)), + ('lvn', (('en', 'credential', ()),)), + ('lvo', (('en', 'credential', ()),)), + ('lvt', (('en', 'credential', ()),)), + ('lyricist', (('en', 'occupation', ()),)), + ('l’', (('fr', 'article', ()), ('it', 'article', ()))), + ('ma', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()), ('fil', 'given', (('same_as', 'maría'),)))), + ('maaa', (('en', 'credential', ()),)), + ('mac', (('en', 'clan', ()), ('en', 'given', ()), ('ga', 'clan', ()), ('gd', 'clan', ()))), + ('madam', (('en', 'salutation', (('after_given', True),)),)), + ('madame', (('en', 'salutation', (('after_given', True),)), ('fr', 'salutation', (('after_given', True),)))), + ('mademoiselle', (('fr', 'salutation', (('after_given', True),)),)), + ('mag', (('en', 'office', ()),)), + ('mag-judge', (('en', 'office', ()),)), + ('mag/judge', (('en', 'office', ()),)), + ('magistrate', (('en', 'office', (('attaches', ('judge',)),)),)), + ('magistrate-judge', (('en', 'office', ()),)), + ('magnate', (('en', 'occupation', ()),)), + ('maharajah', (('hi', 'nobility', ()),)), + ('maharani', (('hi', 'nobility', ()),)), + ('mahdi', (('ar', 'religious', ()), ('ar', 'given', ()), ('ar', 'family', ()))), + ('maid', (('en', 'occupation', (('addresses', 'given'),)),)), + ('maj', (('en', 'military', ()),)), + ('majesty', (('en', 'style', ()),)), + ('majgen', (('en', 'military', ()),)), + ('major', (('en', 'military', (('attaches', ('general',)),)), ('en', 'given', ()), ('en', 'family', ()))), + ('malam', (('ha', 'given', ()), ('ha', 'family', ()))), + ('manager', (('en', 'occupation', ()),)), + ('marcher', (('en', 'modifier', ()),)), + ('marchess', (('en', 'nobility', ()),)), + ('marchioness', (('en', 'nobility', ()),)), + ('marketing', (('en', 'modifier', ()),)), + ('marquess', (('en', 'nobility', ()),)), + ('marquis', (('en', 'nobility', ()), ('en', 'given', ()), ('en', 'family', ()), ('fr', 'nobility', ()))), + ('marquise', (('en', 'nobility', ()), ('en', 'given', ()), ('en', 'family', ()), ('fr', 'nobility', ()))), + ('marshal', (('en', 'military', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('master', (('en', 'salutation', (('addresses', 'given'),)), ('en', 'family', ()))), + ('mathematician', (('en', 'occupation', ()),)), + ('mathematics', (('en', 'modifier', ()),)), + ('matriarch', (('en', 'kinship', ()),)), + ('mayor', (('en', 'office', ()),)), + ('mb', (('en', 'credential', ()),)), + ('mba', (('en', 'credential', ()),)), + ('mbbch', (('en', 'credential', (('cased', 'MBBCh'),)),)), + ('mbchb', (('en', 'credential', (('cased', 'MBChB'),)),)), + ('mbe', (('en', 'credential', ()),)), + ('mbs', (('en', 'credential', ()),)), + ('mc', (('en', 'credential', ()), ('en', 'clan', ()), ('ga', 'clan', ()), ('gd', 'clan', ()))), + ('mcct', (('en', 'credential', ()),)), + ('mcdba', (('en', 'credential', ()),)), + ('mches', (('en', 'credential', ()),)), + ('mcm', (('en', 'credential', ()),)), + ('mcp', (('en', 'credential', ()),)), + ('mcpd', (('en', 'credential', ()),)), + ('mcpo', (('en', 'military', ()),)), + ('mcpoc', (('en', 'military', ()),)), + ('mcpon', (('en', 'military', ()),)), + ('mcsa', (('en', 'credential', ()),)), + ('mcsd', (('en', 'credential', ()),)), + ('mcse', (('en', 'credential', ()),)), + ('mct', (('en', 'credential', ()),)), + ('md', (('en', 'credential', ()),)), + ('mda', (('de', 'credential', ()),)), + ('mdb', (('de', 'credential', ()),)), + ('mdbb', (('de', 'credential', ()),)), + ('mdep', (('de', 'credential', ()),)), + ('mdhb', (('de', 'credential', ()),)), + ('mdiv', (('en', 'credential', (('cased', 'MDiv'),)),)), + ('mdl', (('de', 'credential', ()),)), + ('mdphd', (('en', 'credential', (('cased', 'MDPhD'),)),)), + ('me', (('fr', 'academic', ()),)), + ('mem', (('en', 'credential', ()),)), + ('member', (('en', 'office', ()),)), + ('memoirist', (('en', 'occupation', ()),)), + ('meneer', (('nl', 'salutation', (('after_given', True),)),)), + ('meng', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('merchant', (('en', 'occupation', ()),)), + ('met', (('en', 'religious', ()),)), + ('metropolitan', (('en', 'religious', ()),)), + ('mevr', (('nl', 'salutation', (('after_given', True),)),)), + ('mevrouw', (('nl', 'salutation', (('after_given', True),)),)), + ('mevrouwe', (('nl', 'salutation', (('after_given', True),)),)), + ('mfa', (('en', 'credential', ()),)), + ('mg', (('en', 'military', ()),)), + ('mgr', (('en', 'religious', ()), ('fr', 'religious', ()))), + ('mgysgt', (('en', 'military', ()),)), + ('mhsc', (('en', 'credential', (('cased', 'MHSc'),)),)), + ('micp', (('en', 'credential', ()),)), + ('mieee', (('en', 'credential', ()),)), + ('military', (('en', 'modifier', ()),)), + ('minister', (('en', 'office', ()),)), + ('mirm', (('en', 'credential', ()),)), + ('miss', (('en', 'salutation', (('after_given', True),)), ('en', 'family', ()))), + ('misses', (('en', 'salutation', (('after_given', True),)),)), + ('missionary', (('en', 'occupation', ()),)), + ('mister', (('en', 'salutation', ()), ('en', 'family', ()))), + ('mladšia', (('sk', 'generation', ()),)), + ('mladší', (('cs', 'generation', ()), ('sk', 'generation', ()))), + ('mlajša', (('sl', 'generation', ()),)), + ('mlajši', (('sl', 'generation', ()),)), + ('mlađa', (('bs', 'generation', ()), ('hr', 'generation', ()), ('sr', 'generation', ()))), + ('mlađi', (('bs', 'generation', ()), ('hr', 'generation', ()), ('sr', 'generation', ()))), + ('mle', (('en', 'credential', ()),)), + ('mlle', (('fr', 'salutation', (('after_given', True),)),)), + ('mls', (('en', 'credential', ()),)), + ('mlse', (('en', 'credential', ()),)), + ('mlt', (('en', 'credential', ()),)), + ('mm', (('en', 'credential', ()),)), + ('mmad', (('en', 'credential', ()),)), + ('mmas', (('en', 'credential', ()),)), + ('mme', (('fr', 'salutation', (('after_given', True),)),)), + ('mmed', (('en', 'credential', (('cased', 'MMed'),)),)), + ('mmsc', (('en', 'credential', (('cased', 'MMSc'),)),)), + ('mnaa', (('en', 'credential', ()),)), + ('mnae', (('en', 'credential', ()),)), + ('mobster', (('en', 'occupation', ()),)), + ('model', (('en', 'occupation', ()),)), + ('monk', (('en', 'religious', ()),)), + ('monseigneur', (('fr', 'religious', ()),)), + ('monsieur', (('fr', 'salutation', (('after_given', True),)),)), + ('monsignor', (('en', 'religious', ()),)), + ('most', (('en', 'modifier', ()),)), + ('mother', (('en', 'religious', (('addresses', 'given'),)),)), + ('mountaineer', (('en', 'occupation', ()),)), + ('mp', (('en', 'credential', ()),)), + ('mpa', (('en', 'credential', ()),)), + ('mpco-cg', (('en', 'military', ()),)), + ('mph', (('en', 'credential', ()),)), + ('mphil', (('en', 'credential', (('cased', 'MPhil'),)),)), + ('mpse', (('en', 'credential', ()),)), + ('mr', (('en', 'salutation', (('after_given', True),)),)), + ('mra', (('en', 'credential', ()),)), + ('mrs', (('en', 'salutation', (('after_given', True),)),)), + ('ms', (('en', 'salutation', ()), ('en', 'credential', ()))), + ('msa', (('en', 'credential', ()),)), + ('msc', (('en', 'credential', (('cased', 'MSc'),)),)), + ('mscmsm', (('en', 'credential', ()),)), + ('msg', (('en', 'military', ()),)), + ('msgt', (('en', 'military', ()),)), + ('msm', (('en', 'credential', ()),)), + ('mt', (('en', 'credential', ()),)), + ('mt-bc', (('en', 'credential', ()),)), + ('mts', (('en', 'credential', ()),)), + ('mufti', (('ar', 'religious', ()),)), + ('mullah', (('fa', 'religious', ()),)), + ('municipal', (('en', 'modifier', ()),)), + ('murshid', (('ar', 'religious', ()),)), + ('musician', (('en', 'occupation', ()),)), + ('musicologist', (('en', 'occupation', ()),)), + ('mvo', (('en', 'credential', ()),)), + ('mx', (('en', 'salutation', (('after_given', True),)),)), + ('mystery', (('en', 'modifier', ()),)), + ('nanny', (('en', 'occupation', ()),)), + ('narrator', (('en', 'occupation', ()),)), + ('national', (('en', 'modifier', ()),)), + ('naturalist', (('en', 'occupation', ()),)), + ('navy', (('en', 'modifier', ()),)), + ('nbc-his', (('en', 'credential', ()),)), + ('nbcch', (('en', 'credential', ()),)), + ('nbcch-ps', (('en', 'credential', ()),)), + ('nbcdch', (('en', 'credential', ()),)), + ('nbcdch-ps', (('en', 'credential', ()),)), + ('nbcfch', (('en', 'credential', ()),)), + ('nbcfch-ps', (('en', 'credential', ()),)), + ('nbct', (('en', 'credential', ()),)), + ('ncarb', (('en', 'credential', ()),)), + ('nccp', (('en', 'credential', ()),)), + ('ncidq', (('en', 'credential', ()),)), + ('ncps', (('en', 'credential', ()),)), + ('ncso', (('en', 'credential', ()),)), + ('ncto', (('en', 'credential', ()),)), + ('nd', (('en', 'credential', ()),)), + ('ndtr', (('en', 'credential', ()),)), + ('nee', (('en', 'birth_marker', ()), ('fr', 'birth_marker', ()))), + ('neiti', (('fi', 'salutation', (('after_given', True),)),)), + ('net', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('neta', (('pt', 'agnomen', ()), ('pt', 'given', ()), ('pt', 'family', ()))), + ('neto', (('pt', 'agnomen', ()), ('pt', 'given', ()), ('pt', 'family', ()))), + ('netto', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('neuroscientist', (('en', 'occupation', ()),)), + ('nmd', (('en', 'credential', ()),)), + ('novelist', (('en', 'occupation', ()),)), + ('np', (('en', 'credential', ()),)), + ('np-c', (('en', 'credential', ()),)), + ('nraemt', (('en', 'credential', ()),)), + ('nremr', (('en', 'credential', ()),)), + ('nremt', (('en', 'credential', ()),)), + ('nrp', (('en', 'credential', ()),)), + ('nt', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('nuorempi', (('fi', 'generation', ()),)), + ('nurse', (('en', 'academic', ()),)), + ('né', (('en', 'birth_marker', ()), ('fr', 'birth_marker', ()))), + ('née', (('en', 'birth_marker', ()), ('fr', 'birth_marker', ()))), + ('ní', (('ga', 'clan', (('cased', 'Ní'),)),)), + ('obe', (('en', 'credential', ()),)), + ('obi', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('obstetritian', (('en', 'occupation', ()),)), + ('oca', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('ocm', (('en', 'credential', ()),)), + ('ocp', (('en', 'credential', ()),)), + ('od', (('en', 'credential', ()),)), + ('of', (('en', 'conjunction', ()),)), + ('officer', (('en', 'military', ()),)), + ('om', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('op', (('nl', 'preposition', ()),)), + ('opera', (('en', 'modifier', ()),)), + ('operating', (('en', 'modifier', ()),)), + ('ornithologist', (('en', 'occupation', ()),)), + ('oscp', (('en', 'credential', ()),)), + ('ot', (('en', 'credential', ()),)), + ('otr/l', (('en', 'credential', ()),)), + ('owura', (('ak', 'salutation', (('after_given', True),)),)), + ('pa', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('pa-c', (('en', 'credential', ()),)), + ('painter', (('en', 'occupation', ()),)), + ('paleontologist', (('en', 'occupation', ()),)), + ('paní', (('cs', 'salutation', ()), ('cs', 'family', ()))), + ('pastor', (('en', 'religious', ()),)), + ('patriarch', (('en', 'religious', ()),)), + ('pcc', (('en', 'credential', ()),)), + ('pci', (('en', 'credential', ()),)), + ('pd', (('de', 'academic', ()),)), + ('pe', (('en', 'credential', ()), ('en', 'family', ()))), + ('pediatrician', (('en', 'occupation', ()),)), + ('personality', (('en', 'occupation', ()),)), + ('petty', (('en', 'modifier', ()),)), + ('pfc', (('en', 'military', ()),)), + ('pfmp', (('en', 'credential', ()),)), + ('pg', (('en', 'credential', ()),)), + ('pgmp', (('en', 'credential', ()),)), + ('pharaoh', (('en', 'nobility', ()),)), + ('pharmd', (('en', 'credential', (('cased', 'PharmD'),)),)), + ('phc', (('en', 'credential', (('cased', 'PhC'),)),)), + ('phd', (('en', 'credential', (('cased', 'PhD'),)),)), + ('philantropist', (('en', 'occupation', ()),)), + ('philosopher', (('en', 'occupation', ()),)), + ('photographer', (('en', 'occupation', ()),)), + ('phr', (('en', 'credential', ()),)), + ('phrca', (('en', 'credential', ()),)), + ('physician', (('en', 'occupation', ()),)), + ('physicist', (('en', 'occupation', ()),)), + ('pi', (('en', 'family', ()),)), + ('pianist', (('en', 'occupation', ()),)), + ('pilot', (('en', 'occupation', ()),)), + ('pioneer', (('en', 'occupation', ()),)), + ('pir', (('fa', 'religious', ()),)), + ('pla', (('en', 'credential', ()),)), + ('player', (('en', 'occupation', ()),)), + ('playwright', (('en', 'occupation', ()),)), + ('pls', (('en', 'credential', ()),)), + ('pmc', (('en', 'credential', ()),)), + ('pmhnp-bc', (('en', 'credential', ()),)), + ('pmi-acp', (('en', 'credential', ()),)), + ('pmp', (('en', 'credential', ()),)), + ('po1', (('en', 'military', ()),)), + ('po2', (('en', 'military', ()),)), + ('po3', (('en', 'military', ()),)), + ('poet', (('en', 'occupation', ()),)), + ('police', (('en', 'modifier', ()),)), + ('political', (('en', 'modifier', ()),)), + ('politician', (('en', 'occupation', ()),)), + ('ponas', (('lt', 'salutation', (('after_given', True),)),)), + ('ponia', (('lt', 'salutation', (('after_given', True),)),)), + ('pope', (('en', 'religious', (('addresses', 'given'),)),)), + ('pp', (('en', 'credential', ()),)), + ('pps', (('en', 'credential', ()),)), + ('prefect', (('en', 'office', ()),)), + ('preg.ma', (('it', 'style', (('cased', 'Preg.ma'),)),)), + ('preg.mo', (('it', 'style', (('cased', 'Preg.mo'),)),)), + ('preili', (('et', 'salutation', (('after_given', True),)),)), + ('prelate', (('en', 'religious', ()),)), + ('premier', (('en', 'office', ()),)), + ('pres', (('en', 'office', ()),)), + ('presbyter', (('en', 'religious', ()),)), + ('president', (('en', 'office', ()),)), + ('presiding', (('en', 'modifier', ()),)), + ('priest', (('en', 'religious', ()),)), + ('priestess', (('en', 'religious', ()),)), + ('primate', (('en', 'religious', ()),)), + ('prime', (('en', 'modifier', ()),)), + ('prin', (('en', 'office', ()),)), + ('prince', (('en', 'nobility', (('addresses', 'given'), ('attaches', ('consort',)))), ('en', 'given', ()), ('en', 'family', ()))), + ('princess', (('en', 'nobility', (('addresses', 'given'), ('attaches', ('consort', 'royal')))), ('en', 'given', ()), ('en', 'family', ()))), + ('principal', (('en', 'office', ()),)), + ('printer', (('en', 'occupation', ()),)), + ('printmaker', (('en', 'occupation', ()),)), + ('prinz', (('de', 'nobility', ()),)), + ('prior', (('en', 'religious', ()),)), + ('priv.-doz', (('de', 'academic', ()),)), + ('private', (('en', 'military', ()),)), + ('prm', (('en', 'credential', ()),)), + ('pro', (('en', 'academic', (('attaches', ('chancellor',)),)),)), + ('producer', (('en', 'occupation', ()),)), + ('prof', (('mul', 'academic', ()),)), + ('prof.ssa', (('it', 'academic', (('cased', 'Prof.ssa'),)),)), + ('professor', (('mul', 'academic', ()),)), + ('proua', (('et', 'salutation', (('after_given', True),)),)), + ('provost', (('en', 'office', ()),)), + ('pslc', (('en', 'office', ()),)), + ('psm', (('en', 'credential', ()),)), + ('psp', (('en', 'credential', ()),)), + ('psychiatrist', (('en', 'occupation', ()),)), + ('psychologist', (('en', 'occupation', ()),)), + ('psyd', (('en', 'credential', (('cased', 'PsyD'),)),)), + ('pt', (('en', 'credential', ()),)), + ('pta', (('en', 'credential', ()),)), + ('publisher', (('en', 'occupation', ()),)), + ('pursuivant', (('en', 'office', ()),)), + ('pv2', (('en', 'military', ()),)), + ('pvt', (('en', 'military', ()),)), + ('qam', (('en', 'credential', ()),)), + ('qc', (('en', 'credential', ()),)), + ('qcsw', (('en', 'credential', ()),)), + ('qfsm', (('en', 'credential', ()),)), + ('qgm', (('en', 'credential', ()),)), + ('qpm', (('en', 'credential', ()),)), + ('qsd', (('en', 'credential', ()),)), + ('qsp', (('en', 'credential', ()),)), + ('queen', (('en', 'nobility', (('addresses', 'given'), ('attaches', ('consort', 'mother')))), ('en', 'given', ()), ('en', 'family', ()))), + ("queen's", (('en', 'modifier', ()),)), + ('ra', (('de', 'academic', ()), ('en', 'credential', ()))), + ('rabbi', (('he', 'religious', ()),)), + ('radio', (('en', 'modifier', ()),)), + ('radm', (('en', 'military', ()),)), + ('rangatira', (('mi', 'nobility', ()),)), + ('ranger', (('en', 'occupation', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('rba', (('en', 'credential', ()),)), + ('rci', (('en', 'credential', ()),)), + ('rcp', (('en', 'credential', ()),)), + ('rd', (('en', 'credential', ()),)), + ('rdcs', (('en', 'credential', ()),)), + ('rdh', (('en', 'credential', ()),)), + ('rdml', (('en', 'military', ()),)), + ('rdms', (('en', 'credential', ()),)), + ('rdn', (('en', 'credential', ()),)), + ('rear', (('en', 'modifier', ()),)), + ('rebbe', (('yi', 'religious', ()),)), + ('registrar', (('en', 'office', ()),)), + ('reichsgraf', (('de', 'nobility', ()),)), + ('rep', (('en', 'office', ()),)), + ('representative', (('en', 'office', ()),)), + ('res', (('en', 'credential', ()),)), + ('researcher', (('en', 'occupation', ()),)), + ('resident', (('en', 'occupation', ()),)), + ('ret', (('en', 'status', ()),)), + ('rev', (('en', 'religious', ()),)), + ('revenue', (('en', 'modifier', ()),)), + ('reverend', (('en', 'religious', ()),)), + ('rfp', (('en', 'credential', ()),)), + ('rhca', (('en', 'credential', ()),)), + ('richd', (('en', 'given', (('same_as', 'richard'),)),)), + ('rid', (('en', 'credential', ()),)), + ('right', (('en', 'modifier', ()),)), + ('rinpoche', (('bo', 'courtesy', ()),)), + ('risk', (('en', 'modifier', ()),)), + ('ritter', (('de', 'nobility', ()),)), + ('rls', (('en', 'credential', ()),)), + ('rmsks', (('en', 'credential', ()),)), + ('rn', (('en', 'credential', ()),)), + ('robt', (('en', 'given', (('same_as', 'robert'),)),)), + ('rock', (('en', 'modifier', ()),)), + ('rouva', (('fi', 'salutation', (('after_given', True),)),)), + ('royal', (('en', 'modifier', (('addresses', 'given'),)), ('en', 'given', ()), ('en', 'family', ()))), + ('rozená', (('cs', 'birth_marker', ()),)), + ('rp', (('en', 'credential', ()),)), + ('rpa', (('en', 'credential', ()),)), + ('rpa-c', (('en', 'credential', ()),)), + ('rph', (('en', 'credential', (('cased', 'RPh'),)),)), + ('rpl', (('en', 'credential', ()),)), + ('rrc', (('en', 'credential', ()),)), + ('rrt', (('en', 'credential', ()),)), + ('rrt-accs', (('en', 'credential', ()),)), + ('rrt-nps', (('en', 'credential', ()),)), + ('rrt-sds', (('en', 'credential', ()),)), + ('rt', (('en', 'occupation', ()),)), + ('rtrp', (('en', 'credential', ()),)), + ('rvm', (('en', 'credential', ()),)), + ('rvt', (('en', 'credential', ()),)), + ('sa', (('en', 'given', ()), ('en', 'family', ()))), + ('sailor', (('en', 'occupation', ()),)), + ('saint', (('en', 'religious', ()), ('fr', 'religious', ()))), + ('sainte', (('fr', 'religious', ()),)), + ('same', (('en', 'credential', ()),)), + ('saml', (('en', 'given', (('same_as', 'samuel'),)),)), + ('san', (('es', 'saint', ()), ('es', 'given', ()), ('it', 'saint', ()))), + ('santa', (('es', 'saint', ()), ('es', 'given', ()), ('it', 'saint', ()), ('pt', 'saint', ()))), + ('saoshyant', (('en', 'religious', ()),)), + ('sasm', (('en', 'credential', ()),)), + ('satirist', (('en', 'occupation', ()),)), + ('sccp', (('en', 'credential', ()),)), + ('scd', (('en', 'credential', (('cased', 'ScD'),)),)), + ('scholar', (('en', 'occupation', ()),)), + ('schoolmaster', (('en', 'occupation', ()),)), + ('scientist', (('en', 'occupation', ()),)), + ('scm', (('en', 'credential', (('cased', 'ScM'),)),)), + ('scmp', (('en', 'credential', ()),)), + ('scpo', (('en', 'military', ()),)), + ('screenwriter', (('en', 'occupation', ()),)), + ('se', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('secb', (('en', 'credential', ()),)), + ('secretary', (('en', 'office', ()),)), + ('security', (('en', 'modifier', ()),)), + ('seigneur', (('fr', 'nobility', ()),)), + ('senator', (('en', 'office', ()),)), + ('senhor', (('pt', 'salutation', (('after_given', True),)),)), + ('senhora', (('pt', 'salutation', (('after_given', True),)),)), + ('senhorita', (('pt', 'salutation', (('after_given', True),)),)), + ('senior', (('en', 'modifier', ()),)), + ('senior-judge', (('en', 'office', ()),)), + ('sergeant', (('en', 'military', (('attaches', ('major',)),)),)), + ('servant', (('en', 'occupation', ()),)), + ('señor', (('es', 'salutation', ()), ('es', 'family', ()))), + ('señora', (('es', 'salutation', ()),)), + ('señores', (('es', 'salutation', (('after_given', True),)),)), + ('señorita', (('es', 'salutation', (('after_given', True),)),)), + ('señoritas', (('es', 'salutation', (('after_given', True),)),)), + ('sfc', (('en', 'military', ()),)), + ('sfp', (('en', 'credential', ()),)), + ('sgm', (('en', 'military', ()), ('en', 'credential', ()))), + ('sgt', (('en', 'military', (('attaches', ('major',)),)),)), + ('sgtmaj', (('en', 'military', ()),)), + ('sgtmajmc', (('en', 'military', ()),)), + ('shaik', (('ar', 'style', (('addresses', 'given'),)), ('ar', 'given', ()), ('ar', 'family', ()))), + ('shaikh', (('ar', 'style', (('addresses', 'given'),)), ('ar', 'given', ()), ('ar', 'family', ()))), + ('shayk', (('ar', 'style', (('addresses', 'given'),)),)), + ('shaykh', (('ar', 'style', (('addresses', 'given'),)),)), + ('shehu', (('ha', 'given', ()), ('ha', 'family', ()))), + ('sheik', (('ar', 'style', (('addresses', 'given'),)), ('ar', 'given', ()), ('ar', 'family', ()))), + ('sheikh', (('ar', 'style', (('addresses', 'given'),)), ('ar', 'given', ()), ('ar', 'family', ()))), + ('shekh', (('ar', 'style', (('addresses', 'given'),)), ('ar', 'given', ()))), + ('sheriff', (('en', 'office', ()),)), + ('shrm-cp', (('en', 'credential', ()),)), + ('shrm-scp', (('en', 'credential', ()),)), + ('si', (('en', 'credential', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('siddha', (('en', 'religious', ()), ('en', 'given', ()))), + ('sig.na', (('it', 'salutation', (('after_given', True), ('cased', 'Sig.na'))),)), + ('sig.ra', (('it', 'salutation', (('after_given', True), ('cased', 'Sig.ra'))),)), + ('signor', (('it', 'salutation', ()), ('it', 'family', ()))), + ('signora', (('it', 'salutation', (('after_given', True),)),)), + ('signore', (('it', 'salutation', ()), ('it', 'family', ()))), + ('signorina', (('it', 'salutation', (('after_given', True),)),)), + ('siie', (('en', 'credential', ()),)), + ('singer', (('en', 'occupation', ()),)), + ('singer-songwriter', (('en', 'occupation', ()),)), + ('sir', (('en', 'nobility', (('addresses', 'given'),)),)), + ('sister', (('en', 'religious', (('addresses', 'given'),)),)), + ('slečna', (('cs', 'salutation', (('after_given', True),)),)), + ('sma', (('en', 'military', ()),)), + ('smieee', (('en', 'credential', ()),)), + ('smsgt', (('en', 'military', ()),)), + ('sn', (('en', 'military', ()),)), + ('snr', (('en', 'generation', ()),)), + ('sob', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('sobr', (('pt-BR', 'agnomen', ()), ('pt-BR', 'family', ()))), + ('sobrinha', (('pt', 'agnomen', ()), ('pt', 'family', ()))), + ('sobrinho', (('pt', 'agnomen', ()), ('pt', 'family', ()))), + ('soccer', (('en', 'modifier', ()),)), + ('social', (('en', 'modifier', ()),)), + ('sociologist', (('en', 'occupation', ()),)), + ('software', (('en', 'modifier', ()),)), + ('soldier', (('en', 'occupation', ()),)), + ('solicitor', (('en', 'occupation', ()),)), + ('soprano', (('en', 'occupation', ()),)), + ('spc', (('en', 'military', ()),)), + ('speaker', (('en', 'office', ()),)), + ('special', (('en', 'modifier', ()),)), + ('sphr', (('en', 'credential', ()),)), + ('sr', (('en', 'generation', ()), ('es', 'salutation', ()), ('pt', 'salutation', ()))), + ('sra', (('es', 'salutation', (('after_given', True),)), ('es', 'family', ()), ('pt', 'salutation', (('after_given', True),)))), + ('sres', (('es', 'salutation', (('after_given', True),)),)), + ('srta', (('es', 'salutation', (('after_given', True),)),)), + ('srtas', (('es', 'salutation', (('after_given', True),)),)), + ('sscp', (('en', 'credential', ()),)), + ('ssg', (('en', 'military', ()),)), + ('ssgt', (('en', 'military', ()),)), + ('st', (('cs', 'generation', (('default', False),)), ('en', 'religious', ()), ('en', 'saint', ()), ('en', 'given', ()), ('fr', 'religious', ()), ('fr', 'saint', ()))), + ('staff', (('en', 'modifier', ()),)), + ('starejša', (('sl', 'generation', ()),)), + ('starejši', (('sl', 'generation', ()),)), + ('starija', (('bs', 'generation', ()), ('hr', 'generation', ()), ('sr', 'generation', ()))), + ('stariji', (('bs', 'generation', ()), ('hr', 'generation', ()), ('sr', 'generation', ()))), + ('staršia', (('sk', 'generation', ()),)), + ('starší', (('cs', 'generation', ()), ('sk', 'generation', ()))), + ('state', (('en', 'modifier', ()),)), + ('states', (('en', 'modifier', ()),)), + ('stb', (('de', 'credential', ()),)), + ('ste', (('fr', 'saint', ()),)), + ('stmieee', (('en', 'credential', ()),)), + ('strategy', (('en', 'modifier', ()),)), + ('subaltern', (('en', 'military', ()),)), + ('subedar', (('hi', 'military', ()),)), + ('suffragist', (('en', 'occupation', ()),)), + ('sultan', (('ar', 'nobility', ()), ('ms', 'nobility', ()), ('tr', 'nobility', ()), ('tr', 'given', ()), ('tr', 'family', ()))), + ('sultana', (('ar', 'nobility', ()), ('ar', 'given', ()), ('ar', 'family', ()))), + ('superior', (('en', 'modifier', ()),)), + ('supreme', (('en', 'modifier', ()),)), + ('surgeon', (('en', 'occupation', ()),)), + ('swami', (('bn', 'religious', (('addresses', 'given'),)), ('hi', 'religious', (('addresses', 'given'),)))), + ('swordbearer', (('en', 'office', ()),)), + ('sysselmann', (('nb', 'office', ()),)), + ('tax', (('en', 'modifier', ()),)), + ('tbr-ct', (('en', 'credential', ()),)), + ('td', (('en', 'credential', ()),)), + ('te', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('teacher', (('en', 'occupation', ()),)), + ('technical', (('en', 'modifier', ()),)), + ('technologist', (('en', 'occupation', ()),)), + ('television', (('en', 'modifier', ()),)), + ('tenor', (('en', 'occupation', ()),)), + ('ter', (('nl', 'preposition', ()),)), + ('thd', (('en', 'credential', (('cased', 'ThD'),)),)), + ('the', (('en', 'conjunction', ()),)), + ('theater', (('en', 'modifier', ()),)), + ('theatre', (('en', 'modifier', ()),)), + ('theologian', (('en', 'occupation', ()),)), + ('theorist', (('en', 'occupation', ()),)), + ('thm', (('en', 'credential', (('cased', 'ThM'),)),)), + ('tho', (('de', 'preposition', ()), ('de', 'given', ()), ('nl', 'preposition', ()))), + ('thoe', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('thos', (('en', 'given', (('same_as', 'thomas'),)),)), + ('timi', (('yo', 'nobility', ()),)), + ('tirthankar', (('en', 'religious', ()),)), + ('translator', (('en', 'occupation', ()),)), + ('travel', (('en', 'modifier', ()),)), + ('treasurer', (('en', 'office', ()),)), + ('tsar', (('ru', 'nobility', ()),)), + ('tsarina', (('ru', 'nobility', ()),)), + ('tsgt', (('en', 'military', ()),)), + ('ua', (('ga', 'clan', (('cased', 'Ua'),)), ('ga', 'given', ()))), + ('ud', (('en', 'credential', ()),)), + ('uk', (('en', 'modifier', ()),)), + ('ul-', (('ar', 'article', ()), ('ur', 'article', ()))), + ('umm', (('ar', 'bound_given', ()),)), + ('uncle', (('en', 'kinship', (('addresses', 'given'),)),)), + ('und', (('de', 'conjunction', ()),)), + ('ungfrú', (('is', 'salutation', (('after_given', True),)),)), + ('united', (('en', 'modifier', ()),)), + ('univ.prof', (('de', 'academic', ()),)), + ('us', (('en', 'modifier', ()),)), + ('usa', (('en', 'credential', ()),)), + ('usaf', (('en', 'credential', ()),)), + ('usar', (('en', 'credential', ()),)), + ('uscg', (('en', 'credential', ()),)), + ('usmc', (('en', 'credential', ()),)), + ('usn', (('en', 'credential', ()),)), + ('usnr', (('en', 'credential', ()),)), + ('uxc', (('en', 'credential', ()),)), + ('uxmc', (('en', 'credential', ()),)), + ('v', (('mul', 'generation', ()),)), + ('vadm', (('en', 'military', ()),)), + ('van', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('vande', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('vander', (('nl', 'preposition', ()), ('nl', 'given', ()))), + ('vanhempi', (('fi', 'generation', ()),)), + ('vardapet', (('hy', 'religious', ()),)), + ('vc', (('en', 'office', ()), ('en', 'credential', ()))), + ('vcp', (('en', 'credential', ()),)), + ('vd', (('en', 'credential', ()), ('nl', 'preposition', ()))), + ('vecākais', (('lv', 'generation', ()),)), + ('vecākā', (('lv', 'generation', ()),)), + ('vel', (('pl', 'alias', ()), ('pl', 'given', ()))), + ('venerable', (('en', 'religious', ()),)), + ('verderer', (('en', 'office', ()),)), + ('vet', (('en', 'status', ()),)), + ('vicar', (('en', 'religious', ()),)), + ('vice', (('en', 'modifier', ()),)), + ('viscount', (('en', 'nobility', ()),)), + ('vizier', (('ar', 'office', ()),)), + ('vocalist', (('en', 'occupation', ()),)), + ('voice', (('en', 'modifier', ()),)), + ('vom', (('de', 'preposition', ()),)), + ('von', (('de', 'preposition', ()), ('de', 'given', ()))), + ('vrd', (('en', 'credential', ()),)), + ('vrouwe', (('nl', 'nobility', ()),)), + ('vyresnioji', (('lt', 'generation', ()),)), + ('vyresnysis', (('lt', 'generation', ()),)), + ('warden', (('en', 'office', ()),)), + ('warrant', (('en', 'modifier', ()),)), + ('whnp-bc', (('en', 'credential', ()),)), + ('wing', (('en', 'modifier', ()), ('en', 'given', ()), ('en', 'family', ()))), + ('wm', (('en', 'given', (('same_as', 'william'),)),)), + ('wo-1', (('en', 'military', ()),)), + ('wo1', (('en', 'military', ()),)), + ('wo2', (('en', 'military', ()),)), + ('wo3', (('en', 'military', ()),)), + ('wo4', (('en', 'military', ()),)), + ('wo5', (('en', 'military', ()),)), + ('woodman', (('en', 'occupation', ()),)), + ('wp', (('de', 'academic', ()),)), + ('writer', (('en', 'occupation', ()),)), + ('y', (('es', 'conjunction', ()),)), + ('yngri', (('is', 'generation', ()),)), + ('z domu', (('pl', 'birth_marker', ()),)), + ('zoologist', (('en', 'occupation', ()),)), + ('zu', (('de', 'preposition', ()),)), + ('ó', (('ga', 'clan', (('cased', 'Ó'),)),)), + ('ƙarami', (('ha', 'generation', ()),)), + ('δρ', (('el', 'academic', ()),)), + ('κα', (('el', 'salutation', (('after_given', True),)),)), + ('καθ', (('el', 'academic', ()),)), + ('και', (('el', 'conjunction', ()),)), + ('κος', (('el', 'salutation', (('after_given', True),)),)), + ('σίνιορ', (('el', 'generation', ()),)), + ('τζούνιορ', (('el', 'generation', ()),)), + ('акад', (('bg', 'academic', ()), ('ru', 'academic', ()), ('uk', 'academic', ()))), + ('г-жа', (('bg', 'salutation', (('after_given', True),)), ('ru', 'salutation', (('after_given', True),)))), + ('г-н', (('bg', 'salutation', (('after_given', True),)), ('ru', 'salutation', (('after_given', True),)))), + ('д-р', (('bg', 'academic', ()), ('ru', 'academic', ()), ('uk', 'academic', ()))), + ('и', (('bg', 'conjunction', ()), ('ru', 'conjunction', ()))), + ('й', (('uk', 'conjunction', ()),)), + ('малодшая', (('be', 'generation', ()),)), + ('малодшы', (('be', 'generation', ()),)), + ('мл', (('bg', 'generation', ()), ('ru', 'generation', ()), ('sr', 'generation', ()))), + ('млади', (('bg', 'generation', ()),)), + ('младша', (('bg', 'generation', ()),)), + ('младшая', (('ru', 'generation', ()),)), + ('младши', (('bg', 'generation', ()),)), + ('младший', (('ru', 'generation', ()),)), + ('млађа', (('sr', 'generation', ()),)), + ('млађи', (('sr', 'generation', ()),)), + ('молодша', (('uk', 'generation', ()),)), + ('молодший', (('uk', 'generation', ()),)), + ('пан', (('uk', 'salutation', ()), ('uk', 'family', ()))), + ('пані', (('uk', 'salutation', (('after_given', True),)),)), + ('помладиот', (('mk', 'generation', ()),)), + ('постариот', (('mk', 'generation', ()),)), + ('проф', (('bg', 'academic', ()), ('ru', 'academic', ()), ('uk', 'academic', ()))), + ('стари', (('bg', 'generation', ()),)), + ('старија', (('sr', 'generation', ()),)), + ('старији', (('sr', 'generation', ()),)), + ('старша', (('bg', 'generation', ()), ('uk', 'generation', ()))), + ('старшая', (('ru', 'generation', ()),)), + ('старши', (('bg', 'generation', ()),)), + ('старший', (('ru', 'generation', ()),)), + ('старэйшая', (('be', 'generation', ()),)), + ('старэйшы', (('be', 'generation', ()),)), + ('та', (('uk', 'conjunction', ()),)), + ('урожд', (('ru', 'birth_marker', ()),)), + ('урожденная', (('ru', 'birth_marker', ()),)), + ('урожденный', (('ru', 'birth_marker', ()),)), + ('урождённая', (('ru', 'birth_marker', ()),)), + ('урождённый', (('ru', 'birth_marker', ()),)), + ('і', (('uk', 'conjunction', ()),)), + ('կրտսեր', (('hy', 'generation', ()),)), + ('בן', (('he', 'patronymic', ()),)), + ('בת', (('he', 'patronymic', ()),)), + ("ג'וניור", (('he', 'generation', ()),)), + ("גב'", (('he', 'salutation', (('after_given', True),)),)), + ('גברת', (('he', 'salutation', (('after_given', True),)),)), + ('גב׳', (('he', 'salutation', (('after_given', True),)),)), + ('ג׳וניור', (('he', 'generation', ()),)), + ('ג’וניור', (('he', 'generation', ()),)), + ('ד"ר', (('he', 'academic', ()),)), + ('ד״ר', (('he', 'academic', ()),)), + ('האב', (('he', 'generation', ()),)), + ('הבן', (('he', 'generation', ()),)), + ('הרב', (('he', 'religious', ()),)), + ('ז"ל', (('he', 'status', ()),)), + ('ז״ל', (('he', 'status', ()),)), + ('מר', (('he', 'salutation', (('after_given', True),)),)), + ('עו"ד', (('he', 'academic', ()),)), + ('עו״ד', (('he', 'academic', ()),)), + ("פרופ'", (('he', 'academic', ()),)), + ('פרופסור', (('he', 'academic', ()),)), + ('פרופ׳', (('he', 'academic', ()),)), + ('שליט"א', (('he', 'courtesy', ()),)), + ('שליט״א', (('he', 'courtesy', ()),)), + ('آل', (('ar', 'clan', ()),)), + ('أبو', (('ar', 'teknonymic', ()), ('ar', 'bound_given', ()), ('ar', 'given', ()))), + ('أستاذ', (('ar', 'academic', (('addresses', 'given'),)),)), + ('أستاذة', (('ar', 'academic', (('addresses', 'given'),)),)), + ('أم', (('ar', 'bound_given', ()),)), + ('ابن', (('ar', 'patronymic', ()),)), + ('ابو', (('ar', 'teknonymic', ()), ('ar', 'bound_given', ()), ('ar', 'given', ()))), + ('الأب', (('ar', 'generation', ()),)), + ('الأستاذ', (('ar', 'academic', (('addresses', 'given'),)),)), + ('الأستاذة', (('ar', 'academic', (('addresses', 'given'),)),)), + ('الإبن', (('ar', 'generation', ()),)), + ('الابن', (('ar', 'generation', ()),)), + ('الحاج', (('ar', 'religious', (('addresses', 'given'),)),)), + ('الحاجة', (('ar', 'religious', (('addresses', 'given'),)),)), + ('الدكتور', (('ar', 'academic', (('addresses', 'given'),)),)), + ('الدكتورة', (('ar', 'academic', (('addresses', 'given'),)),)), + ('الشيخ', (('ar', 'style', (('addresses', 'given'),)),)), + ('الشيخة', (('ar', 'style', (('addresses', 'given'),)),)), + ('ام', (('ar', 'bound_given', ()),)), + ('بن', (('ar', 'patronymic', ()),)), + ('بنت', (('ar', 'patronymic', ()),)), + ('جونئیر', (('sd', 'generation', ()), ('ur', 'generation', ()))), + ('جونيئر', (('sd', 'generation', ()), ('ur', 'generation', ()))), + ('جونيور', (('ar', 'generation', ()),)), + ('جونیئر', (('sd', 'generation', ()), ('ur', 'generation', ()))), + ('جونیور', (('fa', 'generation', ()),)), + ('دكتور', (('ar', 'academic', (('addresses', 'given'),)),)), + ('دكتورة', (('ar', 'academic', (('addresses', 'given'),)),)), + ('سينيور', (('ar', 'generation', ()),)), + ('سینئر', (('sd', 'generation', ()), ('ur', 'generation', ()))), + ('سینیئر', (('sd', 'generation', ()), ('ur', 'generation', ()))), + ('سینیور', (('fa', 'generation', ()),)), + ('عبد', (('ar', 'bound_given', ()),)), + ('مهندس', (('ar', 'academic', (('addresses', 'given'),)),)), + ('و', (('ar', 'conjunction', ()),)), + ('गुरु', (('hi', 'religious', (('addresses', 'given'),)),)), + ('जनाब', (('hi', 'salutation', (('after_given', True),)),)), + ('जी', (('hi', 'honorific', ()), ('mr', 'honorific', ()))), + ('जुनियर', (('hi', 'generation', ()),)), + ('जूनियर', (('hi', 'generation', ()),)), + ('ज्युनियर', (('mr', 'generation', ()),)), + ('ज्यूनियर', (('mr', 'generation', ()),)), + ('डा', (('hi', 'academic', ()), ('ne', 'academic', ()))), + ('डॉ', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('डॉक्टर', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('न्यायमूर्ति', (('hi', 'office', ()), ('mr', 'office', ()))), + ('पं', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('पंडित', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('प्रा', (('mr', 'academic', ()),)), + ('प्राध्यापक', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('प्रो', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('प्रोफेसर', (('hi', 'academic', ()), ('mr', 'academic', ()))), + ('बाबा', (('hi', 'religious', (('addresses', 'given'),)),)), + ('बाबू', (('hi', 'salutation', ()), ('hi', 'family', ()))), + ('महात्मा', (('hi', 'style', ()), ('mr', 'style', ()))), + ('महाराज', (('hi', 'courtesy', ()),)), + ('महाराजा', (('hi', 'nobility', ()), ('mr', 'nobility', ()))), + ('मौलाना', (('hi', 'religious', ()),)), + ('श्री', (('hi', 'salutation', ()), ('mr', 'salutation', ()))), + ('श्रीमती', (('hi', 'salutation', (('after_given', True),)), ('mr', 'salutation', (('after_given', True),)))), + ('श्रीमान', (('hi', 'salutation', (('after_given', True),)),)), + ('श्रीयुत', (('hi', 'salutation', (('after_given', True),)),)), + ('संत', (('hi', 'religious', (('addresses', 'given'),)),)), + ('सरदार', (('hi', 'style', ()), ('mr', 'style', ()))), + ('साहब', (('hi', 'honorific', ()),)), + ('साहिब', (('hi', 'honorific', ()), ('pa', 'honorific', ()))), + ('साहेब', (('mr', 'honorific', ()),)), + ('सीनियर', (('hi', 'generation', ()),)), + ('सुश्री', (('hi', 'salutation', (('after_given', True),)),)), + ('सौ', (('mr', 'salutation', (('after_given', True),)),)), + ('स्वामी', (('hi', 'religious', (('addresses', 'given'),)),)), + ('অধ্যাপক', (('bn', 'academic', ()),)), + ('আলহাজ', (('bn', 'style', ()),)), + ('আলহাজ্ব', (('bn', 'style', ()),)), + ('কনিষ্ঠ', (('as', 'generation', ()),)), + ('গুরু', (('bn', 'religious', (('addresses', 'given'),)),)), + ('জনাব', (('bn', 'salutation', (('after_given', True),)),)), + ('জুনিয়র', (('bn', 'generation', ()),)), + ('জুনিয়ৰ', (('as', 'generation', ()),)), + ('জ্যেষ্ঠ', (('as', 'generation', ()),)), + ('ড', (('bn', 'academic', ()),)), + ('ডঃ', (('bn', 'academic', ()),)), + ('ডক্টর', (('bn', 'academic', ()),)), + ('ডা', (('bn', 'academic', ()),)), + ('ডাঃ', (('bn', 'academic', ()),)), + ('ডাক্তার', (('bn', 'academic', ()),)), + ('প্রফেসর', (('bn', 'academic', ()),)), + ('বাবা', (('bn', 'religious', (('addresses', 'given'),)),)), + ('বাবু', (('bn', 'honorific', ()),)), + ('বিচারপতি', (('bn', 'office', ()),)), + ('মহারাজ', (('bn', 'courtesy', ()),)), + ('মাওলানা', (('bn', 'religious', ()),)), + ('মি', (('bn', 'salutation', (('after_given', True),)),)), + ('মিঃ', (('bn', 'salutation', (('after_given', True),)),)), + ('মিসেস', (('bn', 'salutation', (('after_given', True),)),)), + ('মুফতি', (('bn', 'religious', ()),)), + ('মো', (('bn', 'salutation', ()),)), + ('মোঃ', (('bn', 'salutation', ()),)), + ('মোছা', (('bn', 'salutation', (('after_given', True),)),)), + ('মোছাঃ', (('bn', 'salutation', (('after_given', True),)),)), + ('মোসা', (('bn', 'salutation', (('after_given', True),)),)), + ('মোসাঃ', (('bn', 'salutation', (('after_given', True),)),)), + ('শ্রী', (('bn', 'salutation', ()),)), + ('শ্রীমতী', (('bn', 'salutation', (('after_given', True),)),)), + ('শ্রীল', (('bn', 'religious', (('addresses', 'given'),)),)), + ('সাহেব', (('bn', 'honorific', ()),)), + ('সিনিয়র', (('bn', 'generation', ()),)), + ('স্বামী', (('bn', 'religious', (('addresses', 'given'),)),)), + ('ਜੂਨੀਅਰ', (('pa', 'generation', ()),)), + ('ਸੀਨੀਅਰ', (('pa', 'generation', ()),)), + ('જુનિયર', (('gu', 'generation', ()),)), + ('જૂનિયર', (('gu', 'generation', ()),)), + ('સિનિયર', (('gu', 'generation', ()),)), + ('ଜୁନିଅର', (('or', 'generation', ()),)), + ('ଜୁନିୟର୍', (('or', 'generation', ()),)), + ('இளையவர்', (('ta', 'generation', ()),)), + ('ஜூனியர்', (('ta', 'generation', ()),)), + ('மூத்தவர்', (('ta', 'generation', ()),)), + ('జూనియర్', (('te', 'generation', ()),)), + ('సీనియర్', (('te', 'generation', ()),)), + ('ಜೂನಿಯರ್', (('kn', 'generation', ()),)), + ('ജൂനിയർ', (('ml', 'generation', ()),)), + ('സീനിയർ', (('ml', 'generation', ()),)), + ('ජූනියර්', (('si', 'generation', ()),)), + ('จูเนียร์', (('th', 'generation', ()),)), + ('ซีเนียร์', (('th', 'generation', ()),)), + ('ຈູເນຍ', (('lo', 'generation', ()),)), + ('ဂျူနီယာ', (('my', 'generation', ()),)), + ('ၵျူႇၼီႇယႃႇ', (('shn', 'generation', ()),)), + ('უმცროსი', (('ka', 'generation', ()),)), + ('უფროსი', (('ka', 'generation', ()),)), + ('ጁኒየር', (('am', 'generation', ()), ('ti', 'generation', ()))), + ('ጁንየር', (('am', 'generation', ()), ('ti', 'generation', ()))), + ('ជូនៀរ', (('km', 'generation', ()),)), + ('’t', (('nl', 'article', ()),)), + ('くん', (('ja', 'honorific', (('attached', 'after'),)),)), + ('さま', (('ja', 'honorific', (('attached', 'after'),)),)), + ('さん', (('ja', 'honorific', (('attached', 'after'),)),)), + ('ちゃん', (('ja', 'honorific', (('attached', 'after'),)),)), + ('ジュニア', (('ja', 'generation', ()),)), + ('二世', (('ja', 'generation', (('attached', 'after'),)), ('zh', 'generation', (('attached', 'after'),)))), + ('先生', (('ja', 'professional_honorific', (('attached', 'after'),)), ('zh', 'professional_honorific', (('attached', 'after'),)))), + ('医学博士', (('ja', 'credential', (('acronym', False), ('attached', 'after'))), ('zh', 'credential', (('acronym', False), ('attached', 'after'))))), + ('博士', (('ja', 'credential', (('acronym', False),)), ('zh', 'credential', (('acronym', False),)))), + ('国会議員', (('ja', 'professional_honorific', (('attached', 'after'),)),)), + ('國會議員', (('zh', 'professional_honorific', (('attached', 'after'),)),)), + ('太平紳士', (('zh', 'professional_honorific', (('attached', 'after'),)),)), + ('太平绅士', (('zh', 'professional_honorific', (('attached', 'after'),)),)), + ('女士', (('zh', 'honorific', (('attached', 'after'),)),)), + ('小姐', (('zh', 'honorific', (('attached', 'after'),)),)), + ('教授', (('ja', 'professional_honorific', (('attached', 'after'),)), ('zh', 'professional_honorific', (('attached', 'after'),)))), + ('旧姓', (('ja', 'birth_marker', ()),)), + ('様', (('ja', 'honorific', (('attached', 'after'),)),)), + ('殿', (('ja', 'honorific', ()),)), + ('氏', (('ja', 'honorific', ()),)), + ('社長', (('ja', 'professional_honorific', (('attached', 'after'),)), ('zh', 'professional_honorific', (('attached', 'after'),)))), + ('社长', (('zh', 'professional_honorific', (('attached', 'after'),)),)), + ('課長', (('ja', 'professional_honorific', (('attached', 'after'),)), ('zh', 'professional_honorific', (('attached', 'after'),)))), + ('議員', (('ja', 'professional_honorific', (('attached', 'after'),)), ('zh', 'professional_honorific', (('attached', 'after'),)))), + ('议员', (('zh', 'professional_honorific', (('attached', 'after'),)),)), + ('课长', (('zh', 'professional_honorific', (('attached', 'after'),)),)), + ('部長', (('ja', 'professional_honorific', ()), ('zh', 'professional_honorific', ()))), + ('部长', (('zh', 'professional_honorific', ()),)), + ('醫學博士', (('zh', 'credential', (('acronym', False), ('attached', 'after'))),)), + ('교수님', (('ko', 'professional_honorific', (('attached', 'after'),)),)), + ('군', (('ko', 'honorific', ()),)), + ('님', (('ko', 'honorific', (('attached', 'after'),)),)), + ('박사', (('ko', 'credential', (('acronym', False), ('attached', 'after'))),)), + ('박사님', (('ko', 'professional_honorific', (('attached', 'after'),)),)), + ('선생님', (('ko', 'professional_honorific', (('attached', 'after'),)),)), + ('시니어', (('ko', 'generation', ()),)), + ('씨', (('ko', 'honorific', (('attached', 'after'),)),)), + ('양', (('ko', 'honorific', ()),)), + ('주니어', (('ko', 'generation', ()),)), +) diff --git a/nameparser/vocab/README.md b/nameparser/vocab/README.md index 1ad66cf9f..d5fdab493 100644 --- a/nameparser/vocab/README.md +++ b/nameparser/vocab/README.md @@ -2,7 +2,7 @@ This folder holds the parser's default vocabulary: the titles, suffixes, particles, conjunctions, bound given names and maiden markers it recognizes, and the case of the words it capitalizes specially, written one word at a time. It is the place to edit when a word should be recognized, should stop being recognized, or should be recognized differently. -The parser never reads these files. `tools/vocab/build.py` compiles them into `nameparser/config/_compiled.py`, a committed module of sorted `frozenset` literals and one `dict` (`CAPITALIZATION_EXCEPTIONS`), and the `nameparser.config` modules re-export them under their public names (`TITLES`, `PARTICLES`, ...). So an edit here does nothing until you rebuild, and `tests/v2/test_vocab.py` fails, naming the command, when the committed module and the files disagree. +The parser never reads these files. `tools/vocab/build.py` compiles them into `nameparser/config/_compiled.py`, a committed module of sorted `frozenset` literals and one `dict` (`CAPITALIZATION_EXCEPTIONS`), and the `nameparser.config` modules re-export them under their public names (`TITLES`, `PARTICLES`, ...). Beside it the build writes a second committed module from the same readings, the reading index `nameparser/config/_index.py`, which keeps every reading with the tag of the file it is in and the properties the compiler reads, so that a choice of languages can compile from it later (3.0's phase 3). So an edit here does nothing until you rebuild, and the tests fail, naming the command, when either committed module and the files disagree. Why the model looks this way, and what each decision cost, is recorded in [decisions/vocabulary-model](../../docs/design/decisions/vocabulary-model.md). Where it is going (phases 2 to 4) is in [the vocabulary model RFC](../../docs/design/vocabulary-model-rfc.md). @@ -94,7 +94,7 @@ The build checks the order and never rewrites a file, because a comment above an ## Kinds -A reading's key is its **kind**. The table lists the kinds in their canonical order, which is the [word order](#word-order)'s first key and `CATEGORY`'s order in `tools/vocab/build.py`. Each kind belongs to one **category**, and the category decides which compiled set the reading feeds. For title, suffix, particle and conjunction the category's own name is also a kind (`title = {}`: a title not yet classified), so classifying a word later is a data edit that moves no compiled set. The one exception: a generic `suffix = {}` classified as `credential` moves from `SUFFIX_WORDS` to `SUFFIX_ACRONYMS`, because a credential's `acronym` is true unwritten; write `acronym = false` to keep it where it was. The suffix and particle categories are classified (two particles of unrecorded origin, `bon` and `dí`, keep the generic kind in `und.toml`); so are the titles (#750); the others carry their generic kind. +A reading's key is its **kind**. The table lists the kinds in their canonical order, which is the [word order](#word-order)'s first key and `CATEGORY`'s order in `nameparser/_reading_index.py`, the compiler the build shares with the package. Each kind belongs to one **category**, and the category decides which compiled set the reading feeds. For title, suffix, particle and conjunction the category's own name is also a kind (`title = {}`: a title not yet classified), so classifying a word later is a data edit that moves no compiled set. The one exception: a generic `suffix = {}` classified as `credential` moves from `SUFFIX_WORDS` to `SUFFIX_ACRONYMS`, because a credential's `acronym` is true unwritten; write `acronym = false` to keep it where it was. The suffix and particle categories are classified (two particles of unrecorded origin, `bon` and `dí`, keep the generic kind in `und.toml`); so are the titles (#750); the others carry their generic kind. | Kind | Category | Compiled into | |---|---|---| @@ -166,7 +166,7 @@ Unlike the suffix kinds, one word may take a different particle kind in each lan **Classifying a category adds kinds to every check that tests it.** While a category has only its generic kind, a check that asks for the category and one that asks for the generic kind give the same answer, so a test row written in the generic kind cannot tell them apart. When a category gets its first fine kinds, give every check on that category a row in `tests/v2/test_vocab.py` written in a fine kind, and run each row against the check keyed on the kind to see it fail. #749's review found four particle checks pinned only on the generic `particle` kind once the shipped data used it for two words: the required `may_be_given`, an exclusion refusing a sibling kind, the bound given name that is a never-given particle, and the elided particle's two spellings. Each passed every test when keyed on the kind, which would have let a `preposition = {}` with no `may_be_given` compile silently into the never-given set. -The `name` kinds compile to nothing on their own. They record that a word is borne as a given or family name, with counts (see [Bearer counts](#bearer-counts)), as evidence for a reading or an exclusion, and a `given` or `family` reading beside an acronym reading, or a `given` reading beside a particle reading derives a marker (see [Derived markers](#derived-markers)). +The `name` kinds compile to nothing on their own. They record that a word is borne as a given or family name, with counts (see [Bearer counts](#bearer-counts)), as evidence for a reading or an exclusion, and a `given` or `family` reading beside an acronym reading, or a `given` reading beside a particle reading derives a marker (see [Derived markers](#derived-markers)), unless it is written `default = false`. ## Properties @@ -181,7 +181,7 @@ A property is **behavioral** when it compiles into a set and so changes how name | `attached` | behavioral | `suffix` | absent | `GLUED_HONORIFICS`, on a word reading only: `{ side = "after" }` splits the word off the end of a name token, as in `田中さん` ([rules/W2](../../docs/design/rules/W2-glued-honorific-peel.md), the glued-honorific peel) | | `may_be_given` | stated | `particle` | none: required where the word has no `given` reading | `false` is required where the word has no `given` reading in any file, and that absence, not the property, is what puts the word in `NON_GIVEN_NAME_PARTICLES`: a particle opening the name starts the family name ([rules/P1](../../docs/design/rules/P1-lone-particle-fold.md), the lone particle fold). `true` is refused, naming the `given` reading to write instead (check 11) | | `may_be_initial` | behavioral | `conjunction` | false | `CONJUNCTIONS_AMBIGUOUS`, on a cased single letter only: read as an initial in a name written in one case, as in `JOHN E SMITH` ([rules/P8](../../docs/design/rules/P8-single-letter-connective-against-the-initial-it-spells.md), a single-letter connective against the initial it spells) | -| `default` | behavioral | `title`, `suffix`, `particle`, `conjunction`, `bound_given`, `relation` | true | `false` keeps the reading out of every compiled set, `SUFFIX_WORDS` included for a generation: recorded and checked as a reading, for a pack to turn on in phase 3, where its writing offers the default no cue (`h`, Spanish *hijo*, written `H.`, exactly as an initial is). An exclusion does not refuse it, and it cannot read two ways beside the word's live readings (checks 4 to 6); a name reading beside it marks nothing. `true` is refused: it is what every reading is unwritten | +| `default` | behavioral | `title`, `suffix`, `particle`, `conjunction`, `bound_given`, `relation`, `name` | true | `false` keeps the reading out of every compiled set, `SUFFIX_WORDS` included for a generation: recorded and checked as a reading, for a pack to turn on in phase 3, where its writing offers the default no cue (`h`, Spanish *hijo*, written `H.`, exactly as an initial is). An exclusion does not refuse it, and it cannot read two ways beside the word's live readings (checks 4 to 6); a name reading beside it marks nothing. On a `given` or `family` reading (3.0's phase 3a) it derives no marker in the default, so the word's other readings compile as they would without it, until a pack turns the name on. It does not count as the name reading an agnomen on by default needs (check 11), and a particle whose word's only `given` readings are off is refused, which side of check 11 it takes being left to the first pack that needs one; a bound given name that is also a particle is refused with it, before check 9 would ask for its `given` reading. `true` is refused: it is what every reading is unwritten | | `cased` | behavioral | `title`, `suffix`, `particle`, `conjunction`, `bound_given`, `relation` | the word as written | `CAPITALIZATION_EXCEPTIONS`: the case `capitalized()` gives the word (`PhD`), a mask over its letters and digits, never a replacement (#459) | | `note` | descriptive | every category, a `name` reading included | | free text: why the reading is as it is | | `issue` | descriptive | every category but `name` | | an issue number, or a list of them | @@ -198,7 +198,7 @@ Two of these need more than their row: ### Derived markers -Four sets mark a clash with a name, and all four are derived: a word is in one when it carries the clashing reading and a name reading, in any of its files (check 11, since 2026-10-25, [decisions/vocabulary-model](../../docs/design/decisions/vocabulary-model.md)). Phase 1 wrote two of them as flags, `may_be_name` and `may_be_given = true`, which the build now refuses, naming the reading to write; the third, a suffix word that is also a name, arrived with phase 2h, and the fourth, a title that may be a given name, with phase 2i. +Four sets mark a clash with a name, and all four are derived: a word is in one when it carries the clashing reading and a name reading on by default, in any of its files (check 11, since 2026-10-25, [decisions/vocabulary-model](../../docs/design/decisions/vocabulary-model.md)). Phase 1 wrote two of them as flags, `may_be_name` and `may_be_given = true`, which the build now refuses, naming the reading to write; the third, a suffix word that is also a name, arrived with phase 2h, and the fourth, a title that may be a given name, with phase 2i. | Marker | Readings | Compiled into | Effect | |---|---|---|---| @@ -224,7 +224,7 @@ A `given` or `family` reading takes its own keys: | `gender` | `"f"`, `"m"` or `"n"` | | `same_as` | on a `given` reading only: the full given name the word abbreviates, normalized (`chas` is `charles`), which puts the word in `GIVEN_NAME_ABBREVIATIONS`: written with its period at the head of the part of the name that carries the given name, the word is that given name, where an unlisted abbreviation there is a title by its shape ([rules/H2](../../docs/design/rules/H2-leading-abbreviation-title.md), the leading-abbreviation title), so `Chas. Smith` reads given `Chas.`. Anywhere else the reading acts on nothing. A name is no word of the vocabulary, so this `same_as`, unlike a reading's, names no entry, and an entry holding only such a reading is not inert | -A name reading may stand without a count, as a particle kept possibly given under C-i's default with no source counting it does, but it then needs a `note` saying why it is declared (check 3). +A name reading may stand without a count, as a particle kept possibly given under C-i's default with no source counting it does, but it then needs a `note` saying why it is declared (check 3). It may also be written `default = false`, the behavioral property in [Properties](#properties): recorded and checked, it derives no marker until a pack turns it on. ## Exclusions @@ -249,7 +249,7 @@ Record a count on the word as a `given` or `family` reading, with the date you t 2. **Check it against real names, not memory.** `uv run python tools/name_usage.py ` gives the counts (add `--fetch-nppes` for a suffix, to see how often it is written as a credential), and [decisions/vocabulary-collisions](../../docs/design/decisions/vocabulary-collisions.md) is the criterion they are read against: a word that is a real name in the position the reading acts on needs a name reading with those counts, which marks the clash (see [Derived markers](#derived-markers)), or should stay out. `nameparser/vocab/AGENTS.md` lists the traps that have bitten before, filed under these steps. 3. **Check how it is written.** The entry's `text` is the folded spelling (`mbchb`), and `capitalized()` writes an acronym credential in capitals unless the word has a `cased` mask, so one that people write in mixed case (`MBChB`, `DPhil`, `ScD`) needs the mask: `cased = "MBChB"`. Nothing in the build can see this, since no source it reads records case. The ClinicalTrials.gov row of `name_usage.py` lists the spellings as typed ("written MBChB 1,442, MBCHB 72, MbChB 28"), once that local corpus is fetched (`uv run python tools/differential/local_corpora.py fetch ctgov-officials`). Where a mask would cost a name its rendering, refuse it instead, with the reason, as `edd` does (`'edd smith'` would render `'EdD Smith'`). #806 added twelve mixed-case credentials without their masks, and review caught it. 4. **Write the entry** in the file of the language whose names it occurs in (`und.toml` when that is not known), in its place in the [word order](#word-order), with what the checks found on it: `note`, `issue`, a dated name reading, a `cased` mask, or an `excluded` entry if you decided against it. -5. **Rebuild**: `uv run python tools/vocab/build.py`. It refuses what it can detect, naming the word and, where one file is at fault, the file, and rewrites `nameparser/config/_compiled.py`. Commit both. +5. **Rebuild**: `uv run python tools/vocab/build.py`. It refuses what it can detect, naming the word and, where one file is at fault, the file, and rewrites `nameparser/config/_compiled.py` and `nameparser/config/_index.py`. Commit all three. 6. **Run the tests**: `uv run pytest tests/v2/test_vocab.py` for the build, then the full suite, and read what `tests/v2/test_snapshot.py` reports as moved (the root `AGENTS.md`, "Validating a new parsing rule"). `uv run python tools/differential/measure.py --base ` runs that check and the CLDR one against the branch's committed records, and every local-corpus check against the base checkout, in one command, writing one summary (`tools/differential/README.md`, "Measuring a change"). For a word whose reading acts on a name position, such as a title that is also a given name, `uv run python tools/vocab/word_report.py ... --base ` reads it both ways in one run: the word in a grid of shapes (a head, behind a title, after a family comma, after a full name and a comma, past a second comma, trailing with a period, inside a title phrase), each in three cases and under the three name orders, printing the rows the base reads differently; how the names in every fetched local corpus that open with the word, or open the part after the first comma with it, read it (title, given or other, counts only, with the corpora it could not read named); and `name_usage.py`'s rows. Without `--base` it prints every text, one line where the three orders agree and one block where the three cases read alike once lowercased; with it, what moved, folded the same way. `--rank` takes a list of words, the triage for a batch, which needs no change to the tree. It counts the corpus names that read each word as the title at either position, which holds real titles too, and beside them the `psc-individuals` filings opening with the word by the box the filer typed it into: the title box (a real title, what the move costs) and the forename box (a given name the title takes, what it gives back). The words are ordered by the forename box's count, printed beside its share of the two as a fraction (`164/165`), so a word three filers typed as a forename does not outrank one 164 did. On 2026-10-27 `sir` read as the title in 495 names but was filed 277 times as a title and 8 as a forename, while `sheikh`'s 13 title-box filings were the 13 agreement records #912 moved away from agreeing. diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index c5180eab2..60a563241 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,6 +16,7 @@ # a plan, and without this list a renamed existing module would silently # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", + "_reading_index.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_script_segment.py", @@ -57,6 +58,14 @@ # submodule, and since 3.0 the package holds nothing else "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), + # the shared compiler and the reading index's reader (#753, 3a): the + # index, a config DATA module, and nothing else. That the import is + # at call time only, nothing internal at module level, since + # tools/vocab/build.py loads this file by path as it loads + # _lexicon.py, is held by test_reading_index.py's + # test_the_module_imports_nothing_internal_at_module_level, which + # this prefix match cannot see + "_reading_index.py": ("nameparser.config.",), # _types is a downward import (bottom of the graph): Locale shares # the _guarded_getstate/_guarded_setstate pickle functions "_locale.py": ("nameparser._types", "nameparser._lexicon", diff --git a/tests/v2/test_reading_index.py b/tests/v2/test_reading_index.py new file mode 100644 index 000000000..06c8dabd8 --- /dev/null +++ b/tests/v2/test_reading_index.py @@ -0,0 +1,161 @@ +"""The reading index (nameparser/config/_index.py) and the compiler it +shares with the build (nameparser/_reading_index.py), phase 3a of #753. + +The index is the vocabulary as readings, each with the file it sits in, +and _compiled.py is the index compiled with no choice of languages. The +second is the invariant the RFC asks for: a choice of languages (3b) +compiles a subset of these same rows with this same function, so the +default and every pack are one compile apart, never two +implementations. +""" +from __future__ import annotations + +import ast +import importlib.util +import sys +import unicodedata +from pathlib import Path +from types import ModuleType +from typing import Any + +from nameparser import _reading_index as ri +from nameparser.config import _compiled, _index + +_BUILD = Path(__file__).resolve().parents[2] / "tools" / "vocab" / "build.py" + + +def _load_build() -> ModuleType: + spec = importlib.util.spec_from_file_location("vocab_build_for_index", _BUILD) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +build = _load_build() + + +def test_the_module_imports_nothing_internal_at_module_level() -> None: + # tools/vocab/build.py loads this file by path at its own import; a + # module-level import of the package would run nameparser/__init__, + # and so _compiled.py, which a broken rebuild may have left unusable + tree = ast.parse(Path(ri.__file__).read_text(encoding="utf-8")) + top = [n for n in tree.body if isinstance(n, (ast.Import, ast.ImportFrom))] + names = [a.name for n in top if isinstance(n, ast.Import) for a in n.names] + names += [n.module or "" for n in top if isinstance(n, ast.ImportFrom)] + assert names and not [m for m in names if m.startswith("nameparser")] + + +def test_the_index_compiled_with_no_choice_is_the_committed_module() -> None: + # the no-choice invariant: what the parser reads is the index + # compiled, byte for byte as the build writes it + compiled = ri.compile_readings(ri.index()) + assert build.render(compiled) == build.OUTPUT.read_text(encoding="utf-8") + for name, _ in build.CONSTANTS: + assert compiled[name] == getattr(_compiled, name), name # type: ignore[literal-required] + + +#: A value other than the unwritten one for every behavioral property, +#: so a probe reading carrying it compiles differently from one without. +_SET = {"addresses": "given", "after_given": True, "attaches": ["office"], + "cased": "ZzQ", "acronym": None, "attached": {"side": "after"}, + "may_be_initial": True} + + +def _probe_readings() -> list[Any]: + """A build Reading of every kind, carrying every behavioral + property its category takes, set away from its unwritten value + (`acronym` flipped), each word apart; plus a given name written + short, and a name reading written default = false beside a + credential, the two properties a name reading compiles from.""" + out = [] + for i, (kind, category) in enumerate(ri.CATEGORY.items()): + text = "q" if kind == "conjunction" else f"zz{i}" + props: dict[str, object] = {} + for prop in ri.BEHAVIORAL[category] - {"default"}: + value = _SET[prop] + if prop == "acronym": + value = kind != "credential" + elif prop == "cased": + value = text.upper() + props[prop] = value + out.append(build.Reading("en.toml", text, kind, props)) + out.append(build.Reading("en.toml", "chas", "given", {"same_as": "charles"})) + out.append(build.Reading("en.toml", "zzoff", "credential", {})) + out.append(build.Reading("de.toml", "zzoff", "family", {"default": False})) + out.append(build.Reading("de.toml", "zzgen", "generation", {"default": False})) + return out + + +def test_every_property_the_compiler_reads_survives_the_index() -> None: + # an anchor outside the index's own path: compiling the build's + # readings as their files wrote them, and compiling them as the + # index keeps them (indexed, then written and read back as a row), + # give the same sets. A property indexed() drops that the compiler + # reads is lost only on the second side; the no-choice test above + # cannot see one, both its sides going through the index + probes = _probe_readings() + for readings in (probes, build.load(build.read_dir()).readings): + written = ri.compile_readings(readings) + pool_of = build.Pool(readings=list(readings)) + kept = [ri.from_row(r.text, ri.to_row(r)) for r in build.index_rows(pool_of)] + assert ri.compile_readings(kept) == written + # the probes reach the compiler: every word carrying a property + # lands in a compiled constant + probed: dict[str, Any] = dict(ri.compile_readings(probes)) + for r in probes: + if r.props and r.on_by_default: + assert any(r.text in v for v in probed.values()), r + + +def test_the_committed_index_is_what_the_toml_writes() -> None: + pool = build.load(build.read_dir()) + fresh = build.render_index(build.index_rows(pool)) + assert build.index_output().read_text(encoding="utf-8") == fresh, ( + f"nameparser/config/_index.py is stale against nameparser/vocab/; " + f"run: {build.REFRESH}") + + +def test_the_index_holds_every_reading_of_every_file() -> None: + # one row per reading the files hold, on by default or not, so a + # choice can turn on what the default leaves off + pool = build.load(build.read_dir()) + assert sorted((r.text, r.tag, r.kind) for r in pool.readings) == sorted( + (r.text, r.tag, r.kind) for r in ri.index()) + assert any(not r.on_by_default for r in ri.index()), ( + "no reading written default = false: the off half is untested") + + +def test_a_row_round_trips_and_keeps_only_what_the_compiler_reads() -> None: + for text, rows in _index.READINGS: + for row in rows: + r = ri.from_row(text, row) + assert ri.to_row(r) == row + assert set(r.props) <= ri.indexed(r.kind), (text, row) + + +def test_the_index_is_sorted_and_normalized() -> None: + texts = [text for text, _ in _index.READINGS] + assert texts == sorted(set(texts)) + for text, rows in _index.READINGS: + assert unicodedata.normalize("NFC", text) == text == text.lower() + assert list(rows) == sorted( + rows, key=lambda row: (row[0], list(ri.CATEGORY).index(row[1]))) + + +def test_readings_of_gives_a_words_rows_in_every_file() -> None: + for text, rows in _index.READINGS: + got = ri.readings_of(text) + assert [(r.text, r.tag, r.kind) for r in got] == [ + (text, tag, kind) for tag, kind, _ in rows] + assert ri.readings_of("zzq-no-such-word") == () + + +def test_the_accessors_give_the_unwritten_values() -> None: + bare = ri.IndexReading("zzq", "en", "credential", {}) + assert bare.category == "suffix" and bare.acronym and bare.on_by_default + assert bare.addresses == "family" and not bare.flag("after_given") + off = ri.IndexReading("zzq", "en", "family", {"default": False}) + assert off.category == "name" and not off.on_by_default + assert not ri.IndexReading("zzq", "en", "generation", {}).acronym diff --git a/tests/v2/test_vocab.py b/tests/v2/test_vocab.py index 24f7c3ba8..b24f26b9e 100644 --- a/tests/v2/test_vocab.py +++ b/tests/v2/test_vocab.py @@ -11,9 +11,12 @@ #737's review widened check 6 from "the same kind" to "the same set" and the new check quietly stopped refusing the excluded kind itself with `acronym` flipped, a case nothing listed because the old check had -made it impossible. compile_pool's final -acronym-and-word check is a backstop check 5 always reaches first, so -its mutation is refused by check 5. +made it impossible. The compiler's final +acronym-and-word check (nameparser/_reading_index.py) is a backstop +check 5 always reaches first, so its mutation is refused by check 5. +The reading index the build writes beside _compiled.py, and the +invariant that _compiled.py is that index compiled, are +tests/v2/test_reading_index.py's. """ from __future__ import annotations @@ -171,6 +174,34 @@ def test_a_reading_off_by_default_compiles_to_nothing() -> None: assert not any("zzr" in v for v in both.values()) +@pytest.mark.parametrize(("clash", "name", "marker"), [ + ("credential = {}", "family", "SUFFIX_ACRONYMS_AMBIGUOUS"), + ("generation = {}", "given", "SUFFIX_WORDS_AMBIGUOUS"), + ("office = {}", "given", "TITLES_AMBIGUOUS"), +]) +def test_a_name_reading_off_by_default_marks_nothing_until_it_is_on( + clash: str, name: str, marker: str) -> None: + # 3a: a name reading written default = false compiles to nothing, as + # any off reading does, so the reading it clashes with compiles as + # if it were absent; written on, the same reading derives its + # marker -- the turn-on a pack makes (3b), in miniature. A particle + # beside an off given reading is refused instead (the mutation table) + def compiled(default: str) -> dict[str, Any]: + return _compile({"en.toml": _w(f'[[word]]\ntext = "zzq"\n{clash}\n' + f'{name} = {{ note = "x"{default} }}')}) + + assert "zzq" in compiled("")[marker] + off = compiled(", default = false") + assert "zzq" not in off[marker] + assert off == _compile({"en.toml": _w(f'[[word]]\ntext = "zzq"\n{clash}')}) + + +def test_a_given_name_written_short_off_by_default_compiles_to_nothing() -> None: + c = _compile({"tl.toml": _w('[[word]]\ntext = "zzq"\ngiven = { note = "x", same_as = "maria", default = false }\n' + 'excluded = { title = "x" }')}) + assert not any("zzq" in v for v in c.values()) + + def test_a_suffix_word_beside_a_name_reading_is_gated_and_an_agnomen_by_the_count() -> None: # the derived marker for a suffix word (rules/S11): a generation # beside a name reading is gated with the case cue, an agnomen with @@ -297,6 +328,20 @@ def _w(body: str) -> str: ("agnomen-as-an-acronym", {"pt.toml": _w('[[word]]\ntext = "filho"\nagnomen = { acronym = true }\nfamily = { note = "x" }')}, "an agnomen is a word"), ("agnomen-without-a-name-reading", {"pt.toml": _w('[[word]]\ntext = "filho"\nagnomen = {}')}, "an agnomen needs a `given` or `family` reading"), ("agnomen-whose-name-reading-is-elsewhere-only-as-an-exclusion", {"pt.toml": _w('[[word]]\ntext = "filho"\nagnomen = {}\nexcluded = { family = "x" }')}, "an agnomen needs a `given` or `family` reading"), + # a name reading written default = false (3a) is held to the same + # value check as any reading's `default`, gates nothing in the + # default, and so is no name reading for the checks that guard the + # default's compile: an agnomen it alone would gate reads ungated, + # and a particle whose word's only given reading it is waits for a + # pack to decide which side of check 11 it takes + ("name-default-true", {"pt.toml": _w('[[word]]\ntext = "zzq"\nfamily = { note = "x", default = true }\ncredential = {}')}, "write default = false"), + ("name-default-not-a-bool", {"pt.toml": _w('[[word]]\ntext = "zzq"\ngiven = { note = "x", default = 0 }\ncredential = {}')}, "write default = false"), + ("agnomen-whose-only-name-reading-is-off", {"pt.toml": _w('[[word]]\ntext = "filho"\nagnomen = {}\nfamily = { note = "x", default = false }')}, "an agnomen needs a `given` or `family` reading"), + ("agnomen-whose-only-name-reading-is-off-elsewhere", {"pt.toml": _w('[[word]]\ntext = "filho"\nagnomen = {}'), "es.toml": _w('[[word]]\ntext = "filho"\nfamily = { note = "x", default = false }\nexcluded = { title = "x" }')}, "an agnomen needs a `given` or `family` reading"), + ("off-name-reading-beside-an-honorific", {"ja.toml": _w('[[word]]\ntext = "zzt"\nhonorific = {}\nfamily = { note = "x", default = false }')}, "gates nothing"), + ("particle-whose-only-given-reading-is-off", {"nl.toml": _w('[[word]]\ntext = "in"\ngiven = { note = "x", default = false }\npreposition = {}')}, "only given readings are written default = false"), + ("particle-stating-never-given-beside-an-off-given-reading", {"nl.toml": _w('[[word]]\ntext = "in"\ngiven = { note = "x", default = false }\npreposition = { may_be_given = false }')}, "only given readings are written default = false"), + ("bound-given-particle-whose-only-given-reading-is-off", {"und.toml": _w('[[word]]\ntext = "abu"\ngiven = { note = "x", default = false }\nparticle = {}\nbound_given = {}')}, "only given readings are written default = false"), ("decided-not-a-date", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { decided = "2026-10-18" }')}, "TOML date"), # tomllib reads a date-time as a datetime, which isinstance calls a date ("decided-a-date-time", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { decided = 2026-10-18T09:00:00 }')}, "not a string or a date-time"), @@ -444,6 +489,14 @@ def test_every_mutation_starts_from_a_pool_the_checks_accept() -> None: {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = {}\n\n' '[[word]]\ntext = "jnr"\ngeneration = { same_as = "jr" }')}, {"und.toml": _w('[[word]]\ntext = "abu"\ngiven = { note = "x" }\nparticle = {}\nbound_given = {}')}, + # an off name reading beside a reading it would gate compiles, + # and so does an off agnomen beside an off name reading; an off + # given reading beside a particle is fine once another given + # reading of the word is on + {"en.toml": _w('[[word]]\ntext = "zzq"\ncredential = {}\nfamily = { note = "x", default = false }')}, + {"pt-BR.toml": _w('[[word]]\ntext = "zzq"\nagnomen = { default = false }\nfamily = { note = "x", default = false }')}, + {"nl.toml": _w('[[word]]\ntext = "zzq"\ngiven = { note = "x", default = false }\npreposition = {}'), + "en.toml": _w('[[word]]\ntext = "zzq"\ngiven = { note = "x" }\nexcluded = { title = "x" }')}, # check 11 met: a particle beside a given reading states nothing, # in its own file or another, and a count-less one carries a note {"nl.toml": _w('[[word]]\ntext = "van"\ngiven = { note = "x" }\npreposition = {}'), @@ -829,6 +882,8 @@ def test_no_error_exits_1_the_code_for_stale(tmp_path: Path, monkeypatch: pytest assert build.main(["--check"]) == 1 assert "unreadable" in capsys.readouterr().out assert build.main([]) == 0 + # the index is written beside a repointed OUTPUT, never into the package + assert (tmp_path / "_index.py").exists() assert build.main(["--check"]) == 0 # a module that cannot be written is an internal error, exit 3 build.OUTPUT.unlink() @@ -862,6 +917,21 @@ def broken() -> ModuleType: assert build.main(["--check"]) == 3 +@pytest.mark.parametrize(("raised", "code"), [ + (ValueError("unpacking a bug"), 3), # a bug in the compiler is internal + (KeyError("a bug"), 3), + (None, 2), # the backstop: the data's fault +]) +def test_only_the_compilers_backstop_is_a_vocabulary_error( + monkeypatch: pytest.MonkeyPatch, raised: Exception | None, code: int) -> None: + # compile_pool turns the compiler's CompileError into a VocabError; + # catching ValueError instead made any bug a ValueError raises exit 2 + def broken(readings: object) -> None: + raise raised if raised is not None else build._ri.CompileError("['esq']: both") + monkeypatch.setattr(build._ri, "compile_readings", broken) + assert build.main(["--check"]) == code + + def test_a_query_error_exits_3_not_1(monkeypatch: pytest.MonkeyPatch) -> None: query = _load_query() def broken(*args: object, **kwargs: object) -> list[object]: diff --git a/tools/vocab/build.py b/tools/vocab/build.py index 561f140e4..fcbdd3d08 100644 --- a/tools/vocab/build.py +++ b/tools/vocab/build.py @@ -1,23 +1,31 @@ -"""Compile nameparser/vocab/*.toml into nameparser/config/_compiled.py. +"""Compile nameparser/vocab/*.toml into nameparser/config/_compiled.py, +and write the reading index, nameparser/config/_index.py, beside it. Usage:: - uv run python tools/vocab/build.py # rewrite _compiled.py - uv run python tools/vocab/build.py --check # exit 1 if it is stale + uv run python tools/vocab/build.py # rewrite both modules + uv run python tools/vocab/build.py --check # exit 1 if either is stale Either exits 2, printing the failed check, when the vocabulary fails one or a vocabulary file cannot be read, and 3 on an internal error or a module it cannot write, so neither is ever mistaken for exit 1's -"stale". An unreadable _compiled.py IS stale, rebuilding replacing it, -and --check reports it so. +"stale". An unreadable generated module IS stale, rebuilding replacing +it, and --check reports it so. The vocabulary is written per word, in one TOML file per BCP 47 language tag (`und` for words not yet assigned one, `mul` for words shared on purpose). Each word lists its READINGS -- what it can be read as, keyed by kind -- and the compiler folds them into the sets the parser reads (decisions/vocabulary-model). The parser never parses TOML: the build -writes a module of frozenset literals, committed beside the data, and -tests/v2/test_vocab.py fails when the two disagree. +checks the files, turns every reading into an index row with its +file's tag, and writes the rows twice from memory, as the reading index +and, compiled, as a module of frozenset literals, both committed beside +the data (it never reads the index back); tests/v2/test_vocab.py and +tests/v2/test_reading_index.py fail when either disagrees with the +files. The compiler, and the kinds and properties it reads (CATEGORY, +BEHAVIORAL, GATED_WORDS), live in nameparser/_reading_index.py, which +a choice of languages will run too (#753, phase 3); this file holds the +checks. The build checks replace the import-time asserts the data modules used to carry. Lexicon.__post_init__ still runs on the result, and three @@ -65,22 +73,27 @@ `may_be_name` and `may_be_given = true` are refused, each naming the reading to write, a particle reading states `may_be_given = false` exactly where its word has no `given` - reading in any file, an agnomen reading has a name reading on - its word, which gates it (rules/S11), and no other suffix word - kind has one, since the gate reads only those two; + reading in any file, and has no `given` reading only where some + other is on by default, an agnomen reading has a name reading on + its word, which gates it (rules/S11), one on by default where the + agnomen is, and no other suffix word kind has one, since the gate + reads only those two; numbered by subject, not by when they run: 1-3, 8, 7's case mask and 11's refusals run per entry as each file loads, and 4-7, 9, 11's particle statement and agnomen and 10 over the whole pool in check_pool, 6 first and 10 last. A reading written `default = false` is held to 1-3, 7, 8 and 11 like any other and compiles to nothing, so -4, 5, 6 and 9 do not weigh it; +4, 5, 6 and 9 do not weigh it, a name reading included (3a): it derives +no marker in the default, 11's agnomen half counts only the name +readings on by default, and 11's particle half refuses a particle whose +word's only given readings are off, before 9 would read one; plus the data invariants that were asserts: may_be_initial only on a cased single letter, an elided particle in both apostrophes (#700), and after_given only on a salutation reading (#835). No post-nominal is both an acronym and a word, by checks 4 and 5; a -final check in compile_pool is the backstop. The subset relations +final check in the compiler is the backstop. The subset relations (GIVEN_NAME_TITLES and AFTER_GIVEN_TITLES in TITLES and TITLES_AMBIGUOUS together, each marker set in its base) need no check: a flag cannot exist without its reading, and a derived marker is @@ -103,13 +116,17 @@ from dataclasses import dataclass, field from pathlib import Path from types import ModuleType -from typing import Any, TypedDict +from typing import Any ROOT = Path(__file__).resolve().parents[2] # bound at import like VOCAB and OUTPUT below, so a test repointing ROOT # at a fixture tree does not move the parser's own module out from under # a cold _LEXICON cache _LEXICON_PATH = ROOT / "nameparser" / "_lexicon.py" +#: The compiler and the reading index's types (phase 3a, #753), shared +#: with the package so that the build and a choice of languages compile +#: alike; loaded by path, like _lexicon.py, and for the same reason. +_READING_INDEX_PATH = ROOT / "nameparser" / "_reading_index.py" def _load_lexicon_module() -> ModuleType: @@ -123,8 +140,12 @@ def _load_lexicon_module() -> ModuleType: (its config imports sit inside _default_lexicon), so it loads on its own; tests/v2/test_vocab.py's test_the_build_does_not_import_the_package_it_writes holds that.""" + return _load_by_path("_vocab_build_lexicon", _LEXICON_PATH) + + +def _load_by_path(name: str, path: Path) -> ModuleType: import importlib.util - spec = importlib.util.spec_from_file_location("_vocab_build_lexicon", _LEXICON_PATH) + spec = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) # registered before it runs: its dataclasses look their module up @@ -137,6 +158,18 @@ def _load_lexicon_module() -> ModuleType: return module +# Loaded at import, unlike _lexicon.py below: CATEGORY and BEHAVIORAL +# are module constants here, which tools/vocab/query.py and the tests read +# from this module. It is hand-written, never generated, so the stale +# module the build exists to replace cannot break it; a bug in it still +# exits 3, never 1's "stale" -- and so does anything importing this file +# (tools/vocab/query.py, the tests), with SystemExit at import. +try: + _ri = _load_by_path("_vocab_build_reading_index", _READING_INDEX_PATH) +except Exception: + traceback.print_exc() + raise SystemExit(3) from None + _LEXICON: list[ModuleType] = [] @@ -173,49 +206,9 @@ def _alnum(text: str) -> str: #: does. SCHEMA = 1 -#: kind -> category. The category decides which compiled field a reading -#: feeds. For title, suffix, particle and conjunction, the category's own -#: name is also a kind (`title = {}`: a title not yet classified), so -#: classifying a word later moves no set. bound_given's one kind shares -#: its category's name; the relation (birth_marker) and name (given, -#: family) categories have no generic kind. Kind -#: names are unique across categories; a fine kind arrives with its -#: category's classification, in a core PR, as one row here. -CATEGORY: dict[str, str] = { - "title": "title", - "salutation": "title", - "style": "title", - "academic": "title", - "religious": "title", - "nobility": "title", - "military": "title", - "office": "title", - "occupation": "title", - "kinship": "title", - "modifier": "title", - "suffix": "suffix", - "generation": "suffix", - "agnomen": "suffix", - "credential": "suffix", - "honorific": "suffix", - "professional_honorific": "suffix", - "status": "suffix", - "courtesy": "suffix", - "particle": "particle", - "preposition": "particle", - "article": "particle", - "patronymic": "particle", - "teknonymic": "particle", - "clan": "particle", - "rank": "particle", - "saint": "particle", - "alias": "particle", - "conjunction": "conjunction", - "bound_given": "bound_given", - "birth_marker": "relation", - "given": "name", - "family": "name", -} +#: kind -> category, and the canonical order of the kinds: the +#: compiler's, defined in nameparser/_reading_index.py. +CATEGORY = _ri.CATEGORY #: The categories a word-level `excluded` may name besides the kinds. CATEGORIES = frozenset(CATEGORY.values()) #: CATEGORY's order is the canonical order of the kinds, and of the @@ -230,17 +223,10 @@ def _alnum(text: str) -> str: #: those no language file fits yet (nameparser/vocab/README.md). UNORDERED = "und.toml" -#: Behavioral properties per category: each compiles into a field, and -#: its allowed values are exactly what today's pipeline does. -BEHAVIORAL: dict[str, frozenset[str]] = { - "title": frozenset({"addresses", "after_given", "attaches", "cased", - "default"}), - "suffix": frozenset({"acronym", "attached", "cased", "default"}), - "particle": frozenset({"cased", "default"}), - "conjunction": frozenset({"may_be_initial", "cased", "default"}), - "bound_given": frozenset({"cased", "default"}), - "relation": frozenset({"cased", "default"}), -} +#: Behavioral properties per category, each compiling into a field: +#: the compiler's, in nameparser/_reading_index.py. A name reading's one +#: is `default` (3a); its other keys are NAME_KEYS. +BEHAVIORAL = _ri.BEHAVIORAL #: Stated properties per category: compiled to nothing, but held to the #: word's readings (check 11). A particle states `may_be_given = false` #: where its word has no given reading, so a new particle never falls to @@ -254,10 +240,9 @@ def _alnum(text: str) -> str: "may_be_name": "a `given` or `family` reading on the word", "may_be_given": "a `given` reading on the word", } -#: The suffix word kinds a name reading gates (rules/S11): the third -#: derived marker reads these, and a name reading beside another word -#: kind marks nothing, no gate reading that clash yet. -GATED_WORDS = frozenset({"generation", "agnomen"}) +#: The suffix word kinds a name reading gates (rules/S11): the +#: compiler's, in nameparser/_reading_index.py. +GATED_WORDS = _ri.GATED_WORDS #: Descriptive properties: recorded, validated, compiled to nothing -- #: save `same_as` on a `given` reading, which is a NAME_KEYS entry, not #: this set's, and compiles the word into GIVEN_NAME_ABBREVIATIONS. @@ -295,43 +280,19 @@ class VocabWarning(UserWarning): @dataclass(frozen=True) -class Reading: +class Reading(_ri.ReadingProperties): + """A reading as its file wrote it. Its accessors for the unwritten + values (category, acronym, on_by_default, flag, addresses) are the + compiler's, so the checks and the compile read one default.""" file: str text: str kind: str props: dict[str, Any] @property - def category(self) -> str: - return CATEGORY[self.kind] - - # the defaults of the two behavioral properties whose unwritten value - # is not false, each defined once: checks, compile_pool and - # _behavior all read them here - @property - def acronym(self) -> bool: - """Matched with its periods removed: unwritten, true only for a - credential.""" - return bool(self.props.get("acronym", self.kind == "credential")) - - @property - def on_by_default(self) -> bool: - """Compiled into the default vocabulary: unwritten, true. A - reading written `default = false` is recorded and checked as a - reading but compiles to nothing, for a pack to turn on in - phase 3 (decisions/vocabulary-model).""" - return self.props.get("default", True) is not False - - def flag(self, prop: str) -> bool: - """A `may_be_*` flag or `after_given`: unwritten, false.""" - return bool(self.props.get(prop, False)) - - @property - def addresses(self) -> object: - """Which name a title addresses: unwritten, the family name. An - object, not a str, because it is read before check 3 has - refused a value that is not one of the allowed strings.""" - return self.props.get("addresses", "family") + def tag(self) -> str: + """The BCP 47 tag of its file, the reading index's `tag`.""" + return self.file.removesuffix(".toml") def where(self) -> str: return f"{self.file}: {self.text!r} {self.kind}" @@ -760,9 +721,11 @@ def _check_reading(r: Reading, pool: Pool) -> None: where = r.where() p = r.props if r.category == "name": + allowed_here = NAME_KEYS | BEHAVIORAL["name"] for k in p: - if k not in NAME_KEYS: - raise _reserved_or_unknown(where, k, NAME_KEYS) + if k not in allowed_here: + raise _reserved_or_unknown(where, k, allowed_here) + _check_default(where, p) for src in COUNT_SOURCES & p.keys(): if not isinstance(p[src], int) or isinstance(p[src], bool) or p[src] < 1: raise VocabError(f"{where}: {src} must be a positive count " @@ -810,12 +773,7 @@ def _check_reading(r: Reading, pool: Pool) -> None: for flag in ("acronym", "may_be_initial", "after_given"): if flag in p and not isinstance(p[flag], bool): raise VocabError(f"{where}: {flag} must be true or false") - if "default" in p and p["default"] is not False: - # every reading is on by default unwritten, so the one value - # worth writing is the one that turns it off - raise VocabError(f"{where}: default = {_shown(p['default'])}; write " - f"default = false for a reading a pack turns on, " - f"or leave it out") + _check_default(where, p) if r.category == "title": addresses = r.addresses if addresses in ("full", "any"): @@ -914,6 +872,15 @@ def _check_reading(r: Reading, pool: Pool) -> None: pool.readings.append(r) +def _check_default(where: str, p: Mapping[str, Any]) -> None: + if "default" in p and p["default"] is not False: + # every reading is on by default unwritten, so the one value + # worth writing is the one that turns it off + raise VocabError(f"{where}: default = {_shown(p['default'])}; write " + f"default = false for a reading a pack turns on, " + f"or leave it out") + + def _refused_by(r: Reading, excluded: str) -> bool: """Whether a word-level exclusion of `excluded` refuses reading r. @@ -1055,9 +1022,24 @@ def check_pool(pool: Pool) -> None: # silently pick one side of that; a false beside a given reading # would say the opposite of the marker it derives given = {t for t, rs in by_text.items() if any(r.kind == "given" for r in rs)} + # the words with a given reading the default compiles: what the + # derived markers and the default's Lexicon read (checks 9 and 11) + live_given = {t for t, rs in live.items() if any(r.kind == "given" for r in rs)} for r in pool.readings: if r.category != "particle": continue + if r.text in given and r.text not in live_given: + # the default compiles the particle never-given, the given + # reading being off; a pack turning the given reading on + # compiles it possibly given. Which of the two the stated + # property describes is the first such pack's to decide, + # with its measurement (decisions/vocabulary-model, + # 2026-10-29 3a), so nothing reads as decided before then + raise VocabError(f"{r.where()}: the word's only given readings " + f"are written default = false, and whether " + f"this particle then states may_be_given is " + f"undecided; turn the given reading on, or " + f"leave it out until a pack needs it") if r.text in given and "may_be_given" in r.props: raise VocabError(f"{r.where()}: may_be_given = false contradicts " f"the word's given reading, which derives it; " @@ -1073,8 +1055,12 @@ def check_pool(pool: Pool) -> None: # reading would read as the suffix wherever it ends a name, taking # the family name from every two-word name ('Maria Filha') named = {t for t, rs in by_text.items() if any(r.category == "name" for r in rs)} + # a name reading written default = false gates nothing in the + # default, so it leaves an agnomen the default compiles ungated + live_named = {t for t, rs in live.items() if any(r.category == "name" for r in rs)} for r in pool.readings: - if r.kind == "agnomen" and r.text not in named: + if r.kind == "agnomen" and r.text not in ( + live_named if r.on_by_default else named): raise VocabError(f"{r.where()}: an agnomen needs a `given` or " f"`family` reading on its word, which gates it " f"by the count (rules/S11); with dated counts " @@ -1108,6 +1094,8 @@ def check_pool(pool: Pool) -> None: # be given -- the bound join reads it as a given name. Lexicon raises # on the same contradiction; this says which file and word for text, rs in live.items(): + # a word whose only given readings are off never reaches here: + # check 11's particle half has refused it above if any(r.kind == "bound_given" for r in rs) and text not in given: for r in rs: if r.category == "particle": @@ -1185,30 +1173,8 @@ def _behavior(r: Reading, prop: str) -> object: ) -class Compiled(TypedDict): - """What compile_pool returns: one key per constant, in CONSTANTS - order, typed as _compiled.py declares it.""" - GIVEN_NAME_TITLES: frozenset[str] - TITLES: frozenset[str] - AFTER_GIVEN_TITLES: frozenset[str] - TITLES_AMBIGUOUS: frozenset[str] - TITLE_MODIFIERS: frozenset[str] - TITLE_STYLES: frozenset[str] - TITLE_PHRASES: frozenset[str] - NON_GIVEN_NAME_PARTICLES: frozenset[str] - PARTICLES: frozenset[str] - CONJUNCTIONS: frozenset[str] - CONJUNCTIONS_AMBIGUOUS: frozenset[str] - BOUND_GIVEN_NAMES: frozenset[str] - GIVEN_NAME_ABBREVIATIONS: frozenset[str] - MAIDEN_MARKERS: frozenset[str] - SUFFIX_WORDS: frozenset[str] - GLUED_HONORIFICS: frozenset[str] - SUFFIX_WORDS_AMBIGUOUS: frozenset[str] - AGNOMENS: frozenset[str] - SUFFIX_ACRONYMS_AMBIGUOUS: frozenset[str] - SUFFIX_ACRONYMS: frozenset[str] - CAPITALIZATION_EXCEPTIONS: dict[str, str] +#: What compile_pool returns: the compiler's, one key per constant. +Compiled = _ri.Compiled # the one tie between the two: mypy holds compile_pool to Compiled's @@ -1218,136 +1184,32 @@ class Compiled(TypedDict): def compile_pool(pool: Pool) -> Compiled: - """Fold the readings into today's constants.""" + """Run the cross-file checks, then fold the readings into today's + constants -- through the reading index, the same rows + nameparser/config/_index.py writes, so the module the parser reads + is the index compiled, and a choice of languages (phase 3b) compiles + what the build did.""" check_pool(pool) - # the two derived markers (check 11): a clash with a name is a word's - # name reading beside the reading it clashes with - named = {r.text for r in pool.readings if r.category == "name"} - given = {r.text for r in pool.readings if r.kind == "given"} - titles: set[str] = set() - given_titles: set[str] = set() - after_given: set[str] = set() - title_kinds: dict[str, set[str]] = {} - attaches: dict[str, set[str]] = {} - particles: set[str] = set() - never_given: set[str] = set() - conjunctions: set[str] = set() - initial_conjunctions: set[str] = set() - bound: set[str] = set() - abbreviations: set[str] = set() - markers: set[str] = set() - words: set[str] = set() - glued: set[str] = set() - ambiguous_words: set[str] = set() - agnomens: set[str] = set() - ambiguous: set[str] = set() - acronyms: set[str] = set() - cased: dict[str, str] = {} - for r in pool.readings: - t, p = r.text, r.props - if not r.on_by_default: - continue - if "cased" in p: - cased[t] = p["cased"] - if r.category == "title": - titles.add(t) - if r.addresses == "given": - given_titles.add(t) - if r.flag("after_given"): - after_given.add(t) - title_kinds.setdefault(t, set()).add(r.kind) - attaches.setdefault(t, set()).update(p.get("attaches", ())) - elif r.category == "suffix": - if r.acronym: - acronyms.add(t) - if t in named: - ambiguous.add(t) - else: - words.add(t) - if "attached" in p: - glued.add(t) - if t in named and r.kind in GATED_WORDS: - ambiguous_words.add(t) - if r.kind == "agnomen": - agnomens.add(t) - elif r.category == "particle": - particles.add(t) - if t not in given: - never_given.add(t) - elif r.category == "conjunction": - conjunctions.add(t) - if r.flag("may_be_initial"): - initial_conjunctions.add(t) - elif r.category == "bound_given": - bound.add(t) - elif r.category == "relation": - markers.add(t) - elif r.kind == "given" and "same_as" in p: - abbreviations.add(t) - contested = titles & given - particles - out = Compiled( - GIVEN_NAME_TITLES=frozenset(given_titles), - # a title that may be a given name compiles into TITLES_AMBIGUOUS - # INSTEAD: a word in TITLES is a title wherever it stands, which - # is what a caller listing one there has always asked for - TITLES=frozenset(titles - contested), - AFTER_GIVEN_TITLES=frozenset(after_given), - # the fourth derived marker (check 11): a title that may be a - # given name, a title only inside a title phrase (rules/H9), and - # so not in TITLES above. A - # title that is also a particle ('st', 'freiherr') is left to the - # particle rules, which weigh its given reading at the front - # already (rules/P4) - TITLES_AMBIGUOUS=frozenset(contested), - # a word every title reading of which is a modifier: a title only - # with a head title behind it in its run (decision 10, rules/H9) - TITLE_MODIFIERS=frozenset( - t for t, ks in title_kinds.items() if ks == {"modifier"}), - TITLE_STYLES=frozenset( - t for t, ks in title_kinds.items() if "style" in ks), - # `attaches` resolved to the two-word phrases it licenses, the - # word before and each title word of a kind it names - TITLE_PHRASES=frozenset( - f"{t} {h}" for t, kinds in attaches.items() if kinds - for h, hk in title_kinds.items() - if h != t and (hk & kinds or h in kinds)), - NON_GIVEN_NAME_PARTICLES=frozenset(never_given), - PARTICLES=frozenset(particles), - CONJUNCTIONS=frozenset(conjunctions), - CONJUNCTIONS_AMBIGUOUS=frozenset(initial_conjunctions), - BOUND_GIVEN_NAMES=frozenset(bound), - # a given name written short (`chas`, Filipino `ma`): opening the - # given part period-marked, it is the given name, which outranks - # rules/H2's shape (#789) - GIVEN_NAME_ABBREVIATIONS=frozenset(abbreviations), - MAIDEN_MARKERS=frozenset(markers), - SUFFIX_WORDS=frozenset(words), - GLUED_HONORIFICS=frozenset(glued), - SUFFIX_WORDS_AMBIGUOUS=frozenset(ambiguous_words), - AGNOMENS=frozenset(agnomens), - SUFFIX_ACRONYMS_AMBIGUOUS=frozenset(ambiguous), - SUFFIX_ACRONYMS=frozenset(acronyms), - CAPITALIZATION_EXCEPTIONS=dict(sorted(cased.items())), - ) - # One form per post-nominal: the word test strips only edge periods - # and the acronym test strips all of them, so a word in both is - # matched twice by two rules, and which one fired is unreadable from - # outside. The sharper stake is the ambiguous set's: suffix_as_written - # ORs the two branches, so an ambiguous acronym that were also a word - # would be claimed through the word and bypass the period gate the - # marking exists to impose. The single overlap was 'esq', dropped - # 2026-09-08 (decisions/suffix-acronym-collisions). Lexicon guards - # only that ambiguous half (its gate-bypass raise), so a caller may - # still put an unambiguous word in both. In the TOML the overlap is - # unreachable -- check 4 refuses two suffix kinds on one word, check 5 - # one kind with `acronym` differing between files, and check 2 a word - # twice in one file -- so this is a backstop kept because it costs one - # set intersection, not a check any input reaches today. - both = out["SUFFIX_ACRONYMS"] & out["SUFFIX_WORDS"] - if both: - raise VocabError(f"{sorted(both)}: a post-nominal is an acronym or a " - f"word, never both (check `acronym` in every file)") - return out + try: + compiled: Compiled = _ri.compile_readings(index_rows(pool)) + except _ri.CompileError as e: + # only the data's own failure: a bug in the compiler stays an + # exception main reports as internal, exit 3 + raise VocabError(str(e)) from None + return compiled + + +def index_rows(pool: Pool) -> list[Any]: + """The reading index: each reading as an IndexReading holding only + the properties the compiler reads, in index order (text, tag, then + kind in canonical order). Exclusions are left out: they keep a + reading from compiling and are checked here, at build time.""" + rows = [_ri.IndexReading(r.text, r.tag, r.kind, + {k: v for k, v in r.props.items() + if k in _ri.indexed(r.kind)}) + for r in pool.readings] + rows.sort(key=lambda x: (x.text, x.tag, KIND_ORDER[x.kind])) + return rows def read_dir(path: Path = VOCAB) -> dict[str, str]: @@ -1415,6 +1277,45 @@ def render(compiled: Compiled) -> str: return "\n".join(out) + "\n" +INDEX_HEADER = f'''"""The reading index. GENERATED from nameparser/vocab/*.toml by +tools/vocab/build.py -- do not edit; edit the TOML and run + + {REFRESH} + +One line per word in the vocabulary files, holding a row per reading, +on by default or not: (tag, kind, properties), the tag being the file's +BCP 47 tag and the properties the ones the compiler reads, as sorted +(key, value) pairs. +Read it through nameparser._reading_index (index, readings_of), which +documents the row; _compiled.py is this index compiled. +""" +''' + + +def index_output() -> Path: + """Where the index is written: beside OUTPUT, read when the build + runs rather than bound at import, so a test repointing OUTPUT at a + fixture tree moves both generated modules and never writes the + package's own index.""" + return OUTPUT.with_name("_index.py") + + +def render_index(rows: Iterable[Any]) -> str: + """One word per line, its rows in index order, so a change to a + word's readings is a one-line diff; the whole tuple is one constant, + which a cached import loads without building anything, and grouped + by word, so a lookup table is one dict() call.""" + out = [INDEX_HEADER, "", + "READINGS: tuple[tuple[str, tuple[tuple[str, str, " + "tuple[tuple[str, object], ...]], ...]], ...] = ("] + words: dict[str, list[Any]] = {} + for r in rows: + words.setdefault(r.text, []).append(_ri.to_row(r)) + out += [f" ({text!r}, {tuple(rs)!r})," for text, rs in words.items()] + out.append(")") + return "\n".join(out) + "\n" + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--check", action="store_true", @@ -1437,14 +1338,25 @@ def main(argv: list[str] | None = None) -> int: def _run(*, check: bool) -> int: - text = render(compile_files(read_dir())) + pool = load(read_dir()) + compiled = compile_pool(pool) + stale = 0 + for path, text in ((OUTPUT, render(compiled)), + (index_output(), render_index(index_rows(pool)))): + stale += _refresh(path, text, check=check) + return 1 if stale else 0 + + +def _refresh(path: Path, text: str, *, check: bool) -> int: + """Compare one generated module with `text`, writing it unless + `check`; 1 where --check finds it stale.""" try: - current: str | None = OUTPUT.read_text(encoding="utf-8") if OUTPUT.exists() else "" + current: str | None = path.read_text(encoding="utf-8") if path.exists() else "" except (OSError, UnicodeDecodeError): # an unreadable module is stale in the one sense that matters: # rebuilding replaces it current = None - name = OUTPUT.relative_to(ROOT) + name = path.relative_to(ROOT) if check: if current != text: state = "stale" if current is not None else "unreadable" @@ -1455,7 +1367,7 @@ def _run(*, check: bool) -> int: if current == text: print(f"{name} is current") else: - OUTPUT.write_text(text, encoding="utf-8") + path.write_text(text, encoding="utf-8") print(f"wrote {name}") return 0