From e33bb64e1e0dc4a29384feef9bf8b3147d9556ad Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 18 Aug 2026 00:50:53 -0700 Subject: [PATCH 1/3] test(post_rules): make the reduced fixture mirror shipped vocabulary A reduced lexicon is the convention in every pipeline stage module and stays. What it must not do is classify a word DIFFERENTLY from the shipped sets: the test then reads one name and parses another, and passes for a reason its author never sees. Three words did that here, found by the guard this commit adds, not by inspection: dr absent, ships as a title AND a suffix word md suffix_words here, ships as a title and a suffix ACRONYM la never-given here, ships as AMBIGUOUS -- so "de la Vega" was chaining through the wrong kind of particle, in the module whose subject is how far a leading particle run chains `_parsed` now compares its own vocabulary tagging against Lexicon.default() for every input and fails with the divergence named. Mutation-verified: restoring `la` to the never-given half fires it. 'Dr. de MD Mesnil' becomes 'Mr de MD Mesnil'. A period makes any opening abbreviation a title by shape (rules.md#H2), so the dotted title passed whether or not the fixture held the word -- the test could not distinguish vocabulary from shape. Mutation-verified live: removing the piece-skip from _leading_name_piece now fails it, giving family='de', given='MD' exactly as the comment says. Refs #395 Co-Authored-By: Claude Opus 5 --- tests/v2/pipeline/test_post_rules.py | 63 ++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index b4e1d50a..016bf422 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -7,18 +7,59 @@ PatronymicRule, Policy) from nameparser._types import Role +# A reduced lexicon, the convention in every pipeline stage module: a +# stage test should not move when shipped vocabulary does. What it must +# NOT do is classify a word DIFFERENTLY from the shipped sets, which +# makes a test pass by parsing something other than the name it reads +# as -- `_assert_fixture_mirrors_shipped` below holds the line, and +# found three such words when it was written (#395): `dr` and `md` were +# absent while shipping as titles, and `la` sat in the never-given half +# while shipping as ambiguous. _LEX = Lexicon( - titles=frozenset({"mr", "sir"}), + titles=frozenset({"mr", "sir", "dr", "md"}), given_name_titles=frozenset({"sir"}), particles=frozenset({"de", "la", "van"}), - particles_ambiguous=frozenset({"van"}), - suffix_words=frozenset({"md"}), + particles_ambiguous=frozenset({"la", "van"}), + suffix_words=frozenset({"dr"}), + suffix_acronyms=frozenset({"md"}), ) +#: Vocabulary-derived tags. `initial` and the structural tags are +#: excluded on purpose: they come from a token's SHAPE, which no +#: lexicon controls. +_VOCAB_TAGS = frozenset({"particle", "conjunction"}) + + +def _fixture_mirrors_shipped(text: str, policy: Policy) -> str: + """Empty unless `_LEX` classifies a word in `text` differently from + the shipped sets. A reduced fixture is fine -- a MISCLASSIFYING one + is not, because the test then reads one name and parses another, + and passes for a reason its author never sees (#395; the three + words it caught are named on `_LEX`).""" + def vocab(lexicon: Lexicon) -> list[tuple[str, frozenset[str]]]: + state = run(ParseState(original=text, lexicon=lexicon, + policy=policy)) + return [(t.text, frozenset(g for g in t.tags + if g.startswith("vocab:") + or g in _VOCAB_TAGS)) + for t in state.tokens] + mine, shipped = vocab(_LEX), vocab(Lexicon.default()) + if len(mine) != len(shipped): + return f"{text!r}: tokenizes differently under the shipped lexicon" + return "; ".join( + f"{word!r} is {sorted(ours) or 'plain'} here but " + f"{sorted(theirs) or 'plain'} in the shipped sets" + for (word, ours), (_, theirs) in zip(mine, shipped) if ours != theirs) + + def _parsed(text: str, policy: Policy | None = None) -> ParseState: - return run(ParseState(original=text, lexicon=_LEX, - policy=policy or Policy())) + policy = policy or Policy() + divergence = _fixture_mirrors_shipped(text, policy) + assert not divergence, ( + f"{divergence}. Mirror the shipped classification in _LEX, or " + f"pass an explicit lexicon if the test needs this word plain.") + return run(ParseState(original=text, lexicon=_LEX, policy=policy)) def _by_role(state: ParseState, role: Role) -> str: @@ -156,7 +197,7 @@ def test_leading_piece_scan_skips_pieces_that_hold_no_name( policy: Policy) -> None: # `_leading_name_piece` walks PAST pieces carrying no name role # rather than reading piece 0 -- and past the first such piece, not - # only over a single title. 'Mr. de Mesnil' cannot show that: its + # only over a single title. 'Mr de Mesnil' cannot show that: its # particle is chained into one piece with 'Mesnil', so the scan # lands on a two-token piece and the rule declines either way. # Here a mid-name suffix word breaks that chain, leaving the @@ -164,8 +205,12 @@ def test_leading_piece_scan_skips_pieces_that_hold_no_name( # skip, or reading only pieces[0], the scan finds the title (or # nothing) and the name splits: given='MD', middle='Mesnil', # family='de'. - out = _parsed("Dr. de MD Mesnil", policy) - assert _by_role(out, Role.TITLE) == "Dr." + # + # The title is deliberately UNDOTTED: a period makes any opening + # abbreviation a title by shape (rules.md#H2), so a dotted one + # would pass this test with the title vocabulary empty. + out = _parsed("Mr de MD Mesnil", policy) + assert _by_role(out, Role.TITLE) == "Mr" assert _by_role(out, Role.FAMILY) == "de MD Mesnil" assert not _by_role(out, Role.GIVEN) assert not _by_role(out, Role.MIDDLE) @@ -199,7 +244,7 @@ def test_family_first_leading_particle_cases_that_do_not_fold( ("de Mesnil", "", "", "", "de Mesnil", ""), ("de la Vega", "", "", "", "de la Vega", ""), ("de Mesnil Garcia", "", "", "", "de Mesnil Garcia", ""), - ("Dr. de MD Mesnil", "Dr.", "", "", "de MD Mesnil", ""), + ("Mr de MD Mesnil", "Mr", "", "", "de MD Mesnil", ""), ("de Mesnil MD", "", "", "", "de Mesnil", "MD"), ("De Mesnil, MD", "", "", "", "De Mesnil", "MD"), ("Mr. de Mesnil", "Mr.", "", "", "de Mesnil", ""), From f3664ffcc6d8dd991924bd2361864ff70557c184 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 18 Aug 2026 00:57:45 -0700 Subject: [PATCH 2/3] test(cases): pin the two-leftover shape before #395 moves it `de la Cruz Juan Carlos` is the only shape where FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST can disagree with each other -- with one leftover both send it to `given`. It appears in no case row, no unit test and no rule example, which is why PR #394 could discard name_order from the leftover placement and still pass the whole suite. Spanish because the listing is real: "Apellidos Nombres" keeps the particle in place. Dutch would have been wrong for this row -- its family-first listing moves the particle behind the given name ("Jong, Jan Pieter de"), which is rule P6's shape, not this one. The run also reaches 'Cruz' THROUGH ambiguous 'la', so the row doubles as the chain a stop keyed on never-given membership would break. Pinned before the change because what it records is surprising: today all three orders agree, each taking the whole name into the family. The default-order row is the accepted cost of #395's direction and must NOT move when it lands; 1.4.0 gives last 'de la Cruz Juan Carlos' too, so it is parity. The two family-first rows are core-only -- name_order is in _UNTRANSLATED, having no v1 Constants spelling -- so they join _CORE_ONLY_IDS, whose allowlist is what keeps that skip from being silent. Refs #395 Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 45 ++++++++++++++++++++++++++++++++++- tests/v2/test_facade_cases.py | 2 ++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index a55fb4a3..279783ab 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -24,7 +24,8 @@ from dataclasses import dataclass from nameparser import Policy -from nameparser._policy import PatronymicRule +from nameparser._policy import (FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, + PatronymicRule) @dataclass(frozen=True) @@ -161,6 +162,48 @@ def __post_init__(self) -> None: "1b folds the name into the family. 1.4.0 and 2.1 gave " "first 'de Mesnil' with no family, because the chain " "left 1b nothing standing alone to fire on"), + # The two-leftover shape, and the ONLY shape where the two + # family-first orders can disagree with each other: one leftover + # goes to `given` under both. Spanish because the listing is real + # -- "Apellidos Nombres" keeps the particle in place, where Dutch + # moves it behind the given name ("Jong, Jan Pieter de", the + # tussenvoegsel convention rule P6 covers). Pinned here BEFORE #395 + # moves it, because what it records is surprising: today all three + # orders agree, the leading run taking the whole name in each. + Case("leading_never_given_particle_two_leftovers", + "de la Cruz Juan Carlos", + {"family": "de la Cruz Juan Carlos"}, + classification="parity", + notes="the DEFAULT order, which #395 leaves alone: with no " + "order declared there is no evidence that 'Juan Carlos' " + "is anything but more surname, and the same shape reads " + "correctly that way in 'pennie von bergen wessels'. " + "1.4.0 gives last 'de la Cruz Juan Carlos' too, so this " + "row must not move when #395 lands"), + Case("leading_never_given_particle_two_leftovers_family_first", + "de la Cruz Juan Carlos", + {"family": "de la Cruz Juan Carlos"}, + policy=Policy(name_order=FAMILY_FIRST), + classification="feat(name-order)", + notes="core-only: name_order has no v1 spelling. Today " + "identical to the default order, which is the state " + "#395 changes -- the run should stop at 'Cruz', leaving " + "two words for the order to place. The run reaches " + "'Cruz' THROUGH ambiguous 'la', which is the chain a " + "stop keyed on never-given membership would break"), + Case("leading_never_given_particle_two_leftovers_" + "family_first_given_last", + "de la Cruz Juan Carlos", + {"family": "de la Cruz Juan Carlos"}, + policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST), + classification="feat(name-order)", + notes="the row that makes the leftover DISTRIBUTION testable: " + "once the run stops, FAMILY_FIRST and this order place " + "'Juan Carlos' differently, and no other shape can tell " + "them apart. PR #394's review found the code that does " + "the placing passes the whole suite with name_order " + "discarded, which is what an untested divergence looks " + "like"), Case("suffix_word_title_ambiguous_particle", "Jr. Van Johnson", {"title": "Jr.", "given": "Van", "family": "Johnson"}, classification="fix(#367)", diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index 31d73f06..b647ef46 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -71,6 +71,8 @@ #: to the unsubtracted default pushes every parenthesis-maiden row #: into this set and turns nothing red. _CORE_ONLY_IDS = frozenset({ + "leading_never_given_particle_two_leftovers_family_first", + "leading_never_given_particle_two_leftovers_family_first_given_last", "maiden_marker_delimited_beside_a_nickname_clause", "maiden_marker_kyusei_delimited", "ko_honorific_period_under_strict_comma_suffixes", From 4627604fa8856ba9f030f41cbba5bc7f2d1c4d3c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 18 Aug 2026 01:25:38 -0700 Subject: [PATCH 3/3] feat(particles): a family-first order stops the leading run A never-given particle opening the name took every remaining word into the family in all three orders. Under a declared family-first order it now takes one name word and leaves the rest to the order: FAMILY_FIRST "de Mesnil Juan" -> family 'de Mesnil', given 'Juan' FAMILY_FIRST_GIVEN_LAST "de la Cruz Juan Carlos" -> middle 'Juan', given 'Carlos' Declaring a family-first order asserts that what follows the family is not more surname, which is the question the stopping point asks. The DEFAULT order is unchanged and that is the accepted cost: with nothing declared, "de Mesnil Juan" has the shape of "pennie von bergen wessels", whose whole text is the surname. Callers who mean otherwise write the comma, which already parses that way. Where it lives, and why not the two sites that failed before: the fold in post_rules. Grouping was PR #394 and assignment PR #391 -- and the piece is the obstacle, since "de la Cruz Juan Carlos" groups as [de] [la Cruz Juan Carlos] once the ambiguous particle chains, so the stop must cut INSIDE a piece. post_rules can: roles are per token, and nothing downstream reads pieces (measured -- only _assign, which runs before it). Grouping stays order-independent. The order is read, not re-derived. assign records the order it used on ParseState.order and the fold keys on that; policy.name_order would disagree with the roles assign already wrote whenever a script_orders entry overrides it. The run counts UNITS: a conjunction join (P3) and a bound given-name pair (P5) each count once, so "de la Vega y Santos Juan" cannot stop between Vega and Santos, and "abdul Rahman" cannot be halved. Both are read off the tags -- the prefix chain has already merged the joined piece away by then. Measured: one differential corpus name moves, "de Mesnil Garcia", under each family-first order; all 751 are byte-identical in the default order. The corpus cannot see more than that -- it runs under the default policy against 1.4.0, which has no name_order -- so the verification that counts is the two-leftover case rows, which mutation-checking confirms are the only thing in the suite that fails when name_order is discarded from the leftover placement. Closes #395 Co-Authored-By: Claude Opus 5 --- docs/design/decisions.md | 6 +- docs/design/rules.md | 58 ++++-- docs/release_log.rst | 2 + nameparser/_pipeline/_assign.py | 18 +- nameparser/_pipeline/_post_rules.py | 158 ++++++++++++++-- nameparser/_pipeline/_state.py | 21 ++- tests/v2/cases.py | 63 ++++--- tests/v2/pipeline/test_post_rules.py | 257 ++++++++++++++++++++++++--- tests/v2/pipeline/test_state.py | 4 +- tests/v2/rules_doc.py | 2 + 10 files changed, 491 insertions(+), 98 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index f4aaeb50..2c5ef538 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -47,11 +47,15 @@ the 2026-08-16 entries below. The survivor is the degenerate bare family="Juan". given="de" is the reading the vocabulary exists to forbid, and the fold is what prevents it. The all-orders agreement in P1 is deliberate and is W4's shape: a wholly-hangul name reads family="김" under every declared order because the script carries a signal the order does not override, and a leading never-given particle is the Latin-script analogue. decisions.md#O4 already draws the line — "Words no vocabulary has claimed read by position" — so name_order governs the unclaimed remainder, which is most inputs. - 2026-08-17 — SUPERSEDES the order-independence half of the 2026-08-16 order-precedence keystone entry above (#364, #365, #368). That entry says "no name_order moves that stopping point"; it may. Declaring FAMILY_FIRST or FAMILY_FIRST_GIVEN_LAST is precisely an assertion that what follows the family is NOT more surname, and a particle run's stopping point is exactly the question of where surname material ends — so the declaration is evidence about it, not merely about which slot the result lands in. The grouping/assignment split survives everywhere else; what is narrowed is the claim that the stopping point is one of the things order cannot touch. - Concretely: under GIVEN_FIRST a leading particle run keeps joining to the end of the name, which makes "de Mesnil Juan" read exactly like "pennie von bergen wessels" — the same shape, and this session decided the greedy reading is right for that one. Under a family-first order the run stops after its first non-particle name word, because the caller has said the rest is given or middle. + Concretely: under GIVEN_FIRST a leading particle run keeps joining to the end of the name. (This entry originally said that makes "de Mesnil Juan" read "exactly like pennie von bergen wessels — the same shape". Measured while implementing #395, that is false and is corrected here rather than left standing: pennie is the GIVEN name there, von is ambiguous so this fold can never fire on it, and the surname reading comes from P2's mid-name chain. What the real name does support is the weaker claim the argument needs — that a particle followed by several words can be all surname, "von Bergen Wessels" being one.) Under a family-first order the run stops after its first non-particle name word, because the caller has said the rest is given or middle. Accepted, and it is the cost that decided #364 three separate ways before this: "de la Vega Juan" under the DEFAULT order stays wholly a surname. A caller who means family "de la Vega" plus given "Juan" writes the comma. The mixed-language shapes are what make the greedy reading look wrong, and the parser cannot see language. Why the question kept thrashing, worth recording so a fourth attempt does not start from scratch: "how much does a leading particle run take" is a LANGUAGE judgement being forced through a POSITION heuristic, in a parser that has correctly refused to detect language (decisions.md#O4, rules.md's Not-in-scope). Every argument in the thread — `de`/`do` as Vietnamese surnames, `von` as German, `dos` as Portuguese — is really about which tradition the name belongs to. The order declaration is the one place a CALLER supplies that information instead of the parser guessing, which is why keying on it is better than any position rule we tried. Implementation consequence: the stopping point may now read name_order, which the superseded sentence forbade. Whether that lives in grouping (order-aware) or as a split in assignment is open — PR #391 showed assignment cannot split an existing piece today, so choosing assignment means giving it token-level slicing. Supersedes nothing else: #368's reversal, the trailing-orphan rule P6, and the claim that a leading never-given particle takes the FAMILY rather than a given name all stand. +- 2026-08-18 #395 — WHERE THE STOP LIVES, settling what the entry above left open. Not grouping and not assignment: the fold in post_rules, which already retags tokens and is the only site that can express the stop without moving a piece boundary. Grouping was PR #394 (four regressions) and assignment was PR #391, where the finding was that assign cannot split an existing piece — and the piece is the problem: "de la Cruz Juan Carlos" groups as [de][la Cruz Juan Carlos], the ambiguous particle having chained forward, so any rule that stops "after the first name word" must cut INSIDE a piece. post_rules can, because roles are per token and nothing downstream reads pieces (measured: only _assign, which runs before it, and post_rules itself). Grouping therefore stays order-independent, which keeps the 2026-08-16 keystone intact everywhere except the reach of this one fold. + The order is READ, not re-derived: assign now records the order it actually used on ParseState.order, and the fold keys on that. policy.name_order would have been wrong — a script_orders entry can put the family first under a given-first policy, and the roles assign already wrote would then disagree with the roles the fold computes. The same reasoning already appears one function away, where the PARTICLE_OR_GIVEN emitter reads the role off the token rather than assuming given. + A unit, not a token: a particle chain (P2), a conjunction join (P3) and a bound given-name pair (P5) each count once. All three are read off the TAGS rather than the pieces, but for two different reasons, and the difference matters to anyone who tries to simplify this. The conjunction join grouping DID build and the prefix chain then swallowed: the name reaches the fold as [de][la Vega y Santos Juan], the ambiguous particle having chained forward over the join, so the JOIN's boundary is gone. The leading particle keeps its own piece, and must — the fold's site test wants a lone piece, so a one-piece name would not fold at all. The bound-given join grouping never built at all: P5 joins only where the bound word is the first non-title piece, and at a fold site the first piece is the particle, so "ibn Awf abdul Rahman" arrives as four separate pieces. Restoring piece boundaries in group would fix the first and silently split the second. It also makes P5's new implemented: pointer stronger than a citation — this is the only place that join happens in this shape. This is what keeps "de la Vega y Santos Juan" from stopping between Vega and Santos — the case rules.md#P3 was amended for on 2026-08-17, now executable. + Measured, with the denominator this log requires: of 751 corpus names, SEVEN reach P1's leading fold (DE MESNIL, De Groot, de Mesnil, de Mesnil Garcia, de Mesnil Jr., de la Vega, dos Santos) and exactly ONE of those has anything past the run to lose, so one moves (de Mesnil Garcia) under each family-first order and none under the default. The single mover is a fact about the corpus holding no family-first listing with leftovers, not about the blast radius. The differential harness cannot see any of it — it runs the corpus under the default policy against 1.4.0, which has no name_order — so its exit 0 is evidence for the accepted cost and for nothing else. The verification that counts is the two-leftover shape: mutation-checked, discarding name_order from the leftover placement fails three tests across three layers — the FAMILY_FIRST_GIVEN_LAST case row (one of the two, the FAMILY_FIRST row being blind to it by construction), test_post_rules.py's test_the_two_family_first_orders_differ_at_two_leftovers, and rules.md's P1 example for that order. An earlier draft of this sentence said "exactly one, and no other test in the suite": true when measured, false one commit later, because two of the three guards were added after the measurement and it was not re-run. Recording the failure mode as well as the number, since it is this log's recurring one. Declined: diff --git a/docs/design/rules.md b/docs/design/rules.md index 9d964e38..f86b837a 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -95,27 +95,55 @@ P1. Rationale: a never-given particle standing alone cannot be one as the given name, is a surname written out in full. A never-given particle standing alone where the given name would go — or opening the name — marks the name as surname-only: the - particle run and the one name word it attaches to are the - family, and any name words beyond that read by position. It + particle run and the name words it attaches to are the family. It needs another name word to attach to. The run is every particle in sequence, never-given and ambiguous alike ("de la Vega" is one group, not "de" plus a separate "la Vega"). An ambiguous particle keeps - whatever reading its position gives it. The reading holds under - every declared name order: a never-given particle is evidence - about how the name is written, and a declared order governs only - what no vocabulary has claimed (O4) — the same precedence the - script license takes in W4. + whatever reading its position gives it. That the particle claims + the FAMILY rather than a given name holds under every order: a + never-given particle is evidence about how the name is written, + and a declared order governs only what no vocabulary has claimed + (O4) — the same precedence the script license takes in W4. + + How MANY name words it attaches to depends on the order, and on + which of the two positions above the particle stands in. Opening + the name, under a family-first order, it takes exactly ONE — + declaring that order asserts that what follows the family is not + more surname. Opening the name under the default order, or + standing in the given position under any order, it takes the rest + of the name: nothing there marks where the surname ends. One name + word means one UNIT — a particle chain (P2), a conjunction join + (P3) or a bound given-name pair (P5) is taken whole or not at + all. A title does not move the opening position (P4), but a + family comma does end the question: the comma has already fixed + the surname, so there is no positional read left for an order to + narrow, and a particle opening the part AFTER it takes the rest + of that part whatever order is declared. What is left over is not read by O4's rule for a whole name, + which would make the first leftover a second family name; it is + laid out as the positions AFTER the family in the declared order, + the family slot being already filled. "de la Vega" → family="de la Vega" "Sir de Mesnil" → family="de Mesnil" "Mesnil de" family-first → family="Mesnil de" - "de Mesnil Juan" → family="de Mesnil" deviates: #364 (today: family="de Mesnil Juan") - "de Mesnil Juan" → given="Juan" deviates: #364 (today: given="") + "de Mesnil Juan" → family="de Mesnil Juan" + "de Mesnil Juan" family-first → family="de Mesnil" + "de Mesnil Juan" family-first → given="Juan" + "Smith, de Mesnil Juan" family-first → family="Smith de Mesnil Juan" + "de la Vega y Santos Juan" family-first → family="de la Vega y Santos" + "ibn Awf abdul Rahman" family-first → given="abdul Rahman" + "de la Cruz Juan Carlos" family-first-given-last → given="Carlos" "Mc Donald" → family="Mc Donald" "de los Santos" → family="de los Santos" "van Gogh" → given="van" · boundary Accepted: a bare "de" stays the given name — there is nothing to fold into, and inventing a surname would be worse. "de" → given="de" + Accepted: stopping the run leaves a MIDDLE where the fold never + left one before, so O3 has something to claim that it could not + reach until now. The family it then reports is discontiguous in + the input — words 1-3 plus word 5 — and renders the folded word + first, which is R1's order, not this rule's doing. + "de la Cruz Juan Carlos" family-first+middle_as_family → family="Carlos de la Cruz" Accepted: only the OPENING position is this rule's subject. A particle chain standing inside the name is grouped normally (P2) and positioned by the declared order, so a family-first reading @@ -123,7 +151,7 @@ P1. Rationale: a never-given particle standing alone cannot be the bare particle reading as a given name, not any name part that begins with one. "Juan de la Vega" family-first → family="Juan" - history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_post_rules.py + history: decisions.md#P1 · interacts: O3, O4, P2, P3, P4, P5, P6 · implemented: nameparser/_pipeline/_post_rules.py P2. Rationale: a particle is written as part of the surname it precedes, and a title stands outside the name entirely. @@ -132,16 +160,20 @@ P2. Rationale: a particle is written as part of the surname it or the name ends. The final group reads as the family name; earlier groups read by position. The chain begins wherever the name begins, and a preceding title does not move that point. + Where P1's fold has claimed the opening, the fold decides the + family instead — and may take only PART of the final group, + since it counts name words and the group is one part. "John van der Berg" → family="van der Berg" "John van der Berg Smith" → family="van der Berg Smith" "Vincent van Gogh van Beethoven" → middle="van Gogh" "Dr. John van der Berg" → family="van der Berg" "Juan de" → family="de" · boundary + "de la Cruz Juan Carlos" family-first → family="de la Cruz" Accepted: a caller wanting the combined double-surname reading (#132's ask) has it as the surnames view rather than the family field. "Vincent van Gogh van Beethoven" → surnames="van Gogh van Beethoven" - history: decisions.md#P2 · implemented: nameparser/_pipeline/_group.py + history: decisions.md#P2 · interacts: P1, P4 · implemented: nameparser/_pipeline/_group.py, nameparser/_pipeline/_post_rules.py P3. Rationale: connective words ("y", "of the") bind name words into one name part; but a single letter in a short name is more @@ -176,7 +208,7 @@ P3. Rationale: connective words ("y", "of the") bind name words into same two words unjoined are two name words and H1 does not fire. P1's leading run becomes the second once #395 lands — its run must take the "Vega y Santos" join whole or stop before it. - history: decisions.md#P3 · interacts: H1, P1 · implemented: nameparser/_pipeline/_group.py + history: decisions.md#P3 · interacts: H1, P1 · implemented: nameparser/_pipeline/_group.py, nameparser/_pipeline/_post_rules.py P4. Rationale: a particle links forward from inside a name; at the very front there is no name yet to be inside. @@ -197,7 +229,7 @@ P5. Rationale: some given-name words are incomplete alone — "abdul" one given name. "abdul salam ahmed salem" → given="abdul salam" "mohamad ali smith" → given="mohamad" · boundary - history: decisions.md#P5 · implemented: nameparser/_pipeline/_group.py + history: decisions.md#P5 · implemented: nameparser/_pipeline/_group.py, nameparser/_pipeline/_post_rules.py P6. Rationale: a particle ending the name has nothing to link forward to, so it is not doing a particle's work there. A diff --git a/docs/release_log.rst b/docs/release_log.rst index b1c4ed0c..70450955 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -32,6 +32,8 @@ Release Log - Fix a name opening with a particle that is *never* a given name being split at the particle under a family-first name order -- ``Policy(name_order=FAMILY_FIRST)`` and ``Policy(name_order=FAMILY_FIRST_GIVEN_LAST)`` alike, and identically: ``"de Mesnil"`` read as family ``de``, given ``Mesnil``, and ``"de la Vega"`` as family ``de``, given ``la Vega``. Each is now the whole surname, as it has always been in the default order. The rule enforcing it asked for the particle by the ``GIVEN`` role, which under a family-first order belongs to the token *after* the particle, so the test read the wrong word and declined. It now also asks by position -- the piece that opens the name -- so both shapes of the same rule are caught: where such a particle stands alone as a piece, either opening the name or in the given position, the name is left with no given name at all, the given and the middles folding into the family. Standing *alone* is the whole of it, and the rule claims nothing wider: ``"Juan de la Vega"`` under ``FAMILY_FIRST`` still reports given ``de la Vega``, because there the particle chained onto the words after it rather than standing alone, and a bare ``"de"`` with nothing to fold into is still reported as the given name. The decision behind the fix: a word that can never be a given name leaves ``name_order`` nothing to decide, so declaring family-first is not a reason to make ``de`` a surname on its own. A leading particle that *may* be a given name is genuinely order-dependent and is untouched -- ``"van Gogh"`` still reads as family ``van``, given ``Gogh`` under both family-first orders. This is also what gives ``Lexicon.particles_ambiguous`` an effect outside the default order: taking a word out of it now changes the parsed fields under a family-first order, where before it moved only the ambiguity report. Seven of the 751 differential corpus names move, the same seven under each family-first order; default-order output is byte-identical over all 751, at the 1.4.0, 2.0.0 and 2.1.0 differential baselines alike (closes #359) + - Change how far a leading never-given particle takes the surname when a family-first ``name_order`` is declared. ``Policy(name_order=FAMILY_FIRST)`` read ``"de Mesnil Juan"`` as family ``de Mesnil Juan`` -- the whole name -- and now reads family ``de Mesnil``, given ``Juan``. Declaring a family-first order asserts that what follows the family is not more surname, and where the surname run ends is exactly that question, so the declaration settles it. The default order is unchanged, deliberately: with no order declared nothing marks where the surname ends, and a particle followed by several words really can be all surname -- ``von Bergen Wessels`` is one such name. Nothing in ``"de Mesnil Juan"`` distinguishes it from that reading except a declared order or a comma. A caller who means family ``de la Vega`` plus given ``Juan`` in the default order writes the comma, which already parses that way. The run takes one name WORD rather than one token: a conjunction-joined run and a bound given-name pair each count once, so the stop cannot land inside one -- ``"de la Vega y Santos Juan"`` reads family ``de la Vega y Santos``, and ``"ibn Awf abdul Rahman"`` reads given ``abdul Rahman``. Where two or more words are left over the two family-first orders differ from each other for the first time: ``"de la Cruz Juan Carlos"`` reads given ``Juan``, middle ``Carlos`` under ``FAMILY_FIRST`` and middle ``Juan``, given ``Carlos`` under ``FAMILY_FIRST_GIVEN_LAST``. An ambiguous leading particle is untouched in every order -- ``"van Gogh Jan Pieter"`` still reads family ``van`` under both family-first orders -- and so is a family comma, where the comma has already fixed the surname (``"Smith, de Mesnil"`` keeps family ``Smith de Mesnil``). One of the 751 differential corpus names moves, ``"de Mesnil Garcia"`` to family ``de Mesnil``, given ``Garcia``, under each family-first order; default-order output is byte-identical over all 751. This reverses the answer #364 was closed on, and the reasoning is recorded at ``docs/design/decisions.md#P1`` (closes #395) + - Change the ``detail`` text of a ``PARTICLE_OR_GIVEN`` ambiguity to name the role the leading particle was actually given. It said "read as a given name" under every ``name_order``, which is false under ``Policy(name_order=FAMILY_FIRST)`` -- there ``"Van Johnson"`` reads as family ``Van``, given ``Johnson``, and the report described the reading not taken. It now ends "read as a family name" in that case, reading the role off the assigned token the way ``SUFFIX_OR_NAME`` already did -- that kind names both parts (``read as a family name rather than a post-nominal``), while this one names only the part it took. The ``kind`` is unchanged and stays ``PARTICLE_OR_GIVEN``: the fork really is particle-or-given, and only the human-readable text moved. Default-order output is identical (#355) **Deprecations** diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 15b5ddab..1bd599e4 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -174,14 +174,17 @@ def _name_positions(order: tuple[Role, Role, Role], def _assign_main(seg_idx: int, state: ParseState, tokens: list[WorkToken], - ambiguities: list[PendingAmbiguity]) -> None: + ambiguities: list[PendingAmbiguity], + ) -> tuple[Role, Role, Role] | None: + """Returns the order the positional read used, for ParseState.order + -- None on every path that returns before resolving one.""" pieces = state.pieces[seg_idx] ptags = state.piece_tags[seg_idx] has_nickname = any(t.role is Role.NICKNAME for t in tokens) n = _peel_leading_titles(pieces, ptags, tokens) rest = list(range(n, len(pieces))) if not rest: - return + return None # group-flagged suffix pieces (the ph-d merge) are suffixes at ANY # position -- v1's fix_phd extracted the credential from the string # before parsing, so position never mattered (PR review I3) @@ -190,7 +193,7 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[k], Role.SUFFIX) rest = [k for k in rest if "suffix" not in ptags[k]] if not rest: - return + return None # rules.md#N3: "a name that is only a nickname and one name word # reads that word as the family name" (history: decisions.md#N3) # -- v1's p_len == 1 counted @@ -199,7 +202,7 @@ def _assign_main(seg_idx: int, state: ParseState, # name (pinned live 2026-07-17) if len(pieces) == 1 and len(rest) == 1 and has_nickname: _set_roles(tokens, pieces[rest[0]], Role.FAMILY) - return + return None # peel the trailing suffix run: k = first index in rest from which # every piece is a strict suffix (v1's are_suffixes tail rule, with # the roman-numeral special: a final roman numeral after a @@ -304,6 +307,7 @@ def _assign_main(seg_idx: int, state: ParseState, f"leading {token.text!r} may be a family-name " f"particle; read as a {token.role.value} name", tuple(head))) + return order def assign(state: ParseState) -> ParseState: @@ -311,11 +315,12 @@ def assign(state: ParseState) -> ParseState: ambiguities = list(state.ambiguities) if not state.segments: return state + order: tuple[Role, Role, Role] | None = None if state.structure is Structure.NO_COMMA: - _assign_main(0, state, tokens, ambiguities) + order = _assign_main(0, state, tokens, ambiguities) tail = len(state.segments) elif state.structure is Structure.SUFFIX_COMMA: - _assign_main(0, state, tokens, ambiguities) + order = _assign_main(0, state, tokens, ambiguities) tail = 1 else: # FAMILY_COMMA # PARTICLE_OR_GIVEN is deliberately not emitted here: after a @@ -376,4 +381,5 @@ def assign(state: ParseState) -> ParseState: for piece in state.pieces[seg_idx]: _set_roles(tokens, piece, Role.SUFFIX) return dataclasses.replace(state, tokens=tuple(tokens), + order=order, ambiguities=tuple(ambiguities)) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 906db723..f294df32 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -18,6 +18,7 @@ import re from nameparser._lexicon import _title_key +from nameparser._pipeline._assign import _name_positions from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._policy import PatronymicRule from nameparser._types import FOLDED_TAG, Role @@ -69,6 +70,100 @@ def _retag(tokens: list[WorkToken], i: int, role: Role) -> None: tokens[i] = dataclasses.replace(tokens[i], role=role) +# rules.md#P2: "a particle joins the words after it into one name +# part, the join running until the next particle starts a group of +# its own or the name ends" +# rules.md#P3: "the joined part is ONE name word wherever another +# rule counts them" +# rules.md#P5: "a recognized bound given-name word joins the word +# after it into one given name" +def _unit_end(tokens: list[WorkToken], idx: list[int], i: int) -> int: + """One past the end of the unit starting at `idx[i]`. + + RECURSIVE, and that is the whole point: what a conjunction or a + bound given-name word joins is the next UNIT, not the next word. + Absorbing a single index instead strands a particle at the end of + the unit, severed from the words it chains -- "de la Vega y la + Vega" cut between `la` and `Vega`, reporting family + "de la Vega y la", which is the same defect as a bare particle + opening the given name, mirrored.""" + if "particle" in tokens[idx[i]].tags: + j = i + while j + 1 < len(idx) and "particle" in tokens[idx[j + 1]].tags: + j += 1 + # ... then the words it joins, stopping where the next + # particle starts a group of its own, at a suffix word (the + # stop _group's chain uses), or at a conjunction, which the + # shared loop below joins to the whole unit after it rather + # than to the one word after it. + while (j + 1 < len(idx) + and "particle" not in tokens[idx[j + 1]].tags + and "conjunction" not in tokens[idx[j + 1]].tags + and "vocab:suffix" not in tokens[idx[j + 1]].tags): + j += 1 + end = j + 1 + else: + end = i + 1 + if "vocab:bound-given" in tokens[idx[i]].tags and end < len(idx): + end = _unit_end(tokens, idx, end) + while end + 1 < len(idx) and "conjunction" in tokens[idx[end]].tags: + end = _unit_end(tokens, idx, end + 1) + return end + + +def _units(tokens: list[WorkToken], idx: list[int]) -> list[list[int]]: + """`idx` split into the units other rules COUNT: one name word + each, except where another rule has already made several words one + name. Three do -- a particle and the words it chains, a + conjunction-joined run, and a bound given-name word with the word + it completes -- so `van der Berg` and `abdul Rahman` are each one + unit and the fold cannot leave half of either behind. + + Read off the TAGS rather than the pieces, for two different + reasons. A conjunction join grouping DID build and the prefix + chain then swallowed: "de la Vega y Santos Juan" reaches this as + [de][la Vega y Santos Juan], the ambiguous particle having chained + forward over the join, so the join's own boundary is gone. (The + leading particle keeps its piece -- it has to, or the fold's + lone-piece site test would not fire at all.) The bound-given join grouping + never built at all: P5 joins only where the bound word is the + first non-title piece, and at a fold site the first piece is the + particle -- "ibn Awf abdul Rahman" reaches here as four separate + pieces. The tag is the only witness in both cases. + + The particle chain is what keeps this partition agreeing with the + one `assign` reads off pieces: strip the folded run and `assign` + gives the same tail one role, so the fold must not hand its words + out separately.""" + units: list[list[int]] = [] + i = 0 + while i < len(idx): + end = _unit_end(tokens, idx, i) + units.append(list(idx[i:end])) + i = end + return units + + +def _fold_reach(tokens: list[WorkToken], name_idx: list[int]) -> int: + """How many of `name_idx` the fold takes: the particle run, plus + the one name word it attaches to -- one UNIT, so a conjunction + join goes whole ("de la Vega y Santos Juan" keeps Vega y Santos + together). All of them when the name is nothing but particles.""" + i = 0 + while i < len(name_idx) and "particle" in tokens[name_idx[i]].tags: + i += 1 + if i == len(name_idx): + return i + return i + len(_units(tokens, name_idx[i:])[0]) + + +def _is_lone_never_given_particle(site: tuple[int, ...], + tokens: list[WorkToken]) -> bool: + return (len(site) == 1 + and "particle" in tokens[site[0]].tags + and "vocab:particle-ambiguous" not in tokens[site[0]].tags) + + def post_rules(state: ParseState) -> ParseState: tokens = list(state.tokens) titles = _idx(tokens, Role.TITLE) @@ -95,30 +190,55 @@ def post_rules(state: ParseState) -> ParseState: # rules.md#P1: "a never-given particle standing alone where the # given name would go — or opening the name — marks the name as - # surname-only: the particle run and the one name word it - # attaches to are the family, and any name words beyond that - # read by position." (v1 handle_non_first_name_prefix; history: + # surname-only: the particle run and the name words it attaches + # to are the family." (v1 handle_non_first_name_prefix; history: # decisions.md#P1) - # DEVIATION #364: the fold below still takes every remaining name - # word, not just the particle run's own -- de Mesnil Juan gives - # family=de Mesnil Juan where the rule says family=de Mesnil plus - # given=Juan. Pinned by the deviates: markers on P1. - # Values written unquoted deliberately: this note sits INSIDE the - # citation block above (# decisions.md#P1) does not close it -- - # _CITE_RE wants a colon after the ID), and the excerpt check - # takes the first quoted span in the block. + # How far the fold reaches depends on the order the name was READ + # under (#395; decisions.md#P1, 2026-08-17): declaring a + # family-first order asserts that what follows the family is not + # more surname, which is the very question of where the run stops. + # Under that declaration the run takes its own particles and ONE + # name word; under the default order it keeps taking the rest of + # the name, nothing having marked where the surname ends. The + # narrowing is the LEADING site's alone: a particle + # standing in the given slot keeps the old reach. Note what + # actually holds the family-comma shape back, since it is NOT + # that test -- "Smith, de Mesnil Juan" DOES fire the leading site, + # segment 1 opening with the particle. It keeps the old reach + # because assign records no order after a family comma, the comma + # having already fixed the surname, so `order is None` here. + # Anything that later gives that path an order turns the + # narrowing on for it. # Code-local: a lone PIECE is the test at both sites, so a # particle group already chained forward is not a lone particle, # and rule H1 above cannot be what produces the fold's family # reading -- H1 is gated on `not families`. - sites = (_leading_name_piece(state, tokens), tuple(givens)) - if len(givens) + len(middles) + len(families) > 1 and any( - len(site) == 1 - and "particle" in tokens[site[0]].tags - and "vocab:particle-ambiguous" not in tokens[site[0]].tags - for site in sites): - for i in givens + middles: - _retag(tokens, i, Role.FAMILY) + lead = _leading_name_piece(state, tokens) + lead_fires = _is_lone_never_given_particle(lead, tokens) + sites_fire = lead_fires or _is_lone_never_given_particle( + tuple(givens), tokens) + if len(givens) + len(middles) + len(families) > 1 and sites_fire: + order = state.order + if lead_fires and order is not None and order[0] is Role.FAMILY: + # `state.order`, not policy.name_order: a script_orders + # entry can put the family first under a given-first + # policy, and the roles below have to match the read + # assign actually made. + name_idx = sorted(givens + middles + families) + cut = _fold_reach(tokens, name_idx) + for i in name_idx[:cut]: + _retag(tokens, i, Role.FAMILY) + # What is left is a shorter name of the same order: one + # family already placed, so drop that slot and lay the + # rest out as _name_positions would for n + 1 pieces. + rest = _units(tokens, name_idx[cut:]) + for unit, role in zip(rest, _name_positions( + order, len(rest) + 1)[1:]): + for i in unit: + _retag(tokens, i, role) + else: + for i in givens + middles: + _retag(tokens, i, Role.FAMILY) # downstream rules key on the role counts: recompute givens = _idx(tokens, Role.GIVEN) middles = _idx(tokens, Role.MIDDLE) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 16fa5bb8..5382543b 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -78,9 +78,11 @@ class ParseState: pieces, still as sub-slices of the original, and every later index in the segment runs shifts by n); classify -> token tags; group -> pieces/piece_tags/dropped AND maiden token roles; - assign/post_rules -> the remaining token roles. Ambiguities are - recorded by every stage that DECIDES one -- extract (resolved to a - token index by tokenize), segment, script_segment, classify, + assign -> the remaining token roles AND `order`, the effective + order it read them under; post_rules -> roles again. + Ambiguities are recorded by every stage that DECIDES one -- + extract (resolved to a token index by tokenize), segment, + script_segment, classify, group, and assign -- since a fork whose branches are taken in different stages needs an emitter in each. Post-group, segments may retain indices of dropped tokens -- assign iterates pieces, @@ -109,4 +111,17 @@ class ParseState: pieces: tuple[tuple[tuple[int, ...], ...], ...] = () piece_tags: tuple[tuple[frozenset[str], ...], ...] = () dropped: tuple[int, ...] = () # structural tokens (maiden markers) + #: The order `assign` actually READ the name under -- name_order, + #: or the script_orders entry that overrode it. None wherever no + #: positional read happened: after a family comma (which fixes the + #: family, so assign consults no order at all), and on every early + #: return in `_assign_main`, where a segment holds no name piece + #: to position. Recorded rather than recomputed downstream, + #: because the two can differ and a post_rules rule keyed on + #: `policy.name_order` would then disagree with the roles assign + #: already wrote (#395). Reaching that divergence needs a custom + #: lexicon -- every shipped particle is Latin, and Latin has no + #: script_orders entry -- which is why the test for it builds its + #: own (test_post_rules.py). + order: tuple[Role, Role, Role] | None = None ambiguities: tuple[PendingAmbiguity, ...] = () diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 279783ab..2dd749b5 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -162,48 +162,57 @@ def __post_init__(self) -> None: "1b folds the name into the family. 1.4.0 and 2.1 gave " "first 'de Mesnil' with no family, because the chain " "left 1b nothing standing alone to fire on"), - # The two-leftover shape, and the ONLY shape where the two - # family-first orders can disagree with each other: one leftover - # goes to `given` under both. Spanish because the listing is real - # -- "Apellidos Nombres" keeps the particle in place, where Dutch - # moves it behind the given name ("Jong, Jan Pieter de", the - # tussenvoegsel convention rule P6 covers). Pinned here BEFORE #395 - # moves it, because what it records is surprising: today all three - # orders agree, the leading run taking the whole name in each. + # The smallest shape in which the two family-first orders can + # disagree about this rule's leftovers: with one leftover both + # send it to `given`, and two or more is what separates them (the + # orders differ on plenty of names outside this rule -- 186 of the + # 751 corpus names -- so the claim is about the fold, not about + # the orders). Spanish because the listing is real: "Apellidos + # Nombres" keeps the particle in place, where Dutch moves it + # behind the given name ("Jong, Jan Pieter de", the tussenvoegsel + # convention -- rule P6's subject, though P6's attachment is + # deviates: #379, so that spelling does not yet parse the way P6 + # describes). Added before #395 landed, when all three orders + # still agreed, each taking the whole name into the family. Case("leading_never_given_particle_two_leftovers", "de la Cruz Juan Carlos", {"family": "de la Cruz Juan Carlos"}, classification="parity", notes="the DEFAULT order, which #395 leaves alone: with no " "order declared there is no evidence that 'Juan Carlos' " - "is anything but more surname, and the same shape reads " - "correctly that way in 'pennie von bergen wessels'. " - "1.4.0 gives last 'de la Cruz Juan Carlos' too, so this " - "row must not move when #395 lands"), + "is anything but more surname, and a particle followed " + "by several words really can be all surname -- 'von " + "Bergen Wessels' is one. (An earlier draft cited " + "'pennie von bergen wessels' as the same SHAPE, which " + "it is not: pennie is the given name there and von is " + "ambiguous, so this fold cannot fire on it. See " + "decisions.md#P1, 2026-08-17.) 1.4.0 gives last 'de la " + "Cruz Juan Carlos' too, so this row must not move when " + "#395 lands"), Case("leading_never_given_particle_two_leftovers_family_first", "de la Cruz Juan Carlos", - {"family": "de la Cruz Juan Carlos"}, + {"family": "de la Cruz", "given": "Juan", "middle": "Carlos"}, policy=Policy(name_order=FAMILY_FIRST), - classification="feat(name-order)", - notes="core-only: name_order has no v1 spelling. Today " - "identical to the default order, which is the state " - "#395 changes -- the run should stop at 'Cruz', leaving " - "two words for the order to place. The run reaches " + classification="feat(#395)", + notes="core-only: name_order has no v1 spelling. The run " + "stops at 'Cruz' because the declared order says what " + "follows the family is not more surname. It reaches " "'Cruz' THROUGH ambiguous 'la', which is the chain a " "stop keyed on never-given membership would break"), Case("leading_never_given_particle_two_leftovers_" "family_first_given_last", "de la Cruz Juan Carlos", - {"family": "de la Cruz Juan Carlos"}, + {"family": "de la Cruz", "middle": "Juan", "given": "Carlos"}, policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST), - classification="feat(name-order)", - notes="the row that makes the leftover DISTRIBUTION testable: " - "once the run stops, FAMILY_FIRST and this order place " - "'Juan Carlos' differently, and no other shape can tell " - "them apart. PR #394's review found the code that does " - "the placing passes the whole suite with name_order " - "discarded, which is what an untested divergence looks " - "like"), + classification="feat(#395)", + notes="the row that makes the leftover DISTRIBUTION testable, " + "and the divergence is here: FAMILY_FIRST reads 'Juan' " + "as the given name, this order reads 'Carlos'. Nothing " + "with fewer leftovers can tell the two apart. When " + "PR #394 put the placing in grouping, its review found " + "the whole suite passed with name_order discarded from " + "it; on this branch the same mutation fails three " + "tests, this row among them"), Case("suffix_word_title_ambiguous_particle", "Jr. Van Johnson", {"title": "Jr.", "given": "Van", "family": "Johnson"}, classification="fix(#367)", diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index 016bf422..eff38dfd 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -1,11 +1,15 @@ +import dataclasses +import sys + import pytest from nameparser._lexicon import Lexicon from nameparser._pipeline import run from nameparser._pipeline._state import ParseState from nameparser._policy import (FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, - PatronymicRule, Policy) -from nameparser._types import Role + GIVEN_FIRST, PatronymicRule, Policy, + Script) +from nameparser._types import STABLE_TAGS, Role # A reduced lexicon, the convention in every pipeline stage module: a # stage test should not move when shipped vocabulary does. What it must @@ -18,17 +22,21 @@ _LEX = Lexicon( titles=frozenset({"mr", "sir", "dr", "md"}), given_name_titles=frozenset({"sir"}), - particles=frozenset({"de", "la", "van"}), + particles=frozenset({"de", "der", "ibn", "la", "van"}), particles_ambiguous=frozenset({"la", "van"}), suffix_words=frozenset({"dr"}), suffix_acronyms=frozenset({"md"}), + conjunctions=frozenset({"y"}), + bound_given_names=frozenset({"abdul"}), ) -#: Vocabulary-derived tags. `initial` and the structural tags are -#: excluded on purpose: they come from a token's SHAPE, which no -#: lexicon controls. -_VOCAB_TAGS = frozenset({"particle", "conjunction"}) +#: The non-namespaced tags a LEXICON can produce: STABLE_TAGS minus +#: the two that come from a token's SHAPE, which no lexicon controls. +#: Derived rather than listed so that a new stable tag cannot quietly +#: drop out of the comparison below -- STABLE_TAGS is itself pinned, +#: at tests/v2/test_types.py. +_VOCAB_TAGS = STABLE_TAGS - {"initial", "joined"} def _fixture_mirrors_shipped(text: str, policy: Policy) -> str: @@ -48,17 +56,23 @@ def vocab(lexicon: Lexicon) -> list[tuple[str, frozenset[str]]]: if len(mine) != len(shipped): return f"{text!r}: tokenizes differently under the shipped lexicon" return "; ".join( + f"{word!r} is {sorted(ours) or 'plain'} here but " + f"{theirs_word!r} {sorted(theirs) or 'plain'} in the shipped " + f"sets" if word != theirs_word else f"{word!r} is {sorted(ours) or 'plain'} here but " f"{sorted(theirs) or 'plain'} in the shipped sets" - for (word, ours), (_, theirs) in zip(mine, shipped) if ours != theirs) + for (word, ours), (theirs_word, theirs) in zip(mine, shipped) + if (word, ours) != (theirs_word, theirs)) def _parsed(text: str, policy: Policy | None = None) -> ParseState: policy = policy or Policy() divergence = _fixture_mirrors_shipped(text, policy) assert not divergence, ( - f"{divergence}. Mirror the shipped classification in _LEX, or " - f"pass an explicit lexicon if the test needs this word plain.") + f"{divergence}. Mirror the shipped classification in _LEX. A " + f"test that needs the word classified some OTHER way builds " + f"its own state with an explicit lexicon instead of calling " + f"this helper, the way the vocabulary rows below do.") return run(ParseState(original=text, lexicon=_LEX, policy=policy)) @@ -174,10 +188,10 @@ def test_degenerate_bare_particle_stays_given() -> None: # the leading particle chains the rest of the name into the family ("de Mesnil", "de Mesnil", "", ""), ("de la Vega", "de la Vega", "", ""), - # three pieces, so the fold has a MIDDLE to move as well as the - # given -- the `givens + middles` half of the repair, and the only - # no-comma corpus name that reaches it - ("de Mesnil Garcia", "de Mesnil Garcia", "", ""), + # three pieces: the run stops at 'Mesnil' and 'Garcia' is left to + # the order (#395). The only no-comma corpus name that reaches + # this rule, and the one #364 was closed on twice + ("de Mesnil Garcia", "de Mesnil", "Garcia", ""), # ... and the trailing suffix run is peeled before the rule looks, # comma or no comma (NO_COMMA and SUFFIX_COMMA both fold) ("de Mesnil MD", "de Mesnil", "", "MD"), @@ -196,31 +210,40 @@ def test_family_first_folds_leading_never_given_particle( def test_leading_piece_scan_skips_pieces_that_hold_no_name( policy: Policy) -> None: # `_leading_name_piece` walks PAST pieces carrying no name role - # rather than reading piece 0 -- and past the first such piece, not - # only over a single title. 'Mr de Mesnil' cannot show that: its - # particle is chained into one piece with 'Mesnil', so the scan - # lands on a two-token piece and the rule declines either way. - # Here a mid-name suffix word breaks that chain, leaving the - # particle a piece of its own BEHIND a title piece. Without the - # skip, or reading only pieces[0], the scan finds the title (or - # nothing) and the name splits: given='MD', middle='Mesnil', - # family='de'. + # rather than reading piece 0. Without the skip the scan finds the + # title and the name splits: family='de', given='MD'. + # + # An older version of this comment claimed the simpler + # 'Mr de Mesnil' could not show the skip, its particle being + # chained into one piece with 'Mesnil'. That was true before #367, + # when a title displaced the particle out of the name's leading + # position and the chain fired. It is not true now -- the pieces + # are [Mr][de][Mesnil] and the simpler name fails the same way + # under the same mutation. What this input adds is the narrowing + # with a title present: the run stops after 'MD'. Not pinned here, + # despite the scan supporting it: walking past MORE than one + # non-name piece, which no fixture in this module produces. # # The title is deliberately UNDOTTED: a period makes any opening # abbreviation a title by shape (rules.md#H2), so a dotted one # would pass this test with the title vocabulary empty. out = _parsed("Mr de MD Mesnil", policy) assert _by_role(out, Role.TITLE) == "Mr" - assert _by_role(out, Role.FAMILY) == "de MD Mesnil" - assert not _by_role(out, Role.GIVEN) + # 'MD' mid-name is a name word, so it is the ONE word the run + # takes (#395); 'Mesnil' is left to the order. What the skip + # decides is that the run is found at all -- without it the scan + # stops on the title and the family is 'de' alone. + assert _by_role(out, Role.FAMILY) == "de MD" + assert _by_role(out, Role.GIVEN) == "Mesnil" assert not _by_role(out, Role.MIDDLE) @pytest.mark.parametrize("policy", _FAMILY_FIRST) @pytest.mark.parametrize("text,family,given", [ - # a title makes the particle non-leading, so group already chained - # it into one piece -- one name piece, wholly family under both - # family-first orders + # a title is transparent to the fold (#367 removed the title-> + # particle chain, so the pieces are [Mr.][de][Mesnil]), and with + # one name word after the run there is nothing for the stop to + # leave behind: wholly family under both family-first orders ("Mr. de Mesnil", "de Mesnil", ""), # a family comma has already fixed the family; the post-comma part # is the given name and must not be folded into it @@ -380,3 +403,181 @@ def test_article_particles_fold_without_eating_trailing_surnames( assert _by_role(out, Role.GIVEN) == given assert _by_role(out, Role.MIDDLE) == middle assert _by_role(out, Role.FAMILY) == family + + +# --- #395: how far the run reaches under a family-first order ------- + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +@pytest.mark.parametrize("text,family,given,middle,suffix", [ + # one leftover -- where the two family-first orders still agree + ("de la Cruz Juan", "de la Cruz", "Juan", "", ""), + # a trailing suffix is peeled before the rule looks, so it is not + # a leftover and cannot change where the run stops + ("de la Cruz Juan MD", "de la Cruz", "Juan", "", "MD"), + # ... nor can a leading title, which is not a name word either + ("Mr de la Cruz Juan", "de la Cruz", "Juan", "", ""), +]) +def test_family_first_run_stops_at_the_first_name_word( + policy: Policy, text: str, family: str, given: str, + middle: str, suffix: str) -> None: + out = _parsed(text, policy) + assert _by_role(out, Role.FAMILY) == family + assert _by_role(out, Role.GIVEN) == given + assert _by_role(out, Role.MIDDLE) == middle + assert _by_role(out, Role.SUFFIX) == suffix + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +@pytest.mark.parametrize("text,family,rest", [ + # the conjunction join is ONE name word (rules.md#P3), so the run + # takes 'Vega y Santos' whole rather than stopping inside it + ("de la Vega y Santos Juan", "de la Vega y Santos", "Juan"), + # ... and the same clause applies to what is LEFT: 'Juan y Eva' is + # one unit, so it lands in one field rather than being split + ("de la Cruz Juan y Eva", "de la Cruz", "Juan y Eva"), + # a bound given-name word and the word it completes are one unit + # (rules.md#P5), so the fold cannot leave half of the pair behind + ("ibn Awf abdul Rahman", "ibn Awf", "abdul Rahman"), + # a particle in what is LEFT chains forward the same way it would + # if the fold had never run (rules.md#P2). Handing its words out + # separately would report given='van' -- a bare particle as the + # given name, which is the reading P1 exists to forbid -- and + # would disagree with the same tail parsed alone: 'Mesnil van + # Berg Juan' gives given='van Berg Juan' + ("de Mesnil van Berg Juan", "de Mesnil", "van Berg Juan"), + # ... including a never-given particle, where the split output + # would have been given='de' + ("de Mesnil de Berg Juan", "de Mesnil", "de Berg Juan"), + # ... and a two-particle chain + ("de Mesnil van der Berg", "de Mesnil", "van der Berg"), +]) +def test_the_run_counts_units_not_words( + policy: Policy, text: str, family: str, rest: str) -> None: + # one leftover unit, so both family-first orders put it in `given` + out = _parsed(text, policy) + assert _by_role(out, Role.FAMILY) == family + assert _by_role(out, Role.GIVEN) == rest + assert not _by_role(out, Role.MIDDLE) + + +def test_the_two_family_first_orders_differ_at_two_leftovers() -> None: + # The only shape that tells them apart, and the reason + # tests/v2/cases.py carries it: with one leftover both orders send + # it to `given`, so nothing else in the suite distinguishes the + # name_order argument to the leftover placement. + ff = _parsed("de la Cruz Juan Carlos", Policy(name_order=FAMILY_FIRST)) + fgl = _parsed("de la Cruz Juan Carlos", + Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + assert _by_role(ff, Role.GIVEN) == "Juan" + assert _by_role(ff, Role.MIDDLE) == "Carlos" + assert _by_role(fgl, Role.GIVEN) == "Carlos" + assert _by_role(fgl, Role.MIDDLE) == "Juan" + assert _by_role(ff, Role.FAMILY) == _by_role(fgl, Role.FAMILY) + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +@pytest.mark.parametrize("text,family", [ + # an AMBIGUOUS leading particle is out of the rule's scope in every + # order: the fold never fires on it, so there is no run to stop. + # Untouched by #395 + ("van Gogh Jan", "van"), + # a family comma has already fixed the surname, so there is no + # positional read for a declared order to narrow (state.order is + # None on this path) + ("Smith, de Mesnil", "Smith de Mesnil"), + # the particle stands in the GIVEN slot rather than opening the + # name -- the second fold site, and not a leading run + ("Juan de", "Juan de"), +]) +def test_shapes_the_stop_does_not_reach( + policy: Policy, text: str, family: str) -> None: + # the family alone: the rows above place their remaining words by + # order, which is not what these rows are about -- what they pin + # is that the family is not narrowed + out = _parsed(text, policy) + assert _by_role(out, Role.FAMILY) == family + + +def test_the_fixture_guard_fires( + monkeypatch: pytest.MonkeyPatch) -> None: + # The guard is a correctness check on the fixture, so it needs one + # on itself: without this, deleting its body leaves the suite green + # -- the same inertness it exists to prevent. + assert not _fixture_mirrors_shipped("de la Cruz Juan", Policy()) + monkeypatch.setattr(sys.modules[__name__], "_LEX", + dataclasses.replace( + _LEX, particles=frozenset(), + particles_ambiguous=frozenset())) + assert "'de'" in _fixture_mirrors_shipped("de la Cruz Juan", + Policy()) + + +def test_the_stop_reads_the_effective_order_not_the_policy() -> None: + # Why ParseState.order exists. A script_orders entry overrides + # name_order, so the two disagree -- and this is the direction that + # matters: the policy says family-first, the SCRIPT says given- + # first, and the read assign actually made is the given-first one. + # Keying the stop on policy.name_order would narrow a name that was + # never read family-first. Needs a custom lexicon (every shipped + # particle is Latin, and Latin has no script_orders entry), so it + # builds the state directly rather than going through _parsed. + lex = Lexicon(particles=frozenset({"ノ"})) + policy = Policy(name_order=FAMILY_FIRST, + script_orders=((Script.KATAKANA, GIVEN_FIRST),)) + out = run(ParseState(original="ノ クルス フアン カルロス", + lexicon=lex, policy=policy)) + assert out.order == GIVEN_FIRST + assert _by_role(out, Role.FAMILY) == "ノ クルス フアン カルロス" + assert not _by_role(out, Role.GIVEN) + + +@pytest.mark.parametrize("policy", _FAMILY_FIRST) +def test_a_suffix_comma_still_stops_the_run(policy: Policy) -> None: + # SUFFIX_COMMA takes a different assign branch from NO_COMMA + # (tail = 1), while the fold still reads segment 0 + out = _parsed("de la Cruz Juan Carlos, MD", policy) + assert _by_role(out, Role.FAMILY) == "de la Cruz" + assert _by_role(out, Role.SUFFIX) == "MD" + assert {_by_role(out, Role.GIVEN), + _by_role(out, Role.MIDDLE)} == {"Juan", "Carlos"} + + +def test_a_maiden_name_is_not_swept_into_the_run() -> None: + # `name_idx` is givens + middles + families, and MAIDEN is + # deliberately not among them: a regression there moves the maiden + # name silently into the family + out = _parsed("de la Cruz Juan Carlos (Vega)", + Policy(name_order=FAMILY_FIRST, + maiden_delimiters=frozenset({("(", ")")}))) + assert _by_role(out, Role.FAMILY) == "de la Cruz" + assert _by_role(out, Role.MAIDEN) == "Vega" + assert _by_role(out, Role.GIVEN) == "Juan" + + +def test_middle_as_family_folds_a_middle_the_old_reach_never_left() -> None: + # O3 now has a middle to fold, which the greedy reach never left + # it. In TOKEN order the family is 'de la Cruz Carlos'; the field + # renders it 'Carlos de la Cruz', the folded word prepended (R1), + # which is why this asserts roles rather than the rendered field + out = _parsed("de la Cruz Juan Carlos", + Policy(name_order=FAMILY_FIRST, middle_as_family=True)) + assert _by_role(out, Role.GIVEN) == "Juan" + assert not _by_role(out, Role.MIDDLE) + assert _by_role(out, Role.FAMILY) == "de la Cruz Carlos" + + +@pytest.mark.parametrize("policy,given,middle", [ + (Policy(name_order=FAMILY_FIRST), "van Berg", "MD Juan"), + (Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "Juan", "van Berg MD"), +]) +def test_a_suffix_word_mid_run_ends_the_chain( + policy: Policy, given: str, middle: str) -> None: + # `_units` stops a particle chain at a suffix word, which is the + # stop _group's own chain uses (`not prefix(j) and not suffix(j)`). + # Without it the whole leftover is ONE unit and lands in one field. + # Three leftover units here, so this is also the only place the two + # orders are pinned apart at a count other than two. + out = _parsed("de Mesnil van Berg MD Juan", policy) + assert _by_role(out, Role.FAMILY) == "de Mesnil" + assert _by_role(out, Role.GIVEN) == given + assert _by_role(out, Role.MIDDLE) == middle diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index 03676a37..8da7ae8a 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -73,7 +73,9 @@ def test_stage_field_ownership() -> None: # takes, so each stage reports the side it decides "group": {"tokens", "pieces", "piece_tags", "dropped", "ambiguities"}, - "assign": {"tokens", "ambiguities"}, + # assign also records `order`: the effective order it read the + # name under, which post_rules needs and must not re-derive + "assign": {"tokens", "ambiguities", "order"}, "post_rules": {"tokens"}, } assert {s.__name__ for s in STAGES} == set(ownership) diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index cf2781e0..a803d383 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -88,6 +88,8 @@ def has_boundary_or_waiver(self) -> bool: "family-first": Policy(name_order=FAMILY_FIRST), "family-first-given-last": Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "middle_as_family": Policy(middle_as_family=True), + "family-first+middle_as_family": Policy( + name_order=FAMILY_FIRST, middle_as_family=True), "maiden-parens": Policy(maiden_delimiters=frozenset({("(", ")")})), "keep-emoji": Policy(strip_emoji=False), "strict-comma-suffixes": Policy(lenient_comma_suffixes=False),