From b26a8898f446d564e2a62e9307096ff036b56d16 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 16 Aug 2026 22:19:34 -0700 Subject: [PATCH] The leading-particle claim takes its group, not the rest of the name (#390) rules.md#P1 says a leading never-given particle makes the particle run and the ONE name word it attaches to the family, with anything beyond that read by position. The parser swept every remaining name word in, so "de Mesnil Juan" gave family="de Mesnil Juan" where the rule says family="de Mesnil" plus given="Juan". Closes #390 and #365. The claim moves from post_rules into assign, and that relocation is the fix rather than a side effect. decisions.md#P1 diagnosed the inversion in #386: assign handed out positions from _effective_order and post_rules then inspected a fixed list of ROLES, so the fold's reach depended on which role the declared order happened to give the particle. Claiming before positions exist is what makes the reading order-independent by construction instead of by three separate site checks -- which is also why #365 needs no third fold site. The leftover reuses _name_positions(order, count + 1) with FAMILY dropped. Asking for one more slot than there are pieces and discarding the family keeps one definition of "by position"; a plain count would hand the leftover the family a second time under FAMILY_FIRST. P1 is now split across two stages, deliberately. The opening half belongs where the claim happens; the second site -- a lone particle positioning has already dropped into GIVEN ("Mesnil de" under FAMILY_FIRST) -- can only be seen afterwards, so it stays in post_rules. Both docstrings say so, and rules.md's implemented: names both modules. _leading_name_piece retires with the site it served: assign reaches the same piece by peeling titles before it counts. Ledger, and the reason it is three entries rather than none: the new diff was silently absorbed by fix(suffix-routing), the fields-only catch-all that sorts last and "takes whatever nothing narrower named". Its own comment says its prose fits all four of its names, and "Garcia" is not a suffix -- so leaving it there would have made that claim false for a fifth. It gets its own rule in all three ledgers, on the precedent #372 set for the 21 names that fell there before, and suffix-routing is back to four. The recorded-roster guard then caught the claim counts and they are recorded. Verification: differential exit 0 at 1.4.0, 2.0.0 and 2.1.0; 3459 tests, ruff and mypy clean. One corpus name moves, "de Mesnil Garcia", the v1 parity break #364 weighed and accepted. tests/test_particles.py records it; the interaction that test guards (middle_name_as_last has nothing left to do) is unchanged, since middle is empty either way. Co-Authored-By: Claude Opus 5 --- docs/design/rules.md | 6 +- nameparser/_pipeline/_assign.py | 32 ++++++++- nameparser/_pipeline/_post_rules.py | 69 +++++++------------- tests/test_particles.py | 11 +++- tests/v2/pipeline/test_post_rules.py | 39 ++++++----- tests/v2/test_ledger_guards.py | 9 +++ tools/differential/expected_since_1.4.0.toml | 22 +++++++ tools/differential/expected_since_2.0.0.toml | 15 +++++ tools/differential/expected_since_2.1.0.toml | 15 +++++ 9 files changed, 147 insertions(+), 71 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index c5eac187..485b8742 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -108,8 +108,8 @@ P1. Rationale: a never-given particle standing alone cannot be "de la Vega" → family="de la Vega" "Sir de Mesnil" → family="de Mesnil" "Mesnil de" family-first → family="Mesnil de" - "de Mesnil Juan" → family="de Mesnil" deviates: #364 (today: family="de Mesnil Juan") - "de Mesnil Juan" → given="Juan" deviates: #364 (today: given="") + "de Mesnil Juan" → family="de Mesnil" + "de Mesnil Juan" → given="Juan" "van Gogh" → given="van" · boundary Accepted: a bare "de" stays the given name — there is nothing to fold into, and inventing a surname would be worse. @@ -121,7 +121,7 @@ P1. Rationale: a never-given particle standing alone cannot be the bare particle reading as a given name, not any name part that begins with one. "Juan de la Vega" family-first → family="Juan" - history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_post_rules.py + history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_post_rules.py P2. Rationale: a particle is written as part of the surname it precedes, and a title stands outside the name entirely. diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 15b5ddab..57b83a1f 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -268,7 +268,37 @@ def _assign_main(seg_idx: int, state: ParseState, order = _effective_order(state.policy, [pieces[i] for i in name_pieces], tokens, dot_divided=bool(state.interpunct_offsets)) - roles = _name_positions(order, len(name_pieces)) + # rules.md#P1: "a never-given particle standing alone where the + # given name would go — or opening the name — marks the name as + # surname-only: the particle run and the one name word it + # attaches to are the family, and any name words beyond that read + # by position." (history: decisions.md#P1) + # The claim happens HERE, before positions are handed out, because + # the particle is evidence about the name and name_order governs + # only what no vocabulary has claimed (decisions.md#O4). Doing it + # after assignment is what made the fold's reach depend on which + # role the order happened to give the particle (#364, #365). + claimed: list[int] = [] + if len(name_pieces) >= 2: + head = pieces[name_pieces[0]] + if (len(head) == 1 + and "particle" in tokens[head[0]].tags + and "vocab:particle-ambiguous" not in tokens[head[0]].tags): + claimed, name_pieces = name_pieces[:2], name_pieces[2:] + if claimed: + # The family slot is taken, so the remainder positions itself + # as if it did not exist: ask for one more slot than there are + # pieces and drop FAMILY. Reusing _name_positions rather than + # re-deriving the order keeps one definition of "by position" + # -- under FAMILY_FIRST a plain count would hand the leftover + # the family a second time. + roles: list[Role] = [ + r for r in _name_positions(order, len(name_pieces) + 1) + if r is not Role.FAMILY] + for piece_idx in claimed: + _set_roles(tokens, pieces[piece_idx], Role.FAMILY) + else: + roles = list(_name_positions(order, len(name_pieces))) for pos, piece_idx in enumerate(name_pieces): _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 906db723..63c5f5c6 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -1,9 +1,7 @@ """Stage: post_rules. -Consumes: tokens (roles assigned), plus pieces and structure -- the -particle fold reads the opening piece of segment 0, or of segment 1 -under a family comma (#359). structure was always read here, for the -rotation gate. +Consumes: tokens (roles assigned), plus structure for the rotation +gate. Produces: tokens with roles adjusted by the post rules. Reads: Policy.patronymic_rules, Policy.middle_as_family; Lexicon.given_name_titles. @@ -11,6 +9,15 @@ Implements rules H1, P1, O1, O2 and O3 of docs/design/rules.md; each is cited at its code below, and P1/O1/O2's history lives in docs/design/decisions.md. + +P1 is SPLIT across two stages as of #390, and the split is the rule's +own shape rather than an accident: a leading particle CLAIMS the +family before positions exist, so that half lives in assign; a lone +particle that positioning has already dropped into the GIVEN role can +only be seen afterwards, so that half stays here. The pieces scan the +leading half used (`_leading_name_piece`, and with it the reading of +state.pieces) went with it -- assign reaches the same piece by peeling +titles before it counts name pieces. """ from __future__ import annotations @@ -37,34 +44,10 @@ r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) -_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) - - def _idx(tokens: list[WorkToken], role: Role) -> list[int]: return [i for i, t in enumerate(tokens) if t.role is role] -def _leading_name_piece(state: ParseState, - tokens: list[WorkToken]) -> tuple[int, ...]: - """The piece that OPENS the name, whatever role name_order gave it: - the first piece holding a GIVEN, MIDDLE or FAMILY token, in the - segment the positional read governs. Every piece holding none of - those is walked past -- title and suffix pieces, but NICKNAME and - MAIDEN as well, and anything assign left unroled -- and any number - of them, not only a single leading title. The segment is 0, except - under a family comma, where segment 0 is already fixed as the - surname and the name continues in segment 1. Empty on either of - two exits: that segment does not exist, or none of its pieces - holds a name token.""" - seg = 1 if state.structure is Structure.FAMILY_COMMA else 0 - if seg >= len(state.pieces): - return () - for piece in state.pieces[seg]: - if any(tokens[i].role in _NAME_ROLES for i in piece): - return piece - return () - - def _retag(tokens: list[WorkToken], i: int, role: Role) -> None: tokens[i] = dataclasses.replace(tokens[i], role=role) @@ -99,24 +82,22 @@ def post_rules(state: ParseState) -> ParseState: # attaches to are the family, and any name words beyond that # read by position." (v1 handle_non_first_name_prefix; history: # decisions.md#P1) - # DEVIATION #364: the fold below still takes every remaining name - # word, not just the particle run's own -- de Mesnil Juan gives - # family=de Mesnil Juan where the rule says family=de Mesnil plus - # given=Juan. Pinned by the deviates: markers on P1. - # Values written unquoted deliberately: this note sits INSIDE the - # citation block above (# decisions.md#P1) does not close it -- - # _CITE_RE wants a colon after the ID), and the excerpt check - # takes the first quoted span in the block. - # Code-local: a lone PIECE is the test at both sites, so a - # particle group already chained forward is not a lone particle, - # and rule H1 above cannot be what produces the fold's family - # reading -- H1 is gated on `not families`. - sites = (_leading_name_piece(state, tokens), tuple(givens)) - if len(givens) + len(middles) + len(families) > 1 and any( + # This is P1's SECOND site only. The opening-the-name half moved to + # assign in #390, where the claim can happen before positions are + # handed out; what is left here is the case assign cannot see -- + # a lone particle that positional assignment has already dropped + # into the GIVEN role (Mesnil de under FAMILY_FIRST). It has no + # piece after it to attach to, so the fold is backward and the + # whole remainder is one word anyway. + # Code-local: a lone TOKEN in the given role is the test, so a + # particle group already chained forward never reaches it, and + # rule H1 above cannot be what produces this family reading -- + # H1 is gated on `not families`. + site = tuple(givens) + if len(givens) + len(middles) + len(families) > 1 and ( len(site) == 1 and "particle" in tokens[site[0]].tags - and "vocab:particle-ambiguous" not in tokens[site[0]].tags - for site in sites): + and "vocab:particle-ambiguous" not in tokens[site[0]].tags): for i in givens + middles: _retag(tokens, i, Role.FAMILY) # downstream rules key on the role counts: recompute diff --git a/tests/test_particles.py b/tests/test_particles.py index 3edeb863..ca1f87fb 100644 --- a/tests/test_particles.py +++ b/tests/test_particles.py @@ -390,9 +390,14 @@ def test_leading_non_first_name_prefix_with_patronymic_name_order(self) -> None: self.m(hn.last, "de Mesnil", hn) def test_leading_non_first_name_prefix_with_middle_name_as_last(self) -> None: - # handle_non_first_name_prefix runs first and empties middle_list, so + # The leading-particle claim runs first and empties middle_list, so # the later opt-in handle_middle_name_as_last has nothing left to do. + # The claim takes the particle and the ONE piece it attaches to + # (#390), so 'Garcia' survives as the given name -- before that it + # was swept into the family and first was ''. The interaction this + # test guards is unchanged: middle is empty either way. constants = Constants(middle_name_as_last=True) hn = HumanName("de Mesnil Garcia", constants=constants) - self.m(hn.first, "", hn) - self.m(hn.last, "de Mesnil Garcia", hn) + self.m(hn.first, "Garcia", hn) + self.m(hn.middle, "", hn) + self.m(hn.last, "de Mesnil", hn) diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index ad456f60..12e9f1c1 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -133,10 +133,9 @@ def test_degenerate_bare_particle_stays_given() -> None: # the leading particle chains the rest of the name into the family ("de Mesnil", "de Mesnil", "", ""), ("de la Vega", "de la Vega", "", ""), - # three pieces, so the fold has a MIDDLE to move as well as the - # given -- the `givens + middles` half of the repair, and the only - # no-comma corpus name that reaches it - ("de Mesnil Garcia", "de Mesnil Garcia", "", ""), + # three pieces, so a name word survives the claim -- the only + # no-comma corpus name that reaches it, and the one #390 moved + ("de Mesnil Garcia", "de Mesnil", "Garcia", ""), # ... and the trailing suffix run is peeled before the rule looks, # comma or no comma (NO_COMMA and SUFFIX_COMMA both fold) ("de Mesnil MD", "de Mesnil", "", "MD"), @@ -152,21 +151,17 @@ def test_family_first_folds_leading_never_given_particle( @pytest.mark.parametrize("policy", _FAMILY_FIRST) -def test_leading_piece_scan_skips_pieces_that_hold_no_name( +def test_titles_are_peeled_before_the_leading_particle_claim( policy: Policy) -> None: - # `_leading_name_piece` walks PAST pieces carrying no name role - # rather than reading piece 0 -- and past the first such piece, not - # only over a single title. 'Mr. de Mesnil' cannot show that: its - # particle is chained into one piece with 'Mesnil', so the scan - # lands on a two-token piece and the rule declines either way. - # Here a mid-name suffix word breaks that chain, leaving the - # particle a piece of its own BEHIND a title piece. Without the - # skip, or reading only pieces[0], the scan finds the title (or - # nothing) and the name splits: given='MD', middle='Mesnil', - # family='de'. - out = _parsed("Dr. de MD Mesnil", policy) + # The claim reads the first NAME piece, not pieces[0]: a title in + # front must not hide the particle behind it. assign gets this by + # peeling titles before it counts name pieces (#390 moved the claim + # there from post_rules, retiring the _leading_name_piece scan that + # used to walk past non-name pieces). Without the peel the title is + # piece 0, no claim fires, and the name splits by position. + out = _parsed("Dr. de Mesnil", policy) assert _by_role(out, Role.TITLE) == "Dr." - assert _by_role(out, Role.FAMILY) == "de MD Mesnil" + assert _by_role(out, Role.FAMILY) == "de Mesnil" assert not _by_role(out, Role.GIVEN) assert not _by_role(out, Role.MIDDLE) @@ -198,8 +193,12 @@ def test_family_first_leading_particle_cases_that_do_not_fold( @pytest.mark.parametrize("text,title,given,middle,family,suffix", [ ("de Mesnil", "", "", "", "de Mesnil", ""), ("de la Vega", "", "", "", "de la Vega", ""), - ("de Mesnil Garcia", "", "", "", "de Mesnil Garcia", ""), - ("Dr. de MD Mesnil", "Dr.", "", "", "de MD Mesnil", ""), + ("de Mesnil Garcia", "", "Garcia", "", "de Mesnil", ""), + # garbage in, garbage out: MD is suffix vocabulary sitting + # mid-name, so the piece the particle attaches to is "MD". + # Both readings of this input are garbage; pinned only so the + # claim's reach is visible, never as a shape to design around + ("Dr. de MD Mesnil", "Dr.", "Mesnil", "", "de MD", ""), ("de Mesnil MD", "", "", "", "de Mesnil", "MD"), ("De Mesnil, MD", "", "", "", "De Mesnil", "MD"), ("Mr. de Mesnil", "Mr.", "", "", "de Mesnil", ""), @@ -216,7 +215,7 @@ def test_family_first_leading_particle_cases_that_do_not_fold( ("Smith, de Mesnil", "", "", "", "Smith de Mesnil", ""), ("Smith, van Gogh", "", "van", "Gogh", "Smith", ""), ]) -def test_default_order_is_unchanged_by_the_family_first_fold( +def test_leading_particle_claim_reads_alike_in_the_default_order( text: str, title: str, given: str, middle: str, family: str, suffix: str) -> None: out = _parsed(text) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index dc8d7d2e..916d081d 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1020,6 +1020,9 @@ def _claim(rule: dict) -> _Claim: _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(215, ('family', 'given'), "f16a0e79cba3"), + "fix(#390) the leading-particle claim takes the particle's group, " + "not the rest of the name": + _Claim(1, ('family', 'given'), "caf836e3556c"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -1064,10 +1067,16 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('family', 'given', 'suffix', 'title'), "1d45596e6fdb"), "fix(#367) a title no longer displaces a leading particle out of the leading position": _Claim(1, ('family', 'given', 'middle'), "dce0ae6df4be"), + "fix(#390) the leading-particle claim takes the particle's group, " + "not the rest of the name": + _Claim(1, ('family', 'given'), "caf836e3556c"), }, "expected_since_2.1.0.toml": { "fix(#367) a title no longer displaces a leading particle out of the leading position": _Claim(1, ('family', 'given', 'middle'), "dce0ae6df4be"), + "fix(#390) the leading-particle claim takes the particle's group, " + "not the rest of the name": + _Claim(1, ('family', 'given'), "caf836e3556c"), }, } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index dde79d00..5351132b 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -179,6 +179,28 @@ issue = "fix(comma-precomma-family) pre-comma run reads as family, not given" name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "family"] +[[change]] +issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name" +# 'de Mesnil Garcia': 1.4's handle_non_first_name_prefix swept every +# remaining name word into the family, giving last='de Mesnil Garcia' +# with no first. rules.md#P1 says the claim takes the particle and the +# ONE piece it attaches to, so 'Garcia' survives by position. Diff is +# exactly {given, family} -- no suffix moves, nothing is consumed. +# +# Its own rule for the reason #372 gave the three names above one: it +# was falling to the fields-only fix(suffix-routing) below, whose +# prose is "two-token name with unambiguous trailing suffix stays +# suffix" and whose own comment says the prose fits all four of its +# names. 'Garcia' is not a suffix and nothing here routes one, so +# leaving it there would make that claim false for a fifth name. +# +# Narrower than the leading-particle shapes that do NOT diff +# ('de la Vega', 'de Mesnil Jr.'): those are one group, or two name +# words with the third a suffix, so the claim takes everything and +# 1.4 agrees. Only three-or-more-name-word shapes reach this rule. +name_regex = "^de Mesnil Garcia$" +fields = ["given", "family"] + [[change]] issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" # 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index d8cb0fe7..a42f3124 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -283,3 +283,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # classification. name_regex = "(?i)^mr\\.\\s+van\\b" fields = ["given", "middle", "family"] + +[[change]] +issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name" +# 'de Mesnil Garcia': the claim now takes the particle and the ONE +# piece it attaches to (rules.md#P1), so 'Garcia' survives by position +# where it used to be swept into the family. Diff is exactly +# {given, family} -- no suffix moves and nothing is consumed. +# +# The leading-particle shapes that do NOT diff are the check on this +# rule's reach: 'de la Vega' is one group start to finish, and +# 'de Mesnil Jr.' has two name words because Jr. is a suffix, so the +# claim takes everything in both and the baseline agrees. Only +# three-or-more-name-word shapes reach here, and the corpus holds one. +name_regex = "^de Mesnil Garcia$" +fields = ["given", "family"] diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 796b1062..c136315f 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -94,3 +94,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # classification. name_regex = "(?i)^mr\\.\\s+van\\b" fields = ["given", "middle", "family"] + +[[change]] +issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name" +# 'de Mesnil Garcia': the claim now takes the particle and the ONE +# piece it attaches to (rules.md#P1), so 'Garcia' survives by position +# where it used to be swept into the family. Diff is exactly +# {given, family} -- no suffix moves and nothing is consumed. +# +# The leading-particle shapes that do NOT diff are the check on this +# rule's reach: 'de la Vega' is one group start to finish, and +# 'de Mesnil Jr.' has two name words because Jr. is a suffix, so the +# claim takes everything in both and the baseline agrees. Only +# three-or-more-name-word shapes reach here, and the corpus holds one. +name_regex = "^de Mesnil Garcia$" +fields = ["given", "family"]