fix: choose a heteronym's reading from the words around it

CHANGELOG.md

@@ -11,6 +11,24 @@ it needs to read like release notes rather than a commit log.
 
 ## [Unreleased]
 
+### Fixed
+
+- Readings for heteronyms now take the surrounding words into account. 重 in
+  「教学任务很重」 was annotated `chóng` ("to repeat") rather than `zhòng`
+  ("heavy"): with no part-of-speech information the reader fell back to
+  preferring whichever reading the dictionary says most about, and CC-CEDICT
+  says more about `chóng`. A degree adverb or a negator before a character
+  now settles the few cases where that fallback is wrong, and the detail
+  panel's one-line summary follows the reading shown above the word instead
+  of whichever the dictionary happened to list first.
+- A handful of characters whose dictionary evidence points at a reading no
+  living text uses are corrected outright. 价 in 「她叫我给个价」 was read
+  `jiè` ("great, middleman"), which CC-CEDICT documents in four senses and a
+  neutral tone against three senses for `jià` ("price") — so every signal
+  the ranking has pointed the wrong way. The corrections are generated by
+  `Scripts/heteronym-defaults.py` from a criterion the dictionary answers
+  itself: a reading no compound uses, beside one that many do.
+
 ### Added
 
 - Initial repository scaffolding: license boundary between the MIT-licensed

Scripts/heteronym-defaults.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Find characters whose dictionary evidence points at the wrong reading.
+
+`ReadingComposer` has no part-of-speech information, so for a character with
+several readings it prefers a neutral tone and then whichever reading the
+dictionary says most about. Both are proxies, and for a handful of characters
+both point at a reading no living text uses: CC-CEDICT documents 价 as `jiè`
+("great, middleman") in four senses and as `jià` ("price") in three, so the
+reader annotated 她叫我给个价 with `jiè`.
+
+This reports the characters where that happens, by a criterion the dictionary
+can answer on its own: a reading used by *no* multi-character headword, beside
+one used by many. Compounds are the evidence because they are where a
+character's living readings show up — 价格, 物价, 评价, 代价 are all `jià`,
+and nothing is `jiè`.
+
+Readings glossed as grammatical are excluded. A sentence-final particle is
+unattested in compounds for a reason that has nothing to do with being
+archaic: 吧, 呢, 啊 and 吗 would otherwise all be "corrected" to a toned
+reading nobody wants.
+
+Run against the pinned snapshot; paste the result into `HeteronymRules`:
+
+    ./Scripts/heteronym-defaults.py Dictionaries/cc-cedict/cedict_ts.u8
+"""
+
+import argparse
+import collections
+import io
+import re
+import sys
+
+ENTRY = re.compile(r"^(\S+) (\S+) \[([^\]]*)\] /(.*)/$")
+
+# What CC-CEDICT writes when a reading is a function word rather than a word.
+GRAMMATICAL = (
+    "particle",
+    "suffix",
+    "suff.",
+    "prefix",
+    "marker",
+    "interjection",
+    "classifier",
+    "onomatop",
+)
+
+# Below this, a rival reading is not well enough attested to overrule the
+# dictionary's own emphasis. The gap this exists to catch is absolute — many
+# against none — not narrow.
+MINIMUM_RIVAL = 20
+
+
+def is_han(character):
+    return "一" <= character <= "鿿"
+
+
+def read(path):
+    """Single-character entries, and per-character reading counts in compounds."""
+    entries = collections.defaultdict(list)
+    compounds = collections.defaultdict(collections.Counter)
+    with io.open(path, encoding="utf-8") as handle:
+        for line in handle:
+            if line.startswith("#"):
+                continue
+            matched = ENTRY.match(line.strip())
+            if not matched:
+                continue
+            _, simplified, reading, definition = matched.groups()
+            if len(simplified) == 1:
+                entries[simplified].append((reading, definition.split("/")))
+            syllables = reading.split()
+            # Only where every character maps to exactly one syllable, which
+            # is what makes the alignment below sound.
+            if (
+                len(simplified) >= 2
+                and len(simplified) == len(syllables)
+                and all(is_han(c) for c in simplified)
+            ):
+                for character, syllable in zip(simplified, syllables):
+                    compounds[character][syllable.lower()] += 1
+    return entries, compounds
+
+
+def is_neutral(reading):
+    return any(syllable[-1] == "5" for syllable in reading.split())
+
+
+def is_capitalised(reading):
+    return reading[0].isupper()
+
+
+def chosen(readings):
+    """What `ReadingComposer.isWeaker` picks: lowercase, neutral tone, senses."""
+    return sorted(
+        readings,
+        key=lambda r: (is_capitalised(r[0]), not is_neutral(r[0]), -len(r[1]), r[0]),
+    )[0]
+
+
+def report(path):
+    entries, compounds = read(path)
+    rows = []
+    for character, readings in sorted(entries.items()):
+        usable = [r for r in readings if r[0] and "xx5" not in r[0]]
+        if len({r[0].lower() for r in usable}) < 2:
+            continue
+        winner = chosen(usable)
+        if compounds[character].get(winner[0].lower(), 0) != 0:
+            continue
+        rivals = [
+            (reading, count)
+            for reading, count in compounds[character].most_common()
+            if reading != winner[0].lower()
+        ]
+        if not rivals or rivals[0][1] < MINIMUM_RIVAL:
+            continue
+        glosses = [g for r in usable if r[0].lower() == winner[0].lower() for g in r[1]]
+        if any(mark in g.lower() for g in glosses for mark in GRAMMATICAL):
+            continue
+        rows.append((character, rivals[0][0], rivals[0][1], winner[0], glosses))
+
+    for character, reading, count, instead, glosses in rows:
+        print(
+            '            // {} compounds say {}, none say {} "{}"'.format(
+                count, reading, instead, "/".join(glosses)[:48]
+            )
+        )
+        print('            "{}": "{}",'.format(character, reading))
+    return 0 if rows else 1
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("dictionary", help="path to cedict_ts.u8")
+    return report(parser.parse_args().dictionary)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

Sources/HanReaderCore/Dictionary/ReadingComposer.swift

@@ -77,13 +77,19 @@ public protocol DictionaryLookup: Sendable {
 /// the English dictionary.
 public enum ReadingComposer {
     /// The reading to show above `word`, or nil if nothing is known.
+    ///
+    /// - Parameter context: what the neighbouring words say about how this one
+    ///   is being used. Consulted only for a word whose readings actually
+    ///   disagree, so passing `.none` changes nothing for the 99% of tokens
+    ///   that have one reading.
     public static func reading(
         for word: String,
         in lookup: some DictionaryLookup,
+        context: ReadingContext = .none,
     ) throws
         -> TokenReading?
     {
-        if let exact = try exactReading(for: word, in: lookup) {
+        if let exact = try exactReading(for: word, in: lookup, context: context) {
             return exact
         }
         return try composedReading(for: word, in: lookup)
@@ -118,27 +124,56 @@ public enum ReadingComposer {
     /// Lowercase outranks everything, matching `LexiconBuilder`: a capital
     /// marks a surname or a place, rarely the reading wanted in prose.
     ///
-    /// The four it still gets wrong — 为, 长, 重, 相 — have no neutral-tone
-    /// reading and no signal to separate them. Those are exactly the cases a
-    /// tagger exists for, and they are marked `.ambiguous` so the reader can
-    /// see the annotation is a guess.
+    /// ### Where the rule runs out, and what picks up
+    ///
+    /// 为, 长, 重 and 相 have no neutral-tone reading and nothing in the
+    /// dictionary to separate them, so the sense count decides and it decides
+    /// badly: CC-CEDICT says more about 重 as `chóng` ("to repeat") than as
+    /// `zhòng` ("heavy"), which annotated 很重 — "very heavy" — as `chóng`.
+    ///
+    /// No context-free rule can fix that, because both readings are right
+    /// somewhere: 重 is `chóng` in 重复 and `zhòng` in 很重. So `context` is
+    /// consulted first, and `HeteronymRules` answers for the handful of cases
+    /// where a closed-class neighbour settles it grammatically — a degree
+    /// adverb can only modify a stative predicate, so 很 before 重 rules
+    /// `chóng` out rather than merely making it less likely.
+    ///
+    /// What the rules do not reach is still marked `.ambiguous`, so the
+    /// annotation is drawn faintly and the panel lists every reading.
     static func exactReading(
         for word: String,
         in lookup: some DictionaryLookup,
+        context: ReadingContext = .none,
     ) throws
         -> TokenReading?
     {
-        try reading(from: lookup.entries(for: word))
+        try reading(from: lookup.entries(for: word), context: context)
     }
 
     /// The reading to show, given entries already fetched.
     ///
     /// Separate from `exactReading(for:in:)` so a caller that has the entries
     /// in hand — the detail panel has just fetched them to show definitions —
     /// does not query for them a second time.
-    public static func reading(from entries: [DictionaryEntry]) -> TokenReading? {
+    public static func reading(
+        from entries: [DictionaryEntry],
+        context: ReadingContext = .none,
+    )
+        -> TokenReading?
+    {
         let usable = entries.filter(isUsable)
-        guard let best = usable.max(by: isWeaker) else { return nil }
+        guard let fallback = usable.max(by: isWeaker) else { return nil }
+
+        // Three tiers, narrowest first. A grammatical constraint beats
+        // everything below it because it is the only one that knows the
+        // sentence; a corrected default beats the ranking because the ranking
+        // is reading evidence that points the wrong way. Each applies only
+        // where the dictionary actually offers the reading it names, so a
+        // rule this dictionary cannot satisfy falls through to the
+        // context-free answer rather than annotating nothing.
+        let best = preferred(in: usable, context: context)
+            ?? corrected(in: usable)
+            ?? fallback
 
         // Compared case-insensitively. Nearly every common character has a
         // surname entry alongside its ordinary one -- 年 is `nián` and
@@ -166,6 +201,51 @@ public enum ReadingComposer {
         !entry.readingDisplay.isEmpty && !entry.reading.contains(.unknown)
     }
 
+    /// The entry a context rule asks for, if there is a rule and the
+    /// dictionary has the reading it names.
+    ///
+    /// Several entries can share the preferred reading — 和 has four under
+    /// `hé` — so the ordinary ranking still chooses between them. The rule
+    /// narrows the field; it does not replace the ranking.
+    private static func preferred(
+        in entries: [DictionaryEntry],
+        context: ReadingContext,
+    )
+        -> DictionaryEntry?
+    {
+        guard !context.isEmpty,
+              let headword = entries.first?.headword,
+              let wanted = HeteronymRules.preferredReading(for: headword, in: context)
+        else { return nil }
+        return matching(wanted, in: entries)
+    }
+
+    /// The entry a corrected default asks for, for a character whose
+    /// dictionary evidence points at a reading no living text uses.
+    ///
+    /// Below the context rules deliberately: 价 is `jià` for want of any
+    /// reason to think otherwise, whereas 很 before 重 is a reason. A
+    /// character in both tables takes the contextual answer.
+    private static func corrected(in entries: [DictionaryEntry]) -> DictionaryEntry? {
+        guard let headword = entries.first?.headword,
+              let wanted = HeteronymRules.defaultReading(for: headword)
+        else { return nil }
+        return matching(wanted, in: entries)
+    }
+
+    /// Entries whose reading is the one named, best first by the ordinary
+    /// ranking — several can share it, and 和 has four under `hé`.
+    private static func matching(
+        _ reading: String,
+        in entries: [DictionaryEntry],
+    )
+        -> DictionaryEntry?
+    {
+        entries
+            .filter { $0.readingKey.caseInsensitiveCompare(reading) == .orderedSame }
+            .max(by: isWeaker)
+    }
+
     /// Whether `lhs` is the worse choice of reading. See `exactReading` for
     /// where the ordering comes from.
     private static func isWeaker(_ lhs: DictionaryEntry, _ rhs: DictionaryEntry) -> Bool {
@@ -224,7 +304,12 @@ public enum ReadingComposer {
 
         var tokens: [PinyinToken] = []
         for character in word {
-            guard let reading = try lookup.readings(forCharacter: character).first else {
+            // The per-character table is ranked by the same sense count the
+            // whole-word path uses, so it needs the same correction: without
+            // it a word composed from its characters would spell 价 `jiè`
+            // while the same character tapped on its own read `jià`.
+            guard let reading = try correcting(lookup.readings(forCharacter: character))
+            else {
                 return nil
             }
             let parsed = Pinyin.parse(numeric: reading.numeric)
@@ -239,4 +324,19 @@ public enum ReadingComposer {
         guard !display.isEmpty else { return nil }
         return TokenReading(display: display, source: .composed)
     }
+
+    /// The best of a character's candidate readings: the corrected default
+    /// if the table names one and the dictionary has it, otherwise the
+    /// highest-ranked, which is what the caller would have taken anyway.
+    private static func correcting(_ candidates: [CharacterReading]) -> CharacterReading? {
+        guard let character = candidates.first?.character,
+              let wanted = HeteronymRules.defaultReading(
+                  for: Headword(simplified: String(character), traditional: nil),
+              ),
+              let corrected = candidates.first(where: {
+                  $0.numeric.caseInsensitiveCompare(wanted) == .orderedSame
+              })
+        else { return candidates.first }
+        return corrected
+    }
 }