diff --git a/CHANGELOG.md b/CHANGELOG.md index d1ed3db..400504e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,24 @@ it needs to read like release notes rather than a commit log. ## [Unreleased] +### Fixed + +- Readings for heteronyms now take the surrounding words into account. 重 in + 「教学任务很重」 was annotated `chóng` ("to repeat") rather than `zhòng` + ("heavy"): with no part-of-speech information the reader fell back to + preferring whichever reading the dictionary says most about, and CC-CEDICT + says more about `chóng`. A degree adverb or a negator before a character + now settles the few cases where that fallback is wrong, and the detail + panel's one-line summary follows the reading shown above the word instead + of whichever the dictionary happened to list first. +- A handful of characters whose dictionary evidence points at a reading no + living text uses are corrected outright. 价 in 「她叫我给个价」 was read + `jiè` ("great, middleman"), which CC-CEDICT documents in four senses and a + neutral tone against three senses for `jià` ("price") — so every signal + the ranking has pointed the wrong way. The corrections are generated by + `Scripts/heteronym-defaults.py` from a criterion the dictionary answers + itself: a reading no compound uses, beside one that many do. + ### Added - Initial repository scaffolding: license boundary between the MIT-licensed diff --git a/Scripts/heteronym-defaults.py b/Scripts/heteronym-defaults.py new file mode 100755 index 0000000..8a9f6a0 --- /dev/null +++ b/Scripts/heteronym-defaults.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Find characters whose dictionary evidence points at the wrong reading. + +`ReadingComposer` has no part-of-speech information, so for a character with +several readings it prefers a neutral tone and then whichever reading the +dictionary says most about. Both are proxies, and for a handful of characters +both point at a reading no living text uses: CC-CEDICT documents 价 as `jiè` +("great, middleman") in four senses and as `jià` ("price") in three, so the +reader annotated 她叫我给个价 with `jiè`. + +This reports the characters where that happens, by a criterion the dictionary +can answer on its own: a reading used by *no* multi-character headword, beside +one used by many. Compounds are the evidence because they are where a +character's living readings show up — 价格, 物价, 评价, 代价 are all `jià`, +and nothing is `jiè`. + +Readings glossed as grammatical are excluded. A sentence-final particle is +unattested in compounds for a reason that has nothing to do with being +archaic: 吧, 呢, 啊 and 吗 would otherwise all be "corrected" to a toned +reading nobody wants. + +Run against the pinned snapshot; paste the result into `HeteronymRules`: + + ./Scripts/heteronym-defaults.py Dictionaries/cc-cedict/cedict_ts.u8 +""" + +import argparse +import collections +import io +import re +import sys + +ENTRY = re.compile(r"^(\S+) (\S+) \[([^\]]*)\] /(.*)/$") + +# What CC-CEDICT writes when a reading is a function word rather than a word. +GRAMMATICAL = ( + "particle", + "suffix", + "suff.", + "prefix", + "marker", + "interjection", + "classifier", + "onomatop", +) + +# Below this, a rival reading is not well enough attested to overrule the +# dictionary's own emphasis. The gap this exists to catch is absolute — many +# against none — not narrow. +MINIMUM_RIVAL = 20 + + +def is_han(character): + return "一" <= character <= "鿿" + + +def read(path): + """Single-character entries, and per-character reading counts in compounds.""" + entries = collections.defaultdict(list) + compounds = collections.defaultdict(collections.Counter) + with io.open(path, encoding="utf-8") as handle: + for line in handle: + if line.startswith("#"): + continue + matched = ENTRY.match(line.strip()) + if not matched: + continue + _, simplified, reading, definition = matched.groups() + if len(simplified) == 1: + entries[simplified].append((reading, definition.split("/"))) + syllables = reading.split() + # Only where every character maps to exactly one syllable, which + # is what makes the alignment below sound. + if ( + len(simplified) >= 2 + and len(simplified) == len(syllables) + and all(is_han(c) for c in simplified) + ): + for character, syllable in zip(simplified, syllables): + compounds[character][syllable.lower()] += 1 + return entries, compounds + + +def is_neutral(reading): + return any(syllable[-1] == "5" for syllable in reading.split()) + + +def is_capitalised(reading): + return reading[0].isupper() + + +def chosen(readings): + """What `ReadingComposer.isWeaker` picks: lowercase, neutral tone, senses.""" + return sorted( + readings, + key=lambda r: (is_capitalised(r[0]), not is_neutral(r[0]), -len(r[1]), r[0]), + )[0] + + +def report(path): + entries, compounds = read(path) + rows = [] + for character, readings in sorted(entries.items()): + usable = [r for r in readings if r[0] and "xx5" not in r[0]] + if len({r[0].lower() for r in usable}) < 2: + continue + winner = chosen(usable) + if compounds[character].get(winner[0].lower(), 0) != 0: + continue + rivals = [ + (reading, count) + for reading, count in compounds[character].most_common() + if reading != winner[0].lower() + ] + if not rivals or rivals[0][1] < MINIMUM_RIVAL: + continue + glosses = [g for r in usable if r[0].lower() == winner[0].lower() for g in r[1]] + if any(mark in g.lower() for g in glosses for mark in GRAMMATICAL): + continue + rows.append((character, rivals[0][0], rivals[0][1], winner[0], glosses)) + + for character, reading, count, instead, glosses in rows: + print( + ' // {} compounds say {}, none say {} "{}"'.format( + count, reading, instead, "/".join(glosses)[:48] + ) + ) + print(' "{}": "{}",'.format(character, reading)) + return 0 if rows else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dictionary", help="path to cedict_ts.u8") + return report(parser.parse_args().dictionary) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Sources/HanReaderCore/Dictionary/ReadingComposer.swift b/Sources/HanReaderCore/Dictionary/ReadingComposer.swift index df5b8ac..29514a4 100644 --- a/Sources/HanReaderCore/Dictionary/ReadingComposer.swift +++ b/Sources/HanReaderCore/Dictionary/ReadingComposer.swift @@ -77,13 +77,19 @@ public protocol DictionaryLookup: Sendable { /// the English dictionary. public enum ReadingComposer { /// The reading to show above `word`, or nil if nothing is known. + /// + /// - Parameter context: what the neighbouring words say about how this one + /// is being used. Consulted only for a word whose readings actually + /// disagree, so passing `.none` changes nothing for the 99% of tokens + /// that have one reading. public static func reading( for word: String, in lookup: some DictionaryLookup, + context: ReadingContext = .none, ) throws -> TokenReading? { - if let exact = try exactReading(for: word, in: lookup) { + if let exact = try exactReading(for: word, in: lookup, context: context) { return exact } return try composedReading(for: word, in: lookup) @@ -118,17 +124,30 @@ public enum ReadingComposer { /// Lowercase outranks everything, matching `LexiconBuilder`: a capital /// marks a surname or a place, rarely the reading wanted in prose. /// - /// The four it still gets wrong — 为, 长, 重, 相 — have no neutral-tone - /// reading and no signal to separate them. Those are exactly the cases a - /// tagger exists for, and they are marked `.ambiguous` so the reader can - /// see the annotation is a guess. + /// ### Where the rule runs out, and what picks up + /// + /// 为, 长, 重 and 相 have no neutral-tone reading and nothing in the + /// dictionary to separate them, so the sense count decides and it decides + /// badly: CC-CEDICT says more about 重 as `chóng` ("to repeat") than as + /// `zhòng` ("heavy"), which annotated 很重 — "very heavy" — as `chóng`. + /// + /// No context-free rule can fix that, because both readings are right + /// somewhere: 重 is `chóng` in 重复 and `zhòng` in 很重. So `context` is + /// consulted first, and `HeteronymRules` answers for the handful of cases + /// where a closed-class neighbour settles it grammatically — a degree + /// adverb can only modify a stative predicate, so 很 before 重 rules + /// `chóng` out rather than merely making it less likely. + /// + /// What the rules do not reach is still marked `.ambiguous`, so the + /// annotation is drawn faintly and the panel lists every reading. static func exactReading( for word: String, in lookup: some DictionaryLookup, + context: ReadingContext = .none, ) throws -> TokenReading? { - try reading(from: lookup.entries(for: word)) + try reading(from: lookup.entries(for: word), context: context) } /// The reading to show, given entries already fetched. @@ -136,9 +155,25 @@ public enum ReadingComposer { /// Separate from `exactReading(for:in:)` so a caller that has the entries /// in hand — the detail panel has just fetched them to show definitions — /// does not query for them a second time. - public static func reading(from entries: [DictionaryEntry]) -> TokenReading? { + public static func reading( + from entries: [DictionaryEntry], + context: ReadingContext = .none, + ) + -> TokenReading? + { let usable = entries.filter(isUsable) - guard let best = usable.max(by: isWeaker) else { return nil } + guard let fallback = usable.max(by: isWeaker) else { return nil } + + // Three tiers, narrowest first. A grammatical constraint beats + // everything below it because it is the only one that knows the + // sentence; a corrected default beats the ranking because the ranking + // is reading evidence that points the wrong way. Each applies only + // where the dictionary actually offers the reading it names, so a + // rule this dictionary cannot satisfy falls through to the + // context-free answer rather than annotating nothing. + let best = preferred(in: usable, context: context) + ?? corrected(in: usable) + ?? fallback // Compared case-insensitively. Nearly every common character has a // surname entry alongside its ordinary one -- 年 is `nián` and @@ -166,6 +201,51 @@ public enum ReadingComposer { !entry.readingDisplay.isEmpty && !entry.reading.contains(.unknown) } + /// The entry a context rule asks for, if there is a rule and the + /// dictionary has the reading it names. + /// + /// Several entries can share the preferred reading — 和 has four under + /// `hé` — so the ordinary ranking still chooses between them. The rule + /// narrows the field; it does not replace the ranking. + private static func preferred( + in entries: [DictionaryEntry], + context: ReadingContext, + ) + -> DictionaryEntry? + { + guard !context.isEmpty, + let headword = entries.first?.headword, + let wanted = HeteronymRules.preferredReading(for: headword, in: context) + else { return nil } + return matching(wanted, in: entries) + } + + /// The entry a corrected default asks for, for a character whose + /// dictionary evidence points at a reading no living text uses. + /// + /// Below the context rules deliberately: 价 is `jià` for want of any + /// reason to think otherwise, whereas 很 before 重 is a reason. A + /// character in both tables takes the contextual answer. + private static func corrected(in entries: [DictionaryEntry]) -> DictionaryEntry? { + guard let headword = entries.first?.headword, + let wanted = HeteronymRules.defaultReading(for: headword) + else { return nil } + return matching(wanted, in: entries) + } + + /// Entries whose reading is the one named, best first by the ordinary + /// ranking — several can share it, and 和 has four under `hé`. + private static func matching( + _ reading: String, + in entries: [DictionaryEntry], + ) + -> DictionaryEntry? + { + entries + .filter { $0.readingKey.caseInsensitiveCompare(reading) == .orderedSame } + .max(by: isWeaker) + } + /// Whether `lhs` is the worse choice of reading. See `exactReading` for /// where the ordering comes from. private static func isWeaker(_ lhs: DictionaryEntry, _ rhs: DictionaryEntry) -> Bool { @@ -224,7 +304,12 @@ public enum ReadingComposer { var tokens: [PinyinToken] = [] for character in word { - guard let reading = try lookup.readings(forCharacter: character).first else { + // The per-character table is ranked by the same sense count the + // whole-word path uses, so it needs the same correction: without + // it a word composed from its characters would spell 价 `jiè` + // while the same character tapped on its own read `jià`. + guard let reading = try correcting(lookup.readings(forCharacter: character)) + else { return nil } let parsed = Pinyin.parse(numeric: reading.numeric) @@ -239,4 +324,19 @@ public enum ReadingComposer { guard !display.isEmpty else { return nil } return TokenReading(display: display, source: .composed) } + + /// The best of a character's candidate readings: the corrected default + /// if the table names one and the dictionary has it, otherwise the + /// highest-ranked, which is what the caller would have taken anyway. + private static func correcting(_ candidates: [CharacterReading]) -> CharacterReading? { + guard let character = candidates.first?.character, + let wanted = HeteronymRules.defaultReading( + for: Headword(simplified: String(character), traditional: nil), + ), + let corrected = candidates.first(where: { + $0.numeric.caseInsensitiveCompare(wanted) == .orderedSame + }) + else { return candidates.first } + return corrected + } } diff --git a/Sources/HanReaderCore/Dictionary/ReadingContext.swift b/Sources/HanReaderCore/Dictionary/ReadingContext.swift new file mode 100644 index 0000000..3584dba --- /dev/null +++ b/Sources/HanReaderCore/Dictionary/ReadingContext.swift @@ -0,0 +1,292 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// A closed class of function word that constrains how its neighbour is being +/// used. +/// +/// Closed classes only, and that restriction is the whole point. A rule built +/// on an open class — "a noun precedes" — needs a tagger and a lexicon and is +/// wrong at the edges; a rule built on a list of twenty-odd particles and +/// adverbs is a finite, checkable fact about the grammar. These are the ones +/// whose membership can be written down and argued about in a review. +public enum FunctionWordClass: UInt8, Sendable, Hashable, CaseIterable { + /// 很, 太, 非常, 最 — a word whose only job is to intensify. + /// + /// A degree adverb modifies a stative predicate: the thing after it is + /// being used as an adjective, never as a transitive verb, a noun or a + /// classifier. That is a grammatical constraint rather than a frequency + /// guess, which is what makes it worth encoding — it does not merely say + /// which reading is commoner, it says which readings are impossible. + case degreeAdverb + + /// 不, 并不 — a word that negates the predicate after it. + /// + /// A weaker signal than a degree adverb, and deliberately treated as a + /// separate class rather than folded in with it. A negator takes verbs as + /// readily as adjectives — 不去, 不吃 — so it says nothing on its own + /// about which reading applies. It earns its place only because, for a + /// few specific characters, both of the readings it admits happen to + /// point the same way: 不重 is "not heavy" and 不差 is "not bad", because + /// the other reading would be said as 不重复 and 不差别. Its table is + /// correspondingly shorter than the degree adverb's. + case negator + + /// The class of a word, or nil if it is not in one this can act on — + /// which is nearly every word, by design. + public init?(word: String) { + if Self.degreeAdverbs.contains(word) { + self = .degreeAdverb + return + } + if Self.negators.contains(word) { + self = .negator + return + } + return nil + } + + /// Both scripts throughout, because the reader's text may be either and + /// the token is matched as written. A traditional text saying 很長 has to + /// reach the same rule as 很长. + /// + /// 好 and 够 are here as the degree adverbs they are in 好重 ("so heavy") + /// and 够长 ("long enough"). Both are also ordinary words, but a + /// *separate preceding token* 好 before an adjective is the adverb in + /// practically all running text. + static let degreeAdverbs: Set = [ + "很", "太", "挺", "真", "好", "最", "更", "够", "夠", "蛮", "蠻", "颇", "頗", + "非常", "特别", "特別", "比较", "比較", "十分", "相当", "相當", + "极", "極", "极其", "極其", "格外", "尤其", "异常", "異常", + "这么", "這麼", "那么", "那麼", "多么", "多麼", + "稍微", "有点", "有點", "有点儿", "有點兒", "有些", + "过于", "過於", "越来越", "越來越", "不太", + ] + + /// 不 only, plus its emphatic forms. 没 is absent because it negates + /// aspect rather than a predicate, and 没重 is not something anyone says. + static let negators: Set = ["不", "并不", "並不"] +} + +/// What the words around a token say about how it is being used. +/// +/// Deliberately tiny, and deliberately *classified* rather than raw. Holding +/// the neighbouring words verbatim would make this the key to a cache with one +/// entry per phrase in the book; holding their class makes the key space the +/// vocabulary times a handful of cases, which is what lets a reading stay +/// cached per word the way it was before context existed. +public struct ReadingContext: Hashable, Sendable { + /// The class of the word immediately before, where it is one of the few + /// this can act on. Punctuation counts as a word for this purpose, so a + /// context does not reach across 很,重. + public let preceding: FunctionWordClass? + + /// No usable context: the word stands alone, or its neighbours say + /// nothing. The overwhelming majority of tokens. + public static let none = Self(preceding: nil) + + public init(preceding: FunctionWordClass?) { + self.preceding = preceding + } + + /// Classifies a token's neighbour. + public init(precedingWord: String?) { + preceding = precedingWord.flatMap(FunctionWordClass.init(word:)) + } + + public var isEmpty: Bool { + preceding == nil + } + + /// The context of one token, for a caller holding a position rather than + /// a paragraph — a tap, which has no reason to classify every word around + /// it to annotate one. + public static func context(at index: Int, in tokens: [Token]) -> Self { + guard tokens.indices.contains(index) else { return .none } + var cursor = index - 1 + while cursor >= 0, tokens[cursor].kind == .whitespace { + cursor -= 1 + } + return Self(precedingWord: cursor >= 0 ? tokens[cursor].text : nil) + } + + /// The context of every token in a block, aligned one-to-one with + /// `tokens`. + /// + /// Computed over a block rather than a document, which is what stops a + /// context reaching across a paragraph break. Whitespace is skipped — + /// `很 重` with a space between is still a degree adverb modifying 重 — + /// but punctuation is not, so it severs the relation, correctly. + /// + /// Shared by the two callers that need it: the model, deciding what to + /// fetch, and the paragraph view, deciding what to look up. One function + /// so the two can never disagree about what a token's context is, which + /// they would otherwise express as pinyin flickering between two values. + public static func contexts(in tokens: [Token]) -> [Self] { + var contexts: [Self] = [] + contexts.reserveCapacity(tokens.count) + var previous: String? + for token in tokens { + contexts.append(Self(precedingWord: previous)) + if token.kind != .whitespace { + previous = token.text + } + } + return contexts + } +} + +/// A word together with the context it appeared in. +/// +/// The unit a context-sensitive reading is fetched and cached by. It replaces +/// the bare word as a cache key, which it has to: with context in the +/// picture, "what does 重 sound like" no longer has one answer per book. +public struct ReadingRequest: Hashable, Sendable { + public let word: String + public let context: ReadingContext + + public init(word: String, context: ReadingContext = .none) { + self.word = word + self.context = context + } +} + +/// Which reading a character takes in a given grammatical context. +/// +/// ### Why a table at all +/// +/// `ReadingComposer` ranks a character's readings by rules that know nothing +/// about the sentence — prefer lowercase, prefer a neutral tone, then prefer +/// whichever reading the dictionary says more about. That last step is a +/// frequency proxy, and it is wrong for exactly the characters where the +/// readings are different words: CC-CEDICT gives 重 five senses under `chóng` +/// ("to repeat") and three under `zhòng` ("heavy"), so 很重 — plainly "very +/// heavy" — was annotated `chóng`. +/// +/// No context-free default can fix that, because both readings are right +/// somewhere. 重 is `chóng` in 重复 and `zhòng` in 很重, and the only thing +/// that separates them is what sits next to it. +/// +/// ### Why it is small, and why it stays small +/// +/// Each entry is a claim about one character under one grammatical +/// constraint, with the phrase that justifies it. The table holds only the +/// cases where the context *overturns* the context-free answer — 好, 难, 累, +/// 大 and 少 are already ranked correctly, and listing them would be noise +/// that hides the five real claims. `ReadingComposerTests` asserts those stay +/// correct, so if the generic ranking ever changes under them it fails there +/// rather than quietly needing a rule here. +/// +/// This is not a part-of-speech tagger and is not a step towards one. It is a +/// finite set of statements of the form "after 很, 重 is `zhòng`", each of +/// which a reader of Chinese can check in a second. +public enum HeteronymRules { + /// The numeric reading to prefer, or nil if nothing applies. + /// + /// Numeric because that is the cross-dictionary merge key: a rule written + /// as `zhong4` picks the right entry out of CC-CEDICT, which stores + /// numerals, and out of BKRS, which stores diacritics. + public static func preferredReading( + for headword: Headword, + in context: ReadingContext, + ) + -> String? + { + guard let preceding = context.preceding else { return nil } + // Single characters only. A multi-character word's reading comes from + // its own entry, where the dictionary has already resolved which of + // its characters' readings applies — 重复 is `chóngfù` because the + // dictionary says so, not because anything here decided. + for script in headword.bothScripts where script.count == 1 { + guard let character = script.first else { continue } + if let reading = table[preceding]?[character] { + return reading + } + } + return nil + } + + /// The reading a character takes when the sentence says nothing either + /// way, for the few whose dictionary evidence points at a dead reading. + /// + /// Separate from the context rules above, and needed for a different + /// reason. 重 needs context because both its readings are live: it is + /// `chóng` in 重复 and `zhòng` in 很重, and only the neighbours separate + /// them. 价 needs no context at all — `jiè` is simply not a reading of + /// modern Chinese — but every signal available to the ranking points at + /// it, because CC-CEDICT documents the archaic sense more thoroughly + /// than the living one: four glosses for `jiè` "great, middleman" + /// against three for `jià` "price", and a neutral tone besides. So 她叫 + /// 我给个价 was annotated `jiè`. + /// + /// The criterion is mechanical and the table is generated, not curated: + /// a reading used by *no* multi-character headword, beside one used by + /// many. Compounds are the evidence because that is where a character's + /// living readings appear. Readings glossed as grammatical are excluded, + /// or every sentence-final particle would qualify — 吧, 呢, 啊 and 吗 are + /// all absent from compounds, and for a reason that has nothing to do + /// with being archaic. + /// + /// Regenerate with `Scripts/heteronym-defaults.py`, which prints these + /// lines verbatim. + public static func defaultReading(for headword: Headword) -> String? { + for script in headword.bothScripts where script.count == 1 { + if let character = script.first, let reading = defaults[character] { + return reading + } + } + return nil + } + + /// Internal rather than private so a test can check it against the + /// context table: the two must stay disjoint, or the tier order starts + /// deciding answers and someone has to argue it is the right one. + static let defaults: [Character: String] = [ + // 161 compounds say jia4, none say jie5 "great/good/middleman/servant" + "价": "jia4", + "價": "jia4", + // 21 compounds say zhui1, none say chui2 "a hammer" + "椎": "zhui1", + // 65 compounds say pao4, none say bao1 "to sauté/to fry/to dry by heating" + "炮": "pao4", + // 87 compounds say luo4, none say lao4 "small net" + "络": "luo4", + "絡": "luo4", + // 38 compounds say gao1, none say gao4 "to moisten/to grease/to apply" + "膏": "gao1", + ] + + /// A degree adverb forces the stative reading. Each line is the phrase + /// that justifies it, and the reading the context-free ranking picked + /// instead. + static let table: [FunctionWordClass: [Character: String]] = [ + .degreeAdverb: [ + // 很重 "very heavy", not chóng "to repeat" + "重": "zhong4", + // 很长 "very long", not zhǎng "chief, to grow". Both scripts + // spelled out because an entry may carry only one of them, in + // which case `bothScripts` cannot supply the other. + "长": "chang2", + "長": "chang2", + // 很空 "very empty", not kòng "vacancy, free time" + "空": "kong1", + // 很差 "very poor", not chā "difference" + "差": "cha4", + // 很薄 "very thin", not bó "meagre, slight" + "薄": "bao2", + ], + // Only where negating either reading lands on the same answer, which + // is a much shorter list than it looks. 长 and 薄 are absent because + // 不长 can be "does not grow" and 待我不薄 is `bó`; 差 is absent + // because the bundled dictionary settles it against us, recording + // both 不差 [bu4 cha1] "to not lack" and 不差 [bu4 cha4] "not bad" + // as headwords. A degree adverb still settles 差 — 很差 is only + // "very poor" — which is the difference between the two classes. + .negator: [ + // 不重 "not heavy" — "does not repeat" would be 不重复 + "重": "zhong4", + // 不空 "not empty" — free time is 没空, not 不空 + "空": "kong1", + ], + ] +} diff --git a/Sources/HanReaderUI/Models/ReaderModel.swift b/Sources/HanReaderUI/Models/ReaderModel.swift index 8d12af0..fd664f6 100644 --- a/Sources/HanReaderUI/Models/ReaderModel.swift +++ b/Sources/HanReaderUI/Models/ReaderModel.swift @@ -24,6 +24,45 @@ struct WordDetail: Equatable, Sendable { groups.flatMap { $0.entries.map(\.entry) } } + /// The groups, with the one the reader is actually looking at first. + /// + /// Groups arrive in the order their reading was first seen, which for 重 + /// is `chóng` before `zhòng` because that is CC-CEDICT's row order. Once + /// the context rule has chosen `zhòng` for 很重, opening the panel on + /// `chóng` makes the reader hunt for the sense that belongs to the word + /// in front of them. + var orderedGroups: [ReadingGroup] { + let shown = groupsForDisplayedReading + guard !shown.isEmpty else { return groups } + return shown + groups.filter { group in + !shown.contains { $0.readingKey == group.readingKey } + } + } + + /// The entries whose reading is the one shown above the word. + /// + /// The one-line summary has to come from these rather than from whatever + /// group happens to be first: 重 is `chóng` "to repeat" and `zhòng` + /// "heavy", and with the reading chosen by context the panel read + /// "zhòng — to repeat", contradicting the ruby directly above the word. + /// + /// Falls back to every entry when nothing matches, which is the case for + /// a reading composed from characters — there is no group for it, and the + /// word's senses are better than none. + var entriesForDisplayedReading: [DictionaryEntry] { + let matching = groupsForDisplayedReading.flatMap { $0.entries.map(\.entry) } + return matching.isEmpty ? entries : matching + } + + /// The groups spelling the reading on screen, which is normally one and + /// is none when the reading was composed from characters. + private var groupsForDisplayedReading: [ReadingGroup] { + guard let display = reading?.display else { return [] } + return groups.filter { + $0.readingDisplay.caseInsensitiveCompare(display) == .orderedSame + } + } + /// Whether any dictionary defines this word. /// /// A word can have a reading and no entry — composed from its @@ -74,7 +113,11 @@ final class ReaderModel { /// and a few thousand distinct words. Keying by token also implies that /// two instances of a word could be annotated differently, which nothing /// here can decide and so should not be able to express. - private(set) var readings: [String: TokenReading] = [:] + /// Keyed on word *and* context rather than on the word alone, because a + /// heteronym has no single answer: 重 is `zhòng` in 很重 and `chóng` in + /// 重复. Still bounded by the reader's vocabulary rather than by the + /// length of the book, since a context is one of a handful of classes. + private(set) var readings: [ReadingRequest: TokenReading] = [:] private(set) var selection: TokenID? private(set) var reveal: RevealSet @@ -206,15 +249,21 @@ final class ReaderModel { func refreshReadings() async { guard let document else { return } let range = Self.window(around: topBlock ?? 0, in: document) - let words = Set( - document.blocks[range] - .flatMap(\.tokens) - .filter(\.isLookupCandidate) - .map(\.text), - ) + var requests: Set = [] + // Per block, because `ReadingContext.contexts(in:)` is what stops a + // context reaching across a paragraph break — the last word of one + // paragraph does not govern the first word of the next. + for block in document.blocks[range] { + let contexts = ReadingContext.contexts(in: block.tokens) + for (token, context) in zip(block.tokens, contexts) + where token.isLookupCandidate + { + requests.insert(ReadingRequest(word: token.text, context: context)) + } + } // Only what is not already known, so scrolling back over text that // has been read costs nothing. - let missing = words.subtracting(readings.keys) + let missing = requests.subtracting(readings.keys) guard !missing.isEmpty else { return } let fetched = await services.dictionary.readings(for: missing) @@ -253,7 +302,7 @@ final class ReaderModel { selection = id reveal.reveal(token) persistReveal(of: token.text, revealed: true) - loadDetail(for: token.text) + loadDetail(for: token.text, context: context(of: id)) } } @@ -268,7 +317,14 @@ final class ReaderModel { detail = .empty } - private func loadDetail(for word: String) { + /// The grammatical context of one token, so the panel agrees with the + /// ruby above the word rather than showing a different reading for it. + private func context(of id: TokenID) -> ReadingContext { + guard let document, document.blocks.indices.contains(id.block) else { return .none } + return ReadingContext.context(at: id.index, in: document.blocks[id.block].tokens) + } + + private func loadDetail(for word: String, context: ReadingContext) { detailTask?.cancel() detail = .loading(word) detailTask = Task { @@ -283,9 +339,10 @@ final class ReaderModel { // entry is the one composed from its characters. Falling // back to the entries covers a word tapped before its // reading was fetched. - reading: readings[word] + reading: readings[ReadingRequest(word: word, context: context)] ?? ReadingComposer.reading( from: groups.flatMap { $0.entries.map(\.entry) }, + context: context, ), groups: groups, )) diff --git a/Sources/HanReaderUI/Reader/ParagraphView.swift b/Sources/HanReaderUI/Reader/ParagraphView.swift index ea5229c..be09632 100644 --- a/Sources/HanReaderUI/Reader/ParagraphView.swift +++ b/Sources/HanReaderUI/Reader/ParagraphView.swift @@ -20,9 +20,11 @@ import SwiftUI struct ParagraphView: View { let block: TextBlock let style: ReaderStyle - /// Readings keyed by word, not by token: bounded by the reader's - /// vocabulary rather than by the length of the book. - let readings: [String: TokenReading] + /// Readings keyed by word *and context*, not by token: still bounded by + /// the reader's vocabulary rather than by the length of the book, since a + /// context is one of a handful of classes — but 重 after 很 and 重 alone + /// are two questions with two answers, so the word alone is not a key. + let readings: [ReadingRequest: TokenReading] let selection: TokenID? /// Which words are revealed, and in which of the two senses. /// @@ -39,7 +41,7 @@ struct ParagraphView: View { init( block: TextBlock, style: ReaderStyle, - readings: [String: TokenReading], + readings: [ReadingRequest: TokenReading], selection: TokenID?, reveal: RevealSet, onTap: @escaping (TokenID) -> Void, @@ -50,8 +52,16 @@ struct ParagraphView: View { self.selection = selection self.reveal = reveal self.onTap = onTap - flow = zip(block.tokens, ReaderFlow.flowValues(for: block.tokens, style: style)) - .map(FlowToken.init) + // Contexts computed once here, from the same core function the model + // uses to decide what to fetch — so the key this view looks up is by + // construction the key that was stored, and a heteronym cannot end up + // unannotated because the two disagreed about its neighbours. + flow = zip( + zip(block.tokens, ReadingContext.contexts(in: block.tokens)), + ReaderFlow.flowValues(for: block.tokens, style: style), + ).map { pair, flow in + FlowToken(token: pair.0, context: pair.1, flow: flow) + } } var body: some View { @@ -67,7 +77,7 @@ struct ParagraphView: View { ForEach(flow) { entry in TokenView( token: entry.token, - reading: readings[entry.token.text], + reading: readings[entry.request], emphasis: emphasis(of: entry.token), style: style, onTap: onTap, @@ -104,9 +114,16 @@ struct ParagraphView: View { /// A token and the layout values derived from its neighbours. private struct FlowToken: Identifiable { let token: Token + let context: ReadingContext let flow: TokenFlowValue var id: TokenID { token.id } + + /// How this token's reading is keyed, so the view asks the same question + /// the model answered. + var request: ReadingRequest { + ReadingRequest(word: token.text, context: context) + } } diff --git a/Sources/HanReaderUI/Reader/ReaderFixtures.swift b/Sources/HanReaderUI/Reader/ReaderFixtures.swift index fe376ed..9fe1f66 100644 --- a/Sources/HanReaderUI/Reader/ReaderFixtures.swift +++ b/Sources/HanReaderUI/Reader/ReaderFixtures.swift @@ -66,6 +66,29 @@ enum ReaderFixtures { /// Four paragraphs and a blank line. /// + /// The authored table, keyed the way the reading surface asks for it. + /// + /// Derived from a document rather than written out, because a reading is + /// keyed on the word *and* its context: writing the keys by hand would + /// mean every fixture word silently losing its annotation the day someone + /// adds a 很 to the prose. + static func readings(for document: SegmentedDocument) -> [ReadingRequest: TokenReading] { + document.blocks.reduce(into: [:]) { result, block in + result.merge(readings(for: block.tokens)) { current, _ in current } + } + } + + /// One paragraph's worth, for a caller rendering a single block. + static func readings(for tokens: [Token]) -> [ReadingRequest: TokenReading] { + var result: [ReadingRequest: TokenReading] = [:] + for (token, context) in zip(tokens, ReadingContext.contexts(in: tokens)) { + if let reading = readingsByWord[token.text] { + result[ReadingRequest(word: token.text, context: context)] = reading + } + } + return result + } + /// 拍 has no entry in `readingsByWord`, so one token in the last paragraph /// renders with no annotation — the case that must still reserve its ruby /// band, or lines would differ in height according to dictionary coverage. @@ -158,7 +181,7 @@ struct FixtureReader: View { ReaderSurface( document: document, style: style, - readings: ReaderFixtures.readingsByWord, + readings: ReaderFixtures.readings(for: document), selection: selection, reveal: reveal, onTap: select, @@ -312,7 +335,7 @@ private struct FixturePreview: View { ReaderSurface( document: document, style: style, - readings: ReaderFixtures.readingsByWord, + readings: ReaderFixtures.readings(for: document), selection: selection, reveal: ReaderFixtures.reveal(of: revealed), onTap: { selection = selection == $0 ? nil : $0 }, diff --git a/Sources/HanReaderUI/Reader/ReaderSurface.swift b/Sources/HanReaderUI/Reader/ReaderSurface.swift index 875f815..13cd26f 100644 --- a/Sources/HanReaderUI/Reader/ReaderSurface.swift +++ b/Sources/HanReaderUI/Reader/ReaderSurface.swift @@ -17,7 +17,7 @@ import SwiftUI struct ReaderSurface: View { let document: SegmentedDocument let style: ReaderStyle - let readings: [String: TokenReading] + let readings: [ReadingRequest: TokenReading] let selection: TokenID? let reveal: RevealSet let onTap: (TokenID) -> Void diff --git a/Sources/HanReaderUI/Screens/WordDetailContent.swift b/Sources/HanReaderUI/Screens/WordDetailContent.swift index 0584be2..adafe2b 100644 --- a/Sources/HanReaderUI/Screens/WordDetailContent.swift +++ b/Sources/HanReaderUI/Screens/WordDetailContent.swift @@ -23,7 +23,7 @@ struct WordDetailContent: View { VStack(alignment: .leading, spacing: 20) { header if detail.isDefined { - ForEach(Array(detail.groups.enumerated()), id: \.offset) { _, group in + ForEach(Array(detail.orderedGroups.enumerated()), id: \.offset) { _, group in groupView(group) } } else { diff --git a/Sources/HanReaderUI/Screens/WordDetailPanel.swift b/Sources/HanReaderUI/Screens/WordDetailPanel.swift index 4a292bb..e38c4bb 100644 --- a/Sources/HanReaderUI/Screens/WordDetailPanel.swift +++ b/Sources/HanReaderUI/Screens/WordDetailPanel.swift @@ -71,7 +71,7 @@ struct WordDetailPanel: View { word: detail.word, reading: detail.reading, summary: nil, - senses: detail.entries, + senses: detail.entriesForDisplayedReading, ) } } diff --git a/Sources/HanReaderUI/Services/DictionaryService.swift b/Sources/HanReaderUI/Services/DictionaryService.swift index 7d299e4..7786eaa 100644 --- a/Sources/HanReaderUI/Services/DictionaryService.swift +++ b/Sources/HanReaderUI/Services/DictionaryService.swift @@ -32,7 +32,11 @@ actor DictionaryService { /// 77% of BKRS entries carry none. Nil when the service was built from a /// single lookup, as the tests and previews do. private let set: DictionarySet? - private var readingCache: BoundedCache + /// Keyed on the word *and* its context, not the word alone. 重 is + /// `zhòng` after 很 and `chóng` on its own, so a cache keyed on the word + /// would serve whichever of the two was asked for first to every + /// occurrence in the book. + private var readingCache: BoundedCache private var entryCache: BoundedCache /// Whether a database read has already failed. @@ -95,33 +99,41 @@ actor DictionaryService { /// Readings for a window of words, as one batch. /// - /// Returns only the words that have one, so the caller can treat a + /// Returns only the requests that have one, so the caller can treat a /// missing key and a word with no reading identically — which it should, /// because they look identical on the page. - func readings(for words: some Sequence) -> [String: TokenReading] { - var result: [String: TokenReading] = [:] - for word in words { - if let reading = reading(for: word) { - result[word] = reading + func readings( + for requests: some Sequence, + ) + -> [ReadingRequest: TokenReading] + { + var result: [ReadingRequest: TokenReading] = [:] + for request in requests { + if let reading = reading(for: request) { + result[request] = reading } } return result } - func reading(for word: String) -> TokenReading? { - if let cached = readingCache.value(forKey: word) { + func reading(for request: ReadingRequest) -> TokenReading? { + if let cached = readingCache.value(forKey: request) { return cached } + let word = request.word // Routed through `entries(for:)` rather than letting the composer do // its own lookup. The composer would query for exactly the same rows // and throw them away, so annotating a word and then tapping it ran // the same whole-word query twice -- once to find out how it sounds // and once to find out what it means. - let reading = ReadingComposer.reading(from: entries(for: word)) + let reading = ReadingComposer.reading( + from: entries(for: word), + context: request.context, + ) ?? attempt("composed reading for \(word)", default: nil) { try ReadingComposer.composedReading(for: word, in: lookup) } - readingCache.insert(reading, forKey: word) + readingCache.insert(reading, forKey: request) return reading } diff --git a/Tests/HanReaderCoreTests/ReadingContextTests.swift b/Tests/HanReaderCoreTests/ReadingContextTests.swift new file mode 100644 index 0000000..75e51cc --- /dev/null +++ b/Tests/HanReaderCoreTests/ReadingContextTests.swift @@ -0,0 +1,535 @@ +// HanReader — MIT licensed. See LICENSE. + +import Testing +@testable import HanReaderCore + +/// A dictionary written out by hand, so a test about which reading wins says +/// which entries exist rather than building a SQLite container to ask. +private struct StubDictionary: DictionaryLookup { + var words: [String: [DictionaryEntry]] = [:] + var characters: [Character: [CharacterReading]] = [:] + + func entries(for headword: String) throws -> [DictionaryEntry] { + words[headword] ?? [] + } + + func readings(forCharacter character: Character) throws -> [CharacterReading] { + characters[character] ?? [] + } +} + +/// Candidates come out of the container already ranked, so `rank` is the +/// order the composed path would otherwise take them in. +private func character(_ char: Character, _ numeric: String, rank: Int) -> CharacterReading { + CharacterReading( + character: char, + numeric: numeric, + display: Pinyin.display(Pinyin.parse(numeric: numeric)), + rank: rank, + ) +} + +private func entry( + _ simplified: String, + traditional: String? = nil, + _ numeric: String, + senses: Int = 1, +) + -> DictionaryEntry +{ + DictionaryEntry( + headword: Headword(simplified: simplified, traditional: traditional), + reading: Pinyin.parse(numeric: numeric), + senses: (0 ..< senses).map { + Sense(id: $0, kind: .definition, gloss: Gloss(text: "sense \($0)")) + }, + ) +} + +/// The real CC-CEDICT shape for 重, sense counts included, because the sense +/// counts are the whole reason the context-free rule gets it wrong: +/// `chóng` has five senses and `zhòng` three. +private let heavyOrRepeat = [ + entry("重", "chong2", senses: 5), + entry("重", "zhong4", senses: 3), +] + +/// 长 as CC-CEDICT holds it: `zhǎng` has seven senses to `cháng`'s five. +private let longOrChief = [ + entry("长", traditional: "長", "chang2", senses: 5), + entry("长", traditional: "長", "zhang3", senses: 7), +] + +private func tokens(_ words: [String], kinds: [TokenKind]? = nil) -> [Token] { + var offset = 0 + return words.enumerated().map { index, word in + let length = word.utf16.count + defer { offset += length } + return Token( + id: TokenID(block: 0, index: index), + text: word, + range: offset ..< offset + length, + kind: kinds?[index] ?? .han, + ) + } +} + +@Suite("Reading context") +struct ReadingContextTests { + // MARK: - The reported defect + + /// The bug this whole mechanism exists for. 我在一个山村学校当老师,教学任务很重 + /// segments to `… | 任务 | 很 | 重`, so 重 is a lone token, and the + /// context-free ranking annotated it `chóng` ("to repeat") because + /// CC-CEDICT says more about that reading than about `zhòng` ("heavy"). + @Test("A degree adverb makes 很重 zhòng, not chóng") + func degreeAdverbPicksTheStativeReading() { + let context = ReadingContext(precedingWord: "很") + let reading = ReadingComposer.reading(from: heavyOrRepeat, context: context) + #expect(reading?.display == "zhòng") + } + + /// The other half of the same claim, and the reason a context-free + /// default could never have fixed it: standing alone, 重 keeps whatever + /// the ranking says. The rule adds knowledge, it does not replace one + /// blanket default with another. + @Test("Without a degree adverb the ranking is untouched") + func noContextLeavesTheRankingAlone() { + let reading = ReadingComposer.reading(from: heavyOrRepeat) + #expect(reading?.display == "chóng") + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: .none)?.display == "chóng") + } + + @Test( + "Every degree adverb in the class picks the stative reading", + arguments: ["很", "太", "挺", "最", "更", "真", "好", "非常", "比较", "有点", "越来越"], + ) + func everyDegreeAdverbApplies(adverb: String) { + let context = ReadingContext(precedingWord: adverb) + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: context)?.display == "zhòng") + } + + /// 任务不重 is "the task is not heavy". A negator is a weaker signal than + /// a degree adverb and is kept to the characters where both readings it + /// admits point the same way. + @Test("A negator makes 不重 zhòng") + func negatorPicksTheStativeReading() { + let context = ReadingContext(precedingWord: "不") + #expect(context.preceding == .negator) + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: context)?.display == "zhòng") + } + + /// The two classes are not interchangeable, and this is what keeps them + /// apart: 待我不薄 is `bó` and 不长 can be "does not grow", so a negator + /// must not reach the characters a degree adverb does. + @Test("A negator does not settle 差, where a degree adverb does") + func negatorDoesNotSettleDifference() { + // CC-CEDICT records both 不差 [bu4 cha1] "to not lack" and 不差 + // [bu4 cha4] "not bad" as headwords, so a negator decides nothing + // here. 很差 is only ever "very poor". + let entries = [entry("差", "cha1", senses: 6), entry("差", "cha4", senses: 3)] + #expect(ReadingComposer.reading( + from: entries, + context: ReadingContext(precedingWord: "不"), + )?.display == "chā") + #expect(ReadingComposer.reading( + from: entries, + context: ReadingContext(precedingWord: "很"), + )?.display == "chà") + } + + @Test("A negator does not settle 长, where a degree adverb does") + func negatorIsNarrowerThanDegreeAdverb() { + #expect(ReadingComposer.reading( + from: longOrChief, + context: ReadingContext(precedingWord: "不"), + )?.display == "zhǎng") + #expect(ReadingComposer.reading( + from: longOrChief, + context: ReadingContext(precedingWord: "很"), + )?.display == "cháng") + } + + /// 没 negates aspect rather than a predicate, and 没重 is not something + /// anyone says — so it is not a negator for this purpose. + @Test("没 is not treated as a negator") + func aspectualNegatorIsExcluded() { + #expect(FunctionWordClass(word: "没") == nil) + } + + @Test("A degree adverb makes 很长 cháng, not zhǎng") + func longIsNotChief() { + let context = ReadingContext(precedingWord: "很") + #expect(ReadingComposer.reading(from: longOrChief, context: context)?.display == "cháng") + #expect(ReadingComposer.reading(from: longOrChief)?.display == "zhǎng") + } + + /// A traditional text says 很長, and the token matched is 長. The rule has + /// to reach it, or the fix works in one script only. + /// An entry carrying only the simplified form, so the traditional line in + /// the table cannot cover for it. Both spellings are listed there and + /// each has to work alone, or the table silently depends on every + /// dictionary recording both scripts. + @Test("The rule reaches a simplified-only entry") + func simplifiedOnlyEntryIsCovered() { + let simplifiedOnly = [ + entry("长", "chang2", senses: 5), + entry("长", "zhang3", senses: 7), + ] + let context = ReadingContext(precedingWord: "很") + #expect( + ReadingComposer.reading(from: simplifiedOnly, context: context)?.display == "cháng", + ) + } + + @Test("The rule reaches the traditional spelling") + func traditionalScriptIsCovered() { + let traditionalOnly = [ + entry("長", "chang2", senses: 5), + entry("長", "zhang3", senses: 7), + ] + let context = ReadingContext(precedingWord: "很") + #expect( + ReadingComposer.reading(from: traditionalOnly, context: context)?.display == "cháng", + ) + } + + // MARK: - What the rule must not do + + /// 重复 is `chóngfù` and nothing here may interfere: a multi-character + /// word's reading comes from its own entry, where the dictionary has + /// already settled which reading of each character applies. + @Test("A multi-character word is never overridden") + func multiCharacterWordsAreLeftAlone() { + let entries = [entry("重复", "chong2 fu4", senses: 4)] + let context = ReadingContext(precedingWord: "很") + #expect(ReadingComposer.reading(from: entries, context: context)?.display == "chóngfù") + // Asserted on the rule itself as well as on the outcome. A rule that + // matched 重复 on its first character would be caught by nothing + // downstream: the reading it names is one syllable and a two-syllable + // word can never equal it, so the bug would sit there silently until + // some two-character headword turned up with a one-syllable reading. + #expect(HeteronymRules.preferredReading( + for: Headword(simplified: "重复", traditional: nil), + in: context, + ) == nil) + } + + /// A dictionary that simply has no `zhòng` entry must still annotate the + /// word. Falling through to the context-free answer is right; annotating + /// nothing because a rule could not be satisfied is not. + @Test("A rule the dictionary cannot satisfy falls through") + func unsatisfiableRuleFallsThrough() { + let entries = [entry("重", "chong2", senses: 5)] + let context = ReadingContext(precedingWord: "很") + #expect(ReadingComposer.reading(from: entries, context: context)?.display == "chóng") + } + + /// Still a guess, and still drawn faintly. A grammatical constraint makes + /// the choice defensible, not certain — the reader should see that the + /// dictionary holds more than one reading and be able to check the panel. + @Test("A context-chosen reading is still marked as a guess") + func contextDoesNotClaimCertainty() { + let context = ReadingContext(precedingWord: "很") + let reading = ReadingComposer.reading(from: heavyOrRepeat, context: context) + #expect(reading?.source == .ambiguous) + } + + /// Characters the context-free ranking already gets right, listed so that + /// a change to that ranking fails here instead of quietly creating the + /// need for more rules. None of these appears in `HeteronymRules`. + @Test( + "Characters needing no rule stay correct without one", + arguments: [ + ("好", ["hao3", "hao4"], "hǎo"), + ("难", ["nan2", "nan4"], "nán"), + ("累", ["lei4", "lei3", "lei2"], "lèi"), + ("大", ["da4", "dai4"], "dà"), + ("少", ["shao3", "shao4"], "shǎo"), + ], + ) + func unruledCharactersAreAlreadyRight( + word: String, + readings: [String], + expected: String, + ) { + // Sense counts descending, which is the shape that made 重 fail: the + // first reading listed is the one the ranking would pick on count + // alone. + let entries = readings.enumerated().map { + entry(word, $1, senses: readings.count - $0) + } + let context = ReadingContext(precedingWord: "很") + #expect(HeteronymRules.preferredReading( + for: Headword(simplified: word, traditional: nil), + in: context, + ) == nil) + #expect(ReadingComposer.reading(from: entries, context: context)?.display == expected) + } + + /// The rule fires on a single-character *token*, which is only sound if + /// a compound stays one token. Were 很重复 ever segmented 很 | 重 | 复, + /// the degree adverb would reach 重 and spell 重复 as `zhòng fù`. + /// + /// Both shipped segmenters keep it whole — a compound in the lexicon is + /// what maximum matching matches, and the repair pass only ever merges — + /// so this pins the property the rule leans on rather than trusting it. + @Test("A compound after a degree adverb stays one token", arguments: [ + "很重复", "不重复", "这个任务很重复", + ]) + func compoundsSurviveSegmentation(text: String) { + let lexicon = Lexicon(words: ["重复", "任务", "这个"]) + let document = TextSegmenter(words: MaxMatchSegmenter(lexicon: lexicon)) + .segment(text) + let tokens = document.blocks.flatMap(\.tokens) + #expect(tokens.contains { $0.text == "重复" }) + // And so the rule never sees a lone 重 to act on. + let contexts = ReadingContext.contexts(in: tokens) + for (token, context) in zip(tokens, contexts) where token.text == "重" { + #expect( + HeteronymRules.preferredReading( + for: Headword(simplified: token.text, traditional: nil), + in: context, + ) == nil, + ) + } + } + + /// And the repair pass, which is what rescues a segmenter that does + /// split a known compound. It only merges, so it can turn 重 | 复 back + /// into 重复 but can never create the split this rule fears. + @Test("The repair pass rejoins a split compound") + func repairPassRejoinsCompounds() { + // Segmented character by character, which is the worst case the + // repair pass exists for and exactly the split this rule fears. + let split = TextSegmenter(words: CharacterSegmenter()).segment("很重复") + #expect(split.blocks.flatMap(\.tokens).map(\.text) == ["很", "重", "复"]) + + let repaired = DictionaryRepairPass(lexicon: Lexicon(words: ["重复"])).repair(split) + #expect(repaired.blocks.flatMap(\.tokens).map(\.text) == ["很", "重复"]) + } + + // MARK: - Classifying neighbours + + @Test("Punctuation severs the relation") + func punctuationSevers() { + let stream = tokens(["很", ",", "重"], kinds: [.han, .punctuation, .han]) + let contexts = ReadingContext.contexts(in: stream) + #expect(contexts[2] == .none) + // Both classifiers, because a tap uses the other one and the reader + // would see the panel disagree with the ruby above the word. + #expect(ReadingContext.context(at: 2, in: stream) == .none) + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: contexts[2])? + .display == "chóng") + } + + /// `很 重` with a space between is still a degree adverb modifying 重 — + /// which matters because a reader's text may have spaces anywhere. + @Test("Whitespace does not sever the relation") + func whitespaceDoesNotSever() { + let stream = tokens(["很", " ", "重"], kinds: [.han, .whitespace, .han]) + let contexts = ReadingContext.contexts(in: stream) + #expect(contexts[2].preceding == .degreeAdverb) + } + + @Test("The first token of a block has no context") + func blockStartHasNoContext() { + #expect(ReadingContext.contexts(in: tokens(["重", "要"]))[0] == .none) + } + + @Test("A context is produced for every token") + func contextsAlignWithTokens() { + let stream = tokens(["任务", "很", "重"]) + #expect(ReadingContext.contexts(in: stream).count == stream.count) + } + + /// Two implementations of the same question — one walking a paragraph + /// forwards for the model, one scanning backwards from an index for a tap + /// — and they must never disagree. They did not start out agreeing: a + /// divergence shows up as the panel naming a different reading from the + /// ruby directly above the word. + @Test("The single-token and whole-block classifiers agree") + func theTwoClassifiersAgree() { + // 很 sits immediately before a comma, so a classifier that skipped + // punctuation would carry the adverb across it and the two would + // disagree about 重. Without that shape the streams agree whatever + // either one does with punctuation. + let stream = tokens( + ["他", "说", "很", ",", "重", " ", "很", "重", "。"], + kinds: [ + .han, .han, .han, .punctuation, .han, .whitespace, .han, .han, .punctuation, + ], + ) + let all = ReadingContext.contexts(in: stream) + for index in stream.indices { + #expect(ReadingContext.context(at: index, in: stream) == all[index]) + } + } + + @Test("An index outside the block has no context") + func outOfRangeIndexIsSafe() { + let stream = tokens(["重"]) + #expect(ReadingContext.context(at: 5, in: stream) == .none) + #expect(ReadingContext.context(at: -1, in: stream) == .none) + } + + @Test("An ordinary word is in no function-word class") + func ordinaryWordsAreUnclassified() { + #expect(FunctionWordClass(word: "任务") == nil) + #expect(FunctionWordClass(word: "重") == nil) + #expect(FunctionWordClass(word: "很") == .degreeAdverb) + } + + // MARK: - Through the lookup + + /// The whole path, as the reader meets it: the words of the sentence, the + /// dictionary, and the annotation that comes out. + @Test("The reported sentence annotates 重 as zhòng") + func theReportedSentenceIsFixed() throws { + let stub = StubDictionary(words: ["重": heavyOrRepeat, "很": [entry("很", "hen3")]]) + let stream = tokens(["任务", "很", "重"]) + let contexts = ReadingContext.contexts(in: stream) + + let reading = try ReadingComposer.reading(for: "重", in: stub, context: contexts[2]) + #expect(reading?.display == "zhòng") + + // And unchanged where there is no adverb, in the same sentence. + let bare = try ReadingComposer.reading(for: "重", in: stub, context: contexts[0]) + #expect(bare?.display == "chóng") + } +} + +/// Kept apart from the context rules because it answers a different +/// question. A context rule says which of two live readings this sentence +/// wants; a correction says the dictionary's own evidence is pointing at a +/// reading no living sentence wants. +@Suite("Heteronym corrections") +struct HeteronymCorrectionTests { + /// 价 as CC-CEDICT holds it. The archaic reading has both a neutral tone + /// and more senses, so every signal the ranking has points at it. + private var priceOrGreat: [DictionaryEntry] { + [ + entry("价", traditional: "價", "jie5", senses: 4), + entry("价", traditional: "價", "jia4", senses: 3), + ] + } + + /// 她叫我给个价 — "name me a price". `jiè` ("great, middleman") is not a + /// reading of modern Chinese, but it is the one CC-CEDICT says most + /// about, and it carries a neutral tone on top of that. + @Test("价 is jià, against both the neutral tone and the sense count") + func correctedDefaultBeatsTheRanking() { + #expect(ReadingComposer.reading(from: priceOrGreat)?.display == "jià") + } + + /// The correction has to hold with no context at all, which is the case + /// it exists for: unlike 重, there is no neighbour that settles 价 — + /// there is nothing to settle. + @Test( + "价 is jià in every context", + arguments: [nil, "很", "不", "个", "。"], + ) + func correctedDefaultIgnoresContext(preceding: String?) { + let context = ReadingContext(precedingWord: preceding) + #expect(ReadingComposer.reading(from: priceOrGreat, context: context)? + .display == "jià") + } + + @Test( + "Every generated correction is applied", + arguments: [ + ("价", "jie5", "jia4", "jià"), + ("椎", "chui2", "zhui1", "zhuī"), + ("炮", "bao1", "pao4", "pào"), + ("络", "lao4", "luo4", "luò"), + ("膏", "gao4", "gao1", "gāo"), + ], + ) + func everyCorrectionApplies( + word: String, + wrong: String, + right: String, + expected: String, + ) { + // The wrong reading given more senses, which is the shape that made + // the ranking choose it in the first place. + let entries = [ + entry(word, wrong, senses: 9), + entry(word, right, senses: 1), + ] + #expect(ReadingComposer.reading(from: entries)?.display == expected) + } + + /// A character with no correction must be left to the ranking, or the + /// table would be doing the ranking's job everywhere. + @Test("An ordinary character has no correction") + func ordinaryCharactersAreUncorrected() { + #expect(HeteronymRules.defaultReading( + for: Headword(simplified: "重", traditional: nil), + ) == nil) + #expect(HeteronymRules.defaultReading( + for: Headword(simplified: "价格", traditional: nil), + ) == nil) + } + + /// Sentence-final particles are unattested in compounds for a reason + /// that has nothing to do with being archaic, and the generator excludes + /// them. If one ever reached the table, 他来了吧 would read `bā`. + @Test("Particles are not corrected", arguments: ["吧", "呢", "啊", "吗", "了", "的"]) + func particlesAreNotCorrected(particle: String) { + #expect(HeteronymRules.defaultReading( + for: Headword(simplified: particle, traditional: nil), + ) == nil) + } + + /// A correction the dictionary cannot satisfy must not cost the word its + /// annotation. + @Test("A correction the dictionary lacks falls through") + func unsatisfiableCorrectionFallsThrough() { + let entries = [entry("价", "jie5", senses: 4)] + #expect(ReadingComposer.reading(from: entries)?.display == "jie") + } + + /// The composer tries context first and a correction second, but no + /// character is in both tables, so that order decides nothing today and + /// a test asserting it would be asserting an accident. + /// + /// This is the test worth having instead: it fails the moment the two + /// overlap, which is exactly when the order starts to matter and someone + /// has to decide whether context-first is right for that character. + @Test("The context and correction tables are disjoint") + func tablesAreDisjoint() { + let corrected = Set(HeteronymRules.defaults.keys) + #expect(!corrected.isEmpty) + for (functionWord, rules) in HeteronymRules.table { + let overlap = Set(rules.keys).intersection(corrected) + #expect(overlap.isEmpty, "\(functionWord) also corrected: \(overlap)") + } + } + + /// The per-character table is built by the same sense-count ranking, so + /// without the correction a word assembled from its characters spelled + /// 价 `jiè` while the character tapped on its own read `jià` — the same + /// character, two pronunciations, one screen apart. + @Test("A word composed from its characters uses the corrected reading") + func composedReadingIsCorrected() throws { + let stub = StubDictionary(characters: [ + "价": [character("价", "jie5", rank: 0), character("价", "jia4", rank: 1)], + "目": [character("目", "mu4", rank: 0)], + ]) + let reading = try ReadingComposer.composedReading(for: "价目", in: stub) + #expect(reading?.display == "jiàmù") + } + + /// And a character with no correction keeps whatever the container + /// ranked first, or the table would be overriding everywhere. + @Test("An uncorrected character keeps its ranking in a composed word") + func composedReadingKeepsTheRanking() throws { + let stub = StubDictionary(characters: [ + "中": [character("中", "zhong1", rank: 0), character("中", "zhong4", rank: 1)], + "目": [character("目", "mu4", rank: 0)], + ]) + let reading = try ReadingComposer.composedReading(for: "中目", in: stub) + #expect(reading?.display == "zhōngmù") + } +} diff --git a/Tests/HanReaderUITests/DictionaryServiceTests.swift b/Tests/HanReaderUITests/DictionaryServiceTests.swift index 1009457..6b66e33 100644 --- a/Tests/HanReaderUITests/DictionaryServiceTests.swift +++ b/Tests/HanReaderUITests/DictionaryServiceTests.swift @@ -74,21 +74,22 @@ struct DictionaryServiceTests { @Test("Looks a word up") func lookup() async { let service = DictionaryService(lookup: stub()) - let reading = await service.reading(for: "中国") + let reading = await service.reading(for: ReadingRequest(word: "中国")) #expect(reading?.display == "Zhōngguó") } @Test("A whole window of words is one call") func batched() async { let service = DictionaryService(lookup: stub()) - let readings = await service.readings(for: ["中国", "北京", "unknown"]) + let readings = await service + .readings(for: ["中国", "北京", "unknown"].map { ReadingRequest(word: $0) }) - #expect(readings["中国"]?.display == "Zhōngguó") - #expect(readings["北京"]?.source == .composed) + #expect(readings[ReadingRequest(word: "中国")]?.display == "Zhōngguó") + #expect(readings[ReadingRequest(word: "北京")]?.source == .composed) // Words with no reading are absent rather than present-and-nil: on // the page the two are indistinguishable, so the caller should not // have to tell them apart either. - #expect(readings["unknown"] == nil) + #expect(readings[ReadingRequest(word: "unknown")] == nil) #expect(readings.count == 2) } @@ -98,7 +99,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) for _ in 0 ..< 10 { - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) } #expect(dictionary.wordQueries == 1) } @@ -113,7 +114,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) for _ in 0 ..< 10 { - _ = await service.reading(for: "龘龘龘") + _ = await service.reading(for: ReadingRequest(word: "龘龘龘")) } #expect(dictionary.wordQueries == 1) // And it did not keep re-trying the character fallback either. @@ -125,9 +126,9 @@ struct DictionaryServiceTests { let dictionary = stub() let service = DictionaryService(lookup: dictionary) - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) await service.clearCaches() - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) #expect(dictionary.wordQueries == 2) } @@ -171,7 +172,7 @@ struct DictionaryServiceTests { let dictionary = stub() let service = DictionaryService(lookup: dictionary) - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) _ = await service.entries(for: "中国") #expect(dictionary.wordQueries == 1) } @@ -182,7 +183,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) _ = await service.entries(for: "中国") - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) #expect(dictionary.wordQueries == 1) } @@ -198,7 +199,7 @@ struct DictionaryServiceTests { dictionary.failsAlways = true let service = DictionaryService(lookup: dictionary) - #expect(await service.reading(for: "中国") == nil) + #expect(await service.reading(for: ReadingRequest(word: "中国")) == nil) #expect(await service.entries(for: "中国").isEmpty) } @@ -209,7 +210,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) for _ in 0 ..< 10 { - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) } #expect(dictionary.wordQueries == 1) } diff --git a/Tests/HanReaderUITests/ReaderFixtureTests.swift b/Tests/HanReaderUITests/ReaderFixtureTests.swift index f086a1e..221514a 100644 --- a/Tests/HanReaderUITests/ReaderFixtureTests.swift +++ b/Tests/HanReaderUITests/ReaderFixtureTests.swift @@ -42,9 +42,14 @@ struct ReaderFixtureTests { /// lines would differ in height according to dictionary coverage. @Test("Contains at least one word with no reading") func coversAWordWithNoReading() { - let readings = ReaderFixtures.readingsByWord + let readings = ReaderFixtures.readings(for: document) let words = document.blocks.flatMap(\.tokens).filter(\.isLookupCandidate) - #expect(words.contains { readings[$0.text] == nil }) + // Asked the way the surface asks, so a fixture word that has a + // reading but under a different context still counts as uncovered — + // which is what the renderer would see. + #expect(words.contains { token in + !readings.contains { $0.key.word == token.text } + }) } @Test("Every fixture reading is actually used") diff --git a/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift b/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift index f8a9b53..ef3dbd8 100644 --- a/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift +++ b/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift @@ -74,7 +74,7 @@ struct ReaderLayoutRenderTests { let view = ParagraphView( block: block, style: style, - readings: ReaderFixtures.readingsByWord, + readings: ReaderFixtures.readings(for: block.tokens), selection: nil, reveal: reveal, onTap: { _ in }, diff --git a/Tests/HanReaderUITests/ReaderModelTests.swift b/Tests/HanReaderUITests/ReaderModelTests.swift index 6714835..6491f06 100644 --- a/Tests/HanReaderUITests/ReaderModelTests.swift +++ b/Tests/HanReaderUITests/ReaderModelTests.swift @@ -64,10 +64,10 @@ struct ReaderModelTests { let model = fixture.model await model.load() - #expect(model.readings["我"]?.display == "wǒ") - #expect(model.readings["中国"]?.display == "Zhōngguó") + #expect(model.readings[ReadingRequest(word: "我")]?.display == "wǒ") + #expect(model.readings[ReadingRequest(word: "中国")]?.display == "Zhōngguó") // A word the dictionary does not have is simply absent. - #expect(model.readings["。"] == nil) + #expect(model.readings[ReadingRequest(word: "。")] == nil) } /// The window is asymmetric because reading goes downwards, and bounded diff --git a/Tests/HanReaderUITests/WordDetailTests.swift b/Tests/HanReaderUITests/WordDetailTests.swift index 6c88f19..fd18081 100644 --- a/Tests/HanReaderUITests/WordDetailTests.swift +++ b/Tests/HanReaderUITests/WordDetailTests.swift @@ -23,6 +23,60 @@ struct WordDetailTests { ) } + /// 重 as the reader meets it: `chóng` "to repeat" first, because that is + /// CC-CEDICT's row order, and `zhòng` "heavy" second. + private var heavyOrRepeat: [ReadingGroup] { + [ + ReadingGroup(readingKey: "chong2", entries: [ + AttributedEntry( + dictionary: "cc-cedict", + entry: entry("重", "chong2", [(.definition, "to repeat")]), + ), + ]), + ReadingGroup(readingKey: "zhong4", entries: [ + AttributedEntry( + dictionary: "cc-cedict", + entry: entry("重", "zhong4", [(.definition, "heavy")]), + ), + ]), + ] + } + + /// The panel contradicting the word above it. With the reading chosen by + /// context the headword read `zhòng` while the summary beside it, taken + /// from whichever group came first, said "to repeat". + @Test("The summary comes from the reading that is shown") + func summaryFollowsTheReading() { + let detail = WordDetail( + word: "重", + reading: TokenReading(display: "zhòng", source: .ambiguous), + groups: heavyOrRepeat, + ) + #expect(WordDetailPanel.summary(of: detail.entriesForDisplayedReading) == "heavy") + #expect(detail.orderedGroups.first?.readingKey == "zhong4") + // Every reading is still reachable; only the order changed. + #expect(detail.orderedGroups.count == 2) + #expect(Set(detail.orderedGroups.map(\.readingKey)) == ["chong2", "zhong4"]) + } + + @Test("A reading with no group of its own still shows the word's senses") + func composedReadingFallsBackToEverything() { + let detail = WordDetail( + word: "重", + reading: TokenReading(display: "chóngfù", source: .composed), + groups: heavyOrRepeat, + ) + #expect(detail.entriesForDisplayedReading.count == 2) + #expect(detail.orderedGroups.map(\.readingKey) == ["chong2", "zhong4"]) + } + + @Test("With no reading at all nothing is reordered") + func noReadingKeepsDictionaryOrder() { + let detail = WordDetail(word: "重", reading: nil, groups: heavyOrRepeat) + #expect(detail.orderedGroups.map(\.readingKey) == ["chong2", "zhong4"]) + #expect(detail.entriesForDisplayedReading.count == 2) + } + /// A sense that only says "variant of 妳" is a poor one-line summary, /// which is why `SenseKind` exists at all. @Test("The one-line summary prefers an entry that defines something")