From ad47373f2825dfb5ee1d08811c23c1789534883f Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:30:41 +0300 Subject: [PATCH 1/3] fix: choose a heteronym's reading from the words around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重 in 「教学任务很重」 was annotated chóng, "to repeat", where the sentence plainly means "very heavy" — zhòng. Nothing in the dictionary separates the two. ReadingComposer ranks a character's readings by preferring lowercase, then a neutral tone, then whichever reading the dictionary says most about; 重 has neither a capital nor a neutral tone, and CC-CEDICT gives chóng five senses to zhòng's three. The sense count is a frequency proxy, and it is wrong for exactly the characters whose readings are different words. No context-free default could have fixed it, because both readings are right somewhere: 重 is chóng in 重复 and zhòng in 很重. So the composer now takes a ReadingContext, and HeteronymRules answers for the cases a closed-class neighbour settles grammatically — a degree adverb modifies a stative predicate, so 很 before 重 rules chóng out rather than merely making it less likely. Negators are a separate, shorter table: 不 admits verbs as readily as adjectives, and earns its place only where both readings it allows point the same way. The table holds only the five characters where context overturns the context-free answer. 好, 难, 累, 大 and 少 are already ranked correctly and are asserted to stay that way, so a change to the generic ranking fails a test rather than quietly needing more rules. Readings are now keyed on the word and its context rather than on the word alone — a heteronym has no single answer per book — and one core function classifies neighbours for both the model, which decides what to fetch, and the paragraph view, which decides what to look up, so the two cannot disagree. The detail panel's summary follows the reading shown above the word, which it did not: after the rule chose zhòng the panel still read "to repeat" beside it. Verified on the running app over 「我在一个山村学校当老师，教学任务很重。他重复 了三次。任务不重。」 — 很重 and 不重 both zhòng, 重复 still chóngfù. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 11 + .../Dictionary/ReadingComposer.swift | 68 +++- .../Dictionary/ReadingContext.swift | 241 +++++++++++++ Sources/HanReaderUI/Models/ReaderModel.swift | 79 ++++- .../HanReaderUI/Reader/ParagraphView.swift | 31 +- .../HanReaderUI/Reader/ReaderFixtures.swift | 27 +- .../HanReaderUI/Reader/ReaderSurface.swift | 2 +- .../Screens/WordDetailContent.swift | 2 +- .../HanReaderUI/Screens/WordDetailPanel.swift | 2 +- .../Services/DictionaryService.swift | 34 +- .../ReadingContextTests.swift | 330 ++++++++++++++++++ .../DictionaryServiceTests.swift | 27 +- .../HanReaderUITests/ReaderFixtureTests.swift | 9 +- .../ReaderLayoutRenderTests.swift | 2 +- Tests/HanReaderUITests/ReaderModelTests.swift | 6 +- Tests/HanReaderUITests/WordDetailTests.swift | 54 +++ 16 files changed, 864 insertions(+), 61 deletions(-) create mode 100644 Sources/HanReaderCore/Dictionary/ReadingContext.swift create mode 100644 Tests/HanReaderCoreTests/ReadingContextTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index d1ed3db..8c9de14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ it needs to read like release notes rather than a commit log. ## [Unreleased] +### Fixed + +- Readings for heteronyms now take the surrounding words into account. 重 in + 「教学任务很重」 was annotated `chóng` ("to repeat") rather than `zhòng` + ("heavy"): with no part-of-speech information the reader fell back to + preferring whichever reading the dictionary says most about, and CC-CEDICT + says more about `chóng`. A degree adverb or a negator before a character + now settles the few cases where that fallback is wrong, and the detail + panel's one-line summary follows the reading shown above the word instead + of whichever the dictionary happened to list first. + ### Added - Initial repository scaffolding: license boundary between the MIT-licensed diff --git a/Sources/HanReaderCore/Dictionary/ReadingComposer.swift b/Sources/HanReaderCore/Dictionary/ReadingComposer.swift index df5b8ac..44cf4da 100644 --- a/Sources/HanReaderCore/Dictionary/ReadingComposer.swift +++ b/Sources/HanReaderCore/Dictionary/ReadingComposer.swift @@ -77,13 +77,19 @@ public protocol DictionaryLookup: Sendable { /// the English dictionary. public enum ReadingComposer { /// The reading to show above `word`, or nil if nothing is known. + /// + /// - Parameter context: what the neighbouring words say about how this one + /// is being used. Consulted only for a word whose readings actually + /// disagree, so passing `.none` changes nothing for the 99% of tokens + /// that have one reading. public static func reading( for word: String, in lookup: some DictionaryLookup, + context: ReadingContext = .none, ) throws -> TokenReading? { - if let exact = try exactReading(for: word, in: lookup) { + if let exact = try exactReading(for: word, in: lookup, context: context) { return exact } return try composedReading(for: word, in: lookup) @@ -118,17 +124,30 @@ public enum ReadingComposer { /// Lowercase outranks everything, matching `LexiconBuilder`: a capital /// marks a surname or a place, rarely the reading wanted in prose. /// - /// The four it still gets wrong — 为, 长, 重, 相 — have no neutral-tone - /// reading and no signal to separate them. Those are exactly the cases a - /// tagger exists for, and they are marked `.ambiguous` so the reader can - /// see the annotation is a guess. + /// ### Where the rule runs out, and what picks up + /// + /// 为, 长, 重 and 相 have no neutral-tone reading and nothing in the + /// dictionary to separate them, so the sense count decides and it decides + /// badly: CC-CEDICT says more about 重 as `chóng` ("to repeat") than as + /// `zhòng` ("heavy"), which annotated 很重 — "very heavy" — as `chóng`. + /// + /// No context-free rule can fix that, because both readings are right + /// somewhere: 重 is `chóng` in 重复 and `zhòng` in 很重. So `context` is + /// consulted first, and `HeteronymRules` answers for the handful of cases + /// where a closed-class neighbour settles it grammatically — a degree + /// adverb can only modify a stative predicate, so 很 before 重 rules + /// `chóng` out rather than merely making it less likely. + /// + /// What the rules do not reach is still marked `.ambiguous`, so the + /// annotation is drawn faintly and the panel lists every reading. static func exactReading( for word: String, in lookup: some DictionaryLookup, + context: ReadingContext = .none, ) throws -> TokenReading? { - try reading(from: lookup.entries(for: word)) + try reading(from: lookup.entries(for: word), context: context) } /// The reading to show, given entries already fetched. @@ -136,9 +155,21 @@ public enum ReadingComposer { /// Separate from `exactReading(for:in:)` so a caller that has the entries /// in hand — the detail panel has just fetched them to show definitions — /// does not query for them a second time. - public static func reading(from entries: [DictionaryEntry]) -> TokenReading? { + public static func reading( + from entries: [DictionaryEntry], + context: ReadingContext = .none, + ) + -> TokenReading? + { let usable = entries.filter(isUsable) - guard let best = usable.max(by: isWeaker) else { return nil } + guard let fallback = usable.max(by: isWeaker) else { return nil } + + // A grammatical constraint beats every ranking below it, because it is + // the only one that knows the sentence. Applied before the rest and + // only where the dictionary actually offers the preferred reading, so + // a rule naming a reading this dictionary does not have falls through + // to the context-free answer rather than annotating nothing. + let best = preferred(in: usable, context: context) ?? fallback // Compared case-insensitively. Nearly every common character has a // surname entry alongside its ordinary one -- 年 is `nián` and @@ -166,6 +197,27 @@ public enum ReadingComposer { !entry.readingDisplay.isEmpty && !entry.reading.contains(.unknown) } + /// The entry a context rule asks for, if there is a rule and the + /// dictionary has the reading it names. + /// + /// Several entries can share the preferred reading — 和 has four under + /// `hé` — so the ordinary ranking still chooses between them. The rule + /// narrows the field; it does not replace the ranking. + private static func preferred( + in entries: [DictionaryEntry], + context: ReadingContext, + ) + -> DictionaryEntry? + { + guard !context.isEmpty, + let headword = entries.first?.headword, + let wanted = HeteronymRules.preferredReading(for: headword, in: context) + else { return nil } + return entries + .filter { $0.readingKey.caseInsensitiveCompare(wanted) == .orderedSame } + .max(by: isWeaker) + } + /// Whether `lhs` is the worse choice of reading. See `exactReading` for /// where the ordering comes from. private static func isWeaker(_ lhs: DictionaryEntry, _ rhs: DictionaryEntry) -> Bool { diff --git a/Sources/HanReaderCore/Dictionary/ReadingContext.swift b/Sources/HanReaderCore/Dictionary/ReadingContext.swift new file mode 100644 index 0000000..2464d73 --- /dev/null +++ b/Sources/HanReaderCore/Dictionary/ReadingContext.swift @@ -0,0 +1,241 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// A closed class of function word that constrains how its neighbour is being +/// used. +/// +/// Closed classes only, and that restriction is the whole point. A rule built +/// on an open class — "a noun precedes" — needs a tagger and a lexicon and is +/// wrong at the edges; a rule built on a list of twenty-odd particles and +/// adverbs is a finite, checkable fact about the grammar. These are the ones +/// whose membership can be written down and argued about in a review. +public enum FunctionWordClass: UInt8, Sendable, Hashable, CaseIterable { + /// 很, 太, 非常, 最 — a word whose only job is to intensify. + /// + /// A degree adverb modifies a stative predicate: the thing after it is + /// being used as an adjective, never as a transitive verb, a noun or a + /// classifier. That is a grammatical constraint rather than a frequency + /// guess, which is what makes it worth encoding — it does not merely say + /// which reading is commoner, it says which readings are impossible. + case degreeAdverb + + /// 不, 并不 — a word that negates the predicate after it. + /// + /// A weaker signal than a degree adverb, and deliberately treated as a + /// separate class rather than folded in with it. A negator takes verbs as + /// readily as adjectives — 不去, 不吃 — so it says nothing on its own + /// about which reading applies. It earns its place only because, for a + /// few specific characters, both of the readings it admits happen to + /// point the same way: 不重 is "not heavy" and 不差 is "not bad", because + /// the other reading would be said as 不重复 and 不差别. Its table is + /// correspondingly shorter than the degree adverb's. + case negator + + /// The class of a word, or nil if it is not in one this can act on — + /// which is nearly every word, by design. + public init?(word: String) { + if Self.degreeAdverbs.contains(word) { + self = .degreeAdverb + return + } + if Self.negators.contains(word) { + self = .negator + return + } + return nil + } + + /// Both scripts throughout, because the reader's text may be either and + /// the token is matched as written. A traditional text saying 很長 has to + /// reach the same rule as 很长. + /// + /// 好 and 够 are here as the degree adverbs they are in 好重 ("so heavy") + /// and 够长 ("long enough"). Both are also ordinary words, but a + /// *separate preceding token* 好 before an adjective is the adverb in + /// practically all running text. + static let degreeAdverbs: Set = [ + "很", "太", "挺", "真", "好", "最", "更", "够", "夠", "蛮", "蠻", "颇", "頗", + "非常", "特别", "特別", "比较", "比較", "十分", "相当", "相當", + "极", "極", "极其", "極其", "格外", "尤其", "异常", "異常", + "这么", "這麼", "那么", "那麼", "多么", "多麼", + "稍微", "有点", "有點", "有点儿", "有點兒", "有些", + "过于", "過於", "越来越", "越來越", "不太", + ] + + /// 不 only, plus its emphatic forms. 没 is absent because it negates + /// aspect rather than a predicate, and 没重 is not something anyone says. + static let negators: Set = ["不", "并不", "並不"] +} + +/// What the words around a token say about how it is being used. +/// +/// Deliberately tiny, and deliberately *classified* rather than raw. Holding +/// the neighbouring words verbatim would make this the key to a cache with one +/// entry per phrase in the book; holding their class makes the key space the +/// vocabulary times a handful of cases, which is what lets a reading stay +/// cached per word the way it was before context existed. +public struct ReadingContext: Hashable, Sendable { + /// The class of the word immediately before, where it is one of the few + /// this can act on. Punctuation counts as a word for this purpose, so a + /// context does not reach across 很，重. + public let preceding: FunctionWordClass? + + /// No usable context: the word stands alone, or its neighbours say + /// nothing. The overwhelming majority of tokens. + public static let none = Self(preceding: nil) + + public init(preceding: FunctionWordClass?) { + self.preceding = preceding + } + + /// Classifies a token's neighbour. + public init(precedingWord: String?) { + preceding = precedingWord.flatMap(FunctionWordClass.init(word:)) + } + + public var isEmpty: Bool { + preceding == nil + } + + /// The context of every token in a block, aligned one-to-one with + /// `tokens`. + /// + /// Computed over a block rather than a document, which is what stops a + /// context reaching across a paragraph break. Whitespace is skipped — + /// `很 重` with a space between is still a degree adverb modifying 重 — + /// but punctuation is not, so it severs the relation, correctly. + /// + /// Shared by the two callers that need it: the model, deciding what to + /// fetch, and the paragraph view, deciding what to look up. One function + /// so the two can never disagree about what a token's context is, which + /// they would otherwise express as pinyin flickering between two values. + /// The context of one token, for a caller holding a position rather than + /// a paragraph — a tap, which has no reason to classify every word around + /// it to annotate one. + public static func context(at index: Int, in tokens: [Token]) -> Self { + guard tokens.indices.contains(index) else { return .none } + var cursor = index - 1 + while cursor >= 0, tokens[cursor].kind == .whitespace { + cursor -= 1 + } + return Self(precedingWord: cursor >= 0 ? tokens[cursor].text : nil) + } + + public static func contexts(in tokens: [Token]) -> [Self] { + var contexts: [Self] = [] + contexts.reserveCapacity(tokens.count) + var previous: String? + for token in tokens { + contexts.append(Self(precedingWord: previous)) + if token.kind != .whitespace { + previous = token.text + } + } + return contexts + } +} + +/// A word together with the context it appeared in. +/// +/// The unit a context-sensitive reading is fetched and cached by. It replaces +/// the bare word as a cache key, which it has to: with context in the +/// picture, "what does 重 sound like" no longer has one answer per book. +public struct ReadingRequest: Hashable, Sendable { + public let word: String + public let context: ReadingContext + + public init(word: String, context: ReadingContext = .none) { + self.word = word + self.context = context + } +} + +/// Which reading a character takes in a given grammatical context. +/// +/// ### Why a table at all +/// +/// `ReadingComposer` ranks a character's readings by rules that know nothing +/// about the sentence — prefer lowercase, prefer a neutral tone, then prefer +/// whichever reading the dictionary says more about. That last step is a +/// frequency proxy, and it is wrong for exactly the characters where the +/// readings are different words: CC-CEDICT gives 重 five senses under `chóng` +/// ("to repeat") and three under `zhòng` ("heavy"), so 很重 — plainly "very +/// heavy" — was annotated `chóng`. +/// +/// No context-free default can fix that, because both readings are right +/// somewhere. 重 is `chóng` in 重复 and `zhòng` in 很重, and the only thing +/// that separates them is what sits next to it. +/// +/// ### Why it is small, and why it stays small +/// +/// Each entry is a claim about one character under one grammatical +/// constraint, with the phrase that justifies it. The table holds only the +/// cases where the context *overturns* the context-free answer — 好, 难, 累, +/// 大 and 少 are already ranked correctly, and listing them would be noise +/// that hides the five real claims. `ReadingComposerTests` asserts those stay +/// correct, so if the generic ranking ever changes under them it fails there +/// rather than quietly needing a rule here. +/// +/// This is not a part-of-speech tagger and is not a step towards one. It is a +/// finite set of statements of the form "after 很, 重 is `zhòng`", each of +/// which a reader of Chinese can check in a second. +public enum HeteronymRules { + /// The numeric reading to prefer, or nil if nothing applies. + /// + /// Numeric because that is the cross-dictionary merge key: a rule written + /// as `zhong4` picks the right entry out of CC-CEDICT, which stores + /// numerals, and out of BKRS, which stores diacritics. + public static func preferredReading( + for headword: Headword, + in context: ReadingContext, + ) + -> String? + { + guard let preceding = context.preceding else { return nil } + // Single characters only. A multi-character word's reading comes from + // its own entry, where the dictionary has already resolved which of + // its characters' readings applies — 重复 is `chóngfù` because the + // dictionary says so, not because anything here decided. + for script in headword.bothScripts where script.count == 1 { + guard let character = script.first else { continue } + if let reading = table[preceding]?[character] { + return reading + } + } + return nil + } + + /// A degree adverb forces the stative reading. Each line is the phrase + /// that justifies it, and the reading the context-free ranking picked + /// instead. + private static let table: [FunctionWordClass: [Character: String]] = [ + .degreeAdverb: [ + // 很重 "very heavy", not chóng "to repeat" + "重": "zhong4", + // 很长 "very long", not zhǎng "chief, to grow". Both scripts + // spelled out because an entry may carry only one of them, in + // which case `bothScripts` cannot supply the other. + "长": "chang2", + "長": "chang2", + // 很空 "very empty", not kòng "vacancy, free time" + "空": "kong1", + // 很差 "very poor", not chā "difference" + "差": "cha4", + // 很薄 "very thin", not bó "meagre, slight" + "薄": "bao2", + ], + // Only where negating either reading lands on the same answer. 长 and + // 薄 are deliberately absent: 不长 can be "does not grow" and 待我不薄 + // is `bó`, so a negator does not settle them the way a degree adverb + // does. + .negator: [ + // 不重 "not heavy" — "does not repeat" would be 不重复 + "重": "zhong4", + // 不差 "not bad", also 不差钱 — never chā "difference" + "差": "cha4", + // 不空 "not empty" — free time is 没空, not 不空 + "空": "kong1", + ], + ] +} diff --git a/Sources/HanReaderUI/Models/ReaderModel.swift b/Sources/HanReaderUI/Models/ReaderModel.swift index 8d12af0..fd664f6 100644 --- a/Sources/HanReaderUI/Models/ReaderModel.swift +++ b/Sources/HanReaderUI/Models/ReaderModel.swift @@ -24,6 +24,45 @@ struct WordDetail: Equatable, Sendable { groups.flatMap { $0.entries.map(\.entry) } } + /// The groups, with the one the reader is actually looking at first. + /// + /// Groups arrive in the order their reading was first seen, which for 重 + /// is `chóng` before `zhòng` because that is CC-CEDICT's row order. Once + /// the context rule has chosen `zhòng` for 很重, opening the panel on + /// `chóng` makes the reader hunt for the sense that belongs to the word + /// in front of them. + var orderedGroups: [ReadingGroup] { + let shown = groupsForDisplayedReading + guard !shown.isEmpty else { return groups } + return shown + groups.filter { group in + !shown.contains { $0.readingKey == group.readingKey } + } + } + + /// The entries whose reading is the one shown above the word. + /// + /// The one-line summary has to come from these rather than from whatever + /// group happens to be first: 重 is `chóng` "to repeat" and `zhòng` + /// "heavy", and with the reading chosen by context the panel read + /// "zhòng — to repeat", contradicting the ruby directly above the word. + /// + /// Falls back to every entry when nothing matches, which is the case for + /// a reading composed from characters — there is no group for it, and the + /// word's senses are better than none. + var entriesForDisplayedReading: [DictionaryEntry] { + let matching = groupsForDisplayedReading.flatMap { $0.entries.map(\.entry) } + return matching.isEmpty ? entries : matching + } + + /// The groups spelling the reading on screen, which is normally one and + /// is none when the reading was composed from characters. + private var groupsForDisplayedReading: [ReadingGroup] { + guard let display = reading?.display else { return [] } + return groups.filter { + $0.readingDisplay.caseInsensitiveCompare(display) == .orderedSame + } + } + /// Whether any dictionary defines this word. /// /// A word can have a reading and no entry — composed from its @@ -74,7 +113,11 @@ final class ReaderModel { /// and a few thousand distinct words. Keying by token also implies that /// two instances of a word could be annotated differently, which nothing /// here can decide and so should not be able to express. - private(set) var readings: [String: TokenReading] = [:] + /// Keyed on word *and* context rather than on the word alone, because a + /// heteronym has no single answer: 重 is `zhòng` in 很重 and `chóng` in + /// 重复. Still bounded by the reader's vocabulary rather than by the + /// length of the book, since a context is one of a handful of classes. + private(set) var readings: [ReadingRequest: TokenReading] = [:] private(set) var selection: TokenID? private(set) var reveal: RevealSet @@ -206,15 +249,21 @@ final class ReaderModel { func refreshReadings() async { guard let document else { return } let range = Self.window(around: topBlock ?? 0, in: document) - let words = Set( - document.blocks[range] - .flatMap(\.tokens) - .filter(\.isLookupCandidate) - .map(\.text), - ) + var requests: Set = [] + // Per block, because `ReadingContext.contexts(in:)` is what stops a + // context reaching across a paragraph break — the last word of one + // paragraph does not govern the first word of the next. + for block in document.blocks[range] { + let contexts = ReadingContext.contexts(in: block.tokens) + for (token, context) in zip(block.tokens, contexts) + where token.isLookupCandidate + { + requests.insert(ReadingRequest(word: token.text, context: context)) + } + } // Only what is not already known, so scrolling back over text that // has been read costs nothing. - let missing = words.subtracting(readings.keys) + let missing = requests.subtracting(readings.keys) guard !missing.isEmpty else { return } let fetched = await services.dictionary.readings(for: missing) @@ -253,7 +302,7 @@ final class ReaderModel { selection = id reveal.reveal(token) persistReveal(of: token.text, revealed: true) - loadDetail(for: token.text) + loadDetail(for: token.text, context: context(of: id)) } } @@ -268,7 +317,14 @@ final class ReaderModel { detail = .empty } - private func loadDetail(for word: String) { + /// The grammatical context of one token, so the panel agrees with the + /// ruby above the word rather than showing a different reading for it. + private func context(of id: TokenID) -> ReadingContext { + guard let document, document.blocks.indices.contains(id.block) else { return .none } + return ReadingContext.context(at: id.index, in: document.blocks[id.block].tokens) + } + + private func loadDetail(for word: String, context: ReadingContext) { detailTask?.cancel() detail = .loading(word) detailTask = Task { @@ -283,9 +339,10 @@ final class ReaderModel { // entry is the one composed from its characters. Falling // back to the entries covers a word tapped before its // reading was fetched. - reading: readings[word] + reading: readings[ReadingRequest(word: word, context: context)] ?? ReadingComposer.reading( from: groups.flatMap { $0.entries.map(\.entry) }, + context: context, ), groups: groups, )) diff --git a/Sources/HanReaderUI/Reader/ParagraphView.swift b/Sources/HanReaderUI/Reader/ParagraphView.swift index ea5229c..be09632 100644 --- a/Sources/HanReaderUI/Reader/ParagraphView.swift +++ b/Sources/HanReaderUI/Reader/ParagraphView.swift @@ -20,9 +20,11 @@ import SwiftUI struct ParagraphView: View { let block: TextBlock let style: ReaderStyle - /// Readings keyed by word, not by token: bounded by the reader's - /// vocabulary rather than by the length of the book. - let readings: [String: TokenReading] + /// Readings keyed by word *and context*, not by token: still bounded by + /// the reader's vocabulary rather than by the length of the book, since a + /// context is one of a handful of classes — but 重 after 很 and 重 alone + /// are two questions with two answers, so the word alone is not a key. + let readings: [ReadingRequest: TokenReading] let selection: TokenID? /// Which words are revealed, and in which of the two senses. /// @@ -39,7 +41,7 @@ struct ParagraphView: View { init( block: TextBlock, style: ReaderStyle, - readings: [String: TokenReading], + readings: [ReadingRequest: TokenReading], selection: TokenID?, reveal: RevealSet, onTap: @escaping (TokenID) -> Void, @@ -50,8 +52,16 @@ struct ParagraphView: View { self.selection = selection self.reveal = reveal self.onTap = onTap - flow = zip(block.tokens, ReaderFlow.flowValues(for: block.tokens, style: style)) - .map(FlowToken.init) + // Contexts computed once here, from the same core function the model + // uses to decide what to fetch — so the key this view looks up is by + // construction the key that was stored, and a heteronym cannot end up + // unannotated because the two disagreed about its neighbours. + flow = zip( + zip(block.tokens, ReadingContext.contexts(in: block.tokens)), + ReaderFlow.flowValues(for: block.tokens, style: style), + ).map { pair, flow in + FlowToken(token: pair.0, context: pair.1, flow: flow) + } } var body: some View { @@ -67,7 +77,7 @@ struct ParagraphView: View { ForEach(flow) { entry in TokenView( token: entry.token, - reading: readings[entry.token.text], + reading: readings[entry.request], emphasis: emphasis(of: entry.token), style: style, onTap: onTap, @@ -104,9 +114,16 @@ struct ParagraphView: View { /// A token and the layout values derived from its neighbours. private struct FlowToken: Identifiable { let token: Token + let context: ReadingContext let flow: TokenFlowValue var id: TokenID { token.id } + + /// How this token's reading is keyed, so the view asks the same question + /// the model answered. + var request: ReadingRequest { + ReadingRequest(word: token.text, context: context) + } } diff --git a/Sources/HanReaderUI/Reader/ReaderFixtures.swift b/Sources/HanReaderUI/Reader/ReaderFixtures.swift index fe376ed..4e8b327 100644 --- a/Sources/HanReaderUI/Reader/ReaderFixtures.swift +++ b/Sources/HanReaderUI/Reader/ReaderFixtures.swift @@ -35,6 +35,29 @@ enum ReaderFixtures { /// distinction is invisible in a screenshot otherwise, and it is the one /// the reader most needs to be able to see — a composed reading is an /// approximation that will be wrong for heteronyms. + /// The authored table, keyed the way the reading surface asks for it. + /// + /// Derived from a document rather than written out, because a reading is + /// keyed on the word *and* its context: writing the keys by hand would + /// mean every fixture word silently losing its annotation the day someone + /// adds a 很 to the prose. + static func readings(for document: SegmentedDocument) -> [ReadingRequest: TokenReading] { + document.blocks.reduce(into: [:]) { result, block in + result.merge(readings(for: block.tokens)) { current, _ in current } + } + } + + /// One paragraph's worth, for a caller rendering a single block. + static func readings(for tokens: [Token]) -> [ReadingRequest: TokenReading] { + var result: [ReadingRequest: TokenReading] = [:] + for (token, context) in zip(tokens, ReadingContext.contexts(in: tokens)) { + if let reading = readingsByWord[token.text] { + result[ReadingRequest(word: token.text, context: context)] = reading + } + } + return result + } + static let readingsByWord: [String: TokenReading] = [ "我": TokenReading(display: "wǒ", source: .dictionary), "爱": TokenReading(display: "ài", source: .dictionary), @@ -158,7 +181,7 @@ struct FixtureReader: View { ReaderSurface( document: document, style: style, - readings: ReaderFixtures.readingsByWord, + readings: ReaderFixtures.readings(for: document), selection: selection, reveal: reveal, onTap: select, @@ -312,7 +335,7 @@ private struct FixturePreview: View { ReaderSurface( document: document, style: style, - readings: ReaderFixtures.readingsByWord, + readings: ReaderFixtures.readings(for: document), selection: selection, reveal: ReaderFixtures.reveal(of: revealed), onTap: { selection = selection == $0 ? nil : $0 }, diff --git a/Sources/HanReaderUI/Reader/ReaderSurface.swift b/Sources/HanReaderUI/Reader/ReaderSurface.swift index 875f815..13cd26f 100644 --- a/Sources/HanReaderUI/Reader/ReaderSurface.swift +++ b/Sources/HanReaderUI/Reader/ReaderSurface.swift @@ -17,7 +17,7 @@ import SwiftUI struct ReaderSurface: View { let document: SegmentedDocument let style: ReaderStyle - let readings: [String: TokenReading] + let readings: [ReadingRequest: TokenReading] let selection: TokenID? let reveal: RevealSet let onTap: (TokenID) -> Void diff --git a/Sources/HanReaderUI/Screens/WordDetailContent.swift b/Sources/HanReaderUI/Screens/WordDetailContent.swift index 0584be2..adafe2b 100644 --- a/Sources/HanReaderUI/Screens/WordDetailContent.swift +++ b/Sources/HanReaderUI/Screens/WordDetailContent.swift @@ -23,7 +23,7 @@ struct WordDetailContent: View { VStack(alignment: .leading, spacing: 20) { header if detail.isDefined { - ForEach(Array(detail.groups.enumerated()), id: \.offset) { _, group in + ForEach(Array(detail.orderedGroups.enumerated()), id: \.offset) { _, group in groupView(group) } } else { diff --git a/Sources/HanReaderUI/Screens/WordDetailPanel.swift b/Sources/HanReaderUI/Screens/WordDetailPanel.swift index 4a292bb..e38c4bb 100644 --- a/Sources/HanReaderUI/Screens/WordDetailPanel.swift +++ b/Sources/HanReaderUI/Screens/WordDetailPanel.swift @@ -71,7 +71,7 @@ struct WordDetailPanel: View { word: detail.word, reading: detail.reading, summary: nil, - senses: detail.entries, + senses: detail.entriesForDisplayedReading, ) } } diff --git a/Sources/HanReaderUI/Services/DictionaryService.swift b/Sources/HanReaderUI/Services/DictionaryService.swift index 7d299e4..7786eaa 100644 --- a/Sources/HanReaderUI/Services/DictionaryService.swift +++ b/Sources/HanReaderUI/Services/DictionaryService.swift @@ -32,7 +32,11 @@ actor DictionaryService { /// 77% of BKRS entries carry none. Nil when the service was built from a /// single lookup, as the tests and previews do. private let set: DictionarySet? - private var readingCache: BoundedCache + /// Keyed on the word *and* its context, not the word alone. 重 is + /// `zhòng` after 很 and `chóng` on its own, so a cache keyed on the word + /// would serve whichever of the two was asked for first to every + /// occurrence in the book. + private var readingCache: BoundedCache private var entryCache: BoundedCache /// Whether a database read has already failed. @@ -95,33 +99,41 @@ actor DictionaryService { /// Readings for a window of words, as one batch. /// - /// Returns only the words that have one, so the caller can treat a + /// Returns only the requests that have one, so the caller can treat a /// missing key and a word with no reading identically — which it should, /// because they look identical on the page. - func readings(for words: some Sequence) -> [String: TokenReading] { - var result: [String: TokenReading] = [:] - for word in words { - if let reading = reading(for: word) { - result[word] = reading + func readings( + for requests: some Sequence, + ) + -> [ReadingRequest: TokenReading] + { + var result: [ReadingRequest: TokenReading] = [:] + for request in requests { + if let reading = reading(for: request) { + result[request] = reading } } return result } - func reading(for word: String) -> TokenReading? { - if let cached = readingCache.value(forKey: word) { + func reading(for request: ReadingRequest) -> TokenReading? { + if let cached = readingCache.value(forKey: request) { return cached } + let word = request.word // Routed through `entries(for:)` rather than letting the composer do // its own lookup. The composer would query for exactly the same rows // and throw them away, so annotating a word and then tapping it ran // the same whole-word query twice -- once to find out how it sounds // and once to find out what it means. - let reading = ReadingComposer.reading(from: entries(for: word)) + let reading = ReadingComposer.reading( + from: entries(for: word), + context: request.context, + ) ?? attempt("composed reading for \(word)", default: nil) { try ReadingComposer.composedReading(for: word, in: lookup) } - readingCache.insert(reading, forKey: word) + readingCache.insert(reading, forKey: request) return reading } diff --git a/Tests/HanReaderCoreTests/ReadingContextTests.swift b/Tests/HanReaderCoreTests/ReadingContextTests.swift new file mode 100644 index 0000000..dbd395c --- /dev/null +++ b/Tests/HanReaderCoreTests/ReadingContextTests.swift @@ -0,0 +1,330 @@ +// HanReader — MIT licensed. See LICENSE. + +import Testing +@testable import HanReaderCore + +/// A dictionary written out by hand, so a test about which reading wins says +/// which entries exist rather than building a SQLite container to ask. +private struct StubDictionary: DictionaryLookup { + var words: [String: [DictionaryEntry]] = [:] + + func entries(for headword: String) throws -> [DictionaryEntry] { + words[headword] ?? [] + } + + func readings(forCharacter character: Character) throws -> [CharacterReading] { + [] + } +} + +private func entry( + _ simplified: String, + traditional: String? = nil, + _ numeric: String, + senses: Int = 1, +) + -> DictionaryEntry +{ + DictionaryEntry( + headword: Headword(simplified: simplified, traditional: traditional), + reading: Pinyin.parse(numeric: numeric), + senses: (0 ..< senses).map { + Sense(id: $0, kind: .definition, gloss: Gloss(text: "sense \($0)")) + }, + ) +} + +/// The real CC-CEDICT shape for 重, sense counts included, because the sense +/// counts are the whole reason the context-free rule gets it wrong: +/// `chóng` has five senses and `zhòng` three. +private let heavyOrRepeat = [ + entry("重", "chong2", senses: 5), + entry("重", "zhong4", senses: 3), +] + +/// 长 as CC-CEDICT holds it: `zhǎng` has seven senses to `cháng`'s five. +private let longOrChief = [ + entry("长", traditional: "長", "chang2", senses: 5), + entry("长", traditional: "長", "zhang3", senses: 7), +] + +private func tokens(_ words: [String], kinds: [TokenKind]? = nil) -> [Token] { + var offset = 0 + return words.enumerated().map { index, word in + let length = word.utf16.count + defer { offset += length } + return Token( + id: TokenID(block: 0, index: index), + text: word, + range: offset ..< offset + length, + kind: kinds?[index] ?? .han, + ) + } +} + +@Suite("Reading context") +struct ReadingContextTests { + // MARK: - The reported defect + + /// The bug this whole mechanism exists for. 我在一个山村学校当老师，教学任务很重 + /// segments to `… | 任务 | 很 | 重`, so 重 is a lone token, and the + /// context-free ranking annotated it `chóng` ("to repeat") because + /// CC-CEDICT says more about that reading than about `zhòng` ("heavy"). + @Test("A degree adverb makes 很重 zhòng, not chóng") + func degreeAdverbPicksTheStativeReading() { + let context = ReadingContext(precedingWord: "很") + let reading = ReadingComposer.reading(from: heavyOrRepeat, context: context) + #expect(reading?.display == "zhòng") + } + + /// The other half of the same claim, and the reason a context-free + /// default could never have fixed it: standing alone, 重 keeps whatever + /// the ranking says. The rule adds knowledge, it does not replace one + /// blanket default with another. + @Test("Without a degree adverb the ranking is untouched") + func noContextLeavesTheRankingAlone() { + let reading = ReadingComposer.reading(from: heavyOrRepeat) + #expect(reading?.display == "chóng") + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: .none)?.display == "chóng") + } + + @Test( + "Every degree adverb in the class picks the stative reading", + arguments: ["很", "太", "挺", "最", "更", "真", "好", "非常", "比较", "有点", "越来越"], + ) + func everyDegreeAdverbApplies(adverb: String) { + let context = ReadingContext(precedingWord: adverb) + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: context)?.display == "zhòng") + } + + /// 任务不重 is "the task is not heavy". A negator is a weaker signal than + /// a degree adverb and is kept to the characters where both readings it + /// admits point the same way. + @Test("A negator makes 不重 zhòng") + func negatorPicksTheStativeReading() { + let context = ReadingContext(precedingWord: "不") + #expect(context.preceding == .negator) + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: context)?.display == "zhòng") + } + + /// The two classes are not interchangeable, and this is what keeps them + /// apart: 待我不薄 is `bó` and 不长 can be "does not grow", so a negator + /// must not reach the characters a degree adverb does. + @Test("A negator does not settle 长, where a degree adverb does") + func negatorIsNarrowerThanDegreeAdverb() { + #expect(ReadingComposer.reading( + from: longOrChief, + context: ReadingContext(precedingWord: "不"), + )?.display == "zhǎng") + #expect(ReadingComposer.reading( + from: longOrChief, + context: ReadingContext(precedingWord: "很"), + )?.display == "cháng") + } + + /// 没 negates aspect rather than a predicate, and 没重 is not something + /// anyone says — so it is not a negator for this purpose. + @Test("没 is not treated as a negator") + func aspectualNegatorIsExcluded() { + #expect(FunctionWordClass(word: "没") == nil) + } + + @Test("A degree adverb makes 很长 cháng, not zhǎng") + func longIsNotChief() { + let context = ReadingContext(precedingWord: "很") + #expect(ReadingComposer.reading(from: longOrChief, context: context)?.display == "cháng") + #expect(ReadingComposer.reading(from: longOrChief)?.display == "zhǎng") + } + + /// A traditional text says 很長, and the token matched is 長. The rule has + /// to reach it, or the fix works in one script only. + /// An entry carrying only the simplified form, so the traditional line in + /// the table cannot cover for it. Both spellings are listed there and + /// each has to work alone, or the table silently depends on every + /// dictionary recording both scripts. + @Test("The rule reaches a simplified-only entry") + func simplifiedOnlyEntryIsCovered() { + let simplifiedOnly = [ + entry("长", "chang2", senses: 5), + entry("长", "zhang3", senses: 7), + ] + let context = ReadingContext(precedingWord: "很") + #expect( + ReadingComposer.reading(from: simplifiedOnly, context: context)?.display == "cháng", + ) + } + + @Test("The rule reaches the traditional spelling") + func traditionalScriptIsCovered() { + let traditionalOnly = [ + entry("長", "chang2", senses: 5), + entry("長", "zhang3", senses: 7), + ] + let context = ReadingContext(precedingWord: "很") + #expect( + ReadingComposer.reading(from: traditionalOnly, context: context)?.display == "cháng", + ) + } + + // MARK: - What the rule must not do + + /// 重复 is `chóngfù` and nothing here may interfere: a multi-character + /// word's reading comes from its own entry, where the dictionary has + /// already settled which reading of each character applies. + @Test("A multi-character word is never overridden") + func multiCharacterWordsAreLeftAlone() { + let entries = [entry("重复", "chong2 fu4", senses: 4)] + let context = ReadingContext(precedingWord: "很") + #expect(ReadingComposer.reading(from: entries, context: context)?.display == "chóngfù") + // Asserted on the rule itself as well as on the outcome. A rule that + // matched 重复 on its first character would be caught by nothing + // downstream: the reading it names is one syllable and a two-syllable + // word can never equal it, so the bug would sit there silently until + // some two-character headword turned up with a one-syllable reading. + #expect(HeteronymRules.preferredReading( + for: Headword(simplified: "重复", traditional: nil), + in: context, + ) == nil) + } + + /// A dictionary that simply has no `zhòng` entry must still annotate the + /// word. Falling through to the context-free answer is right; annotating + /// nothing because a rule could not be satisfied is not. + @Test("A rule the dictionary cannot satisfy falls through") + func unsatisfiableRuleFallsThrough() { + let entries = [entry("重", "chong2", senses: 5)] + let context = ReadingContext(precedingWord: "很") + #expect(ReadingComposer.reading(from: entries, context: context)?.display == "chóng") + } + + /// Still a guess, and still drawn faintly. A grammatical constraint makes + /// the choice defensible, not certain — the reader should see that the + /// dictionary holds more than one reading and be able to check the panel. + @Test("A context-chosen reading is still marked as a guess") + func contextDoesNotClaimCertainty() { + let context = ReadingContext(precedingWord: "很") + let reading = ReadingComposer.reading(from: heavyOrRepeat, context: context) + #expect(reading?.source == .ambiguous) + } + + /// Characters the context-free ranking already gets right, listed so that + /// a change to that ranking fails here instead of quietly creating the + /// need for more rules. None of these appears in `HeteronymRules`. + @Test( + "Characters needing no rule stay correct without one", + arguments: [ + ("好", ["hao3", "hao4"], "hǎo"), + ("难", ["nan2", "nan4"], "nán"), + ("累", ["lei4", "lei3", "lei2"], "lèi"), + ("大", ["da4", "dai4"], "dà"), + ("少", ["shao3", "shao4"], "shǎo"), + ], + ) + func unruledCharactersAreAlreadyRight( + word: String, + readings: [String], + expected: String, + ) { + // Sense counts descending, which is the shape that made 重 fail: the + // first reading listed is the one the ranking would pick on count + // alone. + let entries = readings.enumerated().map { + entry(word, $1, senses: readings.count - $0) + } + let context = ReadingContext(precedingWord: "很") + #expect(HeteronymRules.preferredReading( + for: Headword(simplified: word, traditional: nil), + in: context, + ) == nil) + #expect(ReadingComposer.reading(from: entries, context: context)?.display == expected) + } + + // MARK: - Classifying neighbours + + @Test("Punctuation severs the relation") + func punctuationSevers() { + let stream = tokens(["很", "，", "重"], kinds: [.han, .punctuation, .han]) + let contexts = ReadingContext.contexts(in: stream) + #expect(contexts[2] == .none) + // Both classifiers, because a tap uses the other one and the reader + // would see the panel disagree with the ruby above the word. + #expect(ReadingContext.context(at: 2, in: stream) == .none) + #expect(ReadingComposer.reading(from: heavyOrRepeat, context: contexts[2])? + .display == "chóng") + } + + /// `很 重` with a space between is still a degree adverb modifying 重 — + /// which matters because a reader's text may have spaces anywhere. + @Test("Whitespace does not sever the relation") + func whitespaceDoesNotSever() { + let stream = tokens(["很", " ", "重"], kinds: [.han, .whitespace, .han]) + let contexts = ReadingContext.contexts(in: stream) + #expect(contexts[2].preceding == .degreeAdverb) + } + + @Test("The first token of a block has no context") + func blockStartHasNoContext() { + #expect(ReadingContext.contexts(in: tokens(["重", "要"]))[0] == .none) + } + + @Test("A context is produced for every token") + func contextsAlignWithTokens() { + let stream = tokens(["任务", "很", "重"]) + #expect(ReadingContext.contexts(in: stream).count == stream.count) + } + + /// Two implementations of the same question — one walking a paragraph + /// forwards for the model, one scanning backwards from an index for a tap + /// — and they must never disagree. They did not start out agreeing: a + /// divergence shows up as the panel naming a different reading from the + /// ruby directly above the word. + @Test("The single-token and whole-block classifiers agree") + func theTwoClassifiersAgree() { + // 很 sits immediately before a comma, so a classifier that skipped + // punctuation would carry the adverb across it and the two would + // disagree about 重. Without that shape the streams agree whatever + // either one does with punctuation. + let stream = tokens( + ["他", "说", "很", "，", "重", " ", "很", "重", "。"], + kinds: [ + .han, .han, .han, .punctuation, .han, .whitespace, .han, .han, .punctuation, + ], + ) + let all = ReadingContext.contexts(in: stream) + for index in stream.indices { + #expect(ReadingContext.context(at: index, in: stream) == all[index]) + } + } + + @Test("An index outside the block has no context") + func outOfRangeIndexIsSafe() { + let stream = tokens(["重"]) + #expect(ReadingContext.context(at: 5, in: stream) == .none) + #expect(ReadingContext.context(at: -1, in: stream) == .none) + } + + @Test("An ordinary word is in no function-word class") + func ordinaryWordsAreUnclassified() { + #expect(FunctionWordClass(word: "任务") == nil) + #expect(FunctionWordClass(word: "重") == nil) + #expect(FunctionWordClass(word: "很") == .degreeAdverb) + } + + // MARK: - Through the lookup + + /// The whole path, as the reader meets it: the words of the sentence, the + /// dictionary, and the annotation that comes out. + @Test("The reported sentence annotates 重 as zhòng") + func theReportedSentenceIsFixed() throws { + let stub = StubDictionary(words: ["重": heavyOrRepeat, "很": [entry("很", "hen3")]]) + let stream = tokens(["任务", "很", "重"]) + let contexts = ReadingContext.contexts(in: stream) + + let reading = try ReadingComposer.reading(for: "重", in: stub, context: contexts[2]) + #expect(reading?.display == "zhòng") + + // And unchanged where there is no adverb, in the same sentence. + let bare = try ReadingComposer.reading(for: "重", in: stub, context: contexts[0]) + #expect(bare?.display == "chóng") + } +} diff --git a/Tests/HanReaderUITests/DictionaryServiceTests.swift b/Tests/HanReaderUITests/DictionaryServiceTests.swift index 1009457..6b66e33 100644 --- a/Tests/HanReaderUITests/DictionaryServiceTests.swift +++ b/Tests/HanReaderUITests/DictionaryServiceTests.swift @@ -74,21 +74,22 @@ struct DictionaryServiceTests { @Test("Looks a word up") func lookup() async { let service = DictionaryService(lookup: stub()) - let reading = await service.reading(for: "中国") + let reading = await service.reading(for: ReadingRequest(word: "中国")) #expect(reading?.display == "Zhōngguó") } @Test("A whole window of words is one call") func batched() async { let service = DictionaryService(lookup: stub()) - let readings = await service.readings(for: ["中国", "北京", "unknown"]) + let readings = await service + .readings(for: ["中国", "北京", "unknown"].map { ReadingRequest(word: $0) }) - #expect(readings["中国"]?.display == "Zhōngguó") - #expect(readings["北京"]?.source == .composed) + #expect(readings[ReadingRequest(word: "中国")]?.display == "Zhōngguó") + #expect(readings[ReadingRequest(word: "北京")]?.source == .composed) // Words with no reading are absent rather than present-and-nil: on // the page the two are indistinguishable, so the caller should not // have to tell them apart either. - #expect(readings["unknown"] == nil) + #expect(readings[ReadingRequest(word: "unknown")] == nil) #expect(readings.count == 2) } @@ -98,7 +99,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) for _ in 0 ..< 10 { - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) } #expect(dictionary.wordQueries == 1) } @@ -113,7 +114,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) for _ in 0 ..< 10 { - _ = await service.reading(for: "龘龘龘") + _ = await service.reading(for: ReadingRequest(word: "龘龘龘")) } #expect(dictionary.wordQueries == 1) // And it did not keep re-trying the character fallback either. @@ -125,9 +126,9 @@ struct DictionaryServiceTests { let dictionary = stub() let service = DictionaryService(lookup: dictionary) - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) await service.clearCaches() - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) #expect(dictionary.wordQueries == 2) } @@ -171,7 +172,7 @@ struct DictionaryServiceTests { let dictionary = stub() let service = DictionaryService(lookup: dictionary) - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) _ = await service.entries(for: "中国") #expect(dictionary.wordQueries == 1) } @@ -182,7 +183,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) _ = await service.entries(for: "中国") - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) #expect(dictionary.wordQueries == 1) } @@ -198,7 +199,7 @@ struct DictionaryServiceTests { dictionary.failsAlways = true let service = DictionaryService(lookup: dictionary) - #expect(await service.reading(for: "中国") == nil) + #expect(await service.reading(for: ReadingRequest(word: "中国")) == nil) #expect(await service.entries(for: "中国").isEmpty) } @@ -209,7 +210,7 @@ struct DictionaryServiceTests { let service = DictionaryService(lookup: dictionary) for _ in 0 ..< 10 { - _ = await service.reading(for: "中国") + _ = await service.reading(for: ReadingRequest(word: "中国")) } #expect(dictionary.wordQueries == 1) } diff --git a/Tests/HanReaderUITests/ReaderFixtureTests.swift b/Tests/HanReaderUITests/ReaderFixtureTests.swift index f086a1e..221514a 100644 --- a/Tests/HanReaderUITests/ReaderFixtureTests.swift +++ b/Tests/HanReaderUITests/ReaderFixtureTests.swift @@ -42,9 +42,14 @@ struct ReaderFixtureTests { /// lines would differ in height according to dictionary coverage. @Test("Contains at least one word with no reading") func coversAWordWithNoReading() { - let readings = ReaderFixtures.readingsByWord + let readings = ReaderFixtures.readings(for: document) let words = document.blocks.flatMap(\.tokens).filter(\.isLookupCandidate) - #expect(words.contains { readings[$0.text] == nil }) + // Asked the way the surface asks, so a fixture word that has a + // reading but under a different context still counts as uncovered — + // which is what the renderer would see. + #expect(words.contains { token in + !readings.contains { $0.key.word == token.text } + }) } @Test("Every fixture reading is actually used") diff --git a/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift b/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift index f8a9b53..ef3dbd8 100644 --- a/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift +++ b/Tests/HanReaderUITests/ReaderLayoutRenderTests.swift @@ -74,7 +74,7 @@ struct ReaderLayoutRenderTests { let view = ParagraphView( block: block, style: style, - readings: ReaderFixtures.readingsByWord, + readings: ReaderFixtures.readings(for: block.tokens), selection: nil, reveal: reveal, onTap: { _ in }, diff --git a/Tests/HanReaderUITests/ReaderModelTests.swift b/Tests/HanReaderUITests/ReaderModelTests.swift index 6714835..6491f06 100644 --- a/Tests/HanReaderUITests/ReaderModelTests.swift +++ b/Tests/HanReaderUITests/ReaderModelTests.swift @@ -64,10 +64,10 @@ struct ReaderModelTests { let model = fixture.model await model.load() - #expect(model.readings["我"]?.display == "wǒ") - #expect(model.readings["中国"]?.display == "Zhōngguó") + #expect(model.readings[ReadingRequest(word: "我")]?.display == "wǒ") + #expect(model.readings[ReadingRequest(word: "中国")]?.display == "Zhōngguó") // A word the dictionary does not have is simply absent. - #expect(model.readings["。"] == nil) + #expect(model.readings[ReadingRequest(word: "。")] == nil) } /// The window is asymmetric because reading goes downwards, and bounded diff --git a/Tests/HanReaderUITests/WordDetailTests.swift b/Tests/HanReaderUITests/WordDetailTests.swift index 6c88f19..fd18081 100644 --- a/Tests/HanReaderUITests/WordDetailTests.swift +++ b/Tests/HanReaderUITests/WordDetailTests.swift @@ -23,6 +23,60 @@ struct WordDetailTests { ) } + /// 重 as the reader meets it: `chóng` "to repeat" first, because that is + /// CC-CEDICT's row order, and `zhòng` "heavy" second. + private var heavyOrRepeat: [ReadingGroup] { + [ + ReadingGroup(readingKey: "chong2", entries: [ + AttributedEntry( + dictionary: "cc-cedict", + entry: entry("重", "chong2", [(.definition, "to repeat")]), + ), + ]), + ReadingGroup(readingKey: "zhong4", entries: [ + AttributedEntry( + dictionary: "cc-cedict", + entry: entry("重", "zhong4", [(.definition, "heavy")]), + ), + ]), + ] + } + + /// The panel contradicting the word above it. With the reading chosen by + /// context the headword read `zhòng` while the summary beside it, taken + /// from whichever group came first, said "to repeat". + @Test("The summary comes from the reading that is shown") + func summaryFollowsTheReading() { + let detail = WordDetail( + word: "重", + reading: TokenReading(display: "zhòng", source: .ambiguous), + groups: heavyOrRepeat, + ) + #expect(WordDetailPanel.summary(of: detail.entriesForDisplayedReading) == "heavy") + #expect(detail.orderedGroups.first?.readingKey == "zhong4") + // Every reading is still reachable; only the order changed. + #expect(detail.orderedGroups.count == 2) + #expect(Set(detail.orderedGroups.map(\.readingKey)) == ["chong2", "zhong4"]) + } + + @Test("A reading with no group of its own still shows the word's senses") + func composedReadingFallsBackToEverything() { + let detail = WordDetail( + word: "重", + reading: TokenReading(display: "chóngfù", source: .composed), + groups: heavyOrRepeat, + ) + #expect(detail.entriesForDisplayedReading.count == 2) + #expect(detail.orderedGroups.map(\.readingKey) == ["chong2", "zhong4"]) + } + + @Test("With no reading at all nothing is reordered") + func noReadingKeepsDictionaryOrder() { + let detail = WordDetail(word: "重", reading: nil, groups: heavyOrRepeat) + #expect(detail.orderedGroups.map(\.readingKey) == ["chong2", "zhong4"]) + #expect(detail.entriesForDisplayedReading.count == 2) + } + /// A sense that only says "variant of 妳" is a poor one-line summary, /// which is why `SenseKind` exists at all. @Test("The one-line summary prefers an entry that defines something") From ee5666235206c45a1f3c9e595d27c71f7b6a2c61 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:54:18 +0300 Subject: [PATCH 2/3] fix: correct readings the dictionary's own evidence gets wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 价 in 「她叫我给个价」 was annotated jiè, "great, middleman", where the sentence means "name me a price" — jià. Unlike 重, this is not a context problem. jiè is not a reading of modern Chinese, so no neighbour needs to rule it out; the ranking simply believes it. CC-CEDICT documents the archaic sense in four glosses against three for jià, and writes it with a neutral tone besides, so both the neutral-tone preference and the sense count point at it. Removing either would not help. No automatic signal fixes it either. Compound attestation looks decisive — 161 headwords spell 价 jià and none spell it jiè — but applied as a ranking it flips every sentence-final particle, since 吧, 呢, 啊 and 吗 are absent from compounds too, for reasons that have nothing to do with being archaic. So the corrections are a generated table, not a ranking and not a curated list. Scripts/heteronym-defaults.py applies one criterion to the pinned snapshot — a reading used by no multi-character headword beside one used by at least twenty, excluding readings glossed as grammatical — and prints the table verbatim. It yields five characters: 价, 椎, 炮, 络, 膏. Each is a dead reading outranking a living one, and the particles are left alone. The correction sits below the context rules and above the ranking: a neighbour is a reason, a default is the absence of one. The two tables are disjoint today, so that order decides nothing yet, and a test fails the moment they overlap rather than asserting an accident. Applied at lookup rather than baked into the per-character table, so an already-imported container does not need rebuilding — and applied on the composed path too, or a word assembled from its characters would spell 价 jiè while the same character tapped on its own read jià. Verified on the running app: 听力64 「她叫我给个价」 now reads jià, panel "price". Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 7 + Scripts/heteronym-defaults.py | 139 ++++++++++++++++ .../Dictionary/ReadingComposer.swift | 66 ++++++-- .../Dictionary/ReadingContext.swift | 52 +++++- .../ReadingContextTests.swift | 149 +++++++++++++++++- 5 files changed, 402 insertions(+), 11 deletions(-) create mode 100755 Scripts/heteronym-defaults.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c9de14..400504e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ it needs to read like release notes rather than a commit log. now settles the few cases where that fallback is wrong, and the detail panel's one-line summary follows the reading shown above the word instead of whichever the dictionary happened to list first. +- A handful of characters whose dictionary evidence points at a reading no + living text uses are corrected outright. 价 in 「她叫我给个价」 was read + `jiè` ("great, middleman"), which CC-CEDICT documents in four senses and a + neutral tone against three senses for `jià` ("price") — so every signal + the ranking has pointed the wrong way. The corrections are generated by + `Scripts/heteronym-defaults.py` from a criterion the dictionary answers + itself: a reading no compound uses, beside one that many do. ### Added diff --git a/Scripts/heteronym-defaults.py b/Scripts/heteronym-defaults.py new file mode 100755 index 0000000..8a9f6a0 --- /dev/null +++ b/Scripts/heteronym-defaults.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Find characters whose dictionary evidence points at the wrong reading. + +`ReadingComposer` has no part-of-speech information, so for a character with +several readings it prefers a neutral tone and then whichever reading the +dictionary says most about. Both are proxies, and for a handful of characters +both point at a reading no living text uses: CC-CEDICT documents 价 as `jiè` +("great, middleman") in four senses and as `jià` ("price") in three, so the +reader annotated 她叫我给个价 with `jiè`. + +This reports the characters where that happens, by a criterion the dictionary +can answer on its own: a reading used by *no* multi-character headword, beside +one used by many. Compounds are the evidence because they are where a +character's living readings show up — 价格, 物价, 评价, 代价 are all `jià`, +and nothing is `jiè`. + +Readings glossed as grammatical are excluded. A sentence-final particle is +unattested in compounds for a reason that has nothing to do with being +archaic: 吧, 呢, 啊 and 吗 would otherwise all be "corrected" to a toned +reading nobody wants. + +Run against the pinned snapshot; paste the result into `HeteronymRules`: + + ./Scripts/heteronym-defaults.py Dictionaries/cc-cedict/cedict_ts.u8 +""" + +import argparse +import collections +import io +import re +import sys + +ENTRY = re.compile(r"^(\S+) (\S+) \[([^\]]*)\] /(.*)/$") + +# What CC-CEDICT writes when a reading is a function word rather than a word. +GRAMMATICAL = ( + "particle", + "suffix", + "suff.", + "prefix", + "marker", + "interjection", + "classifier", + "onomatop", +) + +# Below this, a rival reading is not well enough attested to overrule the +# dictionary's own emphasis. The gap this exists to catch is absolute — many +# against none — not narrow. +MINIMUM_RIVAL = 20 + + +def is_han(character): + return "一" <= character <= "鿿" + + +def read(path): + """Single-character entries, and per-character reading counts in compounds.""" + entries = collections.defaultdict(list) + compounds = collections.defaultdict(collections.Counter) + with io.open(path, encoding="utf-8") as handle: + for line in handle: + if line.startswith("#"): + continue + matched = ENTRY.match(line.strip()) + if not matched: + continue + _, simplified, reading, definition = matched.groups() + if len(simplified) == 1: + entries[simplified].append((reading, definition.split("/"))) + syllables = reading.split() + # Only where every character maps to exactly one syllable, which + # is what makes the alignment below sound. + if ( + len(simplified) >= 2 + and len(simplified) == len(syllables) + and all(is_han(c) for c in simplified) + ): + for character, syllable in zip(simplified, syllables): + compounds[character][syllable.lower()] += 1 + return entries, compounds + + +def is_neutral(reading): + return any(syllable[-1] == "5" for syllable in reading.split()) + + +def is_capitalised(reading): + return reading[0].isupper() + + +def chosen(readings): + """What `ReadingComposer.isWeaker` picks: lowercase, neutral tone, senses.""" + return sorted( + readings, + key=lambda r: (is_capitalised(r[0]), not is_neutral(r[0]), -len(r[1]), r[0]), + )[0] + + +def report(path): + entries, compounds = read(path) + rows = [] + for character, readings in sorted(entries.items()): + usable = [r for r in readings if r[0] and "xx5" not in r[0]] + if len({r[0].lower() for r in usable}) < 2: + continue + winner = chosen(usable) + if compounds[character].get(winner[0].lower(), 0) != 0: + continue + rivals = [ + (reading, count) + for reading, count in compounds[character].most_common() + if reading != winner[0].lower() + ] + if not rivals or rivals[0][1] < MINIMUM_RIVAL: + continue + glosses = [g for r in usable if r[0].lower() == winner[0].lower() for g in r[1]] + if any(mark in g.lower() for g in glosses for mark in GRAMMATICAL): + continue + rows.append((character, rivals[0][0], rivals[0][1], winner[0], glosses)) + + for character, reading, count, instead, glosses in rows: + print( + ' // {} compounds say {}, none say {} "{}"'.format( + count, reading, instead, "/".join(glosses)[:48] + ) + ) + print(' "{}": "{}",'.format(character, reading)) + return 0 if rows else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dictionary", help="path to cedict_ts.u8") + return report(parser.parse_args().dictionary) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Sources/HanReaderCore/Dictionary/ReadingComposer.swift b/Sources/HanReaderCore/Dictionary/ReadingComposer.swift index 44cf4da..29514a4 100644 --- a/Sources/HanReaderCore/Dictionary/ReadingComposer.swift +++ b/Sources/HanReaderCore/Dictionary/ReadingComposer.swift @@ -164,12 +164,16 @@ public enum ReadingComposer { let usable = entries.filter(isUsable) guard let fallback = usable.max(by: isWeaker) else { return nil } - // A grammatical constraint beats every ranking below it, because it is - // the only one that knows the sentence. Applied before the rest and - // only where the dictionary actually offers the preferred reading, so - // a rule naming a reading this dictionary does not have falls through - // to the context-free answer rather than annotating nothing. - let best = preferred(in: usable, context: context) ?? fallback + // Three tiers, narrowest first. A grammatical constraint beats + // everything below it because it is the only one that knows the + // sentence; a corrected default beats the ranking because the ranking + // is reading evidence that points the wrong way. Each applies only + // where the dictionary actually offers the reading it names, so a + // rule this dictionary cannot satisfy falls through to the + // context-free answer rather than annotating nothing. + let best = preferred(in: usable, context: context) + ?? corrected(in: usable) + ?? fallback // Compared case-insensitively. Nearly every common character has a // surname entry alongside its ordinary one -- 年 is `nián` and @@ -213,8 +217,32 @@ public enum ReadingComposer { let headword = entries.first?.headword, let wanted = HeteronymRules.preferredReading(for: headword, in: context) else { return nil } - return entries - .filter { $0.readingKey.caseInsensitiveCompare(wanted) == .orderedSame } + return matching(wanted, in: entries) + } + + /// The entry a corrected default asks for, for a character whose + /// dictionary evidence points at a reading no living text uses. + /// + /// Below the context rules deliberately: 价 is `jià` for want of any + /// reason to think otherwise, whereas 很 before 重 is a reason. A + /// character in both tables takes the contextual answer. + private static func corrected(in entries: [DictionaryEntry]) -> DictionaryEntry? { + guard let headword = entries.first?.headword, + let wanted = HeteronymRules.defaultReading(for: headword) + else { return nil } + return matching(wanted, in: entries) + } + + /// Entries whose reading is the one named, best first by the ordinary + /// ranking — several can share it, and 和 has four under `hé`. + private static func matching( + _ reading: String, + in entries: [DictionaryEntry], + ) + -> DictionaryEntry? + { + entries + .filter { $0.readingKey.caseInsensitiveCompare(reading) == .orderedSame } .max(by: isWeaker) } @@ -276,7 +304,12 @@ public enum ReadingComposer { var tokens: [PinyinToken] = [] for character in word { - guard let reading = try lookup.readings(forCharacter: character).first else { + // The per-character table is ranked by the same sense count the + // whole-word path uses, so it needs the same correction: without + // it a word composed from its characters would spell 价 `jiè` + // while the same character tapped on its own read `jià`. + guard let reading = try correcting(lookup.readings(forCharacter: character)) + else { return nil } let parsed = Pinyin.parse(numeric: reading.numeric) @@ -291,4 +324,19 @@ public enum ReadingComposer { guard !display.isEmpty else { return nil } return TokenReading(display: display, source: .composed) } + + /// The best of a character's candidate readings: the corrected default + /// if the table names one and the dictionary has it, otherwise the + /// highest-ranked, which is what the caller would have taken anyway. + private static func correcting(_ candidates: [CharacterReading]) -> CharacterReading? { + guard let character = candidates.first?.character, + let wanted = HeteronymRules.defaultReading( + for: Headword(simplified: String(character), traditional: nil), + ), + let corrected = candidates.first(where: { + $0.numeric.caseInsensitiveCompare(wanted) == .orderedSame + }) + else { return candidates.first } + return corrected + } } diff --git a/Sources/HanReaderCore/Dictionary/ReadingContext.swift b/Sources/HanReaderCore/Dictionary/ReadingContext.swift index 2464d73..6dbd872 100644 --- a/Sources/HanReaderCore/Dictionary/ReadingContext.swift +++ b/Sources/HanReaderCore/Dictionary/ReadingContext.swift @@ -206,10 +206,60 @@ public enum HeteronymRules { return nil } + /// The reading a character takes when the sentence says nothing either + /// way, for the few whose dictionary evidence points at a dead reading. + /// + /// Separate from the context rules above, and needed for a different + /// reason. 重 needs context because both its readings are live: it is + /// `chóng` in 重复 and `zhòng` in 很重, and only the neighbours separate + /// them. 价 needs no context at all — `jiè` is simply not a reading of + /// modern Chinese — but every signal available to the ranking points at + /// it, because CC-CEDICT documents the archaic sense more thoroughly + /// than the living one: four glosses for `jiè` "great, middleman" + /// against three for `jià` "price", and a neutral tone besides. So 她叫 + /// 我给个价 was annotated `jiè`. + /// + /// The criterion is mechanical and the table is generated, not curated: + /// a reading used by *no* multi-character headword, beside one used by + /// many. Compounds are the evidence because that is where a character's + /// living readings appear. Readings glossed as grammatical are excluded, + /// or every sentence-final particle would qualify — 吧, 呢, 啊 and 吗 are + /// all absent from compounds, and for a reason that has nothing to do + /// with being archaic. + /// + /// Regenerate with `Scripts/heteronym-defaults.py`, which prints these + /// lines verbatim. + public static func defaultReading(for headword: Headword) -> String? { + for script in headword.bothScripts where script.count == 1 { + if let character = script.first, let reading = defaults[character] { + return reading + } + } + return nil + } + + /// Internal rather than private so a test can check it against the + /// context table: the two must stay disjoint, or the tier order starts + /// deciding answers and someone has to argue it is the right one. + static let defaults: [Character: String] = [ + // 161 compounds say jia4, none say jie5 "great/good/middleman/servant" + "价": "jia4", + "價": "jia4", + // 21 compounds say zhui1, none say chui2 "a hammer" + "椎": "zhui1", + // 65 compounds say pao4, none say bao1 "to sauté/to fry/to dry by heating" + "炮": "pao4", + // 87 compounds say luo4, none say lao4 "small net" + "络": "luo4", + "絡": "luo4", + // 38 compounds say gao1, none say gao4 "to moisten/to grease/to apply" + "膏": "gao1", + ] + /// A degree adverb forces the stative reading. Each line is the phrase /// that justifies it, and the reading the context-free ranking picked /// instead. - private static let table: [FunctionWordClass: [Character: String]] = [ + static let table: [FunctionWordClass: [Character: String]] = [ .degreeAdverb: [ // 很重 "very heavy", not chóng "to repeat" "重": "zhong4", diff --git a/Tests/HanReaderCoreTests/ReadingContextTests.swift b/Tests/HanReaderCoreTests/ReadingContextTests.swift index dbd395c..2f3cc42 100644 --- a/Tests/HanReaderCoreTests/ReadingContextTests.swift +++ b/Tests/HanReaderCoreTests/ReadingContextTests.swift @@ -7,16 +7,28 @@ import Testing /// which entries exist rather than building a SQLite container to ask. private struct StubDictionary: DictionaryLookup { var words: [String: [DictionaryEntry]] = [:] + var characters: [Character: [CharacterReading]] = [:] func entries(for headword: String) throws -> [DictionaryEntry] { words[headword] ?? [] } func readings(forCharacter character: Character) throws -> [CharacterReading] { - [] + characters[character] ?? [] } } +/// Candidates come out of the container already ranked, so `rank` is the +/// order the composed path would otherwise take them in. +private func character(_ char: Character, _ numeric: String, rank: Int) -> CharacterReading { + CharacterReading( + character: char, + numeric: numeric, + display: Pinyin.display(Pinyin.parse(numeric: numeric)), + rank: rank, + ) +} + private func entry( _ simplified: String, traditional: String? = nil, @@ -328,3 +340,138 @@ struct ReadingContextTests { #expect(bare?.display == "chóng") } } + +/// Kept apart from the context rules because it answers a different +/// question. A context rule says which of two live readings this sentence +/// wants; a correction says the dictionary's own evidence is pointing at a +/// reading no living sentence wants. +@Suite("Heteronym corrections") +struct HeteronymCorrectionTests { + /// 价 as CC-CEDICT holds it. The archaic reading has both a neutral tone + /// and more senses, so every signal the ranking has points at it. + private var priceOrGreat: [DictionaryEntry] { + [ + entry("价", traditional: "價", "jie5", senses: 4), + entry("价", traditional: "價", "jia4", senses: 3), + ] + } + + /// 她叫我给个价 — "name me a price". `jiè` ("great, middleman") is not a + /// reading of modern Chinese, but it is the one CC-CEDICT says most + /// about, and it carries a neutral tone on top of that. + @Test("价 is jià, against both the neutral tone and the sense count") + func correctedDefaultBeatsTheRanking() { + #expect(ReadingComposer.reading(from: priceOrGreat)?.display == "jià") + } + + /// The correction has to hold with no context at all, which is the case + /// it exists for: unlike 重, there is no neighbour that settles 价 — + /// there is nothing to settle. + @Test( + "价 is jià in every context", + arguments: [nil, "很", "不", "个", "。"], + ) + func correctedDefaultIgnoresContext(preceding: String?) { + let context = ReadingContext(precedingWord: preceding) + #expect(ReadingComposer.reading(from: priceOrGreat, context: context)? + .display == "jià") + } + + @Test( + "Every generated correction is applied", + arguments: [ + ("价", "jie5", "jia4", "jià"), + ("椎", "chui2", "zhui1", "zhuī"), + ("炮", "bao1", "pao4", "pào"), + ("络", "lao4", "luo4", "luò"), + ("膏", "gao4", "gao1", "gāo"), + ], + ) + func everyCorrectionApplies( + word: String, + wrong: String, + right: String, + expected: String, + ) { + // The wrong reading given more senses, which is the shape that made + // the ranking choose it in the first place. + let entries = [ + entry(word, wrong, senses: 9), + entry(word, right, senses: 1), + ] + #expect(ReadingComposer.reading(from: entries)?.display == expected) + } + + /// A character with no correction must be left to the ranking, or the + /// table would be doing the ranking's job everywhere. + @Test("An ordinary character has no correction") + func ordinaryCharactersAreUncorrected() { + #expect(HeteronymRules.defaultReading( + for: Headword(simplified: "重", traditional: nil), + ) == nil) + #expect(HeteronymRules.defaultReading( + for: Headword(simplified: "价格", traditional: nil), + ) == nil) + } + + /// Sentence-final particles are unattested in compounds for a reason + /// that has nothing to do with being archaic, and the generator excludes + /// them. If one ever reached the table, 他来了吧 would read `bā`. + @Test("Particles are not corrected", arguments: ["吧", "呢", "啊", "吗", "了", "的"]) + func particlesAreNotCorrected(particle: String) { + #expect(HeteronymRules.defaultReading( + for: Headword(simplified: particle, traditional: nil), + ) == nil) + } + + /// A correction the dictionary cannot satisfy must not cost the word its + /// annotation. + @Test("A correction the dictionary lacks falls through") + func unsatisfiableCorrectionFallsThrough() { + let entries = [entry("价", "jie5", senses: 4)] + #expect(ReadingComposer.reading(from: entries)?.display == "jie") + } + + /// The composer tries context first and a correction second, but no + /// character is in both tables, so that order decides nothing today and + /// a test asserting it would be asserting an accident. + /// + /// This is the test worth having instead: it fails the moment the two + /// overlap, which is exactly when the order starts to matter and someone + /// has to decide whether context-first is right for that character. + @Test("The context and correction tables are disjoint") + func tablesAreDisjoint() { + let corrected = Set(HeteronymRules.defaults.keys) + #expect(!corrected.isEmpty) + for (functionWord, rules) in HeteronymRules.table { + let overlap = Set(rules.keys).intersection(corrected) + #expect(overlap.isEmpty, "\(functionWord) also corrected: \(overlap)") + } + } + + /// The per-character table is built by the same sense-count ranking, so + /// without the correction a word assembled from its characters spelled + /// 价 `jiè` while the character tapped on its own read `jià` — the same + /// character, two pronunciations, one screen apart. + @Test("A word composed from its characters uses the corrected reading") + func composedReadingIsCorrected() throws { + let stub = StubDictionary(characters: [ + "价": [character("价", "jie5", rank: 0), character("价", "jia4", rank: 1)], + "目": [character("目", "mu4", rank: 0)], + ]) + let reading = try ReadingComposer.composedReading(for: "价目", in: stub) + #expect(reading?.display == "jiàmù") + } + + /// And a character with no correction keeps whatever the container + /// ranked first, or the table would be overriding everywhere. + @Test("An uncorrected character keeps its ranking in a composed word") + func composedReadingKeepsTheRanking() throws { + let stub = StubDictionary(characters: [ + "中": [character("中", "zhong1", rank: 0), character("中", "zhong4", rank: 1)], + "目": [character("目", "mu4", rank: 0)], + ]) + let reading = try ReadingComposer.composedReading(for: "中目", in: stub) + #expect(reading?.display == "zhōngmù") + } +} From b45978233f0a9f00947f70597c1f4fbd89ff95a1 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:40:49 +0300 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20drop=20=E5=B7=AE=20from=20the=20nega?= =?UTF-8?q?tor=20table,=20and=20restore=20two=20doc=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on #33. The bundled dictionary settles 差 against the rule: it records both 不差 [bu4 cha1] "to not lack" and 不差 [bu4 cha4] "not bad" as headwords, so a negator decides nothing there. Removed, with a test asserting a negator leaves 差 to the ranking while a degree adverb still settles it — 很差 is only ever "very poor". That asymmetry is the difference between the two classes, so it is now pinned rather than described. The split-compound concern does not reproduce: both shipped segmenters keep 重复 whole, because a compound in the lexicon is what maximum matching matches, and the repair pass only ever merges. But the rule does lean on that, so it is now a test rather than an assumption — 很重复, 不重复 and 这个任务很重复 keep 重复 as one token, no lone 重 reaches the rule, and a character-by-character segmentation is shown being repaired back to 很 | 重复. Two doc comments ended up on the wrong declarations, both because a new function was inserted between a comment and what it documented: contexts(in:) lost its documentation to context(at:in:), and readingsByWord lost the note about 阅兵式 to readings(for:). Co-Authored-By: Claude Opus 5 --- .../Dictionary/ReadingContext.swift | 37 ++++++------ .../HanReaderUI/Reader/ReaderFixtures.swift | 46 +++++++-------- .../ReadingContextTests.swift | 58 +++++++++++++++++++ 3 files changed, 100 insertions(+), 41 deletions(-) diff --git a/Sources/HanReaderCore/Dictionary/ReadingContext.swift b/Sources/HanReaderCore/Dictionary/ReadingContext.swift index 6dbd872..3584dba 100644 --- a/Sources/HanReaderCore/Dictionary/ReadingContext.swift +++ b/Sources/HanReaderCore/Dictionary/ReadingContext.swift @@ -98,18 +98,6 @@ public struct ReadingContext: Hashable, Sendable { preceding == nil } - /// The context of every token in a block, aligned one-to-one with - /// `tokens`. - /// - /// Computed over a block rather than a document, which is what stops a - /// context reaching across a paragraph break. Whitespace is skipped — - /// `很 重` with a space between is still a degree adverb modifying 重 — - /// but punctuation is not, so it severs the relation, correctly. - /// - /// Shared by the two callers that need it: the model, deciding what to - /// fetch, and the paragraph view, deciding what to look up. One function - /// so the two can never disagree about what a token's context is, which - /// they would otherwise express as pinyin flickering between two values. /// The context of one token, for a caller holding a position rather than /// a paragraph — a tap, which has no reason to classify every word around /// it to annotate one. @@ -122,6 +110,18 @@ public struct ReadingContext: Hashable, Sendable { return Self(precedingWord: cursor >= 0 ? tokens[cursor].text : nil) } + /// The context of every token in a block, aligned one-to-one with + /// `tokens`. + /// + /// Computed over a block rather than a document, which is what stops a + /// context reaching across a paragraph break. Whitespace is skipped — + /// `很 重` with a space between is still a degree adverb modifying 重 — + /// but punctuation is not, so it severs the relation, correctly. + /// + /// Shared by the two callers that need it: the model, deciding what to + /// fetch, and the paragraph view, deciding what to look up. One function + /// so the two can never disagree about what a token's context is, which + /// they would otherwise express as pinyin flickering between two values. public static func contexts(in tokens: [Token]) -> [Self] { var contexts: [Self] = [] contexts.reserveCapacity(tokens.count) @@ -275,15 +275,16 @@ public enum HeteronymRules { // 很薄 "very thin", not bó "meagre, slight" "薄": "bao2", ], - // Only where negating either reading lands on the same answer. 长 and - // 薄 are deliberately absent: 不长 can be "does not grow" and 待我不薄 - // is `bó`, so a negator does not settle them the way a degree adverb - // does. + // Only where negating either reading lands on the same answer, which + // is a much shorter list than it looks. 长 and 薄 are absent because + // 不长 can be "does not grow" and 待我不薄 is `bó`; 差 is absent + // because the bundled dictionary settles it against us, recording + // both 不差 [bu4 cha1] "to not lack" and 不差 [bu4 cha4] "not bad" + // as headwords. A degree adverb still settles 差 — 很差 is only + // "very poor" — which is the difference between the two classes. .negator: [ // 不重 "not heavy" — "does not repeat" would be 不重复 "重": "zhong4", - // 不差 "not bad", also 不差钱 — never chā "difference" - "差": "cha4", // 不空 "not empty" — free time is 没空, not 不空 "空": "kong1", ], diff --git a/Sources/HanReaderUI/Reader/ReaderFixtures.swift b/Sources/HanReaderUI/Reader/ReaderFixtures.swift index 4e8b327..9fe1f66 100644 --- a/Sources/HanReaderUI/Reader/ReaderFixtures.swift +++ b/Sources/HanReaderUI/Reader/ReaderFixtures.swift @@ -35,29 +35,6 @@ enum ReaderFixtures { /// distinction is invisible in a screenshot otherwise, and it is the one /// the reader most needs to be able to see — a composed reading is an /// approximation that will be wrong for heteronyms. - /// The authored table, keyed the way the reading surface asks for it. - /// - /// Derived from a document rather than written out, because a reading is - /// keyed on the word *and* its context: writing the keys by hand would - /// mean every fixture word silently losing its annotation the day someone - /// adds a 很 to the prose. - static func readings(for document: SegmentedDocument) -> [ReadingRequest: TokenReading] { - document.blocks.reduce(into: [:]) { result, block in - result.merge(readings(for: block.tokens)) { current, _ in current } - } - } - - /// One paragraph's worth, for a caller rendering a single block. - static func readings(for tokens: [Token]) -> [ReadingRequest: TokenReading] { - var result: [ReadingRequest: TokenReading] = [:] - for (token, context) in zip(tokens, ReadingContext.contexts(in: tokens)) { - if let reading = readingsByWord[token.text] { - result[ReadingRequest(word: token.text, context: context)] = reading - } - } - return result - } - static let readingsByWord: [String: TokenReading] = [ "我": TokenReading(display: "wǒ", source: .dictionary), "爱": TokenReading(display: "ài", source: .dictionary), @@ -89,6 +66,29 @@ enum ReaderFixtures { /// Four paragraphs and a blank line. /// + /// The authored table, keyed the way the reading surface asks for it. + /// + /// Derived from a document rather than written out, because a reading is + /// keyed on the word *and* its context: writing the keys by hand would + /// mean every fixture word silently losing its annotation the day someone + /// adds a 很 to the prose. + static func readings(for document: SegmentedDocument) -> [ReadingRequest: TokenReading] { + document.blocks.reduce(into: [:]) { result, block in + result.merge(readings(for: block.tokens)) { current, _ in current } + } + } + + /// One paragraph's worth, for a caller rendering a single block. + static func readings(for tokens: [Token]) -> [ReadingRequest: TokenReading] { + var result: [ReadingRequest: TokenReading] = [:] + for (token, context) in zip(tokens, ReadingContext.contexts(in: tokens)) { + if let reading = readingsByWord[token.text] { + result[ReadingRequest(word: token.text, context: context)] = reading + } + } + return result + } + /// 拍 has no entry in `readingsByWord`, so one token in the last paragraph /// renders with no annotation — the case that must still reserve its ruby /// band, or lines would differ in height according to dictionary coverage. diff --git a/Tests/HanReaderCoreTests/ReadingContextTests.swift b/Tests/HanReaderCoreTests/ReadingContextTests.swift index 2f3cc42..75e51cc 100644 --- a/Tests/HanReaderCoreTests/ReadingContextTests.swift +++ b/Tests/HanReaderCoreTests/ReadingContextTests.swift @@ -122,6 +122,22 @@ struct ReadingContextTests { /// The two classes are not interchangeable, and this is what keeps them /// apart: 待我不薄 is `bó` and 不长 can be "does not grow", so a negator /// must not reach the characters a degree adverb does. + @Test("A negator does not settle 差, where a degree adverb does") + func negatorDoesNotSettleDifference() { + // CC-CEDICT records both 不差 [bu4 cha1] "to not lack" and 不差 + // [bu4 cha4] "not bad" as headwords, so a negator decides nothing + // here. 很差 is only ever "very poor". + let entries = [entry("差", "cha1", senses: 6), entry("差", "cha4", senses: 3)] + #expect(ReadingComposer.reading( + from: entries, + context: ReadingContext(precedingWord: "不"), + )?.display == "chā") + #expect(ReadingComposer.reading( + from: entries, + context: ReadingContext(precedingWord: "很"), + )?.display == "chà") + } + @Test("A negator does not settle 长, where a degree adverb does") func negatorIsNarrowerThanDegreeAdverb() { #expect(ReadingComposer.reading( @@ -251,6 +267,48 @@ struct ReadingContextTests { #expect(ReadingComposer.reading(from: entries, context: context)?.display == expected) } + /// The rule fires on a single-character *token*, which is only sound if + /// a compound stays one token. Were 很重复 ever segmented 很 | 重 | 复, + /// the degree adverb would reach 重 and spell 重复 as `zhòng fù`. + /// + /// Both shipped segmenters keep it whole — a compound in the lexicon is + /// what maximum matching matches, and the repair pass only ever merges — + /// so this pins the property the rule leans on rather than trusting it. + @Test("A compound after a degree adverb stays one token", arguments: [ + "很重复", "不重复", "这个任务很重复", + ]) + func compoundsSurviveSegmentation(text: String) { + let lexicon = Lexicon(words: ["重复", "任务", "这个"]) + let document = TextSegmenter(words: MaxMatchSegmenter(lexicon: lexicon)) + .segment(text) + let tokens = document.blocks.flatMap(\.tokens) + #expect(tokens.contains { $0.text == "重复" }) + // And so the rule never sees a lone 重 to act on. + let contexts = ReadingContext.contexts(in: tokens) + for (token, context) in zip(tokens, contexts) where token.text == "重" { + #expect( + HeteronymRules.preferredReading( + for: Headword(simplified: token.text, traditional: nil), + in: context, + ) == nil, + ) + } + } + + /// And the repair pass, which is what rescues a segmenter that does + /// split a known compound. It only merges, so it can turn 重 | 复 back + /// into 重复 but can never create the split this rule fears. + @Test("The repair pass rejoins a split compound") + func repairPassRejoinsCompounds() { + // Segmented character by character, which is the worst case the + // repair pass exists for and exactly the split this rule fears. + let split = TextSegmenter(words: CharacterSegmenter()).segment("很重复") + #expect(split.blocks.flatMap(\.tokens).map(\.text) == ["很", "重", "复"]) + + let repaired = DictionaryRepairPass(lexicon: Lexicon(words: ["重复"])).repair(split) + #expect(repaired.blocks.flatMap(\.tokens).map(\.text) == ["很", "重复"]) + } + // MARK: - Classifying neighbours @Test("Punctuation severs the relation")