From 3d0c786b229d0c09d1f6d3b5720d5a98dc1b4f15 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:44:29 +0300 Subject: [PATCH 1/3] feat: place a transcript in its recording by listening to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M11, second of four. Forced alignment: the words are already known, so what the recogniser supplies is timing, and the characters it gets right carry the clock across the ones it does not. The approach changed after checking what was actually stopping it. The missing Chinese model was not an un-downloaded one — this machine's MDM profile sets `allowDictation = false`, so Dictation cannot be switched on, its languages cannot be added, and `SFSpeechRecognizer(zh-CN) .supportsOnDeviceRecognition` is false with *No Assistant asset for language zh-CN* in the log. `en-US` and `ru-RU` report true, being the two assets that happen to be present. macOS 26's `SpeechTranscriber` manages its own models and is not governed by that restriction: on the same machine it reports zh-CN supported, offers an installation request, and installs. So automatic alignment has a floor of macOS 26 / iOS 26, and older systems place boundaries by hand instead. A second implementation on `SFSpeechRecognizer` would fail precisely where the first was needed, which is not a fallback. Network recognition is refused rather than merely avoided. It would send the reader's own recordings to a server in an app whose premise is that it works offline, and it is capped at about a minute of audio besides. Measured on a real 147-second recording: transcription takes 1.2 seconds, and the transcriber hands back one run per character — 0.24 s for a typical one. Slicing the audio at the resulting boundaries and transcribing each slice on its own reproduces the phrase text, which is the only non-circular way to check that times are right. That run also showed the character budget to be a poor proxy for duration on material with pauses: eighteen characters came to ten seconds, and six phrases in eighteen ran past the seven-second cap with the longest at thirty. Over-long phrases are now cut again once the clock is known. The pauses come free. A recogniser folds the silence before a character into that character's own span, so a character running far longer than its neighbours is one that followed a gap — 1.8 seconds against a typical 0.24. That is silence detection without decoding a sample, and it is what rescues this material, because recognised punctuation is sparse and cutting at punctuation alone left six phrases over. With both: longest 29.8 s to 8.0 s, over-cap six of eighteen to three of twenty-six, slices still 97–100%. `Speech` is fenced to `HanReaderPlayback` by a SwiftLint rule, as `NaturalLanguage` and GRDB already are. Co-Authored-By: Claude Opus 5 --- .swiftlint.yml | 8 + .../Listening/ForcedAlignment.swift | 473 ++++++++++++++++++ .../Listening/PhraseSplitter.swift | 23 + Sources/HanReaderPlayback/SpeechAligner.swift | 210 ++++++++ .../ForcedAlignmentTests.swift | 369 ++++++++++++++ .../SpeechAlignerTests.swift | 91 ++++ 6 files changed, 1174 insertions(+) create mode 100644 Sources/HanReaderCore/Listening/ForcedAlignment.swift create mode 100644 Sources/HanReaderPlayback/SpeechAligner.swift create mode 100644 Tests/HanReaderCoreTests/ForcedAlignmentTests.swift create mode 100644 Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift diff --git a/.swiftlint.yml b/.swiftlint.yml index b36e184..96a2a87 100644 --- a/.swiftlint.yml +++ b/.swiftlint.yml @@ -118,6 +118,14 @@ custom_rules: message: "AppKit/UIKit belongs in HanReaderPlatform only." severity: error + no_speech_outside_playback: + name: "Speech outside HanReaderPlayback" + included: ".*/Sources/(HanReaderCore|HanReaderUI|HanReaderPersistence|HanReaderDictionaryImport|HanReaderTokenization|HanReaderPlatform)/.*\\.swift" + regex: '^\s*(@[A-Za-z_][A-Za-z0-9_]*\s+)*((public|package|internal|private|fileprivate)\s+)?import\s+((typealias|struct|class|enum|protocol|func|let|var)\s+)?Speech\b' + excluded_match_kinds: [comment, doccomment, string] + message: "Speech belongs in HanReaderPlayback; go through HanReaderCore.RecognisedSpeech." + severity: error + no_naturallanguage_outside_tokenization: name: "NaturalLanguage outside HanReaderTokenization" included: ".*/Sources/(HanReaderCore|HanReaderUI|HanReaderPersistence|HanReaderDictionaryImport|HanReaderPlayback|HanReaderPlatform)/.*\\.swift" diff --git a/Sources/HanReaderCore/Listening/ForcedAlignment.swift b/Sources/HanReaderCore/Listening/ForcedAlignment.swift new file mode 100644 index 0000000..59edfba --- /dev/null +++ b/Sources/HanReaderCore/Listening/ForcedAlignment.swift @@ -0,0 +1,473 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// What a speech recogniser heard, and when. +/// +/// Deliberately not the recogniser's own type: this module has no business +/// importing Speech, and the alignment below is worth testing against +/// segments written by hand with deliberate mistakes in them — which is the +/// only way to test the case that matters. +public struct RecognisedSpeech: Hashable, Sendable { + /// One run of recognised text with the span of audio it came from. + public struct Segment: Hashable, Sendable { + public let text: String + public let start: Double + public let end: Double + + public init(text: String, start: Double, end: Double) { + self.text = text + self.start = start + self.end = end + } + } + + public let segments: [Segment] + /// The recording's full length, for the last phrase to end at. + public let duration: Double + + public init(segments: [Segment], duration: Double) { + self.segments = segments + self.duration = duration + } +} + +/// Works out where each phrase of a transcript is said in a recording. +/// +/// ### The recogniser does not have to be right +/// +/// This is forced alignment, not transcription: the words are already known. +/// What the recogniser supplies is *timing*, and it supplies that even where +/// it misheard — a character it got wrong sits between characters it got +/// right, and those carry the clock across it. A recogniser that is 80% +/// accurate still places every phrase, because the 20% it fluffed is bridged +/// rather than trusted. +/// +/// That is also why `confidence` is reported per phrase: it is the share of +/// a phrase's characters the recogniser actually matched, so a phrase whose +/// timing was mostly inferred can be offered to the reader for checking +/// instead of silently believed. +/// +/// ### Why it reuses the dictation scorer's alignment +/// +/// Lining up a transcript against a recognition is the same operation as +/// lining up a transcript against what a learner typed. `SequenceAlignment` +/// does both, which is why the two halves of this milestone are one piece of +/// work rather than two. +public enum ForcedAlignment { + /// Places each phrase in the recording. + /// + /// - Parameters: + /// - phrases: UTF-16 ranges into `transcript`, from `PhraseSplitter`. + /// - recognised: what the recogniser heard, in order. + /// - Returns: one `Phrase` per range, in the same order, with times that + /// never run backwards. + /// - Parameter maximumDuration: a phrase longer than this is cut again + /// at its own punctuation, now that the clock is known. A character + /// budget is only a proxy for duration and a poor one on material with + /// pauses: on a real 听力 recording, eighteen characters came to ten + /// seconds and six phrases in eighteen ran past the cap, the longest + /// at thirty. Pass nil to keep whatever was handed in. + public static func align( + phrases: [Range], + in transcript: String, + to recognised: RecognisedSpeech, + maximumDuration: Double? = 7, + ) + -> [Phrase] + { + let expected = Timeline.scorable(transcript) + let heard = Timeline(recognised) + + guard !expected.isEmpty, !heard.characters.isEmpty else { + return spread(phrases, across: recognised.duration) + } + + let steps = SequenceAlignment.align( + reference: expected.map(\.character), + candidate: heard.characters, + ) + + // Transcript character index -> the time it was heard at. Only + // matched characters are anchors; a substitution is the recogniser + // guessing, and its timing is no better than its reading. + var anchors: [Int: ClosedRange] = [:] + for step in steps where step.operation == .match { + guard let reference = step.reference, let candidate = step.candidate else { continue } + anchors[reference] = heard.span(of: candidate) + } + + var ranges = phrases + if let maximumDuration { + ranges = recut( + ranges, + in: transcript, + expected: expected, + anchors: anchors, + maximumDuration: maximumDuration, + ) + } + return place( + ranges, + expected: expected, + anchors: anchors, + duration: recognised.duration, + ) + } + + // MARK: - Cutting again, by the clock + + /// Re-cuts phrases that turned out too long once their times were known. + /// + /// Only at punctuation the text already has, so a phrase never ends + /// mid-clause — and a phrase with no punctuation left to cut at stays + /// long rather than being chopped somewhere arbitrary. That happens: + /// the vocabulary list read out at the end of an exercise is one + /// unpunctuated minute, and a phrase is a better unit there than a cut + /// in the middle of a word. + static func recut( + _ phrases: [Range], + in transcript: String, + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + maximumDuration: Double, + ) + -> [Range] + { + let typical = medianDuration(of: anchors) + var result: [Range] = [] + for phrase in phrases { + guard span(of: phrase, expected: expected, anchors: anchors) + .map({ $0.upperBound - $0.lowerBound > maximumDuration }) == true + else { + result.append(phrase) + continue + } + let points = cutPoints( + in: phrase, + of: transcript, + expected: expected, + anchors: anchors, + typical: typical, + ) + guard !points.isEmpty else { + result.append(phrase) + continue + } + // Every clause, then packed back up by measured time rather than + // by a character count standing in for it. + var pieces: [Range] = [] + var lower = phrase.lowerBound + for point in points + [phrase.upperBound] where point > lower { + pieces.append(lower ..< point) + lower = point + } + result.append(contentsOf: pack( + pieces, + expected: expected, + anchors: anchors, + maximumDuration: maximumDuration, + )) + } + return result + } + + /// Where a phrase could be cut: at its own punctuation, and wherever + /// the speaker paused. + /// + /// The pauses come free. A recogniser folds the silence before a + /// character into that character's own span, so a character taking far + /// longer than its neighbours is one that followed a gap — on a real + /// 听力 recording a typical character runs 0.24 s and the first one + /// after a pause runs 1.8. That is silence detection without decoding a + /// single sample, and it is what actually rescues this material: + /// recognised punctuation is sparse, so cutting at punctuation alone + /// left six phrases in eighteen over the cap. + static func cutPoints( + in phrase: Range, + of transcript: String, + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + typical: Double, + ) + -> [Int] + { + var points = Set(PhraseSplitter.cutPoints(in: phrase, of: transcript)) + let threshold = max(pauseFloor, typical * pauseMultiple) + for (index, position) in expected.enumerated() + where phrase.contains(position.offset) && position.offset > phrase.lowerBound + { + guard let anchor = anchors[index] else { continue } + if anchor.upperBound - anchor.lowerBound > threshold { + // Cut *before* the character, because the silence belongs to + // the gap and the character starts what comes after it. + points.insert(position.offset) + } + } + return points.sorted() + } + + private static func medianDuration(of anchors: [Int: ClosedRange]) -> Double { + let durations = anchors.values + .map { $0.upperBound - $0.lowerBound } + .filter { $0 > 0 } + .sorted() + guard !durations.isEmpty else { return 0 } + return durations[durations.count / 2] + } + + /// A gap has to be this long to count, however brisk the speaker. Below + /// it the "pause" is the natural lengthening at the end of a clause, + /// which is not somewhere to break. + static let pauseFloor = 0.6 + /// And this much longer than the speaker's own typical character. + static let pauseMultiple = 3.0 + + private static func pack( + _ pieces: [Range], + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + maximumDuration: Double, + ) + -> [Range] + { + var packed: [Range] = [] + for piece in pieces { + guard let last = packed.last else { + packed.append(piece) + continue + } + let combined = last.lowerBound ..< piece.upperBound + let duration = span(of: combined, expected: expected, anchors: anchors) + .map { $0.upperBound - $0.lowerBound } ?? 0 + if duration <= maximumDuration { + packed[packed.count - 1] = combined + } else { + packed.append(piece) + } + } + return packed + } + + /// When a range was heard, from the first anchored character in it to + /// the last. Nil when the recogniser anchored none of it, which is the + /// case re-cutting must leave alone — there is no clock to cut by. + private static func span( + of range: Range, + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + ) + -> ClosedRange? + { + var found: [ClosedRange] = [] + for (index, position) in expected.enumerated() + where range.contains(position.offset) + { + if let anchor = anchors[index] { + found.append(anchor) + } + } + guard let first = found.first, let last = found.last else { return nil } + return first.lowerBound ... max(first.lowerBound, last.upperBound) + } + + // MARK: - Placing + + private static func place( + _ phrases: [Range], + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + duration: Double, + ) + -> [Phrase] + { + // Which scorable characters fall in each phrase. Both lists are in + // order, so this is one walk rather than a search per phrase. + var bounds: [(lower: Int, upper: Int)] = [] + var cursor = 0 + for phrase in phrases { + while cursor < expected.count, expected[cursor].offset < phrase.lowerBound { + cursor += 1 + } + let lower = cursor + var upper = cursor + while upper < expected.count, expected[upper].offset < phrase.upperBound { + upper += 1 + } + bounds.append((lower, upper)) + } + + var placed: [Phrase] = [] + var previousEnd = 0.0 + for (index, phrase) in phrases.enumerated() { + let (lower, upper) = bounds[index] + let matched = (lower ..< upper).compactMap { anchors[$0] } + let confidence = upper > lower ? Double(matched.count) / Double(upper - lower) : 0 + + // A phrase the recogniser matched nothing in gets a slot between + // its neighbours rather than a zero-length one at the start of + // the recording, which would be unplayable and look like a bug. + let start = matched.first?.lowerBound ?? previousEnd + let end = matched.last?.upperBound ?? start + + placed.append(Phrase( + index: index, + start: start, + end: end, + textRange: phrase, + confidence: confidence, + origin: .recognised, + )) + previousEnd = end + } + return tidy(placed, duration: duration) + } + + /// Makes the times playable: ordered, non-overlapping, inside the + /// recording, and with nothing of zero length. + /// + /// The alignment can produce all four problems honestly — a phrase whose + /// only anchor is one character has start equal to end, and two phrases + /// sharing an anchor overlap. A transport handed those would play + /// silence or play the neighbour's words, and the reader would blame + /// their ears. + static func tidy(_ phrases: [Phrase], duration: Double) -> [Phrase] { + guard !phrases.isEmpty else { return [] } + let limit = duration > 0 ? duration : phrases.map(\.end).max() ?? 0 + + var result: [Phrase] = [] + var floor = 0.0 + for (offset, phrase) in phrases.enumerated() { + var start = min(max(phrase.start, floor), limit) + var end = min(max(phrase.end, start), limit) + + // Borrow from the next phrase's start, so a phrase the + // recogniser never anchored is still long enough to hear. + if end <= start { + let next = offset + 1 < phrases.count ? phrases[offset + 1].start : limit + end = min(max(next, start + minimumDuration), limit) + } + // And then once more, unconditionally. Stretching one phrase + // moves the floor under the next, which can squeeze *it* below + // the minimum — so the repair has to run forward, not just on + // the phrase that was short to begin with. Without this a + // recogniser that anchored almost nothing produced a run of + // phrases a transport seeks past without a sound. + if end - start < minimumDuration { + end = min(start + minimumDuration, limit) + } + // At the very end of a short recording there is nothing left to + // stretch into, so take the span out of the silence before it + // rather than hand back something unplayable. + if end - start < minimumDuration { + start = max(0, end - minimumDuration) + } + + result.append(Phrase( + index: offset, + start: start, + end: end, + textRange: phrase.textRange, + confidence: phrase.confidence, + origin: phrase.origin, + grade: phrase.grade, + )) + floor = end + } + return result + } + + /// No recognition at all: hand out equal slices so the reader can still + /// step through and fix the boundaries by hand. + /// + /// Confidence zero throughout, which is what tells the UI to offer the + /// boundary editor rather than present these as answers. + static func spread(_ phrases: [Range], across duration: Double) -> [Phrase] { + guard !phrases.isEmpty else { return [] } + let slice = (duration > 0 ? duration : 0) / Double(phrases.count) + return phrases.enumerated().map { index, range in + Phrase( + index: index, + start: Double(index) * slice, + end: Double(index + 1) * slice, + textRange: range, + confidence: 0, + origin: .recognised, + ) + } + } + + /// Shortest span worth calling a phrase. Below this a transport seeks and + /// stops before any sound reaches the speaker. + static let minimumDuration = 0.25 +} + +// MARK: - Flattening what was heard + +extension ForcedAlignment { + /// The recogniser's segments as one character sequence with a clock + /// attached. + struct Timeline { + /// One character of the transcript, and where it sits in the text. + struct Position: Hashable { + let character: Character + /// UTF-16 offset, so a phrase's range can be compared against it. + let offset: Int + } + + let characters: [Character] + /// Time span of each character in `characters`. + private let spans: [ClosedRange] + + init(_ recognised: RecognisedSpeech) { + var characters: [Character] = [] + var spans: [ClosedRange] = [] + for segment in recognised.segments { + let scorable = Self.scorable(segment.text).map(\.character) + guard !scorable.isEmpty else { continue } + // Spread the segment's span evenly across its characters. + // Apple's Chinese segments are words, so a four-character + // segment covering 0.8 s puts each character within 200 ms — + // far finer than a phrase boundary needs. + let span = max(0, segment.end - segment.start) + let step = span / Double(scorable.count) + for (index, character) in scorable.enumerated() { + let start = segment.start + Double(index) * step + characters.append(character) + spans.append(start ... (start + step)) + } + } + self.characters = characters + self.spans = spans + } + + func span(of index: Int) -> ClosedRange { + spans.indices.contains(index) ? spans[index] : 0 ... 0 + } + + /// Characters worth aligning, with where each sits in the text. + /// + /// Punctuation and spacing are dropped from both sides: a recogniser + /// invents its own punctuation, and matching the transcript's commas + /// against it would be aligning two different authors' habits rather + /// than the same speech. + static func scorable(_ text: String) -> [Position] { + var result: [Position] = [] + var offset = 0 + for character in text { + let width = character.utf16.count + let keep = character.unicodeScalars.contains { scalar in + switch ScalarClass.classify(scalar) { + case .han, .latin, .number: true + case .punctuation, .whitespace, .other: false + } + } + if keep { + result.append(Position(character: character, offset: offset)) + } + offset += width + } + return result + } + } +} diff --git a/Sources/HanReaderCore/Listening/PhraseSplitter.swift b/Sources/HanReaderCore/Listening/PhraseSplitter.swift index 0f3fa51..8a5d239 100644 --- a/Sources/HanReaderCore/Listening/PhraseSplitter.swift +++ b/Sources/HanReaderCore/Listening/PhraseSplitter.swift @@ -268,6 +268,29 @@ public enum PhraseSplitter { return merged } + /// Offsets inside `range` where a clause ends, as somewhere a phrase + /// could be cut. + /// + /// Exposed because a character budget is only a proxy for duration, and + /// a poor one on material with pauses in it: 听力 recordings leave two + /// seconds between sentences, so eighteen characters can be ten seconds + /// of audio. Once an alignment knows the real times, it re-cuts here — + /// at the same places this would have cut, chosen by the clock instead + /// of by a guess at it. + public static func cutPoints(in range: Range, of text: String) -> [Int] { + let metrics = Metrics(text) + var points: [Int] = [] + var offset = range.lowerBound + for character in metrics.text(in: range) { + offset += character.utf16.count + guard offset > range.lowerBound, offset < range.upperBound else { continue } + if separators.contains(character) || terminators.contains(character) { + points.append(offset) + } + } + return points + } + // MARK: - Measuring /// A text's UTF-16 units with a running count of the characters a diff --git a/Sources/HanReaderPlayback/SpeechAligner.swift b/Sources/HanReaderPlayback/SpeechAligner.swift new file mode 100644 index 0000000..0131c13 --- /dev/null +++ b/Sources/HanReaderPlayback/SpeechAligner.swift @@ -0,0 +1,210 @@ +// HanReader — MIT licensed. See LICENSE. + +import AVFoundation +public import Foundation +public import HanReaderCore +import Speech + +/// Why a recording could not be listened to. +public nonisolated enum SpeechAlignmentProblem: Error, Sendable, Equatable { + /// This system is too old for `SpeechTranscriber`. + /// + /// The older `SFSpeechRecognizer` is deliberately not used instead. Its + /// Chinese model arrives with the Dictation language, and on a managed + /// Mac `allowDictation = false` forbids installing one — which is the + /// case this feature most needs to work in. A second implementation that + /// fails exactly where the first was needed is not a fallback. + case systemTooOld + /// The language is not one the system can transcribe at all. + case languageUnsupported(String) + /// The model is not installed and could not be fetched. + case modelUnavailable(String) + case unreadable(String) + + public var isRecoverableByInstalling: Bool { + if case .modelUnavailable = self { + return true + } + return false + } +} + +/// How far a recording has been listened to. +public nonisolated struct SpeechAlignmentProgress: Equatable, Sendable { + /// Zero to one, or nil while the model is downloading and there is + /// nothing yet to measure. + public let fraction: Double? + public let isInstallingModel: Bool + + public init(fraction: Double?, isInstallingModel: Bool = false) { + self.fraction = fraction + self.isInstallingModel = isInstallingModel + } +} + +/// Listens to a recording so a transcript can be placed in it. +/// +/// ### It is not transcribing +/// +/// The words are already known — the reader imported them. What this is for +/// is *timing*: `ForcedAlignment` lines the recogniser's output up against +/// the transcript, and the characters it got right carry the clock across +/// the ones it did not. A recogniser that is 80% accurate still places every +/// phrase. +/// +/// ### Why `SpeechTranscriber` and not `SFSpeechRecognizer` +/// +/// Measured on the machine this was built for, which is managed by an MDM +/// profile setting `allowDictation = false`: +/// +/// - `SFSpeechRecognizer(locale: "zh-CN").supportsOnDeviceRecognition` is +/// **false**, and the system logs *No Assistant asset for language zh-CN*. +/// Its model ships with the Dictation language, and Dictation cannot be +/// switched on, so the language cannot be added. `en-US` and `ru-RU` report +/// true — they are the two assets that happen to be installed. +/// - Falling back to the network is refused outright, not merely avoided: +/// it would send the reader's own recordings to a server, in an app whose +/// whole premise is that it works offline, and server recognition is +/// capped at about a minute of audio anyway. +/// - `SpeechTranscriber` manages its own models and is not governed by that +/// restriction. On the same machine it reports zh-CN `supported` and +/// installs on request. +/// +/// The cost is a floor of macOS 26 / iOS 26 for automatic alignment. Older +/// systems place phrases by hand instead, which is slower but never wrong. +@available(macOS 26.0, iOS 26.0, *) +public nonisolated struct SpeechAligner: Sendable { + public let locale: Locale + + public init(locale: Locale = Locale(identifier: "zh-CN")) { + self.locale = locale + } + + /// Whether this language can be transcribed at all, before any work or + /// any download is attempted. + public func isLanguageSupported() async -> Bool { + await SpeechTranscriber.supportedLocales.contains { $0.matches(locale) } + } + + /// Whether the model is already on the device. + public func isModelInstalled() async -> Bool { + await SpeechTranscriber.installedLocales.contains { $0.matches(locale) } + } + + /// Fetches the model, which on a managed Mac is the only route to one. + /// + /// Reserving it afterwards is what stops the system reclaiming it: there + /// is a per-app allowance, and a model that evaporates would send the + /// reader back to a download they have already done once. + public func installModel( + progress: @escaping @Sendable (Double) -> Void = { _ in }, + ) async throws { + guard await isLanguageSupported() else { + throw SpeechAlignmentProblem.languageUnsupported(locale.identifier) + } + guard let request = try await AssetInventory.assetInstallationRequest( + supporting: [transcriber()], + ) else { + // Nothing to fetch: it is already here. + _ = try? await AssetInventory.reserve(locale: locale) + return + } + let tracked = request.progress + let watcher = Task { + while !Task.isCancelled, !tracked.isFinished { + progress(tracked.fractionCompleted) + try? await Task.sleep(for: .milliseconds(400)) + } + } + defer { watcher.cancel() } + try await request.downloadAndInstall() + _ = try? await AssetInventory.reserve(locale: locale) + } + + /// Listens to a recording and reports what was heard, with timings. + /// + /// - Parameter installIfNeeded: fetch the model when it is missing. + /// Off by default, so nothing downloads hundreds of megabytes without + /// the reader having said so. + public func listen( + to url: URL, + installIfNeeded: Bool = false, + progress: @escaping @Sendable (SpeechAlignmentProgress) -> Void = { _ in }, + ) async throws + -> RecognisedSpeech + { + guard await isLanguageSupported() else { + throw SpeechAlignmentProblem.languageUnsupported(locale.identifier) + } + if await !isModelInstalled() { + guard installIfNeeded else { + throw SpeechAlignmentProblem.modelUnavailable(locale.identifier) + } + try await installModel { fraction in + progress(SpeechAlignmentProgress( + fraction: fraction, + isInstallingModel: true, + )) + } + } + + let file: AVAudioFile + do { + file = try AVAudioFile(forReading: url) + } catch { + throw SpeechAlignmentProblem.unreadable(error.localizedDescription) + } + let duration = file.fileFormat.sampleRate > 0 + ? Double(file.length) / file.fileFormat.sampleRate + : 0 + + let module = transcriber() + let analyzer = SpeechAnalyzer(modules: [module]) + _ = try await analyzer.analyzeSequence(from: file) + try await analyzer.finalizeAndFinishThroughEndOfInput() + + var segments: [RecognisedSpeech.Segment] = [] + for try await result in module.results { + try Task.checkCancellation() + for run in result.text.runs { + guard let range = run.audioTimeRange else { continue } + segments.append(RecognisedSpeech.Segment( + text: String(result.text[run.range].characters), + start: range.start.seconds, + end: range.end.seconds, + )) + } + if duration > 0, let last = segments.last { + progress(SpeechAlignmentProgress(fraction: min(1, last.end / duration))) + } + } + return RecognisedSpeech(segments: segments, duration: duration) + } + + /// Timings are the whole point, so `.audioTimeRange` is the one option + /// that is not optional. The transcriber hands back one run per + /// character for Chinese — measured, not assumed: on a real recording a + /// typical character spans 0.24 s and the first after a pause spans 1.8, + /// which is what `ForcedAlignment` reads pauses out of. + private func transcriber() -> SpeechTranscriber { + SpeechTranscriber( + locale: locale, + transcriptionOptions: [], + reportingOptions: [], + attributeOptions: [.audioTimeRange], + ) + } +} + +extension Locale { + /// Whether two locales name the same language and region. + /// + /// Compared on the language code rather than the identifier: the system + /// lists `zh-CN` and a caller may hold `zh_CN` or `zh-Hans-CN`, and a + /// string comparison would call those different languages. + @available(macOS 26.0, iOS 26.0, *) + fileprivate nonisolated func matches(_ other: Locale) -> Bool { + language.languageCode == other.language.languageCode + && language.region == other.language.region + } +} diff --git a/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift b/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift new file mode 100644 index 0000000..2845d18 --- /dev/null +++ b/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift @@ -0,0 +1,369 @@ +// HanReader — MIT licensed. See LICENSE. + +import Testing +@testable import HanReaderCore + +/// A transcript of three sentences, and a recogniser that heard them +/// perfectly, one word per segment with a tidy clock. +private let transcript = "山村的孩子需要他。我在一个山村学校当老师。教学任务很重。" + +private func segment(_ text: String, _ start: Double, _ end: Double) -> RecognisedSpeech.Segment { + RecognisedSpeech.Segment(text: text, start: start, end: end) +} + +/// Exactly right: each sentence in its own two-second slot. +private let perfect = RecognisedSpeech( + segments: [ + segment("山村的孩子需要他", 0, 2), + segment("我在一个山村学校当老师", 2, 5), + segment("教学任务很重", 5, 7), + ], + duration: 7, +) + +private func phrases(_ text: String = transcript) -> [Range] { + PhraseSplitter.phrases(in: text) +} + +private func aligned(_ speech: RecognisedSpeech, _ text: String = transcript) -> [Phrase] { + ForcedAlignment.align(phrases: phrases(text), in: text, to: speech) +} + +@Suite("Forced alignment") +struct ForcedAlignmentTests { + // MARK: - The invariants a transport depends on + + /// Times a transport can actually play: in order, never overlapping, + /// never zero-length, never past the end of the recording. The alignment + /// can produce all four honestly — a phrase anchored on one character + /// has start equal to end — and a transport handed those plays silence + /// or plays the neighbour's words, and the reader blames their ears. + @Test( + "Placed phrases are ordered, non-overlapping and inside the recording", + arguments: [0, 1, 2, 3], + ) + func timesArePlayable(damage: Int) { + let speech = damaged(by: damage) + let placed = aligned(speech) + #expect(placed.count == phrases().count) + + // Hard guarantees, whatever the recogniser did. + for phrase in placed { + #expect(phrase.start >= 0) + #expect(phrase.end <= speech.duration + 0.000_1) + } + for (earlier, later) in zip(placed, placed.dropFirst()) { + #expect(earlier.end <= later.start + 0.000_1) + #expect(earlier.index < later.index) + } + // And long enough to hear, which is only promised when the recording + // has room for it — three phrases cannot each last a quarter-second + // inside half a second of audio. + if speech.duration >= Double(placed.count) * ForcedAlignment.minimumDuration { + for phrase in placed { + #expect(phrase.duration >= ForcedAlignment.minimumDuration - 0.000_1) + } + } + } + + /// Four recognisers of descending quality, so the invariants above are + /// checked against failure rather than only against success. + private func damaged(by level: Int) -> RecognisedSpeech { + switch level { + case 0: + perfect + case 1: + // Characters misheard: 村→春, 师→狮. Homophone-ish slips, which + // is what a recogniser actually produces. + RecognisedSpeech(segments: [ + segment("山春的孩子需要他", 0, 2), + segment("我在一个山村学校当老狮", 2, 5), + segment("教学任务很重", 5, 7), + ], duration: 7) + case 2: + // A whole sentence lost — a cough, a bad patch of audio. + RecognisedSpeech(segments: [ + segment("山村的孩子需要他", 0, 2), + segment("教学任务很重", 5, 7), + ], duration: 7) + default: + // Almost nothing recognised at all. + RecognisedSpeech(segments: [segment("山", 0, 0.3)], duration: 7) + } + } + + // MARK: - Placing + + @Test("A clean recognition puts each phrase on its own sentence") + func cleanRecognition() { + let placed = aligned(perfect) + #expect(placed.count == 3) + #expect(placed.allSatisfy { $0.confidence > 0.99 }) + + // Each phrase sits inside the segment that said it, give or take the + // width of one character. + #expect(abs(placed[0].start - 0) < 0.3) + #expect(abs(placed[1].start - 2) < 0.4) + #expect(abs(placed[2].start - 5) < 0.4) + #expect(abs(placed[2].end - 7) < 0.4) + } + + /// The whole point of forced alignment: the words are already known, so + /// a recogniser's mistakes cost timing accuracy, not placement. The + /// characters it got right carry the clock across the ones it did not. + @Test("Misheard characters do not move a phrase") + func misheardCharactersAreBridged() { + let placed = aligned(damagedByCharacters) + #expect(placed.count == 3) + #expect(abs(placed[1].start - 2) < 0.5) + #expect(abs(placed[2].start - 5) < 0.5) + // And it says so: not every character was matched. + #expect(placed[0].confidence < 1) + #expect(placed[0].confidence > 0.5) + } + + private var damagedByCharacters: RecognisedSpeech { + RecognisedSpeech(segments: [ + segment("山春的孩子需要他", 0, 2), + segment("我在一个山村学校当老狮", 2, 5), + segment("教学任务很重", 5, 7), + ], duration: 7) + } + + /// A phrase the recogniser lost entirely still gets a slot between its + /// neighbours, so the reader can play it and fix the boundary — rather + /// than a zero-length one at the start of the recording, which is + /// unplayable and looks like a bug. + @Test("A phrase that was never recognised is still placed between its neighbours") + func unrecognisedPhraseIsSlottedIn() { + let speech = RecognisedSpeech(segments: [ + segment("山村的孩子需要他", 0, 2), + segment("教学任务很重", 5, 7), + ], duration: 7) + let placed = aligned(speech) + + #expect(placed.count == 3) + #expect(placed[1].confidence == 0) + #expect(placed[1].start >= placed[0].end - 0.000_1) + #expect(placed[1].end <= placed[2].start + 0.000_1) + #expect(placed[1].duration > 0) + // Its neighbours are unharmed. + #expect(placed[0].confidence > 0.9) + #expect(placed[2].confidence > 0.9) + } + + /// Confidence is what the UI offers for review instead of silently + /// believing, so it has to mean something: the share of a phrase's + /// characters the recogniser actually matched. + @Test("Confidence reports how much was matched rather than inferred") + func confidenceIsMeaningful() { + #expect(aligned(perfect).allSatisfy { $0.confidence > 0.99 }) + let hopeless = aligned(RecognisedSpeech( + segments: [segment("完全不同的内容", 0, 7)], + duration: 7, + )) + #expect(hopeless.allSatisfy { $0.confidence < 0.4 }) + } + + // MARK: - Cutting again, by the clock + + /// A character budget is only a proxy for duration, and a poor one on + /// material with pauses. Measured on a real 听力 recording: eighteen + /// characters came to ten seconds of audio, and six phrases in eighteen + /// ran past the seven-second cap, the longest at thirty. + @Test("A phrase that turns out too long is cut again at its punctuation") + func overlongPhrasesAreRecut() { + // One sentence, read slowly: eight seconds for what the character + // budget took for four. + let text = "我在一个山村学校当老师，教学任务很重。" + let slow = RecognisedSpeech( + segments: Array(text).enumerated().compactMap { index, character in + let start = Double(index) * 0.45 + return .init(text: String(character), start: start, end: start + 0.45) + }, + duration: 9, + ) + let ranges = PhraseSplitter.phrases(in: text) + #expect(ranges.count == 1) + + let placed = ForcedAlignment.align( + phrases: ranges, + in: text, + to: slow, + maximumDuration: 5, + ) + #expect(placed.count > 1) + #expect(placed.allSatisfy { $0.duration <= 5.5 }) + // And it cut at the comma rather than anywhere convenient. + #expect(placed[0].text(in: text).hasSuffix("，")) + } + + /// The pauses come free: a recogniser folds the silence before a + /// character into that character's own span, so a character taking far + /// longer than its neighbours is one that followed a gap. On the real + /// recording this is what rescued the material — recognised punctuation + /// is sparse, and cutting at punctuation alone left six phrases over. + @Test("A phrase with no punctuation is cut where the speaker paused") + func overlongPhrasesAreCutAtPauses() { + // No punctuation at all, and a two-second gap in the middle, folded + // into the span of the character that follows it. + let text = "山村的孩子需要他我在一个山村学校" + var segments: [RecognisedSpeech.Segment] = [] + for (index, character) in Array(text).enumerated() { + let paused = index == 8 + let start = Double(index) * 0.25 + (index > 8 ? 2 : 0) + segments.append(.init( + text: String(character), + start: start, + end: start + (paused ? 2.25 : 0.25), + )) + } + let speech = RecognisedSpeech(segments: segments, duration: 8) + + let placed = ForcedAlignment.align( + phrases: [0 ..< text.utf16.count], + in: text, + to: speech, + maximumDuration: 3, + ) + #expect(placed.count == 2) + #expect(placed[0].text(in: text) == "山村的孩子需要他") + #expect(placed[1].text(in: text) == "我在一个山村学校") + } + + /// A phrase with nothing to cut at stays long rather than being chopped + /// somewhere arbitrary. The vocabulary list read at the end of an + /// exercise is one unpunctuated minute, and a long phrase is a better + /// unit there than a cut in the middle of a word. + @Test("A phrase with nowhere to cut is left alone") + func unsplittablePhrasesSurviveRecutting() { + let text = "山村的孩子需要他" + let speech = RecognisedSpeech( + segments: Array(text).enumerated().map { index, character in + .init(text: String(character), start: Double(index), end: Double(index) + 1) + }, + duration: 8, + ) + let placed = ForcedAlignment.align( + phrases: [0 ..< text.utf16.count], + in: text, + to: speech, + maximumDuration: 2, + ) + #expect(placed.count == 1) + #expect(placed[0].duration > 2) + } + + @Test("Re-cutting can be switched off") + func recuttingIsOptional() { + let text = "我在一个山村学校当老师，教学任务很重。" + let slow = RecognisedSpeech( + segments: Array(text).enumerated().compactMap { index, character in + let start = Double(index) * 0.45 + return .init(text: String(character), start: start, end: start + 0.45) + }, + duration: 9, + ) + let placed = ForcedAlignment.align( + phrases: PhraseSplitter.phrases(in: text), + in: text, + to: slow, + maximumDuration: nil, + ) + #expect(placed.count == 1) + } + + // MARK: - Degenerate input + + /// No recognition at all — the model missing, or a recording of silence. + /// Equal slices let the reader step through and place the boundaries by + /// hand, and confidence zero is what tells the UI to offer that. + @Test("With nothing recognised the phrases are spread evenly") + func nothingRecognised() { + let placed = aligned(RecognisedSpeech(segments: [], duration: 9)) + #expect(placed.count == 3) + #expect(placed.allSatisfy { $0.confidence == 0 }) + #expect(abs(placed[0].end - 3) < 0.000_1) + #expect(abs(placed[2].end - 9) < 0.000_1) + for (earlier, later) in zip(placed, placed.dropFirst()) { + #expect(earlier.end <= later.start + 0.000_1) + } + } + + @Test("A recogniser that heard only punctuation counts as nothing") + func punctuationOnlyRecognition() { + let placed = aligned(RecognisedSpeech( + segments: [segment("，。！", 0, 7)], + duration: 7, + )) + #expect(placed.count == 3) + #expect(placed.allSatisfy { $0.confidence == 0 }) + } + + @Test("An empty transcript produces no phrases") + func emptyTranscript() { + #expect(ForcedAlignment.align(phrases: [], in: "", to: perfect).isEmpty) + } + + /// A recording whose length is unknown must not collapse every phrase to + /// zero, or nothing can be played at all. + @Test("A recording of unknown length still yields playable phrases") + func unknownDuration() { + let placed = aligned(RecognisedSpeech(segments: perfect.segments, duration: 0)) + #expect(placed.count == 3) + #expect(placed.allSatisfy { $0.duration > 0 }) + } + + // MARK: - Punctuation + + /// A recogniser invents its own punctuation, so matching the + /// transcript's commas against it would be aligning two authors' habits + /// rather than the same speech. + @Test("Punctuation differences do not disturb the alignment") + func punctuationIsIgnored() { + let loose = RecognisedSpeech(segments: [ + segment("山村的孩子，需要他！", 0, 2), + segment("我在一个山村学校，当老师", 2, 5), + segment("教学任务很重", 5, 7), + ], duration: 7) + let placed = aligned(loose) + #expect(placed.allSatisfy { $0.confidence > 0.99 }) + #expect(abs(placed[1].start - 2) < 0.4) + } +} + +@Suite("Alignment tidying") +struct ForcedAlignmentTidyTests { + private func phrase(_ index: Int, _ start: Double, _ end: Double) -> Phrase { + Phrase(index: index, start: start, end: end, textRange: index * 5 ..< index * 5 + 4) + } + + @Test("Overlapping phrases are pushed apart") + func overlapsAreRemoved() { + let tidied = ForcedAlignment.tidy( + [phrase(0, 0, 5), phrase(1, 3, 8), phrase(2, 6, 10)], + duration: 10, + ) + for (earlier, later) in zip(tidied, tidied.dropFirst()) { + #expect(earlier.end <= later.start + 0.000_1) + } + } + + @Test("A phrase running backwards is given a forward span") + func backwardsPhraseIsRepaired() { + let tidied = ForcedAlignment.tidy([phrase(0, 5, 2), phrase(1, 6, 9)], duration: 10) + #expect(tidied[0].end > tidied[0].start) + #expect(tidied.allSatisfy { $0.duration >= ForcedAlignment.minimumDuration - 0.000_1 }) + } + + @Test("Times past the end of the recording are pulled back") + func beyondTheEndIsClamped() { + let tidied = ForcedAlignment.tidy([phrase(0, 0, 4), phrase(1, 8, 99)], duration: 10) + #expect(tidied.allSatisfy { $0.end <= 10.000_1 }) + } + + @Test("Tidying nothing yields nothing") + func empty() { + #expect(ForcedAlignment.tidy([], duration: 10).isEmpty) + } +} diff --git a/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift b/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift new file mode 100644 index 0000000..03cb3ce --- /dev/null +++ b/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift @@ -0,0 +1,91 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import HanReaderCore +import Testing +@testable import HanReaderPlayback + +/// What can be checked without a recording and without the model. +/// +/// The parts that need both — that a real 147-second 听力 recording yields +/// per-character timings, and that slicing the audio at the resulting +/// boundaries re-transcribes to the same words — are verified by hand +/// against the reader's own material and reported in the pull request. A +/// test that downloaded a speech model would be a test that fails on a +/// fresh clone and on CI, which is worse than one that says what it covers. +@Suite("Speech aligner") +struct SpeechAlignerTests { + @Test("Chinese is a language the system can transcribe") + func chineseIsSupported() async { + guard #available(macOS 26.0, iOS 26.0, *) else { return } + #expect(await SpeechAligner().isLanguageSupported()) + } + + /// A locale nobody speaks must be refused before any work is attempted, + /// rather than producing an empty transcription that looks like silence. + @Test("An unsupported language is refused up front") + func unsupportedLanguage() async throws { + guard #available(macOS 26.0, iOS 26.0, *) else { return } + let aligner = SpeechAligner(locale: Locale(identifier: "xx-XX")) + #expect(await aligner.isLanguageSupported() == false) + + await #expect(throws: SpeechAlignmentProblem.languageUnsupported("xx-XX")) { + try await aligner.listen(to: URL(fileURLWithPath: "/dev/null")) + } + } + + /// A file that is not audio, or is not there, is the reader's mistake + /// and has to come back as something the UI can show them. + @Test("An unreadable recording reports itself as unreadable") + func unreadableRecording() async throws { + guard #available(macOS 26.0, iOS 26.0, *) else { return } + let aligner = SpeechAligner() + guard await aligner.isModelInstalled() else { return } + + let missing = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("not-there-\(UUID().uuidString).m4a") + do { + _ = try await aligner.listen(to: missing) + Issue.record("expected a failure") + } catch let problem as SpeechAlignmentProblem { + guard case .unreadable = problem else { + Issue.record("expected .unreadable, got \(problem)") + return + } + } + } + + /// Only a missing model is worth offering the reader a download for. + /// Telling them to install something for a language that does not exist, + /// or for a file that is not audio, would be a dead end dressed up as a + /// remedy. + @Test("Only a missing model is offered as installable") + func recoverability() { + #expect(SpeechAlignmentProblem.modelUnavailable("zh-CN").isRecoverableByInstalling) + #expect(!SpeechAlignmentProblem.languageUnsupported("xx").isRecoverableByInstalling) + #expect(!SpeechAlignmentProblem.unreadable("…").isRecoverableByInstalling) + #expect(!SpeechAlignmentProblem.systemTooOld.isRecoverableByInstalling) + } + + /// Nothing downloads hundreds of megabytes because a reader opened a + /// text. The install is a thing they ask for. + @Test("A missing model is reported rather than fetched unasked") + func modelIsNotFetchedWithoutAsking() async throws { + guard #available(macOS 26.0, iOS 26.0, *) else { return } + // A language the system supports but is most unlikely to have + // installed, so this exercises the refusal rather than the cache. + let aligner = SpeechAligner(locale: Locale(identifier: "ko-KR")) + guard await aligner.isLanguageSupported(), await !aligner.isModelInstalled() else { + return + } + await #expect(throws: SpeechAlignmentProblem.modelUnavailable("ko-KR")) { + try await aligner.listen(to: URL(fileURLWithPath: "/dev/null")) + } + } + + @Test("Progress distinguishes downloading from listening") + func progressShape() { + #expect(SpeechAlignmentProgress(fraction: 0.5).isInstallingModel == false) + #expect(SpeechAlignmentProgress(fraction: nil, isInstallingModel: true).fraction == nil) + } +} From 63996ec66943cdb7114f1350d260e8a717074263 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:50:23 +0300 Subject: [PATCH 2/3] fix: stop a test asserting a fact about this machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SpeechAlignerTests` asserted that the system can transcribe Chinese. True here, where the model had just been installed; false on a headless CI runner, which reports no supported languages at all. So the test passed locally and failed in CI for the only reason it could — it was checking the environment rather than the code. It is conditional now, on something that *is* a property of the code: wherever the system can transcribe anything, it can transcribe Chinese, because that is the language this app exists for and the one the availability check names. Where it transcribes nothing there is nothing to assert, and saying so is better than a green tick meaning "this machine happens to be set up". `SpeechAligner.supportedLanguages()` comes with it, because the same distinction matters above: "this Mac has no speech support" and "it has speech support but not Chinese" need different things said to the reader. Co-Authored-By: Claude Opus 5 --- Sources/HanReaderPlayback/SpeechAligner.swift | 10 ++++++++++ .../SpeechAlignerTests.swift | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Sources/HanReaderPlayback/SpeechAligner.swift b/Sources/HanReaderPlayback/SpeechAligner.swift index 0131c13..9416e71 100644 --- a/Sources/HanReaderPlayback/SpeechAligner.swift +++ b/Sources/HanReaderPlayback/SpeechAligner.swift @@ -86,6 +86,16 @@ public nonisolated struct SpeechAligner: Sendable { await SpeechTranscriber.supportedLocales.contains { $0.matches(locale) } } + /// Every language this system can transcribe. + /// + /// Exposed so a caller can tell "this machine has no speech support at + /// all" from "it has speech support but not for Chinese" — a headless CI + /// runner is the first, a Japanese-only install the second, and they + /// want different things said to the reader. + public static func supportedLanguages() async -> [String] { + await SpeechTranscriber.supportedLocales.map { $0.identifier(.bcp47) } + } + /// Whether the model is already on the device. public func isModelInstalled() async -> Bool { await SpeechTranscriber.installedLocales.contains { $0.matches(locale) } diff --git a/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift b/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift index 03cb3ce..8cbdb81 100644 --- a/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift +++ b/Tests/HanReaderPlaybackTests/SpeechAlignerTests.swift @@ -15,9 +15,19 @@ import Testing /// fresh clone and on CI, which is worse than one that says what it covers. @Suite("Speech aligner") struct SpeechAlignerTests { - @Test("Chinese is a language the system can transcribe") - func chineseIsSupported() async { + /// Conditional on purpose. A headless CI runner reports *no* supported + /// languages at all, so asserting Chinese outright asserts a fact about + /// the machine rather than about the code — which is how this test first + /// passed here and failed in CI. + /// + /// What *is* a fact about the code: wherever the system can transcribe + /// anything, it can transcribe Chinese, because that is the language + /// this app exists for and the one the availability check names. + @Test("Where the system transcribes anything, Chinese is among it") + func chineseIsSupportedWhereAnythingIs() async { guard #available(macOS 26.0, iOS 26.0, *) else { return } + let languages = await SpeechAligner.supportedLanguages() + guard !languages.isEmpty else { return } #expect(await SpeechAligner().isLanguageSupported()) } From fb1cd217dfbe3d33e27a76fa00bd2e808137e79a Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:20:43 +0300 Subject: [PATCH 3/3] fix: answer five review findings on the alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Misheard characters at a phrase's edge were cut out of its audio. A phrase anchored on its first and last *matched* characters begins after a mis-recognised opening character has already been spoken — the character is in the recording, the reader simply does not hear it. A phrase now reaches out to where the neighbouring content ends and begins. Stretching a short final phrase backwards to meet the minimum ignored the phrase before it, so two could replay the same audio and the reader would transcribe the same words twice without being told why. A cut point offered immediately after a terminator stranded the closing quote following it — the same defect as one found in the first part of this milestone, reproduced in new code. Re-cutting a long sentence could hand back a practice phrase that was one 」. Re-cutting rebuilt the document's metrics once per over-long phrase, which is also the same performance mistake as last time in new code: the whole transcript scanned again for an answer concerning one range of it. And listening never said it had finished. The fraction came from the last recognised segment, so a recording trailing into silence stopped at eight-tenths and looked stuck, and one the recogniser heard nothing in never reported at all. **A correction, from testing rather than reasoning.** The 97% slice in the previous commit was attributed to the boundary clipping the onset of 白, and a lead-in was proposed for the transport. Measured: with a 0.25 s lead-in the slice reads 黑白电影里, because the boundary falls *inside the word* 黑白 — the recogniser had put a full stop there, 「只有在黑。白电影里」. So it is neither clipping nor a timing error but a misplaced stop in an ASR-derived transcript, which is an artefact of verifying against a transcript the recogniser wrote. A human transcript punctuates correctly, and the transport needs no lead-in on that evidence. Re-measured after the fixes: progress reaches 1.0 exactly, mean slice accuracy 99% over six phrases, twenty-six phrases with a median of 5.4 s and a longest of 8.0 s. Co-Authored-By: Claude Opus 5 --- .../Listening/ForcedAlignment+Recut.swift | 169 ++++++++++++ .../Listening/ForcedAlignment.swift | 193 +++---------- .../Listening/PhraseSplitter.swift | 31 ++- Sources/HanReaderPlayback/SpeechAligner.swift | 6 + .../ForcedAlignmentTests.swift | 255 +++++++++++------- .../PhraseSplitterTests.swift | 36 +++ 6 files changed, 434 insertions(+), 256 deletions(-) create mode 100644 Sources/HanReaderCore/Listening/ForcedAlignment+Recut.swift diff --git a/Sources/HanReaderCore/Listening/ForcedAlignment+Recut.swift b/Sources/HanReaderCore/Listening/ForcedAlignment+Recut.swift new file mode 100644 index 0000000..3d7b887 --- /dev/null +++ b/Sources/HanReaderCore/Listening/ForcedAlignment+Recut.swift @@ -0,0 +1,169 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// Cutting phrases again once the clock is known. +/// +/// Split from `ForcedAlignment` itself because it is a separate idea that +/// only became possible after measuring real material: a character budget +/// stands in for duration while nothing better exists, and once the audio +/// has been listened to there *is* something better. +extension ForcedAlignment { + // MARK: - Cutting again, by the clock + + /// Re-cuts phrases that turned out too long once their times were known. + /// + /// Only at punctuation the text already has, so a phrase never ends + /// mid-clause — and a phrase with no punctuation left to cut at stays + /// long rather than being chopped somewhere arbitrary. That happens: + /// the vocabulary list read out at the end of an exercise is one + /// unpunctuated minute, and a phrase is a better unit there than a cut + /// in the middle of a word. + static func recut( + _ phrases: [Range], + in transcript: String, + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + maximumDuration: Double, + ) + -> [Range] + { + let typical = medianDuration(of: anchors) + // Once for the document, not once per over-long phrase. + let metrics = PhraseSplitter.Metrics(transcript) + var result: [Range] = [] + for phrase in phrases { + guard span(of: phrase, expected: expected, anchors: anchors) + .map({ $0.upperBound - $0.lowerBound > maximumDuration }) == true + else { + result.append(phrase) + continue + } + let points = cutPoints( + in: phrase, + metrics: metrics, + expected: expected, + anchors: anchors, + typical: typical, + ) + guard !points.isEmpty else { + result.append(phrase) + continue + } + // Every clause, then packed back up by measured time rather than + // by a character count standing in for it. + var pieces: [Range] = [] + var lower = phrase.lowerBound + for point in points + [phrase.upperBound] where point > lower { + pieces.append(lower ..< point) + lower = point + } + result.append(contentsOf: pack( + pieces, + expected: expected, + anchors: anchors, + maximumDuration: maximumDuration, + )) + } + return result + } + + /// Where a phrase could be cut: at its own punctuation, and wherever + /// the speaker paused. + /// + /// The pauses come free. A recogniser folds the silence before a + /// character into that character's own span, so a character taking far + /// longer than its neighbours is one that followed a gap — on a real + /// 听力 recording a typical character runs 0.24 s and the first one + /// after a pause runs 1.8. That is silence detection without decoding a + /// single sample, and it is what actually rescues this material: + /// recognised punctuation is sparse, so cutting at punctuation alone + /// left six phrases in eighteen over the cap. + static func cutPoints( + in phrase: Range, + metrics: PhraseSplitter.Metrics, + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + typical: Double, + ) + -> [Int] + { + var points = Set(PhraseSplitter.cutPoints(in: phrase, metrics: metrics)) + let threshold = max(pauseFloor, typical * pauseMultiple) + for (index, position) in expected.enumerated() + where phrase.contains(position.offset) && position.offset > phrase.lowerBound + { + guard let anchor = anchors[index] else { continue } + if anchor.upperBound - anchor.lowerBound > threshold { + // Cut *before* the character, because the silence belongs to + // the gap and the character starts what comes after it. + points.insert(position.offset) + } + } + return points.sorted() + } + + private static func medianDuration(of anchors: [Int: ClosedRange]) -> Double { + let durations = anchors.values + .map { $0.upperBound - $0.lowerBound } + .filter { $0 > 0 } + .sorted() + guard !durations.isEmpty else { return 0 } + return durations[durations.count / 2] + } + + /// A gap has to be this long to count, however brisk the speaker. Below + /// it the "pause" is the natural lengthening at the end of a clause, + /// which is not somewhere to break. + static let pauseFloor = 0.6 + /// And this much longer than the speaker's own typical character. + static let pauseMultiple = 3.0 + + private static func pack( + _ pieces: [Range], + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + maximumDuration: Double, + ) + -> [Range] + { + var packed: [Range] = [] + for piece in pieces { + guard let last = packed.last else { + packed.append(piece) + continue + } + let combined = last.lowerBound ..< piece.upperBound + let duration = span(of: combined, expected: expected, anchors: anchors) + .map { $0.upperBound - $0.lowerBound } ?? 0 + if duration <= maximumDuration { + packed[packed.count - 1] = combined + } else { + packed.append(piece) + } + } + return packed + } + + /// When a range was heard, from the first anchored character in it to + /// the last. Nil when the recogniser anchored none of it, which is the + /// case re-cutting must leave alone — there is no clock to cut by. + private static func span( + of range: Range, + expected: [Timeline.Position], + anchors: [Int: ClosedRange], + ) + -> ClosedRange? + { + var found: [ClosedRange] = [] + for (index, position) in expected.enumerated() + where range.contains(position.offset) + { + if let anchor = anchors[index] { + found.append(anchor) + } + } + guard let first = found.first, let last = found.last else { return nil } + return first.lowerBound ... max(first.lowerBound, last.upperBound) + } +} diff --git a/Sources/HanReaderCore/Listening/ForcedAlignment.swift b/Sources/HanReaderCore/Listening/ForcedAlignment.swift index 59edfba..413d6e7 100644 --- a/Sources/HanReaderCore/Listening/ForcedAlignment.swift +++ b/Sources/HanReaderCore/Listening/ForcedAlignment.swift @@ -115,162 +115,6 @@ public enum ForcedAlignment { ) } - // MARK: - Cutting again, by the clock - - /// Re-cuts phrases that turned out too long once their times were known. - /// - /// Only at punctuation the text already has, so a phrase never ends - /// mid-clause — and a phrase with no punctuation left to cut at stays - /// long rather than being chopped somewhere arbitrary. That happens: - /// the vocabulary list read out at the end of an exercise is one - /// unpunctuated minute, and a phrase is a better unit there than a cut - /// in the middle of a word. - static func recut( - _ phrases: [Range], - in transcript: String, - expected: [Timeline.Position], - anchors: [Int: ClosedRange], - maximumDuration: Double, - ) - -> [Range] - { - let typical = medianDuration(of: anchors) - var result: [Range] = [] - for phrase in phrases { - guard span(of: phrase, expected: expected, anchors: anchors) - .map({ $0.upperBound - $0.lowerBound > maximumDuration }) == true - else { - result.append(phrase) - continue - } - let points = cutPoints( - in: phrase, - of: transcript, - expected: expected, - anchors: anchors, - typical: typical, - ) - guard !points.isEmpty else { - result.append(phrase) - continue - } - // Every clause, then packed back up by measured time rather than - // by a character count standing in for it. - var pieces: [Range] = [] - var lower = phrase.lowerBound - for point in points + [phrase.upperBound] where point > lower { - pieces.append(lower ..< point) - lower = point - } - result.append(contentsOf: pack( - pieces, - expected: expected, - anchors: anchors, - maximumDuration: maximumDuration, - )) - } - return result - } - - /// Where a phrase could be cut: at its own punctuation, and wherever - /// the speaker paused. - /// - /// The pauses come free. A recogniser folds the silence before a - /// character into that character's own span, so a character taking far - /// longer than its neighbours is one that followed a gap — on a real - /// 听力 recording a typical character runs 0.24 s and the first one - /// after a pause runs 1.8. That is silence detection without decoding a - /// single sample, and it is what actually rescues this material: - /// recognised punctuation is sparse, so cutting at punctuation alone - /// left six phrases in eighteen over the cap. - static func cutPoints( - in phrase: Range, - of transcript: String, - expected: [Timeline.Position], - anchors: [Int: ClosedRange], - typical: Double, - ) - -> [Int] - { - var points = Set(PhraseSplitter.cutPoints(in: phrase, of: transcript)) - let threshold = max(pauseFloor, typical * pauseMultiple) - for (index, position) in expected.enumerated() - where phrase.contains(position.offset) && position.offset > phrase.lowerBound - { - guard let anchor = anchors[index] else { continue } - if anchor.upperBound - anchor.lowerBound > threshold { - // Cut *before* the character, because the silence belongs to - // the gap and the character starts what comes after it. - points.insert(position.offset) - } - } - return points.sorted() - } - - private static func medianDuration(of anchors: [Int: ClosedRange]) -> Double { - let durations = anchors.values - .map { $0.upperBound - $0.lowerBound } - .filter { $0 > 0 } - .sorted() - guard !durations.isEmpty else { return 0 } - return durations[durations.count / 2] - } - - /// A gap has to be this long to count, however brisk the speaker. Below - /// it the "pause" is the natural lengthening at the end of a clause, - /// which is not somewhere to break. - static let pauseFloor = 0.6 - /// And this much longer than the speaker's own typical character. - static let pauseMultiple = 3.0 - - private static func pack( - _ pieces: [Range], - expected: [Timeline.Position], - anchors: [Int: ClosedRange], - maximumDuration: Double, - ) - -> [Range] - { - var packed: [Range] = [] - for piece in pieces { - guard let last = packed.last else { - packed.append(piece) - continue - } - let combined = last.lowerBound ..< piece.upperBound - let duration = span(of: combined, expected: expected, anchors: anchors) - .map { $0.upperBound - $0.lowerBound } ?? 0 - if duration <= maximumDuration { - packed[packed.count - 1] = combined - } else { - packed.append(piece) - } - } - return packed - } - - /// When a range was heard, from the first anchored character in it to - /// the last. Nil when the recogniser anchored none of it, which is the - /// case re-cutting must leave alone — there is no clock to cut by. - private static func span( - of range: Range, - expected: [Timeline.Position], - anchors: [Int: ClosedRange], - ) - -> ClosedRange? - { - var found: [ClosedRange] = [] - for (index, position) in expected.enumerated() - where range.contains(position.offset) - { - if let anchor = anchors[index] { - found.append(anchor) - } - } - guard let first = found.first, let last = found.last else { return nil } - return first.lowerBound ... max(first.lowerBound, last.upperBound) - } - // MARK: - Placing private static func place( @@ -297,6 +141,7 @@ public enum ForcedAlignment { bounds.append((lower, upper)) } + let anchored = anchors.keys.sorted() var placed: [Phrase] = [] var previousEnd = 0.0 for (index, phrase) in phrases.enumerated() { @@ -307,8 +152,31 @@ public enum ForcedAlignment { // A phrase the recogniser matched nothing in gets a slot between // its neighbours rather than a zero-length one at the start of // the recording, which would be unplayable and look like a bug. - let start = matched.first?.lowerBound ?? previousEnd - let end = matched.last?.upperBound ?? start + var start = matched.first?.lowerBound ?? previousEnd + var end = matched.last?.upperBound ?? start + + // Characters the recogniser misheard at a phrase's edge are + // still *said* in the recording, and anchoring only on matches + // cuts them off: a phrase whose first character was misheard + // begins after that character has been spoken, and the reader + // hears it clipped. Reach out to where the neighbouring content + // actually ends and begins instead. + if !matched.isEmpty { + if let first = (lower ..< upper).first(where: { anchors[$0] != nil }), + first > lower, + let previous = anchored.last(where: { $0 < lower }), + let span = anchors[previous] + { + start = min(start, span.upperBound) + } + if let last = (lower ..< upper).last(where: { anchors[$0] != nil }), + last < upper - 1, + let next = anchored.first(where: { $0 >= upper }), + let span = anchors[next] + { + end = max(end, span.lowerBound) + } + } placed.append(Phrase( index: index, @@ -357,10 +225,13 @@ public enum ForcedAlignment { end = min(start + minimumDuration, limit) } // At the very end of a short recording there is nothing left to - // stretch into, so take the span out of the silence before it - // rather than hand back something unplayable. + // stretch into, so take the span out of the silence before it — + // but never back past the phrase before, or two phrases replay + // the same audio and the reader transcribes the same words + // twice without being told why. if end - start < minimumDuration { - start = max(0, end - minimumDuration) + start = max(floor, max(0, end - minimumDuration)) + end = max(end, start) } result.append(Phrase( diff --git a/Sources/HanReaderCore/Listening/PhraseSplitter.swift b/Sources/HanReaderCore/Listening/PhraseSplitter.swift index 8a5d239..6f38f3a 100644 --- a/Sources/HanReaderCore/Listening/PhraseSplitter.swift +++ b/Sources/HanReaderCore/Listening/PhraseSplitter.swift @@ -278,16 +278,41 @@ public enum PhraseSplitter { /// at the same places this would have cut, chosen by the clock instead /// of by a guess at it. public static func cutPoints(in range: Range, of text: String) -> [Int] { - let metrics = Metrics(text) + cutPoints(in: range, metrics: Metrics(text)) + } + + /// The same, for a caller that already has the metrics. + /// + /// Building them is proportional to the whole document, so a caller + /// re-cutting many phrases of a chapter must not rebuild them per + /// phrase — that is the document scanned once per over-long phrase, for + /// an answer that only concerns one range of it. + static func cutPoints(in range: Range, metrics: Metrics) -> [Int] { var points: [Int] = [] var offset = range.lowerBound + var pending: Int? for character in metrics.text(in: range) { offset += character.utf16.count - guard offset > range.lowerBound, offset < range.upperBound else { continue } + // A closing quote after a full stop belongs to the clause it + // closes, so the cut moves past it. Otherwise re-cutting a long + // sentence hands back a practice phrase that is one 」. + if pending != nil, trailing.contains(character) { + pending = offset + continue + } + if let point = pending { + if point > range.lowerBound, point < range.upperBound { + points.append(point) + } + pending = nil + } if separators.contains(character) || terminators.contains(character) { - points.append(offset) + pending = offset } } + if let point = pending, point > range.lowerBound, point < range.upperBound { + points.append(point) + } return points } diff --git a/Sources/HanReaderPlayback/SpeechAligner.swift b/Sources/HanReaderPlayback/SpeechAligner.swift index 9416e71..2937ad2 100644 --- a/Sources/HanReaderPlayback/SpeechAligner.swift +++ b/Sources/HanReaderPlayback/SpeechAligner.swift @@ -188,6 +188,12 @@ public nonisolated struct SpeechAligner: Sendable { progress(SpeechAlignmentProgress(fraction: min(1, last.end / duration))) } } + // Said plainly at the end, rather than left at whatever the last + // segment implied. A recording that trails off into silence would + // otherwise stop at eight-tenths and look stuck, and one the + // recogniser heard nothing in would never report at all — which is + // the case a reader most needs to see finish. + progress(SpeechAlignmentProgress(fraction: 1)) return RecognisedSpeech(segments: segments, duration: duration) } diff --git a/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift b/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift index 2845d18..130e79f 100644 --- a/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift +++ b/Tests/HanReaderCoreTests/ForcedAlignmentTests.swift @@ -165,112 +165,55 @@ struct ForcedAlignmentTests { #expect(hopeless.allSatisfy { $0.confidence < 0.4 }) } - // MARK: - Cutting again, by the clock - - /// A character budget is only a proxy for duration, and a poor one on - /// material with pauses. Measured on a real 听力 recording: eighteen - /// characters came to ten seconds of audio, and six phrases in eighteen - /// ran past the seven-second cap, the longest at thirty. - @Test("A phrase that turns out too long is cut again at its punctuation") - func overlongPhrasesAreRecut() { - // One sentence, read slowly: eight seconds for what the character - // budget took for four. - let text = "我在一个山村学校当老师，教学任务很重。" - let slow = RecognisedSpeech( - segments: Array(text).enumerated().compactMap { index, character in - let start = Double(index) * 0.45 - return .init(text: String(character), start: start, end: start + 0.45) - }, - duration: 9, - ) - let ranges = PhraseSplitter.phrases(in: text) - #expect(ranges.count == 1) - - let placed = ForcedAlignment.align( - phrases: ranges, - in: text, - to: slow, - maximumDuration: 5, - ) - #expect(placed.count > 1) - #expect(placed.allSatisfy { $0.duration <= 5.5 }) - // And it cut at the comma rather than anywhere convenient. - #expect(placed[0].text(in: text).hasSuffix("，")) - } - - /// The pauses come free: a recogniser folds the silence before a - /// character into that character's own span, so a character taking far - /// longer than its neighbours is one that followed a gap. On the real - /// recording this is what rescued the material — recognised punctuation - /// is sparse, and cutting at punctuation alone left six phrases over. - @Test("A phrase with no punctuation is cut where the speaker paused") - func overlongPhrasesAreCutAtPauses() { - // No punctuation at all, and a two-second gap in the middle, folded - // into the span of the character that follows it. - let text = "山村的孩子需要他我在一个山村学校" + /// Review finding: anchoring a phrase on its first and last *matched* + /// characters cuts off any it misheard at the edges — the character is + /// still spoken in the recording, so the reader hears the phrase + /// clipped. This is what a 97% round trip looked like on real audio. + @Test("A misheard first character is still inside the phrase's audio") + func misheardEdgesAreCovered() { + let text = "山村的孩子。教学任务很重。" + // 教 misheard as 交, so the second phrase's first character is not + // an anchor. Its audio must still begin where 。 left off. var segments: [RecognisedSpeech.Segment] = [] - for (index, character) in Array(text).enumerated() { - let paused = index == 8 - let start = Double(index) * 0.25 + (index > 8 ? 2 : 0) - segments.append(.init( - text: String(character), - start: start, - end: start + (paused ? 2.25 : 0.25), - )) + for (index, character) in Array("山村的孩子交学任务很重").enumerated() { + let start = Double(index) * 0.5 + segments.append(.init(text: String(character), start: start, end: start + 0.5)) } - let speech = RecognisedSpeech(segments: segments, duration: 8) - + let speech = RecognisedSpeech(segments: segments, duration: 6) let placed = ForcedAlignment.align( - phrases: [0 ..< text.utf16.count], + phrases: PhraseSplitter.phrases(in: text), in: text, to: speech, - maximumDuration: 3, + maximumDuration: nil, ) - #expect(placed.count == 2) - #expect(placed[0].text(in: text) == "山村的孩子需要他") - #expect(placed[1].text(in: text) == "我在一个山村学校") - } - /// A phrase with nothing to cut at stays long rather than being chopped - /// somewhere arbitrary. The vocabulary list read at the end of an - /// exercise is one unpunctuated minute, and a long phrase is a better - /// unit there than a cut in the middle of a word. - @Test("A phrase with nowhere to cut is left alone") - func unsplittablePhrasesSurviveRecutting() { - let text = "山村的孩子需要他" - let speech = RecognisedSpeech( - segments: Array(text).enumerated().map { index, character in - .init(text: String(character), start: Double(index), end: Double(index) + 1) - }, - duration: 8, - ) - let placed = ForcedAlignment.align( - phrases: [0 ..< text.utf16.count], - in: text, - to: speech, - maximumDuration: 2, - ) - #expect(placed.count == 1) - #expect(placed[0].duration > 2) + #expect(placed.count == 2) + // 交 is spoken over 2.5–3.0, and the phrase must include it rather + // than starting at 3.0 where 学 was matched. + #expect(placed[1].start <= 2.51) + #expect(placed[1].confidence < 1) } - @Test("Re-cutting can be switched off") - func recuttingIsOptional() { - let text = "我在一个山村学校当老师，教学任务很重。" - let slow = RecognisedSpeech( - segments: Array(text).enumerated().compactMap { index, character in - let start = Double(index) * 0.45 - return .init(text: String(character), start: start, end: start + 0.45) - }, - duration: 9, - ) + /// And at the other edge. + @Test("A misheard last character is still inside the phrase's audio") + func misheardTrailingEdgeIsCovered() { + let text = "山村的孩子。教学任务很重。" + var segments: [RecognisedSpeech.Segment] = [] + for (index, character) in Array("山村的孩紫教学任务很重").enumerated() { + let start = Double(index) * 0.5 + segments.append(.init(text: String(character), start: start, end: start + 0.5)) + } + let speech = RecognisedSpeech(segments: segments, duration: 6) let placed = ForcedAlignment.align( phrases: PhraseSplitter.phrases(in: text), in: text, - to: slow, + to: speech, maximumDuration: nil, ) - #expect(placed.count == 1) + #expect(placed.count == 2) + // 紫 stands where 子 was said, over 2.0–2.5, so the first phrase has + // to reach 2.5 rather than stopping at 2.0. + #expect(placed[0].end >= 2.49) } // MARK: - Degenerate input @@ -362,8 +305,136 @@ struct ForcedAlignmentTidyTests { #expect(tidied.allSatisfy { $0.end <= 10.000_1 }) } + /// Review finding: stretching a short final phrase backwards to reach + /// the minimum could reach back over its predecessor, so two phrases + /// replayed the same audio and the reader transcribed the same words + /// twice without being told why. + @Test("Reaching back for a minimum never overlaps the phrase before") + func backwardStretchRespectsItsNeighbour() { + // The last phrase is a sliver at the very end of the recording, so + // the only room left is behind it. + let tidied = ForcedAlignment.tidy( + [phrase(0, 0, 9.95), phrase(1, 9.96, 9.97)], + duration: 10, + ) + for (earlier, later) in zip(tidied, tidied.dropFirst()) { + #expect(earlier.end <= later.start + 0.000_1) + } + #expect(tidied.allSatisfy { $0.end <= 10.000_1 }) + } + @Test("Tidying nothing yields nothing") func empty() { #expect(ForcedAlignment.tidy([], duration: 10).isEmpty) } } + +/// Re-cutting a phrase that turned out too long once its times were known. +@Suite("Alignment re-cutting") +struct ForcedAlignmentRecutTests { + /// A character budget is only a proxy for duration, and a poor one on + /// material with pauses. Measured on a real 听力 recording: eighteen + /// characters came to ten seconds of audio, and six phrases in eighteen + /// ran past the seven-second cap, the longest at thirty. + @Test("A phrase that turns out too long is cut again at its punctuation") + func overlongPhrasesAreRecut() { + // One sentence, read slowly: eight seconds for what the character + // budget took for four. + let text = "我在一个山村学校当老师，教学任务很重。" + let slow = RecognisedSpeech( + segments: Array(text).enumerated().compactMap { index, character in + let start = Double(index) * 0.45 + return .init(text: String(character), start: start, end: start + 0.45) + }, + duration: 9, + ) + let ranges = PhraseSplitter.phrases(in: text) + #expect(ranges.count == 1) + + let placed = ForcedAlignment.align( + phrases: ranges, + in: text, + to: slow, + maximumDuration: 5, + ) + #expect(placed.count > 1) + #expect(placed.allSatisfy { $0.duration <= 5.5 }) + // And it cut at the comma rather than anywhere convenient. + #expect(placed[0].text(in: text).hasSuffix("，")) + } + + /// The pauses come free: a recogniser folds the silence before a + /// character into that character's own span, so a character taking far + /// longer than its neighbours is one that followed a gap. On the real + /// recording this is what rescued the material — recognised punctuation + /// is sparse, and cutting at punctuation alone left six phrases over. + @Test("A phrase with no punctuation is cut where the speaker paused") + func overlongPhrasesAreCutAtPauses() { + // No punctuation at all, and a two-second gap in the middle, folded + // into the span of the character that follows it. + let text = "山村的孩子需要他我在一个山村学校" + var segments: [RecognisedSpeech.Segment] = [] + for (index, character) in Array(text).enumerated() { + let paused = index == 8 + let start = Double(index) * 0.25 + (index > 8 ? 2 : 0) + segments.append(.init( + text: String(character), + start: start, + end: start + (paused ? 2.25 : 0.25), + )) + } + let speech = RecognisedSpeech(segments: segments, duration: 8) + + let placed = ForcedAlignment.align( + phrases: [0 ..< text.utf16.count], + in: text, + to: speech, + maximumDuration: 3, + ) + #expect(placed.count == 2) + #expect(placed[0].text(in: text) == "山村的孩子需要他") + #expect(placed[1].text(in: text) == "我在一个山村学校") + } + + /// A phrase with nothing to cut at stays long rather than being chopped + /// somewhere arbitrary. The vocabulary list read at the end of an + /// exercise is one unpunctuated minute, and a long phrase is a better + /// unit there than a cut in the middle of a word. + @Test("A phrase with nowhere to cut is left alone") + func unsplittablePhrasesSurviveRecutting() { + let text = "山村的孩子需要他" + let speech = RecognisedSpeech( + segments: Array(text).enumerated().map { index, character in + .init(text: String(character), start: Double(index), end: Double(index) + 1) + }, + duration: 8, + ) + let placed = ForcedAlignment.align( + phrases: [0 ..< text.utf16.count], + in: text, + to: speech, + maximumDuration: 2, + ) + #expect(placed.count == 1) + #expect(placed[0].duration > 2) + } + + @Test("Re-cutting can be switched off") + func recuttingIsOptional() { + let text = "我在一个山村学校当老师，教学任务很重。" + let slow = RecognisedSpeech( + segments: Array(text).enumerated().compactMap { index, character in + let start = Double(index) * 0.45 + return .init(text: String(character), start: start, end: start + 0.45) + }, + duration: 9, + ) + let placed = ForcedAlignment.align( + phrases: PhraseSplitter.phrases(in: text), + in: text, + to: slow, + maximumDuration: nil, + ) + #expect(placed.count == 1) + } +} diff --git a/Tests/HanReaderCoreTests/PhraseSplitterTests.swift b/Tests/HanReaderCoreTests/PhraseSplitterTests.swift index 863c4b1..3964766 100644 --- a/Tests/HanReaderCoreTests/PhraseSplitterTests.swift +++ b/Tests/HanReaderCoreTests/PhraseSplitterTests.swift @@ -190,6 +190,42 @@ struct PhraseSplitterTests { #expect(result.joined() == source) } + // MARK: - Cut points + + /// Review finding: offering a cut immediately after a terminator + /// stranded the closing quote that follows it, and when the clause + /// before was already over the duration cap nothing could merge it back + /// — so re-cutting a long sentence handed back a practice phrase that + /// was one 」. + @Test("A cut after a terminator carries its closing quote along") + func cutPointsCarryClosingMarks() { + let source = "他说：“这是什么意思？”我不知道，真的不知道。" + let points = PhraseSplitter.cutPoints(in: 0 ..< source.utf16.count, of: source) + let utf16 = Array(source.utf16) + for point in points { + // Nothing after a cut may begin with a closing mark. + guard point < utf16.count, let scalar = Unicode.Scalar(utf16[point]) else { continue } + #expect(!PhraseSplitter.trailing.contains(Character(scalar))) + } + #expect(!points.isEmpty) + } + + @Test("Cut points sit strictly inside the range they are asked for") + func cutPointsStayInside() { + let source = "第一句，第二句。第三句。" + let range = 0 ..< source.utf16.count + for point in PhraseSplitter.cutPoints(in: range, of: source) { + #expect(point > range.lowerBound) + #expect(point < range.upperBound) + } + } + + @Test("A range with no punctuation offers nowhere to cut") + func noCutPoints() { + let source = "山村的孩子需要他" + #expect(PhraseSplitter.cutPoints(in: 0 ..< source.utf16.count, of: source).isEmpty) + } + // MARK: - Ranges /// Phrases are consumed as UTF-16 ranges into the stored text, the same