From e576e63c2d64b5201b167fbe1b4fe13c1ada95d6 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:40:46 -0800 Subject: [PATCH] Read a tone's pitch past breaks in its waveform (CHALLENGES 390) Fixes #334. An emulator started by hand breaks a tone's phase through its host audio output, 151 breaks in 48 s against none with -no-audio, same app. The FFT read 430 Hz for 440, which split one sound in two. Not a dropout, as the issue guessed: the tone is unbroken, and packets were neither late nor short. Measured by hand: audio.sh main 1 of 6, this 6 of 6. Real captures at -42 to -49 dBFS split 6 of 68 on main and none here; at -62 dBFS, 5 of 14 and 2 of 14. Also: -port leaves an emulator with no control port, so SETUP.md no longer says to pass it and the refusal names it. audio.sh took a tone's first instant for a sound in the pause; a sound inside the pause still fails it. --- docs/CHALLENGES.md | 56 ++++++++++++++++++++++++++- docs/RELEASE-CHECKLIST.md | 2 +- docs/SETUP.md | 14 +++++-- docs/checks/audio.sh | 5 ++- internal/audio/audio.go | 75 +++++++++++++++++++++++++++++++++++- internal/audio/audio_test.go | 70 +++++++++++++++++++++++++++++++++ internal/device/emuaudio.go | 2 +- 7 files changed, 215 insertions(+), 9 deletions(-) diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 5e0313f1..39cad8ea 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -7,7 +7,7 @@ the source. The pattern across them is the reason the document exists: **almost nothing here was found by reading code or by a test written from imagination.** Of -389 defects, 311 were found only by running against a real device. Seventeen +390 defects, 312 were found only by running against a real device. Seventeen more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386, 387 and 388 — were met while running against one, though the fault lay in a check, the host or Mobium's own design rather than in the platform. The other sixty-one — 4, 5, 6, 14, 23, @@ -8590,6 +8590,60 @@ tap on the emulator. `press back` and `press home` on a sleeping Android answer after 20 s, saying what they could not read. A read on a sleeping Android takes 10 s to fail, so these refusals do too. +### 390. A steady tone was heard as two sounds + +**Found by:** a sweep of `audio.sh` (mobiumdev/mobium#334), which passed 1 of 6 on +an Android 15 emulator started by hand, on `main` as much as on a branch. `440 Hz +0.4-0.7s`, then `440 Hz 0.8-2.4s`, for one 2-second tone. + +Four explanations were measured and dropped. Packets were not late: none arrived +more than 60ms after the last, and no silence was padded in. They were not short: +the stream ran at the full 48 kHz. There was no dropout, which the issue guessed: +the tone sounded unbroken, and only 2 of 31 off-pitch windows held a run of zeros. +The packet's `timestamp` could not have helped: it tracks arrival to within a +millisecond, so it says when a packet was sent. + +The samples hold the cause. The waveform has pairs of breaks 33 samples apart, the +tone's phase jumping across them, 113 degrees at 440 Hz. An FFT across them read +430 Hz for 440, and 2%, 8.8 Hz there, is what holds one sound together. + +**The breaks are the emulator's host audio output.** The same app and the same +tones gave 151 breaks in 48 s on an emulator started by hand, and none on one from +`mobium boot`, which passes `-no-audio`. They cluster after audio starts: captures +in a row held 26, 10, 2, 0, 0, 0. That is why a loop of captures rarely split and +the check, whose tones come early, usually did. `-no-audio` is not a cure that +reaches everyone: `docs/SETUP.md` showed the hand start. + +**Fixed:** `steadyPitch` cuts a window at each break and takes the median of the +stretches, which holds if one hides a break too small to see. Stretches that are +not one sound are two notes, read whole as before, so a melody and a real pitch +change still split. `docs/SETUP.md` now boots with `mobium boot`, and says what a +hand start costs. + +Two more, found on the way. `-port` leaves an emulator with no discovery file and +no control port, even with `-grpc-use-token`, measured on emulator 37.1.11; two +`mobium boot`s take 5554 and 5556 by themselves and both capture. `SETUP.md` said to +pass it, and the refusal blamed `-no-grpc` alone. And `audio.sh` took a tone's own +first instant, a pitchless sound touching it, for a sound in the pause, failing +both versions now and then; a sound inside the pause still fails it. + +**Measured:** `audio.sh` by hand, `main` 1 of 6 and this 6 of 6, in turn on one +emulator before the pause fix made it 5 of 6 and after. On `-no-audio`, 4 of 4, and +Android 17 passes. `audio-sim.sh` passes. Real captures by hand, `main` then this: + +| Level | Split captures | Windows over 0.5 Hz off | +| --- | --- | --- | +| -42 dBFS, 36 captures | 1, then 0 | 85 of 747, then 14 | +| -46 dBFS, 16 captures | 2, then 0 | 32 of 327, then 6 | +| -49 dBFS, 16 captures | 3, then 0 | 50 of 329, then 10 | +| -62 dBFS, 14 captures | 5, then 2 | 32 of 283, then 10 | + +**Still so:** at -62 dBFS, the quietest step that sounds here, a tone by hand still +splits 2 in 14: breaks that small go unseen. A synthetic spliced tone holds to 18 +kHz at -42 dBFS, and above 1320 Hz neither version split, since a break bends a +higher tone less. Not measured: a tone from anything but MobiumApp (Chrome played +nothing), and any other host. Not run on a phone, which captures no samples. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index 38b0269d..ec5f290e 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -4,7 +4,7 @@ [CI workflow](../.github/workflows/ci.yml) runs on every push — formatting, vet, both modules' tests, the four non-Go clients' own tests, and six cross-compile targets. This file is the rest, and the rest is where most -serious defects in this project have come from: 311 of 389 were found only by +serious defects in this project have come from: 312 of 390 were found only by running against a real device. Work through it before tagging a release, on three substrates: an Android diff --git a/docs/SETUP.md b/docs/SETUP.md index 2b80f67a..88b9272b 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -257,8 +257,8 @@ Wear OS images are on the stable channel, Wear OS 3 to 6.1, and 7.0 at API sdkmanager "system-images;android-35-ext15;android-wear;arm64-v8a" # Wear OS 5.1 avdmanager create avd -n mobium-wear -d wearos_large_round \ -k "system-images;android-35-ext15;android-wear;arm64-v8a" -emulator -avd mobium-wear -port 5558 -no-snapshot-load -no-boot-anim & -docs/checks/wear-os.sh emulator-5558 +mobium boot mobium-wear +docs/checks/wear-os.sh ``` `avdmanager` prints "Could not load devices" for another image's @@ -269,10 +269,16 @@ too. The crown has no command yet. ### Running two at once ```sh -emulator -avd mobium-test -no-snapshot-save -no-boot-anim & # 5554 -emulator -avd mobium-test-17 -no-snapshot-save -no-boot-anim -port 5556 & # 5556 +mobium boot mobium-test # emulator-5554 +mobium boot mobium-test-17 # emulator-5556 ``` +Each takes the next free port by itself. Started by hand, do not pass `-port`: +measured on emulator 37, it leaves no control port, so `mobium audio` is refused. +By hand, also pass `-no-audio`, as `mobium boot` does. Without it the host's audio +output breaks up what a capture hears: 151 breaks in 48 s of the same tones where +`-no-audio` had none. CHALLENGES 390. + Then pass `--device emulator-5556` to every command, or give a client the device option. With two ready, a call that names no device is refused, so `MOBIUM_SESSION` alone does not choose one: it gives each run its own daemon diff --git a/docs/checks/audio.sh b/docs/checks/audio.sh index 0786a777..f9d60071 100755 --- a/docs/checks/audio.sh +++ b/docs/checks/audio.sh @@ -191,7 +191,10 @@ import json, sys segs = json.load(open(sys.argv[1]))["timeline"] tones = [s for s in segs if s["sound"] and (s["to"] - s["from"]) / 1e9 > 0.2] gap = (tones[1]["from"] - tones[0]["to"]) / 1e9 -quiet = [s for s in segs if s["from"] >= tones[0]["to"] and s["to"] <= tones[1]["from"]] +# A tone starting at the end of a tenth of a second is a sound with no pitch +# there, touching the tone. It is the tone, not a sound in the pause. +quiet = [s for s in segs if s["from"] >= tones[0]["to"] and s["to"] <= tones[1]["from"] + and not (s["to"] - s["from"] <= 100000000 and (s["to"] == tones[1]["from"] or s["from"] == tones[0]["to"]))] if not (0.8 <= gap <= 1.2 and all(not s["sound"] for s in quiet)): sys.exit("heard " + ", ".join("%.1f-%.1fs %s" % (s["from"] / 1e9, s["to"] / 1e9, ("%.0f Hz" % s.get("hz", 0)) if s["sound"] else "silence") for s in segs)) diff --git a/internal/audio/audio.go b/internal/audio/audio.go index 49ed9a21..bd000adf 100644 --- a/internal/audio/audio.go +++ b/internal/audio/audio.go @@ -156,9 +156,82 @@ func read(x []int16, rate int) window { if first < 0 || last-first < rate/200 { return window{level: level} } - return window{level: level, hz: pitch(x[first:last+1], rate)} + return window{level: level, hz: steadyPitch(x[first:last+1], rate)} } +// steadyPitch is the pitch of x, read past breaks in its waveform. The Android +// 15 emulator's stream breaks a tone's phase every 35ms or so, and an FFT across +// the breaks bends the pitch past what holds one sound together. Stretches of one +// sound give their median, since one may hold a break too small to see. Stretches +// that are not one sound are two notes, and x is read whole, as it always was. +// CHALLENGES 390. +func steadyPitch(x []int16, rate int) float64 { + shortest := rate / minStretch + var read []float64 + for _, part := range stretches(x) { + if len(part) < shortest { + break + } + hz := pitch(part, rate) + if hz == 0 { + return pitch(x, rate) + } + read = append(read, hz) + } + if len(read) < 2 { + return pitch(x, rate) + } + sort.Float64s(read) + if !samePitch(read[0], read[len(read)-1]) { + return pitch(x, rate) + } + return read[len(read)/2] +} + +// stretches cuts x at each break in its waveform, longest first. +func stretches(x []int16) [][]int16 { + if len(x) < 3 { + return [][]int16{x} + } + bends := make([]int, len(x)-2) + for i := range bends { + b := int(x[i+2]) - 2*int(x[i+1]) + int(x[i]) + if b < 0 { + b = -b + } + bends[i] = b + } + sorted := append([]int(nil), bends...) + sort.Ints(sorted) + limit := max(minBreak, breakOverTypical*sorted[len(sorted)/2]) + + var parts [][]int16 + from := 0 + for i, b := range bends { + if b > limit { + if i > from { + parts = append(parts, x[from:i]) + } + from = i + 3 + } + } + if from < len(x) { + parts = append(parts, x[from:]) + } + sort.SliceStable(parts, func(a, b int) bool { return len(parts[a]) > len(parts[b]) }) + return parts +} + +const ( + // A break bends the waveform this many times more than a typical sample + // of it does, and by at least minBreak counts: rounding reaches a few. + breakOverTypical = 8 + minBreak = 6 + // minStretch is the shortest stretch read for a pitch, as a fraction of a + // second: 20ms. + minStretch = 50 +) + // edge is the amplitude that marks where sound begins inside a window: well // above the quiet floor's few counts, and below a tone at the sound floor. const edge = 8 diff --git a/internal/audio/audio_test.go b/internal/audio/audio_test.go index b303ccc4..7aa22c25 100644 --- a/internal/audio/audio_test.go +++ b/internal/audio/audio_test.go @@ -186,3 +186,73 @@ func TestAnalyzeAOneSidedClickHasNoPitch(t *testing.T) { t.Errorf("60 Hz: %+v", got) } } + +// spliced is a tone as the Android 15 emulator's stream carries it: every 1666 +// samples, 33 samples that break its phase. CHALLENGES 390. +func spliced(hz float64, seconds, offset, amp float64, rate int) []int16 { + out := make([]int16, 0, int(seconds*float64(rate))) + phase := 2 * math.Pi * offset + step := 2 * math.Pi * hz / float64(rate) + for i := 0; i < int(seconds*float64(rate)); i++ { + if i%1666 == 0 && i > 0 { + phase += 33 * step + } + out = append(out, int16(amp*math.Sin(phase))) + phase += step + } + return out +} + +func TestASplicedToneIsOneSound(t *testing.T) { + for _, hz := range []float64{220, 440, 660, 880, 1320, 3520, 10000} { + for _, offset := range []float64{0, 0.13, 0.37, 0.71} { + segs := Analyze(spliced(hz, 2, offset, 387, 48000), 48000) + var sounds []Segment + for _, s := range segs { + if s.Sound { + sounds = append(sounds, s) + } + } + if len(sounds) != 1 { + t.Errorf("%.0f Hz, offset %.2f: heard %d sounds: %+v", hz, offset, len(sounds), sounds) + continue + } + if d := math.Abs(sounds[0].Hz - hz); d > hz*0.005 { + t.Errorf("%.0f Hz, offset %.2f: heard %.1f Hz", hz, offset, sounds[0].Hz) + } + } + } +} + +// A quiet tone's breaks are too small to see them all, so its reading stays bent +// by some of them. It must stay within a semitone's half of the tone. +func TestAQuietSplicedToneStaysNearItsPitch(t *testing.T) { + for _, amp := range []float64{60, 30} { + for _, hz := range []float64{220, 440, 880} { + for _, offset := range []float64{0, 0.13, 0.37, 0.71} { + x := spliced(hz, 2, offset, amp, 48000) + for at := 0; at+4800 <= len(x); at += 4800 { + if got := steadyPitch(x[at:at+4800], 48000); math.Abs(got-hz) > hz*0.03 { + t.Fatalf("%.0f Hz at amplitude %.0f: read %.1f", hz, amp, got) + } + } + } + } + } +} + +// Two notes in one window are not a splice to read past: a real change still +// splits the sound. +func TestAPitchChangeStillSplitsTheSound(t *testing.T) { + a := spliced(440, 1, 0, 387, 48000) + b := spliced(880, 1, 0, 387, 48000) + var sounds []Segment + for _, s := range Analyze(append(a, b...), 48000) { + if s.Sound { + sounds = append(sounds, s) + } + } + if len(sounds) < 2 || math.Abs(sounds[0].Hz-440) > 3 || math.Abs(sounds[len(sounds)-1].Hz-880) > 5 { + t.Errorf("heard %+v", sounds) + } +} diff --git a/internal/device/emuaudio.go b/internal/device/emuaudio.go index 68cb341e..3083cc7a 100644 --- a/internal/device/emuaudio.go +++ b/internal/device/emuaudio.go @@ -149,7 +149,7 @@ func findEmulatorEndpoint(serial string, dirs []string) (emulatorEndpoint, error // it says it is running — so the remedy names the stop as well. // CHALLENGES 284. return emulatorEndpoint{}, mobiumerr.New(mobiumerr.DeviceNotReady, "%s has no control port Mobium "+ - "can find (looked in %s) — it was started with -no-grpc, or by an emulator older than 33", + "can find (looked in %s) — it was started with -port or -no-grpc, or by an emulator older than 33", serial, strings.Join(dirs, ", ")). WithRemedy(fmt.Sprintf("restart it: `mobium shutdown %s`, then `mobium boot `, which "+ "starts it with the control port on", serial))