diff --git a/tools/kokoro/include/kokoro-decoder.h b/tools/kokoro/include/kokoro-decoder.h index 312e13ed5..e95e84786 100644 --- a/tools/kokoro/include/kokoro-decoder.h +++ b/tools/kokoro/include/kokoro-decoder.h @@ -50,6 +50,19 @@ bool decoder_forward( std::vector & audio, std::string & err); +// The decoder up to the generator (F0/N convs, encode, asr_res, decode[0..3]): +// the AdaIN trunk on its own, for heads other than the iSTFTNet generator +// (Paradee-8M v1.1). Same inputs as decoder_forward; x is +// [decode[3].Cout, 2*T_frame] channel-major. +bool decoder_front_forward( + ggml_context * ctx, + const std::string & ns, bool instnorm_affine, + const float * asr_ct, int asr_dim, int T_frame, + const float * F0, const float * N, + const float * style, + std::vector & x, + std::string & err); + // Run the full decoder. Inputs come from kokoro_predictor_forward: // asr_ct : [512, T_frame] channel-major (transpose of PredictorOut.asr [T,512]) // F0, N : [2*T_frame] (PredictorOut.F0_pred / N_pred — the up-2x curves) diff --git a/tools/kokoro/src/kokoro-decoder.cpp b/tools/kokoro/src/kokoro-decoder.cpp index eaec858b0..1b9dd697d 100644 --- a/tools/kokoro/src/kokoro-decoder.cpp +++ b/tools/kokoro/src/kokoro-decoder.cpp @@ -96,25 +96,8 @@ bool fill_gen_block(const Lk & L, GenAdaResBlockWeights & b, const std::string & return true; } -} // namespace - -bool decoder_forward( - ggml_context * ctx, - DecoderConfig & cfg, - const float * asr_ct, int asr_dim, int T_frame, - const float * F0, const float * N, - const float * style, - std::vector & audio, - std::string & err) { - audio.clear(); - cfg.cancelled = false; - if (!ctx) { err = "null model context"; return false; } - if (T_frame <= 0) { err = "non-positive T_frame"; return false; } - Lk L{ctx, cfg.ns}; - const bool affine = cfg.instnorm_affine; - - // --- decoder_front weights --- - DecoderFrontWeights W; +// decoder_front weights from ".decoder.*". Checks the asr width. +bool fill_front(const Lk & L, bool affine, int asr_dim, DecoderFrontWeights & W, std::string & err) { W.F0_conv_w = L.get(".decoder.F0_conv.weight"); W.F0_conv_b = L.get(".decoder.F0_conv.bias"); W.N_conv_w = L.get(".decoder.N_conv.weight"); @@ -130,7 +113,7 @@ bool decoder_forward( /*upsample*/ b == 3, affine); } if (!ok || !W.F0_conv_w || !W.asr_res_w || !W.encode.conv1_w || !W.decode[3].pool_w) { - err = "missing decoder weights (is the GGUF a full " + cfg.ns + " model?)"; + err = "missing decoder weights (is the GGUF a full " + L.ns + " model?)"; return false; } if (W.encode.Cin != asr_dim + 2 || (int) asr_res_t->ne[1] != asr_dim) { @@ -138,6 +121,50 @@ bool decoder_forward( " != asr width " + std::to_string(asr_dim); return false; } + return true; +} + +} // namespace + +bool decoder_front_forward( + ggml_context * ctx, + const std::string & ns, bool instnorm_affine, + const float * asr_ct, int asr_dim, int T_frame, + const float * F0, const float * N, + const float * style, + std::vector & x, + std::string & err) { + x.clear(); + if (!ctx) { err = "null model context"; return false; } + if (T_frame <= 0) { err = "non-positive T_frame"; return false; } + Lk L{ctx, ns}; + DecoderFrontWeights W; + if (!fill_front(L, instnorm_affine, asr_dim, W, err)) return false; + std::vector F0_down, N_down; + kokoro_phase_timer t("decoder front"); + decoder_front(W, asr_ct, asr_dim, T_frame, F0, N, style, x, F0_down, N_down); + return true; +} + +bool decoder_forward( + ggml_context * ctx, + DecoderConfig & cfg, + const float * asr_ct, int asr_dim, int T_frame, + const float * F0, const float * N, + const float * style, + std::vector & audio, + std::string & err) { + audio.clear(); + cfg.cancelled = false; + if (!ctx) { err = "null model context"; return false; } + if (T_frame <= 0) { err = "non-positive T_frame"; return false; } + Lk L{ctx, cfg.ns}; + const bool affine = cfg.instnorm_affine; + + // --- decoder_front weights --- + DecoderFrontWeights W; + if (!fill_front(L, affine, asr_dim, W, err)) return false; + bool ok = true; // --- generator weights --- GeneratorWeights G; diff --git a/tools/paradee/CMakeLists.txt b/tools/paradee/CMakeLists.txt index 0cf849b45..143396379 100644 --- a/tools/paradee/CMakeLists.txt +++ b/tools/paradee/CMakeLists.txt @@ -1,11 +1,12 @@ # tools/paradee/CMakeLists.txt # -# Paradee-8M v1.0 TTS (sahilmahendrakar/Paradee-8M-v1.0, Apache-2.0): a +# Paradee-8M TTS (sahilmahendrakar/Paradee-8M-v1.0 and -v1.1, Apache-2.0): a # width-scaled, single-voice distillation of Kokoro-82M. Built when # LLAMA_BUILD_PARADEE=ON (see the top-level CMakeLists.txt). # # Produces: -# paradee_lib static library (loader, synthesis, phase-locking filter). +# paradee_lib static library (loader, synthesis, phase-locking filter, +# v1.1 source-filter head). # Reuses kokoro_lib's StyleTTS-2 / iSTFTNet layers. # paradee-tts CLI: misaki phonemes -> 24 kHz WAV (host only). # paradee-stage-dump writes every stage for the PyTorch parity gate. @@ -18,6 +19,8 @@ endif() add_library(paradee_lib STATIC src/paradee.cpp src/paradee-phase-lock.cpp + src/paradee-head.cpp + src/paradee-head.h include/paradee.h) set_target_properties(paradee_lib PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(paradee_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/tools/paradee/README.md b/tools/paradee/README.md index 67f3b6ac6..56c28b875 100644 --- a/tools/paradee/README.md +++ b/tools/paradee/README.md @@ -1,4 +1,4 @@ -# tools/paradee: Paradee-8M v1.0 TTS +# tools/paradee: Paradee-8M TTS (v1.0 and v1.1) Paradee-8M v1.0 ([sahilmahendrakar/Paradee-8M-v1.0](https://huggingface.co/sahilmahendrakar/Paradee-8M-v1.0), Apache-2.0) is Kokoro-82M's StyleTTS-2 + iSTFTNet code at smaller widths. It is distilled to one voice (`af_heart`), has @@ -13,6 +13,7 @@ natively, with no ONNX: - `paradee_lib` runs it on the `tools/kokoro` layers. Those layers now read their widths from the tensor shapes, and Kokoro-82M's output is unchanged bit for bit. - `src/paradee-phase-lock.cpp` is a native implementation of the model's phase-locking filter. +- `src/paradee-head.cpp` runs Paradee-8M v1.1's source-filter head. See [v1.1](#v11-source-filter-head). ## Pipeline @@ -52,10 +53,12 @@ namespace eliza_paradee { It checks `cancel` between stages and inside the generator and phase-lock loops. Calls on one model run one at a time. - `paradee_synthesize` takes the same arguments in two structs (`paradee_synth_options` and - `paradee_audio`). + `paradee_audio`). For v1.1, `paradee_synth_options::noise_seed` picks the head's noise. +- `paradee_get_head` and `paradee_version` say which release a GGUF holds. - Diagnostics: - `paradee_synthesize_ids_staged` and `paradee_decode_staged` return every stage. - `paradee_phase_lock` runs the filter alone. + - `paradee_head_staged` runs the v1.1 head alone; `paradee_stages::noise_in` replaces its noise. ## Build and use @@ -65,6 +68,8 @@ and `paradee-stage-dump`. ```sh python tools/paradee/convert_paradee_pt_to_gguf.py --model-dir \ --output paradee-8m-v1.0-f32.gguf --check-output +python tools/paradee/convert_paradee_pt_to_gguf.py --version 1.1 --model-dir \ + --output paradee-8m-v1.1-f32.gguf --check-output build/bin/paradee-tts --model paradee-8m-v1.0-f32.gguf --phonemes "həlˈO wˈɜɹld." --output hello.wav ``` @@ -82,6 +87,13 @@ front end. - **Reference environment:** torch 2.14.0, kokoro 0.9.4 and misaki 0.9.4, from `training/uv.lock`. - **GGUF:** the expected sha256 of the F32 converter output. The converter is byte-reproducible. `--check-output` enforces this sha256. +- **v1.1:** the `"v1.1"` section pins the same things for sahilmahendrakar/Paradee-8M-v1.1 at `330617ad` + (the `v1.1` tag) and github.com/sahilmahendrakar/paradee at `606e16e1` (`vhead.py` and + `export_paradee_v1_1.py` as well as the v1.0 scripts, which are unchanged there). The reference + environment is the same: `training/uv.lock` did not change. +- **v1.0 GGUF sha256:** writing `paradee.version` and `paradee.decoder.head` changed the converter, and so the + v1.0 output's sha256 (the tensors are unchanged). Files from the earlier converter, such as `357477…` + (32,072,768 bytes), still load as v1.0 and give byte-identical audio. - **No F16 pin:** `--outtype f16` halves the file (16 MB) but fails the parity gate (end-to-end SNR 16–23 dB), so it is not pinned or shipped. @@ -112,7 +124,7 @@ ctest --test-dir build -R paradee --output-on-failure |---|---|---| | `test-paradee-phase-lock` | nothing (committed fixture) | Lock within 1e-4 of `phase_lock.lock()` evaluated in float64; no further from the shipped float32 `phase_lock.lock()` than float32 is from float64; pass-through, length and cancel behaviour | | `test-paradee-api` | `PARADEE_GGUF` | ids, vocabulary and length errors, input validation, determinism, speed, cancellation at several points | -| `test-paradee-parity` | `PARADEE_GGUF`, `PARADEE_REF_DIR` | integer durations identical; bert/d/dur/F0/N/asr within 1e-3 × RMS; source (from the reference F0) and decoder output (from the reference asr/F0/N/source) within 1e-3 × RMS; lock alone within 1e-4 of float64 `phase_lock.lock()`; final waveform SNR ≥ 35 dB | +| `test-paradee-parity` | `PARADEE_GGUF`, `PARADEE_REF_DIR` | integer durations identical; bert/d/dur/F0/N/asr within 1e-3 × RMS; source (from the reference F0) and decoder output (from the reference asr/F0/N/source) within 1e-3 × RMS; lock alone within 1e-4 of float64 `phase_lock.lock()`; final waveform SNR ≥ 35 dB. For v1.1, see [v1.1 parity](#v11-parity) | The model-backed tests exit 77 (CTest SKIP) when the model or the reference is not configured. @@ -123,3 +135,94 @@ source phase error and be charged to the wrong stage. The end-to-end SNR covers The shipped float32 `phase_lock.lock()` differs from its own float64 evaluation by up to about 1e-3, at bins where the source is nearly silent. No independent implementation can match it within 1e-4. The gate therefore uses the float64 evaluation and reports the float32 distance. + +## v1.1: source-filter head + +[sahilmahendrakar/Paradee-8M-v1.1](https://huggingface.co/sahilmahendrakar/Paradee-8M-v1.1) (`330617ad`, +Apache-2.0) keeps v1.0's text side (the same `text_side.pt`) and the decoder's AdaIN trunk (same layout, +retrained weights). It replaces the iSTFTNet generator and the phase-locking filter with `SourceFilterHead2` +from `training/scripts/vhead.py` (github.com/sahilmahendrakar/paradee `606e16e1`). `src/paradee-head.cpp` +implements that head: + +| Part | What it does | +|---|---| +| ConvNeXt stack | Conv1d 128→160 (k 7), LayerNorm, 4 × (depthwise conv 7, LayerNorm, Linear 160→480, GELU, Linear 480→160, gamma, residual), LayerNorm. 80 fps, on the kokoro layers. | +| Output layers | Per STFT frame (n_fft 600, hop 150, 2 frames per trunk frame): harmonic log-gain, phase (cos, sin), noise log-gain and 4 pulsing-noise gains at 80 mel-spaced points, linearly interpolated to the 301 bins; one pulse shift (0.5 tanh, in pitch periods). Noise-burst gains at 24 points for a 150/75 STFT. | +| Sources | A 64-harmonic sine bank from F0 (harmonics above 11 kHz and unvoiced frames muted), white noise, and noise × voicing × cos/sin(2πm·phase) for m = 1, 2. | +| Output | iSTFT(E · gain · phase · e^{i2π(f/F0)·shift} + (noise + Σ pulsing noise · gain) · noise gain) + iSTFT_150/75(STFT_150/75(reversed noise) · burst gain). | + +The STFTs are mixed-radix FFTs (600 = 4·2·3·5·5, 150 = 2·3·5·5). The main path runs one STFT frame at a time, +so no spectrogram is kept. + +- **Routing.** The converter writes `paradee.version` and `paradee.decoder.head` (`istftnet_lock` for v1.0, + `source_filter_v2` for v1.1). `paradee_load_model` picks the graph from that key. A file without it is v1.0, + so v1.0 GGUFs written before this change still load and give byte-identical audio (checked with the + `357477…` file). The v1.1 file has no generator or lock tensors and no `paradee.lock.*` keys. + `paradee_get_lock_params` returns null for it. +- **Noise.** The head draws Gaussian noise. Natively, that is xoshiro256** seeded by splitmix64, with + Box-Muller in double. The seed is `PARADEE_DEFAULT_NOISE_SEED` (0) unless `paradee_synth_options::noise_seed` + says otherwise, so the same input gives the same audio. `paradee_stages::noise_in` replaces the noise, and + the parity test uses that to feed in the reference's noise. libelizainference needs no change: it calls + `paradee_synthesize` with the default options. +- **Rounding.** The native head follows the module's float32 arithmetic, with three deliberate differences: + - The harmonics come from a rotation recurrence on exp(i2π·phase), computed in double, instead of one + float32 `sin(2πk·phase)` per harmonic. That float32 argument reaches about 440 rad, where one float32 step + is about 3e-5 rad. + - The pulse shift's phase steps across the bins in double. + - The running phase `c + fr·d` is rounded twice, exactly as torch rounds it, and never fused into one FMA. + +### v1.1 parity + +`dump_paradee_reference.py dump --version 1.1` runs the upstream module (`VDecoder(head="v2")` with +`load_ckpt`, as `export_paradee_v1_1.py` builds it) in float32, with no other changes. The noise is +`torch.randn` from `Generator().manual_seed(--noise-seed + index)`, or `--noise-dir/.noise.f32`, and it is +written out as the `noise` stage. + +```sh +python tools/paradee/tools/dump_paradee_reference.py dump --version 1.1 --model-dir \ + --paradee-src --out ref11 +PARADEE_GGUF=paradee-8m-v1.1-f32.gguf PARADEE_REF_DIR=ref11 PARADEE_GGUF_V10=paradee-8m-v1.0-f32.gguf \ + build/bin/test-paradee-parity +``` + +| Gate | Input | Limit | Measured (5 sentences, M4 Max and TB336FU) | +|---|---|---|---| +| ids, integer durations | – | identical | identical | +| text side unchanged | `PARADEE_GGUF_V10` | bert/d/dur/dur_int/F0/N/asr bit-identical between the v1.0 and v1.1 GGUFs | bit-identical | +| bert, d, dur, F0, N, asr | – | max\|err\| ≤ 1e-3 × RMS | ≤ 6e-6 × RMS | +| trunk | reference asr/F0/N | ≤ 1e-3 × RMS | ≤ 9e-6 × RMS | +| hn (ConvNeXt stack) | reference trunk | ≤ 1e-3 × RMS | ≤ 1e-5 × RMS | +| sine bank | reference F0 | ≤ 1e-4 × RMS of the float64 bank; no further from float32 `sine_bank` than float32 is from float64 (+1e-4) | 1.3–1.4e-5; float32 itself is 2.6–3.1e-4 away | +| main path, burst path, head output | reference trunk, F0, noise | ≤ 1e-3 × RMS | ≤ 9.3e-5 × RMS | +| end to end | reference noise | SNR ≥ 35 dB | 68.5–81.9 dB (Mac), 69.0–77.3 dB (tablet) | + +The end-to-end figure is set by the text side's float32 rounding, as in v1.0. F0 differs by about 1e-6 relative, +and the harmonic phase integrates that error. + +### v1.1 against v1.0 + +Measured October 19, 2026, with the 40 sentences of the F16 and v1.1 evaluation in elizaOS/llama.cpp#85: 20 +assistant prompts and 20 Harvard sentences (130.1 s of speech), phonemized with `tools/misaki-g2p`. Both releases give the same durations, because the text side is the same. +Both GGUFs are F32 and read into memory, as on this branch. RTF is the sum of the best of 3 runs per sentence +from `paradee-tts`, divided by the audio length. + +| | v1.0 | v1.1 | +|---|---|---| +| GGUF (F32) | 32,072,864 B (deflate -9: 29.7 MB) | 31,275,232 B (deflate -9: 29.0 MB) | +| RTF, M4 Max, 1 thread | 0.025 | 0.0083 | +| RTF, M4 Max, default threads | 0.025 | 0.0081 | +| RTF, TB336FU (MT6835), 8 threads | 0.263 | 0.091 | +| RTF, TB336FU, 1 thread on an A76 core | 0.533 | 0.105 | +| WER, Phonon-2 / whisper.cpp `base.en` | 3.90% / 3.90% | 3.90% / 3.12% | +| Peak RssAnon, TB336FU, 19.8 s utterance | 202 MB | 69–73 MB | +| Peak footprint, M4 Max, 19.8 s utterance | 300 MB | 110–115 MB | + +- The WER figures equal upstream's ONNX graphs on the same sentences (v1.1 fp32: 3.90% / 3.12%). Phonon-2's + transcripts of the native v1.1 audio are identical to its transcripts of the ONNX v1.1 audio for all 40 + sentences. That holds even though the two draw different noise. +- On the tablet, v1.1 is 2.9× faster with 8 threads and 5× faster on one core. The head runs on one thread, + so most of the 8-thread gain is the text side's. +- The Mac was busy with other work (load average ~50) and the tablet had load ~11–14 from its own background + work. Treat the absolute RTFs as approximate; the two releases were measured back to back. +- Peak memory is lower mainly because the head never builds the generator's 24 kHz feature maps. With + #85's memory mapping, both files would also stop counting their 31–32 MB of weights as private memory. diff --git a/tools/paradee/convert_paradee_pt_to_gguf.py b/tools/paradee/convert_paradee_pt_to_gguf.py index b8396ffc3..86a30d19a 100644 --- a/tools/paradee/convert_paradee_pt_to_gguf.py +++ b/tools/paradee/convert_paradee_pt_to_gguf.py @@ -29,9 +29,17 @@ records the input pins and the git blob sha of this script (not the commit sha, which would make the expected output depend on the commit that pins it). +--version 1.1 converts sahilmahendrakar/Paradee-8M-v1.1 (pins.json "v1.1"). +v1.1 keeps v1.0's text side (the same text_side.pt) and the decoder's AdaIN +trunk (retrained weights, same layout), and replaces the iSTFTNet generator +and the phase-locking filter with a frame-rate source-filter head +(training/scripts/vhead.py SourceFilterHead2). The GGUF says which decoder +head it holds in paradee.decoder.head ("istftnet_lock" or "source_filter_v2"); +the runtime picks the graph from that key (a file without it is v1.0). + usage: convert_paradee_pt_to_gguf.py --model-dir \ - --output paradee-8m-v1.0-f32.gguf [--outtype f32|f16] [--check-output] + --output paradee-8m-v1.0-f32.gguf [--version 1.0|1.1] [--outtype f32|f16] [--check-output] """ from __future__ import annotations @@ -58,6 +66,7 @@ # Architecture of Paradee-8M v1.0, read from the checkpoint shapes and from # training/scripts/{student.py, student_decoder.py} at the pinned source # commit. The converter checks every tensor against these values. +# HPARAMS_V11 below is the v1.1 file's set. HPARAMS: dict[str, Any] = { # CustomAlbert (transformers AlbertConfig, num_hidden_groups=1). "text.n_layer": 6, @@ -104,6 +113,35 @@ "input.max_ids": 512, } +# Paradee-8M v1.1: the same text side and AdaIN trunk, then SourceFilterHead2 +# (training/scripts/vhead.py at the pinned v1.1 source commit; constructor +# defaults, as VDecoder(head="v2") builds it). No generator, no phase lock. +_V10_GENERATOR_KEYS = ("decoder.upsample_rates", "decoder.upsample_kernels", "decoder.resblock_kernels", + "decoder.resblock_dilations", "decoder.istft_n_fft", "decoder.istft_hop", + "decoder.harmonic_num", "decoder.sine_amp", "decoder.voiced_threshold") +HPARAMS_V11: dict[str, Any] = { + **{k: v for k, v in HPARAMS.items() if not k.startswith("lock.") and k not in _V10_GENERATOR_KEYS}, + "head.dim": 160, # ConvNeXt width + "head.blocks": 4, # ConvNeXt blocks (depthwise 7, LayerNorm, Linear x3, GELU, Linear, gamma) + "head.kernel": 7, # inp and depthwise kernel + "head.mult": 3, # ConvNeXt expansion + "head.f0_hop": 300, # samples per trunk frame / F0 value (80 fps) + "head.n_fft": 600, # main STFT: periodic Hann, centred, reflect + "head.hop": 150, # 2 STFT frames per trunk frame + "head.harmonics": 64, # sine bank K + "head.sine_amp": 0.1, + "head.max_harmonic_hz": 11000.0, + "head.voiced_f0_hz": 10.0, # sine bank / pulsing noise mute below this + "head.min_f0_hz": 60.0, # F0 floor of the pulse-shift ratio + "head.mod": 2, # pulsing noise: cos/sin(m * phi), m = 1..mod + "head.points": 80, # G: gains predicted at 80 mel-spaced points + "head.gain_clamp": 6.0, # exp(min(x, 6)) + "head.shift_scale": 0.5, # pulse shift = 0.5 * tanh(x) cycles + "head.burst_n_fft": 150, # noise-burst STFT (window 150, hop 75) + "head.burst_hop": 75, + "head.burst_points": 24, # Gb +} + def sha256_file(path: Path) -> str: h = hashlib.sha256() @@ -132,10 +170,12 @@ def check_pins(model_dir: Path, pins: dict) -> None: raise SystemExit(f"{rel}: sha256 {digest} != pinned {pin['sha256']}") -def load_vocab(model_dir: Path) -> list[tuple[str, int]]: +def load_vocab(model_dir: Path, version: str = "1.0") -> list[tuple[str, int]]: config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) tok = json.loads((model_dir / "tokenizer.json").read_text(encoding="utf-8")) vocab = config["vocab"] + if config.get("version") != version: + raise SystemExit(f"config.json version {config.get('version')!r} != {version!r}") if config.get("sample_rate") != HPARAMS["audio.sample_rate"]: raise SystemExit(f"config.json sample_rate {config.get('sample_rate')} != 24000") if config.get("n_token") != HPARAMS["text.vocab_size"]: @@ -283,6 +323,38 @@ def decoder_map(sd: dict) -> list[tuple[str, str]]: return m +def decoder_map_v11(sd: dict) -> list[tuple[str, str]]: + """v1.1 decoder.pt: v1.0's AdaIN trunk + head.* (SourceFilterHead2).""" + m = [p for p in decoder_map(sd) if not p[0].startswith("generator.")] + hp = HPARAMS_V11 + for part in ("inp.weight", "inp.bias", "norm0.weight", "norm0.bias", "norm1.weight", "norm1.bias", + "out.weight", "out.bias", "out_b.weight", "out_b.bias"): + m.append((f"head.{part}", f"{NS}.head.{part}")) + for i in range(hp["head.blocks"]): + for part in ("dw.weight", "dw.bias", "norm.weight", "norm.bias", "pw1.weight", "pw1.bias", + "pw2.weight", "pw2.bias", "gamma"): + m.append((f"head.blocks.{i}.{part}", f"{NS}.head.blocks.{i}.{part}")) + return m + + +def expected_shapes_v11() -> dict[str, tuple[int, ...]]: + h = HPARAMS_V11 + D, uic, K = h["head.dim"], h["decoder.initial_ch"], h["head.kernel"] + up = h["head.f0_hop"] // h["head.hop"] + C = 4 + 2 * h["head.mod"] + shapes = {k: v for k, v in expected_shapes().items() if ".gen." not in k} + shapes.update({ + f"{NS}.head.inp.weight": (D, uic, K), + f"{NS}.head.blocks.0.dw.weight": (D, 1, K), + f"{NS}.head.blocks.0.pw1.weight": (D * h["head.mult"], D), + f"{NS}.head.blocks.0.pw2.weight": (D, D * h["head.mult"]), + f"{NS}.head.blocks.0.gamma": (D,), + f"{NS}.head.out.weight": (up * (C * h["head.points"] + 1), D), + f"{NS}.head.out_b.weight": ((h["head.f0_hop"] // h["head.burst_hop"]) * h["head.burst_points"], D), + }) + return shapes + + def expected_shapes() -> dict[str, tuple[int, ...]]: """Shapes (PyTorch order) of the tensors whose widths the C++ graph reads from hparams. Everything else is checked for consistency at load time.""" @@ -343,17 +415,27 @@ def collect(sd_raw: dict, mapping: list[tuple[str, str]], skipped: tuple[str, .. return out +# paradee.decoder.head values. The runtime treats a file without the key as +# HEAD_V10 (GGUFs written before v1.1 support). +HEAD_V10 = "istftnet_lock" +HEAD_V11 = "source_filter_v2" + + def write_gguf(out_path: Path, tensors: dict[str, np.ndarray], vocab: list[tuple[str, int]], pins: dict, - outtype: str) -> None: + outtype: str, version: str = "1.0") -> None: + v11 = version == "1.1" writer = gguf.GGUFWriter(str(out_path), arch=ARCH) - writer.add_string("general.name", "Paradee-8M v1.0") + writer.add_string("general.name", f"Paradee-8M v{version}") writer.add_string("general.license", "apache-2.0") writer.add_string("general.source.url", f"https://huggingface.co/{pins['model']['repo']}") writer.add_string("general.base_model.0.name", "Kokoro-82M") writer.add_string("general.description", - "Paradee-8M v1.0 single-voice (af_heart) TTS, distilled from Kokoro-82M. " - "StyleTTS-2 text side + iSTFTNet decoder + phase-locking filter.") - for key, val in HPARAMS.items(): + f"Paradee-8M v{version} single-voice (af_heart) TTS, distilled from Kokoro-82M. " + + ("StyleTTS-2 text side + AdaIN decoder trunk + frame-rate source-filter head." if v11 else + "StyleTTS-2 text side + iSTFTNet decoder + phase-locking filter.")) + writer.add_string(f"{NS}.version", version) + writer.add_string(f"{NS}.decoder.head", HEAD_V11 if v11 else HEAD_V10) + for key, val in (HPARAMS_V11 if v11 else HPARAMS).items(): full = f"{NS}.{key}" if isinstance(val, list): writer.add_array(full, [int(x) for x in val]) @@ -395,6 +477,9 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--model-dir", required=True, type=Path, help="local snapshot of the pinned HF revision (pytorch/*.pt, config.json, tokenizer.json)") ap.add_argument("--output", required=True, type=Path) + ap.add_argument("--version", choices=("1.0", "1.1"), default="1.0", + help="Paradee-8M release: 1.0 (default) or 1.1 (source-filter head); " + "selects the pins.json section the inputs are checked against") ap.add_argument("--outtype", choices=("f32", "f16"), default="f32", help="f32 (default, the parity-gated file) or f16 for >=2-D weights " "(experimental: fails test-paradee-parity, not pinned)") @@ -403,8 +488,10 @@ def main(argv: list[str] | None = None) -> int: args = ap.parse_args(argv) pins = json.loads(PINS_PATH.read_text(encoding="utf-8")) + if args.version == "1.1": + pins = pins["v1.1"] check_pins(args.model_dir, pins) - vocab = load_vocab(args.model_dir) + vocab = load_vocab(args.model_dir, args.version) import torch @@ -415,25 +502,26 @@ def main(argv: list[str] | None = None) -> int: text_sd, dec_sd = text_sd["model"], dec_sd["model"] tensors = collect(text_sd, text_side_map(text_sd), TEXT_SIDE_SKIPPED, "text_side.pt") - dec = collect(dec_sd, decoder_map(dec_sd), (), "decoder.pt") + dec_map = decoder_map_v11(dec_sd) if args.version == "1.1" else decoder_map(dec_sd) + dec = collect(dec_sd, dec_map, (), "decoder.pt") overlap = set(tensors) & set(dec) if overlap: raise SystemExit(f"tensor name collision: {sorted(overlap)}") tensors.update(dec) - for name, shape in expected_shapes().items(): + for name, shape in (expected_shapes_v11() if args.version == "1.1" else expected_shapes()).items(): if tuple(tensors[name].shape) != shape: raise SystemExit(f"{name}: shape {tuple(tensors[name].shape)} != expected {shape}") n_params = sum(int(a.size) for a in tensors.values()) args.output.parent.mkdir(parents=True, exist_ok=True) - write_gguf(args.output, tensors, vocab, pins, args.outtype) + write_gguf(args.output, tensors, vocab, pins, args.outtype, args.version) digest = sha256_file(args.output) print(f"wrote {args.output}: {len(tensors)} tensors, {n_params} parameters, " f"{args.output.stat().st_size} bytes, sha256 {digest}") if args.check_output: - want = pins["gguf"][args.outtype]["sha256"] + want = pins["gguf"].get(args.outtype, {}).get("sha256") if digest != want: print(f"error: sha256 {digest} != pinned {want}", file=sys.stderr) return 1 diff --git a/tools/paradee/include/paradee.h b/tools/paradee/include/paradee.h index 49b38d157..74aefe2dc 100644 --- a/tools/paradee/include/paradee.h +++ b/tools/paradee/include/paradee.h @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // -// paradee.h — Paradee-8M v1.0 text-to-speech, native in the elizaOS llama.cpp -// fork (tools/paradee). +// paradee.h — Paradee-8M (v1.0 and v1.1) text-to-speech, native in the +// elizaOS llama.cpp fork (tools/paradee). // // Paradee (sahilmahendrakar/Paradee-8M-v1.0, Apache-2.0) is Kokoro-82M's // StyleTTS-2 + iSTFTNet code at smaller widths, distilled to the single @@ -22,6 +22,21 @@ // harmonic source (STFT 1024 / hop 256, 33-frame smoothing) // -> 24 kHz mono float PCM // +// Paradee-8M v1.1 (sahilmahendrakar/Paradee-8M-v1.1) keeps the text side and +// the decoder's AdaIN trunk and replaces the generator and the phase lock +// with a frame-rate source-filter head: +// +// -> decoder : AdaIN res blocks (256) -> 128 channels at 80 fps +// -> head : ConvNeXt x4 (160) -> per STFT frame (n_fft 600, hop 150) +// gains / phase / pulse shift on a 64-harmonic sine bank +// from F0, gains on white and pitch-pulsed noise, plus a +// 3 ms noise-burst path (STFT 150 / 75) -> iSTFT +// +// The GGUF's paradee.decoder.head says which graph to run ("istftnet_lock" or +// "source_filter_v2"; a file without the key is v1.0). The v1.1 head draws +// Gaussian noise: from a fixed seed (PARADEE_DEFAULT_NOISE_SEED unless the +// options say otherwise), so the same input gives the same audio. +// // Input phonemes must be misaki's spelling (Kokoro's G2P); eSpeak IPA must be // converted to misaki spelling first. @@ -39,7 +54,7 @@ struct paradee_model; struct paradee_model_deleter { void operator()(paradee_model *) const noexcept; }; using paradee_model_ptr = std::unique_ptr; -// Input contract of Paradee-8M v1.0: at most 512 ids, the pad (0) at each end. +// Input contract of Paradee-8M (v1.0 and v1.1): at most 512 ids, the pad (0) at each end. constexpr int PARADEE_MAX_INPUT_IDS = 512; constexpr int PARADEE_PAD_ID = 0; @@ -61,14 +76,28 @@ enum paradee_status { const char * paradee_status_str(paradee_status st) noexcept; +// Which decoder head a loaded model runs (from paradee.decoder.head). +enum paradee_head_kind { + PARADEE_HEAD_ISTFTNET_LOCK = 0, // v1.0: iSTFTNet generator + phase-locking filter + PARADEE_HEAD_SOURCE_FILTER = 1, // v1.1: vhead.py SourceFilterHead2 +}; + +// Seed of the v1.1 head's noise when the caller does not choose one. +constexpr uint64_t PARADEE_DEFAULT_NOISE_SEED = 0; + // Load a Paradee GGUF written by convert_paradee_pt_to_gguf.py. The file must // declare general.architecture == "paradee"; every hparam the graph reads is // required. Returns nullptr and sets `err` on failure. paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string & err) noexcept; -// 24000 for Paradee-8M v1.0 (0 for a null model). +// 24000 for Paradee-8M (0 for a null model). int paradee_sample_rate(const paradee_model * model) noexcept; +// The head the model runs (PARADEE_HEAD_ISTFTNET_LOCK for a null model) and +// the release it was converted from ("1.0", "1.1"; "" for a null model). +paradee_head_kind paradee_get_head(const paradee_model * model) noexcept; +const char * paradee_version(const paradee_model * model) noexcept; + // Map a misaki phoneme string (UTF-8, one vocabulary symbol per code point) // to model input ids: [0] + ids + [0]. Rejects empty input, invalid UTF-8, // any symbol outside the vocabulary (PARADEE_E_UNKNOWN_SYMBOL, the code point @@ -125,6 +154,7 @@ struct paradee_synth_options { float speed = 1.0f; paradee_cancel_fn cancel = nullptr; void * cancel_user_data = nullptr; + uint64_t noise_seed = PARADEE_DEFAULT_NOISE_SEED; // v1.1 head only }; struct paradee_audio { @@ -154,12 +184,24 @@ struct paradee_stages { std::vector F0; // [2*T_frame] std::vector N; // [2*T_frame] std::vector asr; // [asr_dim, T_frame] aligned asr features (channel-major) - std::vector dec; // [2*T_frame*300] generator output before the phase lock - std::vector src; // [2*T_frame*300] harmonic source (m_source output) + std::vector dec; // [2*T_frame*300] generator output before the phase lock (v1.0) + std::vector src; // [2*T_frame*300] harmonic source (m_source output, v1.0) std::vector wav; // final waveform + // v1.1 (source-filter head) only, n = 2*T_frame: + std::vector trunk; // [128, n] AdaIN trunk output (channel-major) + std::vector hn; // [n, 160] head ConvNeXt stack after norm1 + std::vector sine; // [n*300] sine bank (vhead.sine_bank) + std::vector main_path; // [n*300] the n_fft 600 path + std::vector burst_path; // [n*300] the noise-burst path + std::vector noise; // [n*300] the noise the head used + // Input (v1.1): when set, the head uses this noise ([n*300] samples) + // instead of drawing it from the seed. The parity test feeds the + // reference's noise here. + const std::vector * noise_in = nullptr; }; -// paradee_synthesize_ids that also fills `stages` (when non-null). +// paradee_synthesize_ids that also fills `stages` (when non-null; its +// noise_in, if set, replaces the v1.1 head's noise). paradee_status paradee_synthesize_ids_staged( const paradee_model * model, const int32_t * ids, size_t n, @@ -174,7 +216,9 @@ paradee_status paradee_synthesize_ids_staged( // its own). asr is [asr_dim, T_frame] channel-major, F0 and N are // [2*T_frame]. `src` (optional, [2*T_frame*300]) replaces the harmonic // source the generator would compute. Fills stages->dec / src / wav (stages -// may not be null). +// may not be null). For a v1.1 model: trunk + head, filling stages->trunk, +// hn, sine, main_path, burst_path, noise and wav (`src` must be null; the +// noise is stages->noise_in or the default seed's). paradee_status paradee_decode_staged( const paradee_model * model, const float * asr, int asr_dim, int T_frame, @@ -184,6 +228,16 @@ paradee_status paradee_decode_staged( paradee_stages * stages, std::string & err) noexcept; +// v1.1 only: the source-filter head alone from a given trunk output +// (trunk [C, n] channel-major, F0 [n], noise [n*300]). Fills stages->hn, +// sine, main_path, burst_path and wav. +paradee_status paradee_head_staged( + const paradee_model * model, + const float * trunk, int channels, int n, + const float * F0, const float * noise, + paradee_stages * stages, + std::string & err) noexcept; + // Phase-locking filter parameters (read from the GGUF; defaults are v1.0). struct paradee_lock_params { int n_fft = 1024; @@ -210,6 +264,7 @@ bool paradee_phase_lock( std::vector & out, paradee_cancel_fn cancel = nullptr, void * user = nullptr); +// nullptr for a null model and for a v1.1 model (it has no phase lock). const paradee_lock_params * paradee_get_lock_params(const paradee_model * model) noexcept; } // namespace eliza_paradee diff --git a/tools/paradee/pins.json b/tools/paradee/pins.json index 88d2c76c1..3a55c5c09 100644 --- a/tools/paradee/pins.json +++ b/tools/paradee/pins.json @@ -75,9 +75,9 @@ "gguf": { "f32": { "file": "paradee-8m-v1.0-f32.gguf", - "size": 32072768, - "sha256": "357477651abb53b3fb1911a6e932026f3f1697fd772668326152e3ca04c9be27", - "note": "convert_paradee_pt_to_gguf.py --outtype f32 output; byte-reproducible (check with --check-output). Changes whenever the converter changes, because its git blob sha is in the metadata." + "size": 32072864, + "sha256": "c0d8f2010571c01015277bc11c9901d4d4cfbbfb25145932cdd86398464281d2", + "note": "convert_paradee_pt_to_gguf.py --outtype f32 output; byte-reproducible (check with --check-output). Changes whenever the converter changes, because its git blob sha is in the metadata. A v1.0 file written by an earlier converter (for example 357477651abb…, 32072768 bytes, before paradee.version / paradee.decoder.head existed) still loads as v1.0 and gives byte-identical audio." } }, "fixtures": { @@ -89,5 +89,78 @@ "sha256": "691b032dd084e0c9c28f363acef8fb3029880041c9e5a282452626733ef3ee34", "made_by": "tools/dump_paradee_reference.py lock-fixture" } + }, + "v1.1": { + "model": { + "repo": "sahilmahendrakar/Paradee-8M-v1.1", + "revision": "330617ad1f097b4f6b02c103e54b39006c090b45", + "tag": "v1.1", + "tag_commit": "330617ad1f097b4f6b02c103e54b39006c090b45", + "tag_note": "The v1.1 tag and main both resolve to 330617ad.", + "license": "Apache-2.0" + }, + "files": { + "pytorch/text_side.pt": { + "size": 16998347, + "sha256": "0995bee746043e4f06f72be68fb8634530b95d6467b4a141759e721122648062" + }, + "pytorch/decoder.pt": { + "size": 14655657, + "sha256": "143f9678a8cc2a53ca72e06a2f8a8dd67424a5395eb939c60ec820a1122c8a13" + }, + "config.json": { + "size": 1774, + "sha256": "c3f420e8a5aea41b97f698404a1b1e28e403c662f65927345335fc5493f2e075" + }, + "tokenizer.json": { + "size": 3497, + "sha256": "77a02c8e164413299b4b4c403b14f8e0e1c1b727db4d46a09d6327b861060a34" + }, + "samples/sentences.txt": { + "size": 621, + "sha256": "2d411facb6b0429ad05f6ea3abf9c6a79ba6613990cadf54e2213e11a8c9fd1b" + }, + "samples/1_paradee.wav": { + "size": 468044, + "sha256": "6fdb16483025c0349c0faaf82b646c54478df5edca415c5f47d651548bb5857b" + }, + "samples/2_paradee.wav": { + "size": 283244, + "sha256": "15a9d34c5e0b976a2735286bb8a9fc342e1ad328ff550de8e9205b520acde1db" + }, + "samples/3_paradee.wav": { + "size": 361244, + "sha256": "ea5de64b47ffb680a41c7a20043178ebf827acaaf915fb430f0fa98cc123e160" + }, + "samples/4_paradee.wav": { + "size": 457244, + "sha256": "01994be7988be80b83ca9457519c106d75cf89f3570cf0e0906a46f8063d3d9b" + }, + "samples/5_paradee.wav": { + "size": 512444, + "sha256": "a028892e7470f9c5e969dd845a4882043a40fa81481b523014247b60cc0b3376" + } + }, + "text_side_note": "pytorch/text_side.pt is byte-identical to v1.0's. decoder.pt keeps v1.0's AdaIN trunk layout (retrained weights) and replaces the generator with head.* (SourceFilterHead2).", + "source": { + "repo": "https://github.com/sahilmahendrakar/paradee", + "commit": "606e16e12f53bc6dfb814fe89fef338726687161", + "license": "Apache-2.0", + "files": { + "training/scripts/student.py": "90456f28ac3d6dd06e76ad8f253f9bcadd9060cdecb6880268651e0ff8ba0f8b", + "training/scripts/student_decoder.py": "2443f14fe8ff27be5702d88dc80f8c566e92c03273b6216825fa7787cd66b97c", + "training/scripts/common.py": "76faab4b54f633c1dfee28bfb08c981d4adb3f4ee99b42271cf80a3fc0e9fc9c", + "training/scripts/vhead.py": "c3b6be6f8b4d5b883a1afd59ccd924712e690842a1723142703851ab8c152b63", + "training/scripts/export_paradee_v1_1.py": "b5e52cb27324c3bf728b41bf57d89ce549b9bae55316687aed27bf2fcca47173" + } + }, + "gguf": { + "f32": { + "file": "paradee-8m-v1.1-f32.gguf", + "size": 31275232, + "sha256": "f205ab055f0120d571aa6fda2deadde1e6fbc88e8d8bd56a592917fd0e50892e", + "note": "convert_paradee_pt_to_gguf.py --version 1.1 --outtype f32 output; byte-reproducible (check with --check-output). Changes whenever the converter changes, because its git blob sha is in the metadata." + } + } } } diff --git a/tools/paradee/src/paradee-head.cpp b/tools/paradee/src/paradee-head.cpp new file mode 100644 index 000000000..07b5b5456 --- /dev/null +++ b/tools/paradee/src/paradee-head.cpp @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: MIT +// +// paradee-head.cpp — Paradee-8M v1.1 source-filter head (see paradee-head.h). +// +// Port of training/scripts/vhead.py SourceFilterHead2.forward and sine_bank +// (sahilmahendrakar/paradee 606e16e1, Apache-2.0). Same operations, same +// order, float32 like the module, with three implementation choices that only +// change rounding: +// +// - The sine bank's harmonics come from a rotation recurrence in float on +// exp(i 2 pi cyc) (computed in double), not one sin() per harmonic. The +// recurrence is closer to the exact value than torch's float32 argument +// 2 pi k cyc (|arg| up to 400 rad, one float32 step ~3e-5 rad). +// - The pulse shift's exp(i 2 pi (f / f0) shift) steps over the bins with a +// double recurrence instead of one sincos per bin. +// - STFTs are mixed-radix FFTs (600 = 4*2*3*5*5, 150 = 2*3*5*5) with two +// real signals packed into one complex transform. The iSTFT is torch.istft: +// C2R per frame (imaginary DC / Nyquist ignored), periodic Hann, overlap- +// add, divided by the summed squared window, centre trimmed. + +#include "paradee-head.h" + +#include "kokoro-layers.h" +#include "kokoro-profile.h" + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include + +namespace eliza_paradee { +namespace head { + +namespace { + +using cf = std::complex; +constexpr double K_PI = 3.14159265358979323846; + +// Complex FFT for n with factors 2, 3, 4 and 5: iterative Stockham +// (decimation in frequency, natural order in and out, ping-pong buffers). +// inverse = conj(FFT(conj(x))), no 1/n scaling. +class MixedFft { +public: + explicit MixedFft(int n) : n_(n) { + int m = n; + for (int f : { 4, 2, 3, 5 }) { + while (m % f == 0) { fac_.push_back(f); m /= f; } + } + ok_ = m == 1 && n > 1; + tw_.resize((size_t) n); + for (int k = 0; k < n; ++k) { + const double a = -2.0 * K_PI * (double) k / (double) n; + tw_[(size_t) k] = cf((float) std::cos(a), (float) std::sin(a)); + } + work_.resize((size_t) n); + } + bool ok() const { return ok_; } + int size() const { return n_; } + // out may not alias in. Not thread-safe (one scratch buffer per object). + void run(const cf * in, cf * out, bool inverse) { + const size_t N = (size_t) n_; + if (inverse) for (size_t i = 0; i < N; ++i) work_[i] = std::conj(in[i]); + else std::copy(in, in + N, work_.begin()); + // Each stage reads x and writes y; the last stage must write `out`. + cf * bufs[2] = { work_.data(), out }; + int cur = 0; + if (fac_.size() % 2 == 0) { // even stage count: start from `out` + std::copy(work_.begin(), work_.end(), out); + cur = 1; + } + size_t n = N, st = 1; + for (int p : fac_) { + stage(p, n, st, bufs[cur], bufs[cur ^ 1]); + cur ^= 1; + n /= (size_t) p; + st *= (size_t) p; + } + if (inverse) for (size_t i = 0; i < N; ++i) out[i] = std::conj(out[i]); + } +private: + static inline cf mul_mi(cf z) { return cf(z.imag(), -z.real()); } // -i z + // One radix-p step: sequence length n, stride st. + void stage(int p, size_t n, size_t st, const cf * x, cf * y) const { + const size_t m = n / (size_t) p, tws = (size_t) n_ / n; + constexpr float h3 = 0.86602540378443865f; // sin(2 pi / 3) + const float c1 = 0.30901699437494742f, c2 = -0.80901699437494742f; // cos(2 pi / 5), cos(4 pi / 5) + const float s1 = 0.95105651629515357f, s2 = 0.58778525229247313f; // sin(2 pi / 5), sin(4 pi / 5) + for (size_t j = 0; j < m; ++j) { + const cf * w = tw_.data(); + const size_t wj = j * tws; + for (size_t q = 0; q < st; ++q) { + cf * o = y + q + st * (size_t) p * j; + const cf * a = x + q + st * j; + const size_t sm = st * m; + switch (p) { + case 2: { + const cf a0 = a[0], a1 = a[sm]; + o[0] = a0 + a1; + o[st] = (a0 - a1) * w[wj]; + break; + } + case 3: { + const cf a0 = a[0], a1 = a[sm], a2 = a[2 * sm]; + const cf t = a1 + a2, u = a0 - 0.5f * t, v = mul_mi(h3 * (a1 - a2)); + o[0] = a0 + t; + o[st] = (u + v) * w[wj]; + o[2 * st] = (u - v) * w[2 * wj]; + break; + } + case 4: { + const cf a0 = a[0], a1 = a[sm], a2 = a[2 * sm], a3 = a[3 * sm]; + const cf t0 = a0 + a2, t1 = a0 - a2, t2 = a1 + a3, t3 = mul_mi(a1 - a3); + o[0] = t0 + t2; + o[st] = (t1 + t3) * w[wj]; + o[2 * st] = (t0 - t2) * w[2 * wj]; + o[3 * st] = (t1 - t3) * w[3 * wj]; + break; + } + default: { // 5 + const cf a0 = a[0], a1 = a[sm], a2 = a[2 * sm], a3 = a[3 * sm], a4 = a[4 * sm]; + const cf t1 = a1 + a4, t2 = a2 + a3, t3 = a1 - a4, t4 = a2 - a3; + const cf r1 = a0 + c1 * t1 + c2 * t2, r2 = a0 + c2 * t1 + c1 * t2; + const cf i1 = mul_mi(s1 * t3 + s2 * t4), i2 = mul_mi(s2 * t3 - s1 * t4); + o[0] = a0 + t1 + t2; + o[st] = (r1 + i1) * w[wj]; + o[2 * st] = (r2 + i2) * w[2 * wj]; + o[3 * st] = (r2 - i2) * w[3 * wj]; + o[4 * st] = (r1 - i1) * w[4 * wj]; + break; + } + } + } + } + } + int n_; + bool ok_ = false; + std::vector fac_; + std::vector tw_; + std::vector work_; +}; + +// torch.hann_window(N) (periodic), float32. +std::vector hann(int N) { + std::vector w((size_t) N); + for (int j = 0; j < N; ++j) w[(size_t) j] = (float) (0.5 - 0.5 * std::cos(2.0 * K_PI * (double) j / (double) N)); + return w; +} + +// Reflect-padded sample j - pad of x (torch.stft center=True, pad_mode="reflect"). +inline float reflect_at(const float * x, int L, long j) { + if (j < 0) j = -j; + else if (j >= L) j = 2L * (L - 1) - j; + return x[j]; +} + +// Frame f of torch.stft(x, N, H, window=hann, center=True) for one or two +// real signals of length L (> N/2): o1 (and o2) get N/2+1 bins. Two signals +// share one complex FFT. z, Z: N-sample scratch. +void stft_frame(MixedFft & fft, const std::vector & w, int H, const float * x1, const float * x2, int L, + int f, std::vector & z, std::vector & Z, cf * o1, cf * o2) { + const int N = fft.size(), nb = N / 2 + 1, pad = N / 2; + const long base = (long) f * H - pad; + if (base >= 0 && base + N <= L) { + const float * p1 = x1 + base; + const float * p2 = x2 ? x2 + base : nullptr; + for (int j = 0; j < N; ++j) z[(size_t) j] = cf(p1[j] * w[(size_t) j], p2 ? p2[j] * w[(size_t) j] : 0.0f); + } else { + for (int j = 0; j < N; ++j) { + const float wj = w[(size_t) j]; + z[(size_t) j] = cf(reflect_at(x1, L, base + j) * wj, x2 ? reflect_at(x2, L, base + j) * wj : 0.0f); + } + } + fft.run(z.data(), Z.data(), false); + if (!o2) { + std::copy(Z.begin(), Z.begin() + nb, o1); + return; + } + for (int k = 0; k < nb; ++k) { + const cf a = Z[(size_t) k], b = std::conj(Z[(size_t) ((N - k) % N)]); + o1[k] = (a + b) * 0.5f; + o2[k] = (a - b) * cf(0.0f, -0.5f); + } +} + +// torch.stft of one real signal, all F = L/H + 1 frames: X [F][N/2+1]. +void stft(MixedFft & fft, const std::vector & w, int H, const float * x, int L, std::vector & X) { + const int N = fft.size(), nb = N / 2 + 1, F = L / H + 1; + X.assign((size_t) F * nb, cf()); + std::vector z((size_t) N), Z((size_t) N); + for (int f = 0; f < F; ++f) stft_frame(fft, w, H, x, nullptr, L, f, z, Z, X.data() + (size_t) f * nb, nullptr); +} + +// torch.istft(Y, N, H, window=hann, center=True, length=L), fed frame pairs +// as they are made: C2R per frame (imaginary DC / Nyquist dropped), window, +// overlap-add, divide by the summed squared window, trim the centre padding. +// Two frames share one complex inverse FFT. +class Ola { +public: + Ola(MixedFft & fft, const std::vector & w, int H, int F) + : fft_(fft), w_(w), H_(H), N_(fft.size()), nb_(fft.size() / 2 + 1), + acc_((size_t) fft.size() + (size_t) H * (size_t) (F - 1), 0.0f), env_(acc_.size(), 0.0f), + z_((size_t) fft.size()), r_((size_t) fft.size()) {} + // Frames f (A) and f + 1 (B, may be null for the last odd frame). + void add(const cf * A, const cf * B, int f) { + for (int k = 0; k < N_; ++k) { + const cf a = full(A, k); + z_[(size_t) k] = B ? a + cf(0.0f, 1.0f) * full(B, k) : a; + } + fft_.run(z_.data(), r_.data(), true); + const float inv_n = 1.0f / (float) N_; + for (int fi = 0; fi < (B ? 2 : 1); ++fi) { + const size_t off = (size_t) (f + fi) * (size_t) H_; + for (int j = 0; j < N_; ++j) { + const float v = (fi == 0 ? r_[(size_t) j].real() : r_[(size_t) j].imag()) * inv_n; + acc_[off + (size_t) j] += v * w_[(size_t) j]; + env_[off + (size_t) j] += w_[(size_t) j] * w_[(size_t) j]; + } + } + } + void finish(int L, std::vector & y) const { + const size_t pad = (size_t) N_ / 2; + y.assign((size_t) L, 0.0f); + for (int t = 0; t < L; ++t) { + const float e = env_[(size_t) t + pad]; + y[(size_t) t] = e > 1e-11f ? acc_[(size_t) t + pad] / e : 0.0f; + } + } +private: + cf full(const cf * A, int k) const { // Hermitian extension + if (k == 0) return cf(A[0].real(), 0.0f); + if (2 * k == N_) return cf(A[k].real(), 0.0f); + return k < nb_ ? A[k] : std::conj(A[N_ - k]); + } + MixedFft & fft_; + const std::vector & w_; + int H_, N_, nb_; + std::vector acc_, env_; + std::vector z_, r_; +}; + +void istft(MixedFft & fft, const std::vector & w, int H, const cf * Y, int F, int L, std::vector & y) { + const size_t nb = (size_t) fft.size() / 2 + 1; + Ola ola(fft, w, H, F); + for (int f = 0; f < F; f += 2) ola.add(Y + (size_t) f * nb, f + 1 < F ? Y + (size_t) (f + 1) * nb : nullptr, f); + ola.finish(L, y); +} + +// LayerNorm over channels of a channel-major [C, n] map (nn.LayerNorm(C) on +// x.transpose(1, 2)), eps 1e-5. In place. +void layer_norm_cm(float * x, int C, int n, const float * w, const float * b) { + std::vector sum((size_t) n, 0.0), sq((size_t) n, 0.0); + for (int c = 0; c < C; ++c) { + const float * xc = x + (size_t) c * n; + for (int t = 0; t < n; ++t) { sum[(size_t) t] += xc[t]; sq[(size_t) t] += (double) xc[t] * xc[t]; } + } + std::vector mean((size_t) n), inv((size_t) n); + for (int t = 0; t < n; ++t) { + const double m = sum[(size_t) t] / C; + const double var = std::max(0.0, sq[(size_t) t] / C - m * m); + mean[(size_t) t] = (float) m; + inv[(size_t) t] = (float) (1.0 / std::sqrt(var + 1e-5)); + } + for (int c = 0; c < C; ++c) { + float * xc = x + (size_t) c * n; + const float wc = w[c], bc = b[c]; + for (int t = 0; t < n; ++t) xc[t] = (xc[t] - mean[(size_t) t]) * inv[(size_t) t] * wc + bc; + } +} + +// vhead.warp_matrix(nb, G) as (index, weight) pairs: bin b takes +// (1 - w) * g[i] + w * g[i + 1]. float32 as the module computes it. +void warp(int nb, int G, int sr, std::vector & idx, std::vector & wt) { + idx.resize((size_t) nb); + wt.resize((size_t) nb); + const float top = std::log1p((float) sr / 2.0f / 700.0f); + for (int b = 0; b < nb; ++b) { + const float f = nb > 1 ? (float) ((double) sr / 2.0 * b / (nb - 1)) : 0.0f; + const float pos = std::log1p(f / 700.0f) / top * (float) (G - 1); + const int i = std::min(std::max((int) std::floor(pos), 0), G - 2); + idx[(size_t) b] = i; + wt[(size_t) b] = pos - (float) i; + } +} + +// Consecutive phase timings under KOKORO_PROFILE=1 (kokoro-profile.h). +class laps { +public: + explicit laps(const char * first) : label_(first), t0_(std::chrono::steady_clock::now()) {} + ~laps() { next(nullptr); } + void next(const char * label) { + const auto now = std::chrono::steady_clock::now(); + if (label_ && eliza_kokoro::kokoro_profile_enabled()) { + eliza_kokoro::kokoro_profile_emit(label_, std::chrono::duration(now - t0_).count()); + } + label_ = label; + t0_ = now; + } +private: + const char * label_; + std::chrono::steady_clock::time_point t0_; +}; + +inline float gelu(float x) { return 0.5f * x * (1.0f + std::erf(x * 0.70710678118654752f)); } + +bool get_i32(gguf_context * g, const char * key, int & out, std::string & err) { + const int64_t id = gguf_find_key(g, key); + if (id < 0) { err = std::string("missing GGUF key ") + key; return false; } + switch (gguf_get_kv_type(g, id)) { + case GGUF_TYPE_INT32: out = gguf_get_val_i32(g, id); return true; + case GGUF_TYPE_UINT32: out = (int) gguf_get_val_u32(g, id); return true; + default: err = std::string("GGUF key ") + key + " is not an integer"; return false; + } +} + +bool get_f32(gguf_context * g, const char * key, float & out, std::string & err) { + const int64_t id = gguf_find_key(g, key); + if (id < 0) { err = std::string("missing GGUF key ") + key; return false; } + if (gguf_get_kv_type(g, id) != GGUF_TYPE_FLOAT32) { err = std::string("GGUF key ") + key + " is not f32"; return false; } + out = gguf_get_val_f32(g, id); + return true; +} + +// ggml shape check against the PyTorch shape (ne reversed); 0 = any. +const float * tensor(ggml_context * ctx, const std::string & name, std::initializer_list torch_shape, + std::string & err) { + ggml_tensor * t = ggml_get_tensor(ctx, name.c_str()); + if (!t || t->type != GGML_TYPE_F32) { err = "missing F32 tensor " + name; return nullptr; } + const int nd = (int) torch_shape.size(); + int i = 0; + for (int64_t d : torch_shape) { + if (t->ne[nd - 1 - i] != d) { err = "tensor " + name + " has an unexpected shape"; return nullptr; } + ++i; + } + for (int k = nd; k < GGML_MAX_DIMS; ++k) { + if (t->ne[k] != 1) { err = "tensor " + name + " has an unexpected shape"; return nullptr; } + } + return (const float *) t->data; +} + +} // namespace + +bool load(gguf_context * g, ggml_context * ctx, params & p, weights & w, std::string & err) { + bool ok = + get_i32(g, "paradee.head.dim", p.dim, err) && + get_i32(g, "paradee.head.blocks", p.blocks, err) && + get_i32(g, "paradee.head.kernel", p.kernel, err) && + get_i32(g, "paradee.head.mult", p.mult, err) && + get_i32(g, "paradee.head.f0_hop", p.f0_hop, err) && + get_i32(g, "paradee.head.n_fft", p.n_fft, err) && + get_i32(g, "paradee.head.hop", p.hop, err) && + get_i32(g, "paradee.head.harmonics", p.K, err) && + get_i32(g, "paradee.head.mod", p.mod, err) && + get_i32(g, "paradee.head.points", p.G, err) && + get_i32(g, "paradee.head.burst_n_fft", p.burst_n_fft, err) && + get_i32(g, "paradee.head.burst_hop", p.burst_hop, err) && + get_i32(g, "paradee.head.burst_points", p.Gb, err) && + get_f32(g, "paradee.head.sine_amp", p.sine_amp, err) && + get_f32(g, "paradee.head.max_harmonic_hz", p.max_harmonic_hz, err) && + get_f32(g, "paradee.head.voiced_f0_hz", p.voiced_f0_hz, err) && + get_f32(g, "paradee.head.min_f0_hz", p.min_f0_hz, err) && + get_f32(g, "paradee.head.gain_clamp", p.gain_clamp, err) && + get_f32(g, "paradee.head.shift_scale", p.shift_scale, err) && + get_i32(g, "paradee.audio.sample_rate", p.sample_rate, err) && + get_i32(g, "paradee.decoder.initial_ch", p.in_ch, err); + if (!ok) return false; + // Bounds, so a corrupt file cannot ask for absurd buffers, and the + // structure the graph relies on (whole STFT frames per trunk frame, + // FFT sizes this file's MixedFft handles). + const bool sane = + p.dim > 0 && p.dim <= 4096 && p.blocks >= 0 && p.blocks <= 64 && p.kernel > 0 && p.kernel % 2 == 1 && + p.kernel <= 31 && p.mult > 0 && p.mult <= 16 && p.in_ch > 0 && p.in_ch <= 4096 && + p.f0_hop > 0 && p.hop > 0 && p.f0_hop % p.hop == 0 && p.burst_hop > 0 && p.f0_hop % p.burst_hop == 0 && + p.n_fft >= 2 && p.n_fft % 2 == 0 && p.n_fft <= 8192 && p.burst_n_fft >= 2 && p.burst_n_fft % 2 == 0 && + p.burst_n_fft <= 8192 && p.n_fft / 2 < 2 * p.f0_hop && p.burst_n_fft / 2 < 2 * p.f0_hop && + p.K > 0 && p.K <= 512 && p.mod >= 0 && p.mod <= 8 && p.G >= 2 && p.G <= 4096 && p.Gb >= 2 && p.Gb <= 4096 && + p.sample_rate > 0 && p.sample_rate <= 192000 && std::isfinite(p.sine_amp) && std::isfinite(p.gain_clamp) && + std::isfinite(p.shift_scale) && std::isfinite(p.max_harmonic_hz) && std::isfinite(p.voiced_f0_hz) && + std::isfinite(p.min_f0_hz) && p.min_f0_hz > 0.0f && + MixedFft(p.n_fft).ok() && MixedFft(p.burst_n_fft).ok(); + if (!sane) { err = "unsupported Paradee head hparams"; return false; } + + const int64_t D = p.dim, F = (int64_t) p.mult * p.dim, K = p.kernel; + const int64_t up = p.f0_hop / p.hop, C = 4 + 2 * (int64_t) p.mod; + const int64_t R = up * (C * p.G + 1), Rb = (int64_t) (p.f0_hop / p.burst_hop) * p.Gb; + const std::string h = "paradee.head."; + w = weights(); + ok = (w.inp_w = tensor(ctx, h + "inp.weight", { D, p.in_ch, K }, err)) && + (w.inp_b = tensor(ctx, h + "inp.bias", { D }, err)) && + (w.norm0_w = tensor(ctx, h + "norm0.weight", { D }, err)) && + (w.norm0_b = tensor(ctx, h + "norm0.bias", { D }, err)) && + (w.norm1_w = tensor(ctx, h + "norm1.weight", { D }, err)) && + (w.norm1_b = tensor(ctx, h + "norm1.bias", { D }, err)) && + (w.out_w = tensor(ctx, h + "out.weight", { R, D }, err)) && + (w.out_b = tensor(ctx, h + "out.bias", { R }, err)) && + (w.outb_w = tensor(ctx, h + "out_b.weight", { Rb, D }, err)) && + (w.outb_b = tensor(ctx, h + "out_b.bias", { Rb }, err)); + if (!ok) return false; + w.blocks.resize((size_t) p.blocks); + for (int i = 0; i < p.blocks; ++i) { + const std::string b = h + "blocks." + std::to_string(i) + "."; + block_weights & bw = w.blocks[(size_t) i]; + ok = (bw.dw_w = tensor(ctx, b + "dw.weight", { D, 1, K }, err)) && + (bw.dw_b = tensor(ctx, b + "dw.bias", { D }, err)) && + (bw.norm_w = tensor(ctx, b + "norm.weight", { D }, err)) && + (bw.norm_b = tensor(ctx, b + "norm.bias", { D }, err)) && + (bw.pw1_w = tensor(ctx, b + "pw1.weight", { F, D }, err)) && + (bw.pw1_b = tensor(ctx, b + "pw1.bias", { F }, err)) && + (bw.pw2_w = tensor(ctx, b + "pw2.weight", { D, F }, err)) && + (bw.pw2_b = tensor(ctx, b + "pw2.bias", { D }, err)) && + (bw.gamma = tensor(ctx, b + "gamma", { D }, err)); + if (!ok) return false; + } + return true; +} + +void conv_stack(const params & p, const weights & w, const float * x, int n, std::vector & hn) { + using eliza_kokoro::conv1d_forward; + const int D = p.dim, F = p.mult * p.dim, K = p.kernel; + hn.assign((size_t) D * n, 0.0f); + conv1d_forward(x, p.in_ch, n, w.inp_w, w.inp_b, D, K, 1, K / 2, 1, hn.data(), n); + layer_norm_cm(hn.data(), D, n, w.norm0_w, w.norm0_b); + std::vector d((size_t) D * n), u((size_t) F * n), v((size_t) D * n); + for (const block_weights & b : w.blocks) { + eliza_kokoro::conv1d_grouped_forward(hn.data(), D, n, b.dw_w, b.dw_b, D, K, 1, K / 2, 1, D, d.data(), n); + layer_norm_cm(d.data(), D, n, b.norm_w, b.norm_b); + conv1d_forward(d.data(), D, n, b.pw1_w, b.pw1_b, F, 1, 1, 0, 1, u.data(), n); + for (float & e : u) e = gelu(e); + conv1d_forward(u.data(), F, n, b.pw2_w, b.pw2_b, D, 1, 1, 0, 1, v.data(), n); + for (int c = 0; c < D; ++c) { + const float g = b.gamma[c]; + float * hc = hn.data() + (size_t) c * n; + const float * vc = v.data() + (size_t) c * n; + for (int t = 0; t < n; ++t) hc[t] += g * vc[t]; + } + } + layer_norm_cm(hn.data(), D, n, w.norm1_w, w.norm1_b); +} + +namespace { + +// sine_bank plus the fundamental's exp(i 2 pi m cyc) for m = 1..mod (the +// pulsing noise carriers) and the voicing mask, all per sample. +void sine_bank_full(const params & p, const float * f0, int n, std::vector & e, + std::vector * carriers, std::vector * uv) { + const int H = p.f0_hop; + const size_t L = (size_t) n * H; + e.assign(L, 0.0f); + if (carriers) carriers->assign(L * 2 * (size_t) p.mod, 0.0f); // [2*mod][L]: cos m, sin m + if (uv) uv->assign(L, 0.0f); + // c: running phase at each frame (cycles, float64 cumsum, wrapped); + // d: advance to the next frame's centre. + std::vector c((size_t) n), d((size_t) n); + double acc = 0.0; + for (int j = 0; j < n; ++j) { + const double r = (double) f0[j] * H / p.sample_rate; + acc += r; + c[(size_t) j] = (float) (acc - std::floor(acc)); + if (j > 0) d[(size_t) j - 1] = (float) r; + } + d[(size_t) n - 1] = 0.0f; + std::vector cs((size_t) H), sn((size_t) H), ck((size_t) H), sk((size_t) H), sum((size_t) H); + for (int j = 0; j < n; ++j) { + const float f = f0[j]; + const bool voiced = f > p.voiced_f0_hz; + int kmax = 0; + if (voiced) while (kmax < p.K && f * (float) (kmax + 1) < p.max_harmonic_hz) ++kmax; + for (int s = 0; s < H; ++s) { + const size_t t = (size_t) j * H + (size_t) s; + const float pf = ((float) t + 0.5f) / (float) H - 0.5f; + const int i = std::min(std::max((int) std::floor(pf), 0), n - 1); + const float fr = std::min(std::max(pf - (float) i, 0.0f), 1.0f); + // torch rounds fr * d and the sum separately; spell both roundings + // out so no compiler contracts them into one fused multiply-add + // (the product of two floats is exact in double). + const float step = (float) ((double) fr * (double) d[(size_t) i]); + const float cyc = (float) ((double) c[(size_t) i] + (double) step); + const double a = 2.0 * K_PI * (double) cyc; + const std::complex z1(std::cos(a), std::sin(a)); + cs[(size_t) s] = (float) z1.real(); + sn[(size_t) s] = (float) z1.imag(); + if (carriers) { + std::complex zm = z1; // exp(i 2 pi m cyc) + for (int m = 1; m <= p.mod; ++m, zm *= z1) { + (*carriers)[(size_t) (2 * (m - 1)) * L + t] = (float) zm.real(); + (*carriers)[(size_t) (2 * (m - 1) + 1) * L + t] = (float) zm.imag(); + } + } + if (uv) (*uv)[t] = voiced ? 1.0f : 0.0f; + } + if (kmax == 0) continue; + std::copy(cs.begin(), cs.end(), ck.begin()); + std::copy(sn.begin(), sn.end(), sk.begin()); + std::copy(sn.begin(), sn.end(), sum.begin()); + for (int k = 2; k <= kmax; ++k) { + for (int s = 0; s < H; ++s) { + const float c2 = ck[(size_t) s] * cs[(size_t) s] - sk[(size_t) s] * sn[(size_t) s]; + const float s2 = sk[(size_t) s] * cs[(size_t) s] + ck[(size_t) s] * sn[(size_t) s]; + ck[(size_t) s] = c2; + sk[(size_t) s] = s2; + sum[(size_t) s] += s2; + } + } + float * ej = e.data() + (size_t) j * H; + for (int s = 0; s < H; ++s) ej[s] = sum[(size_t) s] * p.sine_amp; + } +} + +} // namespace + +void sine_bank(const params & p, const float * f0, int n, std::vector & e) { + sine_bank_full(p, f0, n, e, nullptr, nullptr); +} + +bool forward(const params & p, const weights & w, + const float * x, int n, const float * f0, const float * noise, + std::vector & y, stages * st, + cancel_fn cancel, void * user, bool & cancelled, std::string & err) { + y.clear(); + cancelled = false; + if (!x || !f0 || !noise || n < 2) { err = "head: bad input"; return false; } + auto stop = [&]() { if (cancel && cancel(user)) { cancelled = true; y.clear(); return true; } return false; }; + using eliza_kokoro::conv1d_forward; + const int L = n * p.f0_hop, up = p.f0_hop / p.hop, T = n * up, nf = T + 1; + const int nb = p.n_fft / 2 + 1, C = 4 + 2 * p.mod, R = C * p.G + 1; + const int upb = p.f0_hop / p.burst_hop, Tb = n * upb, nfb = Tb + 1, nbb = p.burst_n_fft / 2 + 1; + + // ConvNeXt stack and the two output projections (channel-major). + laps lap("head convnext"); + std::vector hn; + conv_stack(p, w, x, n, hn); + if (st) { + st->hn.assign((size_t) n * p.dim, 0.0f); + for (int c = 0; c < p.dim; ++c) + for (int t = 0; t < n; ++t) st->hn[(size_t) t * p.dim + c] = hn[(size_t) c * n + t]; + } + std::vector o((size_t) up * R * n), ob((size_t) upb * p.Gb * n); + conv1d_forward(hn.data(), p.dim, n, w.out_w, w.out_b, up * R, 1, 1, 0, 1, o.data(), n); + conv1d_forward(hn.data(), p.dim, n, w.outb_w, w.outb_b, upb * p.Gb, 1, 1, 0, 1, ob.data(), n); + if (stop()) return false; + + // Sources: sine bank and the pulsing noise (noise * uv * cos/sin(2 pi m phi)). + lap.next("head sources"); + std::vector e; + std::vector> nm((size_t) 2 * p.mod, std::vector((size_t) L)); + { + std::vector carriers, uv; + sine_bank_full(p, f0, n, e, &carriers, &uv); + for (int j = 0; j < 2 * p.mod; ++j) { + const float * cj = carriers.data() + (size_t) j * L; + float * mj = nm[(size_t) j].data(); + for (int t = 0; t < L; ++t) mj[t] = noise[t] * uv[(size_t) t] * cj[t]; + } + } + if (st) st->sine = e; + if (stop()) return false; + + // Main path, one STFT frame at a time (no spectrogram is kept): the + // spectra of e, the noise and the pulsing noise, the predicted filter, + // and the inverse STFT's overlap-add. + lap.next("head main path"); + MixedFft fft(p.n_fft); + const std::vector win = hann(p.n_fft); + std::vector wi; + std::vector ww; + warp(nb, p.G, p.sample_rate, wi, ww); + const double bin_hz = (double) p.sample_rate / 2.0 / (nb - 1); + const int M = 2 * p.mod; + std::vector z((size_t) p.n_fft), Z((size_t) p.n_fft); + std::vector Ef((size_t) nb), Nf((size_t) nb), Mf((size_t) (M + 1) * nb), Yp((size_t) 2 * nb); + std::vector gk((size_t) C * nb); + Ola ola(fft, win, p.hop, nf); + for (int f0i = 0; f0i < nf; f0i += 2) { + for (int fi = 0; fi < 2 && f0i + fi < nf; ++fi) { + const int f = f0i + fi; + stft_frame(fft, win, p.hop, e.data(), noise, L, f, z, Z, Ef.data(), Nf.data()); + for (int j = 0; j < M; j += 2) { + stft_frame(fft, win, p.hop, nm[(size_t) j].data(), j + 1 < M ? nm[(size_t) j + 1].data() : nullptr, L, + f, z, Z, Mf.data() + (size_t) j * nb, j + 1 < M ? Mf.data() + (size_t) (j + 1) * nb : nullptr); + } + const int ff = std::min(f, T - 1), t = ff / up, s = ff % up; + auto oc = [&](int r) { return o[((size_t) s * R + (size_t) r) * n + (size_t) t]; }; + for (int c = 0; c < C; ++c) { + float * g = gk.data() + (size_t) c * nb; + for (int b = 0; b < nb; ++b) { + const int i = wi[(size_t) b]; + const float a = ww[(size_t) b]; + g[b] = (1.0f - a) * oc(c * p.G + i) + a * oc(c * p.G + i + 1); + } + } + const float shift = p.shift_scale * std::tanh(oc(C * p.G)); + const float f0f = std::max(f0[std::min(f / up, n - 1)], p.min_f0_hz); + // exp(i 2 pi (b * bin_hz / f0f) shift), stepped over the bins. + const double step = 2.0 * K_PI * bin_hz / (double) f0f * (double) shift; + const std::complex rot(std::cos(step), std::sin(step)); + std::complex ph(1.0, 0.0); + cf * Yf = Yp.data() + (size_t) fi * nb; + for (int b = 0; b < nb; ++b) { + const float gh = std::exp(std::min(gk[(size_t) b], p.gain_clamp)); + const float gn = std::exp(std::min(gk[(size_t) 3 * nb + b], p.gain_clamp)); + const float ur = gk[(size_t) nb + b], ui = gk[(size_t) 2 * nb + b]; + const float um = std::max(std::hypot(ur, ui), 1e-4f); + const cf hh = cf(ur / um, ui / um) * cf((float) ph.real(), (float) ph.imag()) * gh; + cf ntot = Nf[(size_t) b]; + for (int j = 0; j < M; ++j) ntot += gk[(size_t) (4 + j) * nb + b] * Mf[(size_t) j * nb + b]; + Yf[b] = Ef[(size_t) b] * hh + ntot * gn; + ph *= rot; + } + } + ola.add(Yp.data(), f0i + 1 < nf ? Yp.data() + nb : nullptr, f0i); + if ((f0i & 511) == 0 && stop()) return false; + } + std::vector main; + ola.finish(L, main); + if (stop()) return false; + + // Noise-burst path: STFT 150/75 of the reversed noise, gains at Gb points. + lap.next("head burst"); + MixedFft fftb(p.burst_n_fft); + const std::vector winb = hann(p.burst_n_fft); + std::vector rev(noise, noise + L); + std::reverse(rev.begin(), rev.end()); + std::vector Nb; + stft(fftb, winb, p.burst_hop, rev.data(), L, Nb); + warp(nbb, p.Gb, p.sample_rate, wi, ww); + for (int f = 0; f < nfb; ++f) { + const int ff = std::min(f, Tb - 1), t = ff / upb, s = ff % upb; + cf * Bf = Nb.data() + (size_t) f * nbb; + for (int b = 0; b < nbb; ++b) { + const int i = wi[(size_t) b]; + const float a = ww[(size_t) b]; + const float v = (1.0f - a) * ob[((size_t) s * p.Gb + (size_t) i) * n + (size_t) t] + + a * ob[((size_t) s * p.Gb + (size_t) i + 1) * n + (size_t) t]; + Bf[b] *= std::exp(std::min(v, p.gain_clamp)); + } + } + std::vector burst; + istft(fftb, winb, p.burst_hop, Nb.data(), nfb, L, burst); + + y.resize((size_t) L); + for (int t = 0; t < L; ++t) y[(size_t) t] = main[(size_t) t] + burst[(size_t) t]; + if (st) { st->main = std::move(main); st->burst = std::move(burst); } + if (stop()) return false; + return true; +} + +void gaussian_noise(uint64_t seed, size_t count, std::vector & out) { + auto splitmix = [](uint64_t & z) { + uint64_t r = (z += 0x9e3779b97f4a7c15ull); + r = (r ^ (r >> 30)) * 0xbf58476d1ce4e5b9ull; + r = (r ^ (r >> 27)) * 0x94d049bb133111ebull; + return r ^ (r >> 31); + }; + uint64_t sm = seed, s[4]; + for (uint64_t & v : s) v = splitmix(sm); + auto rotl = [](uint64_t v, int k) { return (v << k) | (v >> (64 - k)); }; + auto next = [&]() { + const uint64_t r = rotl(s[1] * 5, 7) * 9, t = s[1] << 17; + s[2] ^= s[0]; s[3] ^= s[1]; s[1] ^= s[2]; s[0] ^= s[3]; s[2] ^= t; s[3] = rotl(s[3], 45); + return r; + }; + out.resize(count); + for (size_t i = 0; i < count; i += 2) { + const double u1 = (double) ((next() >> 11) + 1) * 0x1.0p-53; // (0, 1] + const double u2 = (double) (next() >> 11) * 0x1.0p-53; // [0, 1) + const double r = std::sqrt(-2.0 * std::log(u1)); + out[i] = (float) (r * std::cos(2.0 * K_PI * u2)); + if (i + 1 < count) out[i + 1] = (float) (r * std::sin(2.0 * K_PI * u2)); + } +} + +} // namespace head +} // namespace eliza_paradee diff --git a/tools/paradee/src/paradee-head.h b/tools/paradee/src/paradee-head.h new file mode 100644 index 000000000..c76780703 --- /dev/null +++ b/tools/paradee/src/paradee-head.h @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MIT +// +// paradee-head.h — Paradee-8M v1.1's frame-rate source-filter head +// (training/scripts/vhead.py SourceFilterHead2, sahilmahendrakar/paradee +// 606e16e1, Apache-2.0). Internal to paradee_lib. +// +// The head replaces v1.0's iSTFTNet generator and phase lock. It never +// upsamples: from the AdaIN trunk's 80 fps features x [in_ch, n] and the F0 +// curve [n] it predicts, for every frame of an STFT with n_fft 600 / hop 150 +// (2 frames per trunk frame), +// +// - a harmonic gain, a phase (cos, sin) and a noise gain at G = 80 +// mel-spaced points, linearly interpolated to the 301 bins, +// - gains of 2*mod pulsing noise sources noise*uv*cos/sin(2 pi m phi), +// - one pulse shift per frame (0.5 tanh, in pitch periods), +// +// applies them to the STFTs of a 64-harmonic sine bank built from F0 and of +// white noise, and inverts: +// +// y = iSTFT(E * gh * u * exp(i 2 pi (f / f0) shift) + (Nz + sum_j a_j Nm_j) * gn) +// + iSTFT_150/75(STFT_150/75(reverse(noise)) * gb) (3 ms noise bursts) +// +// The ConvNeXt stack is ConvNeXt(dim) x blocks between LayerNorms: depthwise +// conv 7, LayerNorm, Linear(dim, 3 dim), exact GELU, Linear(3 dim, dim), +// per-channel gamma, residual. +// +// The noise is an input. paradee.cpp draws it with paradee_head_noise (a +// seeded Gaussian), the parity test feeds the reference's noise. + +#pragma once + +#include +#include +#include +#include + +struct ggml_context; +struct gguf_context; + +namespace eliza_paradee { +namespace head { + +struct params { + int in_ch = 128, dim = 160, blocks = 4, kernel = 7, mult = 3; + int f0_hop = 300; // samples per trunk frame / F0 value + int n_fft = 600, hop = 150; // main STFT + int K = 64; // sine bank harmonics + int mod = 2; // pulsing noise orders + int G = 80; // gain points (mel-spaced) + int burst_n_fft = 150, burst_hop = 75, Gb = 24; + float sine_amp = 0.1f, max_harmonic_hz = 11000.0f, voiced_f0_hz = 10.0f, min_f0_hz = 60.0f; + float gain_clamp = 6.0f, shift_scale = 0.5f; + int sample_rate = 24000; +}; + +struct block_weights { + const float * dw_w = nullptr; // [dim, 1, kernel] + const float * dw_b = nullptr; + const float * norm_w = nullptr; // [dim] + const float * norm_b = nullptr; + const float * pw1_w = nullptr; // [mult*dim, dim] + const float * pw1_b = nullptr; + const float * pw2_w = nullptr; // [dim, mult*dim] + const float * pw2_b = nullptr; + const float * gamma = nullptr; // [dim] +}; + +struct weights { + const float * inp_w = nullptr; // [dim, in_ch, kernel] + const float * inp_b = nullptr; + const float * norm0_w = nullptr, * norm0_b = nullptr; + std::vector blocks; + const float * norm1_w = nullptr, * norm1_b = nullptr; + const float * out_w = nullptr; // [up * (C*G + 1), dim], C = 4 + 2*mod, up = f0_hop / hop + const float * out_b = nullptr; + const float * outb_w = nullptr; // [(f0_hop / burst_hop) * Gb, dim] + const float * outb_b = nullptr; +}; + +// Read the hparams ("paradee.head.*") and tensors ("paradee.head.*") and +// check every shape. Returns false with err set. +bool load(gguf_context * g, ggml_context * ctx, params & p, weights & w, std::string & err); + +// Optional intermediate outputs, in the reference dump's layouts. +struct stages { + std::vector hn; // [n, dim] ConvNeXt stack after norm1 (time-major) + std::vector sine; // [n*f0_hop] sine bank + std::vector main; // [n*f0_hop] the n_fft 600 path + std::vector burst; // [n*f0_hop] the noise-burst path +}; + +using cancel_fn = bool (*)(void *); + +// x [in_ch, n] channel-major (the trunk output), f0 [n], noise [n*f0_hop]. +// y gets n*f0_hop samples. Returns false with y empty when cancel() returned +// true (cancelled = true) or on bad input (err set). +bool forward(const params & p, const weights & w, + const float * x, int n, const float * f0, const float * noise, + std::vector & y, stages * st, + cancel_fn cancel, void * user, bool & cancelled, std::string & err); + +// The ConvNeXt stack alone: hn [dim, n] (channel-major) from x [in_ch, n]. +void conv_stack(const params & p, const weights & w, const float * x, int n, std::vector & hn); + +// vhead.sine_bank: e [n*f0_hop] from f0 [n]. +void sine_bank(const params & p, const float * f0, int n, std::vector & e); + +// Seeded standard-normal noise (xoshiro256** seeded by splitmix64, then +// Box-Muller in double). The same seed gives the same samples on every +// platform up to the last bit of libm's log/sin/cos. +void gaussian_noise(uint64_t seed, size_t count, std::vector & out); + +} // namespace head +} // namespace eliza_paradee diff --git a/tools/paradee/src/paradee.cpp b/tools/paradee/src/paradee.cpp index a00d72323..51fc3df5b 100644 --- a/tools/paradee/src/paradee.cpp +++ b/tools/paradee/src/paradee.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT // -// paradee.cpp — Paradee-8M v1.0 loader and synthesis (see include/paradee.h). +// paradee.cpp — Paradee-8M loader and synthesis (see include/paradee.h). // // The text side and decoder are Kokoro's modules at smaller widths, so this // file only binds the GGUF to the generalised tools/kokoro layers @@ -14,8 +14,13 @@ // - torch.round (half to even) for the integer durations, // - the canonical source STFT (exact zeros for near-empty bins), // - the phase-locking filter after the generator. +// +// A v1.1 GGUF (paradee.decoder.head = "source_filter_v2") runs the same text +// side and the decoder's AdaIN trunk (kokoro decoder_front_forward), then the +// source-filter head in paradee-head.cpp instead of the generator and lock. #include "paradee.h" +#include "paradee-head.h" #include "kokoro-decoder.h" #include "kokoro-layers.h" @@ -69,7 +74,11 @@ struct paradee_hparams { struct paradee_model { paradee_hparams hp; - paradee_lock_params lock; + paradee_head_kind head_kind = PARADEE_HEAD_ISTFTNET_LOCK; + std::string version; + paradee_lock_params lock; // v1.0 + head::params head_p; // v1.1 + head::weights head_w; std::unordered_map vocab; // code point -> id gguf_context * gguf = nullptr; ggml_context * file_ctx = nullptr; // tensors as stored @@ -195,6 +204,17 @@ paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string return nullptr; } } + { + const int64_t id = gguf_find_key(g, "paradee.decoder.head"); + const std::string kind = id >= 0 && gguf_get_kv_type(g, id) == GGUF_TYPE_STRING ? gguf_get_val_str(g, id) + : "istftnet_lock"; + if (kind == "istftnet_lock") m->head_kind = PARADEE_HEAD_ISTFTNET_LOCK; + else if (kind == "source_filter_v2") m->head_kind = PARADEE_HEAD_SOURCE_FILTER; + else { err = "unsupported paradee.decoder.head '" + kind + "'"; return nullptr; } + const int64_t vid = gguf_find_key(g, "paradee.version"); + m->version = vid >= 0 && gguf_get_kv_type(g, vid) == GGUF_TYPE_STRING ? gguf_get_val_str(g, vid) : "1.0"; + } + const bool v10 = m->head_kind == PARADEE_HEAD_ISTFTNET_LOCK; paradee_hparams & h = m->hp; std::vector rates; bool ok = @@ -211,24 +231,31 @@ paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string read_i32(g, "paradee.style.text_dim", h.style_text_dim, err) && read_i32(g, "paradee.style.decoder_dim", h.style_decoder_dim, err) && read_i32(g, "paradee.asr.dim", h.asr_dim, err) && - read_i32_array(g, "paradee.decoder.upsample_rates", rates, err) && - read_i32(g, "paradee.decoder.istft_hop", h.istft_hop, err) && read_i32(g, "paradee.audio.sample_rate", h.sample_rate, err) && read_i32(g, "paradee.input.pad_id", h.pad_id, err) && - read_i32(g, "paradee.input.max_ids", h.max_ids, err) && - read_i32(g, "paradee.lock.n_fft", m->lock.n_fft, err) && - read_i32(g, "paradee.lock.hop", m->lock.hop, err) && - read_i32(g, "paradee.lock.smooth_frames", m->lock.smooth_frames, err) && - read_i32(g, "paradee.lock.f0_hop", m->lock.f0_hop, err) && - read_f32(g, "paradee.lock.voiced_f0_hz", m->lock.voiced_f0_hz, err); + read_i32(g, "paradee.input.max_ids", h.max_ids, err); if (!ok) return nullptr; - int band_lo = 0, band_hi = 0; - if (!read_i32(g, "paradee.lock.band_lo_hz", band_lo, err) || - !read_i32(g, "paradee.lock.band_hi_hz", band_hi, err)) { - return nullptr; + // The generator and the phase lock are v1.0's; v1.1 has neither. + if (v10) { + ok = read_i32_array(g, "paradee.decoder.upsample_rates", rates, err) && + read_i32(g, "paradee.decoder.istft_hop", h.istft_hop, err) && + read_i32(g, "paradee.lock.n_fft", m->lock.n_fft, err) && + read_i32(g, "paradee.lock.hop", m->lock.hop, err) && + read_i32(g, "paradee.lock.smooth_frames", m->lock.smooth_frames, err) && + read_i32(g, "paradee.lock.f0_hop", m->lock.f0_hop, err) && + read_f32(g, "paradee.lock.voiced_f0_hz", m->lock.voiced_f0_hz, err); + if (!ok) return nullptr; + int band_lo = 0, band_hi = 0; + if (!read_i32(g, "paradee.lock.band_lo_hz", band_lo, err) || + !read_i32(g, "paradee.lock.band_hi_hz", band_hi, err)) { + return nullptr; + } + m->lock.band_lo_hz = (float) band_lo; + m->lock.band_hi_hz = (float) band_hi; + } else { + rates = { 1, 1 }; // unused; keeps the shared bounds check below simple + h.istft_hop = 1; } - m->lock.band_lo_hz = (float) band_lo; - m->lock.band_hi_hz = (float) band_hi; m->lock.sample_rate = h.sample_rate; const int widths[] = { h.text_n_layer, h.text_n_head, h.text_d_model, h.text_d_ff, h.text_vocab_size, @@ -303,7 +330,7 @@ paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string "paradee.bert_encoder.weight", "paradee.predictor.duration_proj.weight", "paradee.predictor.F0_proj.weight", "paradee.text_encoder.embd.weight", "paradee.asr_proj.0.weight", "paradee.asr_proj.1.weight", - "paradee.decoder.encode.conv1.weight", "paradee.gen.conv_post.weight", + "paradee.decoder.encode.conv1.weight", "paradee.decoder.decode.3.conv1.weight", }; for (const char * name : required) { if (!ggml_get_tensor(m->ctx, name)) { @@ -311,6 +338,19 @@ paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string return nullptr; } } + if (v10 && !ggml_get_tensor(m->ctx, "paradee.gen.conv_post.weight")) { + err = "required tensor missing: paradee.gen.conv_post.weight"; + return nullptr; + } + if (!v10) { + if (!head::load(g, m->ctx, m->head_p, m->head_w, err)) return nullptr; + // The head reads the trunk's output channels and runs at the F0 rate. + ggml_tensor * d3 = ggml_get_tensor(m->ctx, "paradee.decoder.decode.3.conv1.weight"); // [Cout, Cin, 3] + if ((int) d3->ne[2] != m->head_p.in_ch || m->head_p.sample_rate != h.sample_rate) { + err = "the head's input width does not match the decoder trunk"; + return nullptr; + } + } ggml_tensor * be = ggml_get_tensor(m->ctx, "paradee.bert_encoder.weight"); ggml_tensor * ap = ggml_get_tensor(m->ctx, "paradee.asr_proj.0.weight"); ggml_tensor * tok = ggml_get_tensor(m->ctx, "paradee.bert.token_embd.weight"); @@ -330,8 +370,16 @@ int paradee_sample_rate(const paradee_model * model) noexcept { return model ? model->hp.sample_rate : 0; } +paradee_head_kind paradee_get_head(const paradee_model * model) noexcept { + return model ? model->head_kind : PARADEE_HEAD_ISTFTNET_LOCK; +} + +const char * paradee_version(const paradee_model * model) noexcept { + return model ? model->version.c_str() : ""; +} + const paradee_lock_params * paradee_get_lock_params(const paradee_model * model) noexcept { - return model ? &model->lock : nullptr; + return model && model->head_kind == PARADEE_HEAD_ISTFTNET_LOCK ? &model->lock : nullptr; } namespace { @@ -452,6 +500,57 @@ bool cancel_trampoline(void * p) { namespace { +// v1.1: AdaIN trunk + source-filter head. noise_in (optional) replaces the +// seeded noise. Caller holds model->mu. +paradee_status decode_head_locked( + const paradee_model * model, + const float * asr_ct, int A, int T, + const float * F0, const float * N, + paradee_cancel_fn cancel, void * user, + const std::vector * noise_in, uint64_t seed, + std::vector & pcm, paradee_stages * stages, std::string & err) { + std::vector x; + if (!eliza_kokoro::decoder_front_forward(model->ctx, "paradee", /*instnorm_affine*/ true, asr_ct, A, T, F0, N, + model->style_dec.data(), x, err)) { + if (err.empty()) err = "decoder trunk failed"; + return PARADEE_E_RUNTIME; + } + if (cancel && cancel(user)) { err = "cancelled"; return PARADEE_CANCELLED; } + const int n = 2 * T; + const head::params & hp = model->head_p; + if (x.size() != (size_t) hp.in_ch * (size_t) n) { err = "decoder trunk output width mismatch"; return PARADEE_E_RUNTIME; } + const size_t L = (size_t) n * (size_t) hp.f0_hop; + std::vector drawn; + if (noise_in) { + if (noise_in->size() != L) { + err = "noise must have " + std::to_string(L) + " samples, got " + std::to_string(noise_in->size()); + return PARADEE_E_INVALID_ARG; + } + } else { + head::gaussian_noise(seed, L, drawn); + } + const std::vector & noise = noise_in ? *noise_in : drawn; + head::stages hs; + bool cancelled = false; + if (!head::forward(hp, model->head_w, x.data(), n, F0, noise.data(), pcm, stages ? &hs : nullptr, + cancel, user, cancelled, err)) { + pcm.clear(); + if (cancelled) { err = "cancelled"; return PARADEE_CANCELLED; } + if (err.empty()) err = "head failed"; + return PARADEE_E_RUNTIME; + } + if (stages) { + stages->trunk = std::move(x); + stages->hn = std::move(hs.hn); + stages->sine = std::move(hs.sine); + stages->main_path = std::move(hs.main); + stages->burst_path = std::move(hs.burst); + stages->noise = noise; + } + if (cancel && cancel(user)) { pcm.clear(); err = "cancelled"; return PARADEE_CANCELLED; } + return PARADEE_OK; +} + // Decoder + generator + phase lock. Caller holds model->mu. paradee_status decode_locked( const paradee_model * model, @@ -492,10 +591,12 @@ paradee_status decode_locked( } // namespace -paradee_status paradee_synthesize_ids_staged( +namespace { + +paradee_status synthesize_impl( const paradee_model * model, const int32_t * ids, size_t n, - float speed, + float speed, uint64_t seed, paradee_cancel_fn cancel, void * user, std::vector & pcm, std::string & err, @@ -567,10 +668,14 @@ paradee_status paradee_synthesize_ids_staged( } if (cancel && cancel(user)) { err = "cancelled"; return PARADEE_CANCELLED; } - // 2. Decoder + generator, 3. phase lock. + // 2. Decoder + generator, 3. phase lock (v1.0); or trunk + head (v1.1). std::vector src, dec; - const paradee_status dst = decode_locked(model, asr_ct.data(), A, T, pred.F0_pred.data(), - pred.N_pred.data(), cancel, user, dec, src, pcm, err); + const paradee_status dst = + model->head_kind == PARADEE_HEAD_SOURCE_FILTER + ? decode_head_locked(model, asr_ct.data(), A, T, pred.F0_pred.data(), pred.N_pred.data(), cancel, + user, stages ? stages->noise_in : nullptr, seed, pcm, stages, err) + : decode_locked(model, asr_ct.data(), A, T, pred.F0_pred.data(), pred.N_pred.data(), cancel, user, + dec, src, pcm, err); if (dst != PARADEE_OK) { pcm.clear(); return dst; } if (stages) { @@ -603,6 +708,19 @@ paradee_status paradee_synthesize_ids_staged( } } +} // namespace + +paradee_status paradee_synthesize_ids_staged( + const paradee_model * model, + const int32_t * ids, size_t n, + float speed, + paradee_cancel_fn cancel, void * user, + std::vector & pcm, + std::string & err, + paradee_stages * stages) noexcept { + return synthesize_impl(model, ids, n, speed, PARADEE_DEFAULT_NOISE_SEED, cancel, user, pcm, err, stages); +} + paradee_status paradee_synthesize_ids( const paradee_model * model, const int32_t * ids, size_t n, @@ -610,7 +728,7 @@ paradee_status paradee_synthesize_ids( paradee_cancel_fn cancel, void * user, std::vector & pcm, std::string & err) noexcept { - return paradee_synthesize_ids_staged(model, ids, n, speed, cancel, user, pcm, err, nullptr); + return synthesize_impl(model, ids, n, speed, PARADEE_DEFAULT_NOISE_SEED, cancel, user, pcm, err, nullptr); } } // namespace eliza_paradee @@ -624,8 +742,8 @@ paradee_status paradee_synthesize( paradee_audio & out, std::string & err) noexcept { out.sample_rate = model ? model->hp.sample_rate : 24000; - return paradee_synthesize_ids_staged(model, input_ids.data(), input_ids.size(), opts.speed, - opts.cancel, opts.cancel_user_data, out.samples, err, nullptr); + return synthesize_impl(model, input_ids.data(), input_ids.size(), opts.speed, opts.noise_seed, + opts.cancel, opts.cancel_user_data, out.samples, err, nullptr); } } // namespace eliza_paradee @@ -650,6 +768,11 @@ paradee_status paradee_decode_staged( eliza_kokoro::detail::KokoroLayerOptionsScope layer_scope(fast_layers()); stages->T_frame = T_frame; stages->asr_dim = asr_dim; + if (model->head_kind == PARADEE_HEAD_SOURCE_FILTER) { + if (src) { err = "a v1.1 model has no harmonic source to replace"; return PARADEE_E_INVALID_ARG; } + return decode_head_locked(model, asr, asr_dim, T_frame, F0, N, cancel, user, stages->noise_in, + PARADEE_DEFAULT_NOISE_SEED, stages->wav, stages, err); + } const paradee_hparams & h = model->hp; const size_t n_src = (size_t) T_frame * 2 * (size_t) (h.up_rates[0] * h.up_rates[1] * h.istft_hop); if (src && src->size() != n_src) { @@ -668,3 +791,49 @@ paradee_status paradee_decode_staged( } } // namespace eliza_paradee + +namespace eliza_paradee { + +paradee_status paradee_head_staged( + const paradee_model * model, + const float * trunk, int channels, int n, + const float * F0, const float * noise, + paradee_stages * stages, + std::string & err) noexcept { + err.clear(); + if (!model || !trunk || !F0 || !noise || !stages || n < 2) { + err = "invalid head arguments"; + return PARADEE_E_INVALID_ARG; + } + if (model->head_kind != PARADEE_HEAD_SOURCE_FILTER) { + err = "not a v1.1 (source-filter head) model"; + return PARADEE_E_INVALID_ARG; + } + if (channels != model->head_p.in_ch) { + err = "trunk must have " + std::to_string(model->head_p.in_ch) + " channels"; + return PARADEE_E_INVALID_ARG; + } + try { + std::lock_guard lk(const_cast(model)->mu); + eliza_kokoro::detail::KokoroLayerOptionsScope layer_scope(fast_layers()); + head::stages hs; + bool cancelled = false; + if (!head::forward(model->head_p, model->head_w, trunk, n, F0, noise, stages->wav, &hs, nullptr, nullptr, + cancelled, err)) { + return PARADEE_E_RUNTIME; + } + stages->hn = std::move(hs.hn); + stages->sine = std::move(hs.sine); + stages->main_path = std::move(hs.main); + stages->burst_path = std::move(hs.burst); + return PARADEE_OK; + } catch (const std::bad_alloc &) { + err = "out of memory"; + return PARADEE_E_OOM; + } catch (const std::exception & e) { + err = e.what(); + return PARADEE_E_RUNTIME; + } +} + +} // namespace eliza_paradee diff --git a/tools/paradee/tests/test_paradee_api.cpp b/tools/paradee/tests/test_paradee_api.cpp index d058371f5..6312cfcf8 100644 --- a/tools/paradee/tests/test_paradee_api.cpp +++ b/tools/paradee/tests/test_paradee_api.cpp @@ -3,7 +3,8 @@ // test-paradee-api — the eliza_paradee C++ API contract on the real model: // loading, phoneme mapping (pads, unknown symbols, length limit, UTF-8), // input validation, determinism, speed, and cancellation (before the call, -// between stages and inside the generator loops). +// between stages and inside the generator loops). Runs on a v1.0 or a v1.1 +// GGUF; for v1.1 also the noise seed and injected noise. // // Needs PARADEE_GGUF (environment); exits 77 (CTest SKIP) without it. @@ -44,10 +45,18 @@ int main() { if (!model) { std::fprintf(stderr, "load failed: %s\n", err.c_str()); return 1; } CHECK(paradee_sample_rate(model.get()) == 24000, "sample rate %d", paradee_sample_rate(model.get())); CHECK(paradee_sample_rate(nullptr) == 0, "null model sample rate"); + const bool v11 = paradee_get_head(model.get()) == PARADEE_HEAD_SOURCE_FILTER; + CHECK(std::string(paradee_version(model.get())) == (v11 ? "1.1" : "1.0"), "version %s", paradee_version(model.get())); + CHECK(paradee_get_head(nullptr) == PARADEE_HEAD_ISTFTNET_LOCK && std::string(paradee_version(nullptr)).empty(), + "null model head / version"); const paradee_lock_params * lp = paradee_get_lock_params(model.get()); - CHECK(lp && lp->n_fft == 1024 && lp->hop == 256 && lp->smooth_frames == 33 && lp->f0_hop == 300 && - lp->band_lo_hz == 2000.0f && lp->band_hi_hz == 8000.0f && lp->voiced_f0_hz == 60.0f, - "lock parameters not read from the GGUF"); + if (v11) { + CHECK(!lp, "a v1.1 model has no phase lock"); + } else { + CHECK(lp && lp->n_fft == 1024 && lp->hop == 256 && lp->smooth_frames == 33 && lp->f0_hop == 300 && + lp->band_lo_hz == 2000.0f && lp->band_hi_hz == 8000.0f && lp->voiced_f0_hz == 60.0f, + "lock parameters not read from the GGUF"); + } // Phonemes -> ids. std::vector ids; @@ -151,6 +160,45 @@ int main() { audio.sample_rate == 24000, "paradee_synthesize differs"); } + // v1.1: the noise comes from the seed (the default one unless the options + // choose), or from the caller. + if (v11) { + paradee_synth_options opts; + paradee_audio s0, s1, s1b; + opts.noise_seed = PARADEE_DEFAULT_NOISE_SEED; + CHECK(paradee_synthesize(model.get(), ids, opts, s0, err) == PARADEE_OK && s0.samples == a, "default seed"); + opts.noise_seed = 1; + CHECK(paradee_synthesize(model.get(), ids, opts, s1, err) == PARADEE_OK && s1.samples.size() == a.size() && + s1.samples != a, "another seed must change the noise"); + CHECK(paradee_synthesize(model.get(), ids, opts, s1b, err) == PARADEE_OK && s1b.samples == s1.samples, + "the same seed must give the same audio"); + paradee_stages st; + const std::vector wrong(a.size() + 1, 0.0f); + st.noise_in = &wrong; + CHECK(paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, pcm, err, &st) == + PARADEE_E_INVALID_ARG && pcm.empty(), "noise of the wrong length accepted"); + paradee_stages s2; + CHECK(paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, pcm, err, &s2) == + PARADEE_OK && pcm == a && s2.noise.size() == a.size(), "staged synthesis differs"); + paradee_stages s3; + s3.noise_in = &s2.noise; + CHECK(paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, pcm, err, &s3) == + PARADEE_OK && pcm == a, "the seeded noise fed back in must give the same audio"); + const std::vector silent(a.size(), 0.0f); + paradee_stages s4; + s4.noise_in = &silent; + CHECK(paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, pcm, err, &s4) == + PARADEE_OK && pcm != a, "zero noise must change the audio"); + CHECK(paradee_head_staged(model.get(), s2.trunk.data(), (int) (s2.trunk.size() / (2 * s2.T_frame)) + 1, + 2 * s2.T_frame, s2.F0.data(), s2.noise.data(), &s4, err) == PARADEE_E_INVALID_ARG, + "head with the wrong trunk width accepted"); + } else { + paradee_stages st; + const std::vector zeros(128 * 4 * 300, 0.0f); + CHECK(paradee_head_staged(model.get(), zeros.data(), 128, 4, zeros.data(), zeros.data(), &st, err) == + PARADEE_E_INVALID_ARG, "a v1.0 model has no source-filter head"); + } + // Concurrent calls on one model are serialised and give the same audio. { std::vector t1, t2; diff --git a/tools/paradee/tests/test_paradee_parity.cpp b/tools/paradee/tests/test_paradee_parity.cpp index 73b916312..becf655c9 100644 --- a/tools/paradee/tests/test_paradee_parity.cpp +++ b/tools/paradee/tests/test_paradee_parity.cpp @@ -17,6 +17,21 @@ // float32 evaluation is from float64 (+1e-4) // end to end final waveform SNR >= 35 dB against the reference // +// For a v1.1 GGUF (source-filter head; reference from `dump --version 1.1`) +// the text-side gates are the same, and the acoustic half is gated as: +// trunk AdaIN trunk output from the reference asr / F0 / N: +// max|err| <= 1e-3 * RMS +// head from the reference trunk, F0 and noise: hn (ConvNeXt stack), +// main (n_fft 600 path), burst (noise-burst path) and the head +// output, each max|err| <= 1e-3 * RMS +// sine sine bank from the reference F0: max|err| <= 1e-4 * RMS +// against the bank evaluated in float64, and no further from +// the float32 vhead.sine_bank than that is from float64 +// (+1e-4 * RMS). float32's own error reaches ~9e-4 * RMS. +// end to end the whole chain with the reference's noise: SNR >= 35 dB +// PARADEE_GGUF_V10= also checks that the text side is unchanged: +// the two GGUFs' text-side stages must be bit-identical. +// // Each acoustic stage is fed the reference's inputs, so an error is charged // to the stage that made it: the harmonic source integrates F0 over seconds, // and float32 rounding differences in F0 (~1e-6 relative) would otherwise @@ -107,13 +122,27 @@ int main() { const char * out_dir = std::getenv("PARADEE_PARITY_OUT"); const json manifest = read_json(ref_dir + "/manifest.json"); - const json fixture = read_json(std::string(PARADEE_FIXTURE_DIR) + "/sentences.misaki.json"); + // PARADEE_FIXTURE_DIR in the environment overrides the build-time path + // (for runs on another machine, such as a device over adb). + const char * fixture_dir = std::getenv("PARADEE_FIXTURE_DIR"); + const json fixture = read_json(std::string(fixture_dir && *fixture_dir ? fixture_dir : PARADEE_FIXTURE_DIR) + + "/sentences.misaki.json"); const float speed = manifest.value("speed", 1.0f); std::string err; auto model = paradee_load_model(gguf, err); if (!model) { std::fprintf(stderr, "load failed: %s\n", err.c_str()); return 1; } - const paradee_lock_params lock = *paradee_get_lock_params(model.get()); + const bool v11 = paradee_get_head(model.get()) == PARADEE_HEAD_SOURCE_FILTER; + std::printf("model: Paradee-8M v%s (%s head)\n", paradee_version(model.get()), v11 ? "source-filter" : "iSTFTNet + lock"); + CHECK(manifest.value("version", std::string("1.0")) == (v11 ? "1.1" : "1.0"), + "the reference was dumped for another Paradee version"); + const paradee_lock_params lock = v11 ? paradee_lock_params() : *paradee_get_lock_params(model.get()); + paradee_model_ptr model_v10; + if (const char * g10 = std::getenv("PARADEE_GGUF_V10"); v11 && g10 && *g10) { + model_v10 = paradee_load_model(g10, err); + CHECK(model_v10 && paradee_get_head(model_v10.get()) == PARADEE_HEAD_ISTFTNET_LOCK, + "PARADEE_GGUF_V10 is not a v1.0 GGUF: %s", err.c_str()); + } CHECK(manifest["sentences"].size() == fixture["sentences"].size(), "reference and fixture sentence counts differ"); double total_audio = 0, total_secs = 0; @@ -132,8 +161,11 @@ int main() { CHECK(ids == fx["ids"].get>(), "sentence %d: ids differ from the fixture", sent); CHECK(ids == read_raw(file("ids")), "sentence %d: ids differ from the reference", sent); - // full synthesis + // full synthesis (v1.1: with the reference's noise) + std::vector ref_noise; + if (v11) ref_noise = read_raw(file("noise")); paradee_stages st; + if (v11) st.noise_in = &ref_noise; std::vector pcm; const auto t0 = std::chrono::steady_clock::now(); const paradee_status rc = paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), speed, @@ -153,6 +185,59 @@ int main() { gate_stage(sent, "N", read_raw(file("N")), st.N); gate_stage(sent, "asr", read_raw(file("asr")), st.asr); + if (model_v10) { + paradee_stages s10; + std::vector p10; + CHECK(paradee_synthesize_ids_staged(model_v10.get(), ids.data(), ids.size(), speed, nullptr, nullptr, + p10, err, &s10) == PARADEE_OK, "%s", err.c_str()); + const bool same = s10.bert == st.bert && s10.d == st.d && s10.dur == st.dur && s10.dur_int == st.dur_int && + s10.F0 == st.F0 && s10.N == st.N && s10.asr == st.asr; + std::printf(" text side bit-identical to the v1.0 GGUF: %s\n", same ? "yes" : "NO"); + CHECK(same, "sentence %d: v1.1 text side differs from v1.0", sent); + } + + if (v11) { + const std::vector ref_asr = read_raw(file("asr")); + const std::vector ref_F0 = read_raw(file("F0")); + const std::vector ref_N = read_raw(file("N")); + const std::vector ref_trunk = read_raw(file("trunk")); + const std::vector ref_wav = read_raw(file("wav")); + const int T = (int) ref_F0.size() / 2; + const int n = (int) ref_F0.size(); + paradee_stages a; + a.noise_in = &ref_noise; + CHECK(paradee_decode_staged(model.get(), ref_asr.data(), 512, T, ref_F0.data(), ref_N.data(), nullptr, + nullptr, nullptr, &a, err) == PARADEE_OK, "%s", err.c_str()); + gate_stage(sent, "trunk", ref_trunk, a.trunk); + const int C = n > 0 ? (int) (ref_trunk.size() / (size_t) n) : 0; + paradee_stages b; + CHECK(paradee_head_staged(model.get(), ref_trunk.data(), C, n, ref_F0.data(), ref_noise.data(), &b, err) + == PARADEE_OK, "%s", err.c_str()); + gate_stage(sent, "hn", read_raw(file("hn")), b.hn); + { + const std::vector s32 = read_raw(file("sine")), s64 = read_raw(file("sine64")); + const err_stats e64 = compare(s64, b.sine), e32 = compare(s32, b.sine), f32 = compare(s64, s32); + const double rms = std::max(e64.rms_ref, 1e-30); + std::printf(" sine max|err| %9.3g x rms vs float64 (gate 1e-4); %9.3g vs float32 sine_bank " + "(float32 itself: %.3g)\n", e64.max_abs / rms, e32.max_abs / rms, f32.max_abs / rms); + CHECK(e64.same_size && e64.max_abs <= 1e-4 * rms, "sentence %d sine: %.3g > 1e-4 * rms", sent, + e64.max_abs / rms); + CHECK(e32.same_size && e32.max_abs <= f32.max_abs + 1e-4 * rms, "sentence %d sine vs float32", sent); + } + gate_stage(sent, "main", read_raw(file("main")), b.main_path); + gate_stage(sent, "burst", read_raw(file("burst")), b.burst_path); + gate_stage(sent, "head", ref_wav, b.wav); + const err_stats e = compare(ref_wav, pcm); + std::printf(" end to end: snr %.1f dB, max|err| %.3g, %zu samples, %.3f s synth (%.1fx realtime)\n", + e.snr_db, e.max_abs, pcm.size(), secs, (double) pcm.size() / 24000.0 / secs); + CHECK(e.same_size && e.snr_db >= 35.0, "sentence %d: end-to-end SNR %.1f dB < 35 dB", sent, e.snr_db); + if (out_dir && *out_dir) { + std::ofstream f(std::string(out_dir) + "/" + std::to_string(sent) + ".wav.f32", std::ios::binary); + f.write((const char *) pcm.data(), (std::streamsize) (pcm.size() * sizeof(float))); + } + continue; + } + // acoustic half, teacher-forced stage by stage const std::vector ref_asr = read_raw(file("asr")); const std::vector ref_F0 = read_raw(file("F0")); diff --git a/tools/paradee/tools/dump_paradee_reference.py b/tools/paradee/tools/dump_paradee_reference.py index 3561d7836..eb6972b4e 100644 --- a/tools/paradee/tools/dump_paradee_reference.py +++ b/tools/paradee/tools/dump_paradee_reference.py @@ -41,6 +41,19 @@ The final waveform applies phase_lock.lock() in float64. lock_f32 is the unmodified float32 phase_lock.lock() on the same decoder output, for the Lock-only check. +`dump --version 1.1` runs Paradee-8M v1.1 (pins.json "v1.1"): the same text side, then +vhead.VDecoder(head="v2") loaded with load_ckpt, as export_paradee_v1_1.py builds it. The +v1.1 head draws Gaussian noise, so the reference fixes it: each sentence's noise is +torch.randn from torch.Generator().manual_seed(--noise-seed + index), or read from +--noise-dir/.noise.f32, and is written out as the `noise` stage for the native +test to feed back in. Nothing else in the v1.1 reference is changed; it runs the +upstream module in float32. Extra v1.1 stages: trunk (AdaIN trunk output), hn (the +head's ConvNeXt stack after norm1), sine (vhead.sine_bank), sine64 (the same sine bank +with the harmonics evaluated in float64: float32's 2*pi*k*cyc reaches ~440 rad, where one +float32 step is ~3e-5 rad, so the float32 bank is up to ~9e-4 x RMS from its exact value), +main (the n_fft 600 path, head run with burst=False), burst (the 150/75 noise-burst path) +and wav (head output). + usage: dump_paradee_reference.py phonemes --model-dir HF_SNAPSHOT --out tools/paradee/tests/fixtures/sentences.misaki.json dump_paradee_reference.py dump --model-dir HF_SNAPSHOT --paradee-src PARADEE_CHECKOUT --out REF_DIR @@ -145,6 +158,138 @@ def cmd_phonemes(args) -> int: # reference model # --------------------------------------------------------------------------- +def load_reference_v11(model_dir: Path, paradee_src: Path): + """Paradee-8M v1.1 in float32, as export_paradee_v1_1.py builds it.""" + pins = PINS["v1.1"] + check_versions("torch", "kokoro", "misaki") + for rel, digest in pins["source"]["files"].items(): + check_file(paradee_src, rel, digest) + for rel in ("pytorch/text_side.pt", "pytorch/decoder.pt"): + check_file(model_dir, rel, pins["files"][rel]) + import torch + sys.path.insert(0, str(paradee_src / "training" / "scripts")) + student = importlib.import_module("student") + vhead = importlib.import_module("vhead") + from torch.nn.utils import remove_weight_norm + from torch.nn.utils.parametrize import is_parametrized, remove_parametrizations + torch.set_grad_enabled(False) + + def strip(m): # export_paradee_v1_1.py strip() + for x in m.modules(): + try: + remove_weight_norm(x) + except Exception: + pass + if is_parametrized(x, "weight"): + try: + remove_parametrizations(x, "weight", leave_parametrized=True) + except Exception: + pass + + ts = student.TextStudent("s+mlp").eval() + ts.load_state_dict(torch.load(model_dir / "pytorch/text_side.pt", map_location="cpu", weights_only=True)["model"]) + vd = vhead.VDecoder(head="v2").eval() + vd.load_ckpt(str(model_dir / "pytorch/decoder.pt")) + if not vd.head.burst: + raise SystemExit("v1.1 decoder.pt has no noise-burst path") + strip(ts) + strip(vd) + return torch, ts, vd, vhead, student + + +def text_side(torch, ts, student, ids: list[int], speed: float): + """export_paradee.py Paradee.text() (the same in export_paradee_v1_1.py), with the stages kept.""" + import torch.nn as nn + x_ids = torch.tensor([ids], dtype=torch.long) + p, s = ts.predictor, ts.style + bert = ts.bert(x_ids, attention_mask=torch.ones_like(x_ids)) # [1,T,256] + d_en = ts.bert_encoder(bert).transpose(-1, -2) + T = x_ids.shape[1] + sx = s[:, :, None].expand(-1, -1, T) + x = torch.cat([d_en, sx], 1) + for b in p.text_encoder.lstms: + if isinstance(b, nn.LSTM): + x = b(x.transpose(-1, -2))[0].transpose(-1, -2) + else: + x = torch.cat([b(x.transpose(-1, -2), s).transpose(-1, -2), sx], 1) + d = x.transpose(-1, -2) # [1,T,224] + dur = torch.sigmoid(p.duration_proj(p.lstm(d)[0])).sum(-1)[0] # [T] + e = ts.text_encoder + h = e.embedding(x_ids).transpose(1, 2) + for c in e.cnn: + h = c(h) + t_en = e.lstm(h.transpose(1, 2))[0] + asr_tok = ts.asr_proj(t_en).transpose(-1, -2) # [1,512,T] + n = torch.clamp(torch.round(dur / speed), min=1).long() + aln = student.alignment(n)[None] # [1,T,F] + F0, N = p.F0Ntrain(d.transpose(-1, -2) @ aln, s) + asr = asr_tok @ aln # [1,512,F] + return bert, d, dur, n, F0, N, asr + + +def sine_bank64(torch, F0, K=64, hop=300, sr=24000): + """vhead.sine_bank with sin(2 pi k cyc) and the harmonic sum in float64. The running + phase c, its step d, the interpolation weight and cyc are float32 as in the module.""" + import torch.nn.functional as F + n = F0.shape[1] + r = F0.double() * hop / sr + c = (torch.cumsum(r, 1) % 1).float() + d = F.pad(r[:, 1:], (0, 1)).float() + p = (torch.arange(n * hop) + 0.5) / hop - 0.5 + i = p.floor().clamp(0, n - 1).long() + fr = (p - i).clamp(0, 1) + cyc = c[:, i] + fr * d[:, i] + f0s = F0.repeat_interleave(hop, 1) + k = torch.arange(1, K + 1)[None, :, None] + amp = ((f0s[:, None] * k < 11000) & (f0s[:, None] > 10)).double() + return (torch.sin(2 * math.pi * k.double() * cyc.double()[:, None]) * amp).sum(1) * 0.1 + + +def run_reference_v11(torch, ts, vd, vhead, student, ids: list[int], speed: float, noise) -> dict: + """export_paradee_v1_1.py Paradee2.forward with the torch head (vhead.SourceFilterHead2) + and the given noise, with the stages kept.""" + bert, d, dur, n, F0, N, asr = text_side(torch, ts, student, ids, speed) + h = vd.head + L = F0.shape[1] * vhead.FHOP + if noise is None: + raise SystemExit("v1.1 reference needs its noise") + nz = torch.as_tensor(noise, dtype=torch.float32)[None] + if nz.shape[1] != L: + raise SystemExit(f"noise has {nz.shape[1]} samples, the sentence needs {L}") + cap = {} + hook = h.norm1.register_forward_hook(lambda m, i, o: cap.__setitem__("hn", o)) + s = vd.style.expand(1, -1) # VDecoder.forward up to the head + F0c = vd.F0_conv(F0.unsqueeze(1)); Nc = vd.N_conv(N.unsqueeze(1)) + x = vd.encode(torch.cat([asr, F0c, Nc], 1), s); asr_res = vd.asr_res(asr); res = True + for block in vd.decode: + if res: + x = torch.cat([x, asr_res, F0c, Nc], 1) + x = block(x, s) + if block.upsample_type != "none": + res = False + wav = h(x, F0, nz) + gb = h.last["gb"] + hook.remove() + h.burst = False + main = h(x, F0, nz) + h.burst = True + Nb = torch.stft(nz.flip(-1), 150, 75, window=h.wb, return_complex=True) + burst = torch.istft(Nb * gb, 150, 75, window=h.wb, length=L) + sine = vhead.sine_bank(F0, h.K)[0] + sine64 = sine_bank64(torch, F0, h.K) + full = vd(asr, F0, N, noise=nz) # VDecoder.forward itself + if not torch.equal(full, wav): + raise SystemExit("VDecoder.forward differs from the staged trunk + head") + f = lambda t: t.detach().contiguous().float().numpy() + return { + "ids": np.asarray(ids, dtype=np.int32), + "bert": f(bert[0]), "d": f(d[0]), "dur": f(dur / speed), "dur_int": n.numpy().astype(np.int32), + "F0": f(F0[0]), "N": f(N[0]), "asr": f(asr[0]), "noise": f(nz[0]), + "trunk": f(x[0]), "hn": f(cap["hn"][0]), "sine": f(sine[0]), "sine64": f(sine64[0]), "main": f(main[0]), "burst": f(burst[0]), + "wav": f(wav[0]), + } + + def load_reference(model_dir: Path, paradee_src: Path): check_versions("torch", "kokoro", "misaki") for rel, digest in PINS["source"]["files"].items(): @@ -269,31 +414,7 @@ def lock64(torch, phase_lock, s, src, f0, L): def run_reference(torch, ts, ds, cap, phase_lock, student, ids: list[int], speed: float) -> dict: """export_paradee.py Paradee.text() + forward(), with the stages kept.""" - import torch.nn as nn - x_ids = torch.tensor([ids], dtype=torch.long) - p, s = ts.predictor, ts.style - bert = ts.bert(x_ids, attention_mask=torch.ones_like(x_ids)) # [1,T,256] - d_en = ts.bert_encoder(bert).transpose(-1, -2) - T = x_ids.shape[1] - sx = s[:, :, None].expand(-1, -1, T) - x = torch.cat([d_en, sx], 1) - for b in p.text_encoder.lstms: - if isinstance(b, nn.LSTM): - x = b(x.transpose(-1, -2))[0].transpose(-1, -2) - else: - x = torch.cat([b(x.transpose(-1, -2), s).transpose(-1, -2), sx], 1) - d = x.transpose(-1, -2) # [1,T,224] - dur = torch.sigmoid(p.duration_proj(p.lstm(d)[0])).sum(-1)[0] # [T] - e = ts.text_encoder - h = e.embedding(x_ids).transpose(1, 2) - for c in e.cnn: - h = c(h) - t_en = e.lstm(h.transpose(1, 2))[0] - asr_tok = ts.asr_proj(t_en).transpose(-1, -2) # [1,512,T] - n = torch.clamp(torch.round(dur / speed), min=1).long() - aln = student.alignment(n)[None] # [1,T,F] - F0, N = p.F0Ntrain(d.transpose(-1, -2) @ aln, s) - asr = asr_tok @ aln # [1,512,F] + bert, d, dur, n, F0, N, asr = text_side(torch, ts, student, ids, speed) dec = ds(asr, F0, N).squeeze() # [L] src = cap["src"].squeeze() k = min(len(dec), len(src)) @@ -310,18 +431,35 @@ def run_reference(torch, ts, ds, cap, phase_lock, student, ids: list[int], speed def cmd_dump(args) -> int: fixture = json.loads(args.phonemes.read_text(encoding="utf-8")) - torch, ts, ds, cap, phase_lock, student = load_reference(args.model_dir, args.paradee_src) + v11 = args.version == "1.1" + pins = PINS["v1.1"] if v11 else PINS + if v11: + torch, ts, vd, vhead, student = load_reference_v11(args.model_dir, args.paradee_src) + else: + torch, ts, ds, cap, phase_lock, student = load_reference(args.model_dir, args.paradee_src) args.out.mkdir(parents=True, exist_ok=True) manifest = { - "model": PINS["model"], "source_commit": PINS["source"]["commit"], + "model": pins["model"], "source_commit": pins["source"]["commit"], "version": args.version, "phonemes_fixture_sha256": sha256_file(args.phonemes), "speed": args.speed, "versions": {k: importlib.metadata.version(k) for k in ("torch", "kokoro", "misaki")}, - "reference": "deterministic: SineGen rand_ini=0, no noise, float64; canonical float64 source STFT; float64 phase lock", + "reference": ("float32 upstream modules; head noise fixed (the noise stage), " + f"{'read from ' + str(args.noise_dir) if args.noise_dir else 'torch.randn seeded with %d + index' % args.noise_seed}" + if v11 else + "deterministic: SineGen rand_ini=0, no noise, float64; canonical float64 source STFT; float64 phase lock"), "sentences": [], } for item in fixture["sentences"]: - out = run_reference(torch, ts, ds, cap, phase_lock, student, item["ids"], args.speed) + if v11: + frames = int(text_side(torch, ts, student, item["ids"], args.speed)[3].sum()) + L = 2 * frames * 300 + if args.noise_dir: + noise = np.fromfile(args.noise_dir / f"{item['index']}.noise.f32", dtype=" int: p.add_argument("--phonemes", type=Path, default=DEFAULT_FIXTURE) p.add_argument("--speed", type=float, default=1.0) p.add_argument("--out", type=Path, required=True) + p.add_argument("--version", choices=("1.0", "1.1"), default="1.0", + help="1.1: Paradee-8M v1.1 (source-filter head; --model-dir and --paradee-src at the v1.1 pins)") + p.add_argument("--noise-seed", type=int, default=0, help="v1.1: noise = torch.randn seeded with this + index") + p.add_argument("--noise-dir", type=Path, help="v1.1: read .noise.f32 from here instead") p = sub.add_parser("lock-fixture") p.add_argument("--paradee-src", type=Path, required=True) p.add_argument("--out", type=Path, default=PARADEE_DIR / "tests" / "fixtures" / "phase_lock_synthetic.f32") diff --git a/tools/paradee/tools/paradee-stage-dump.cpp b/tools/paradee/tools/paradee-stage-dump.cpp index fd453c908..fba4831ca 100644 --- a/tools/paradee/tools/paradee-stage-dump.cpp +++ b/tools/paradee/tools/paradee-stage-dump.cpp @@ -11,7 +11,8 @@ // outputs (asr [512, T] channel-major, F0 / N [2T]). // // writes .{bert,d,dur,F0,N,asr,dec,src,wav}.f32, .dur_int.i32 -// and .ids.i32. +// and .ids.i32. For a v1.1 GGUF (source-filter head) dec and src are +// empty and .{trunk,hn,sine,main,burst,noise}.f32 are written too. #include "paradee.h" @@ -29,6 +30,13 @@ static bool write_raw(const std::string & path, const std::vector & v) { return (bool) f; } +static bool write_head(const std::string & pre, const eliza_paradee::paradee_stages & st) { + if (st.trunk.empty()) return true; // v1.0 + return write_raw(pre + ".trunk.f32", st.trunk) && write_raw(pre + ".hn.f32", st.hn) && + write_raw(pre + ".sine.f32", st.sine) && write_raw(pre + ".main.f32", st.main_path) && + write_raw(pre + ".burst.f32", st.burst_path) && write_raw(pre + ".noise.f32", st.noise); +} + static std::vector read_f32(const char * path) { std::ifstream f(path, std::ios::binary | std::ios::ate); std::vector v((size_t) f.tellg() / sizeof(float)); @@ -52,7 +60,7 @@ int main(int argc, char ** argv) { } const std::string pre = argv[6]; return write_raw(pre + ".dec.f32", st.dec) && write_raw(pre + ".src.f32", st.src) && - write_raw(pre + ".wav.f32", st.wav) ? 0 : 1; + write_raw(pre + ".wav.f32", st.wav) && write_head(pre, st) ? 0 : 1; } if (argc < 4) { std::fprintf(stderr, "usage: %s [speed]\n", argv[0]); @@ -85,7 +93,7 @@ int main(int argc, char ** argv) { write_raw(pre + ".dur_int.i32", st.dur_int) && write_raw(pre + ".F0.f32", st.F0) && write_raw(pre + ".N.f32", st.N) && write_raw(pre + ".asr.f32", st.asr) && write_raw(pre + ".dec.f32", st.dec) && write_raw(pre + ".src.f32", st.src) && - write_raw(pre + ".wav.f32", st.wav); + write_raw(pre + ".wav.f32", st.wav) && write_head(pre, st); std::printf("T_phon=%d T_frame=%d samples=%zu\n", st.T_phon, st.T_frame, st.wav.size()); return ok ? 0 : 1; }