tools/paradee: native Paradee-8M v1.0 TTS with phase lock and PyTorch parity gate

CMakeLists.txt

@@ -131,6 +131,16 @@ option(OMNIVOICE_SHARED      "Build libomnivoice for ABI consumers" OFF)
 # Kokoro is a first-class voice surface in the eliza-1 stack.
 option(LLAMA_BUILD_KOKORO    "Build the Kokoro-82M TTS subtree (tools/kokoro/)" ON)
 
+# Paradee-8M v1.0 (single-voice distillation of Kokoro-82M, Apache-2.0).
+# Produces paradee_lib + paradee-tts. Runs on kokoro_lib's layers, so it is
+# built only together with LLAMA_BUILD_KOKORO. Default ON: Paradee is the
+# shipped mobile voice.
+option(LLAMA_BUILD_PARADEE   "Build the Paradee-8M TTS subtree (tools/paradee/)" ON)
+
+# Native misaki grapheme-to-phoneme (Paradee/Kokoro phoneme spelling). The
+# subtree lives in tools/misaki-g2p/ and is added when present.
+option(LLAMA_BUILD_MISAKI_G2P "Build the misaki G2P subtree (tools/misaki-g2p/)" ON)
+
 # W3-3 deprecation: ELIZA_FUSE_OMNIVOICE=ON is the pre-W3-3 flag for the
 # fork-root `omnivoice/` graft. The graft is gone (see the deprecation
 # block below); turn the flag into a redirect now so the option ON state
@@ -281,6 +291,27 @@ if (LLAMA_BUILD_KOKORO AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
     add_subdirectory(tools/kokoro)
 endif()
 
+# misaki G2P and Paradee — embed-as-library hooks (mirror kokoro above). Both
+# precede tools/omnivoice so the fused `elizainference` target can fold them
+# in with `if(TARGET ...)`.
+if (LLAMA_BUILD_MISAKI_G2P AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
+    if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/misaki-g2p/CMakeLists.txt")
+        add_subdirectory(tools/misaki-g2p)
+    else()
+        message(STATUS "LLAMA_BUILD_MISAKI_G2P: tools/misaki-g2p/ is not in this checkout; skipped")
+    endif()
+endif()
+if (LLAMA_BUILD_PARADEE AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
+    # Paradee runs on the tools/kokoro layers. A build that turns Kokoro off
+    # (backend-only CI, for example) skips Paradee instead of failing; the
+    # fused library then reports paradee_supported() == 0.
+    if (LLAMA_BUILD_KOKORO)
+        add_subdirectory(tools/paradee)
+    else()
+        message(STATUS "LLAMA_BUILD_PARADEE: skipped because LLAMA_BUILD_KOKORO=OFF")
+    endif()
+endif()
+
 if (LLAMA_BUILD_OMNIVOICE)
     if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/omnivoice/CMakeLists.txt")
         message(FATAL_ERROR

tools/CMakeLists.txt

@@ -41,6 +41,26 @@ else()
         # this ordering for the embed-as-library (LLAMA_BUILD_TOOLS=OFF) case.
         add_subdirectory(kokoro)
     endif()
+    if (LLAMA_BUILD_MISAKI_G2P)
+        # Native misaki G2P (the phoneme spelling Paradee and Kokoro were
+        # trained on). Added when the subtree is present.
+        if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/misaki-g2p/CMakeLists.txt")
+            add_subdirectory(misaki-g2p)
+        else()
+            message(STATUS "LLAMA_BUILD_MISAKI_G2P: tools/misaki-g2p/ is not in this checkout; skipped")
+        endif()
+    endif()
+    if (LLAMA_BUILD_PARADEE)
+        # Paradee-8M (width-scaled Kokoro student, single voice). Needs
+        # kokoro_lib; like kokoro it precedes omnivoice so the fused
+        # `elizainference` target can fold paradee_lib in.
+        # A build with Kokoro off skips Paradee rather than failing.
+        if (LLAMA_BUILD_KOKORO)
+            add_subdirectory(paradee)
+        else()
+            message(STATUS "LLAMA_BUILD_PARADEE: skipped because LLAMA_BUILD_KOKORO=OFF")
+        endif()
+    endif()
     if (LLAMA_BUILD_OMNIVOICE)
         # The merged omnivoice subtree lives under tools/omnivoice/. The
         # top-level CMakeLists.txt has a parallel add_subdirectory hook for

tools/kokoro/include/kokoro-decoder-front.h

@@ -1,5 +1,9 @@
 // SPDX-License-Identifier: MIT
 // kokoro-decoder-front.h — Decoder.forward up to the generator (validated port, #9588).
+//
+// Channel widths come from the weights struct (filled from the GGUF tensor
+// shapes by kokoro-decoder.cpp), so width-scaled students of Kokoro-82M
+// (Paradee-8M: AdaIN width 256, style 16) share this code.
 #pragma once
 #include <cmath>
 #include <cstring>
@@ -24,6 +28,12 @@ struct DecAdainResBlk {
     const float * conv1x1_b  = nullptr;  // [Cout] (null — conv1x1 bias=False)
     const float * pool_w     = nullptr;  // [Cin, 1, 3] (upsample only)
     const float * pool_b     = nullptr;  // [Cin] (upsample only)
+    // Trained InstanceNorm affine inside AdaIN1d (students only; null keeps
+    // Kokoro-82M's identity affine).
+    const float * norm1_w    = nullptr;  // [Cin]
+    const float * norm1_b    = nullptr;  // [Cin]
+    const float * norm2_w    = nullptr;  // [Cout]
+    const float * norm2_b    = nullptr;  // [Cout]
 };
 
 // AdainResBlk1d (decode-block flavor: leaky_relu 0.2; pool/shortcut;
@@ -35,7 +45,7 @@ inline void dec_adainresblk1d_forward(
 
     // residual branch: norm1 -> leaky_relu(0.2) -> [pool] -> conv1 -> norm2 -> leaky_relu -> conv2
     std::vector<float> r(x, x + (size_t)Cin * T_in);
-    adain1d_forward(r.data(), Cin, T_in, s, Sdim, w.norm1_fc_w, w.norm1_fc_b);
+    adain1d_forward(r.data(), Cin, T_in, s, Sdim, w.norm1_fc_w, w.norm1_fc_b, w.norm1_w, w.norm1_b);
     for (size_t i = 0; i < r.size(); ++i) if (r[i] < 0) r[i] *= 0.2f;
 
     int T_pool = T_in;
@@ -47,7 +57,7 @@ inline void dec_adainresblk1d_forward(
     }
     std::vector<float> r3((size_t)Cout * T_pool);
     conv1d_forward(r.data(), Cin, T_pool, w.conv1_w, w.conv1_b, Cout, 3, 1, 1, 1, r3.data(), T_pool);
-    adain1d_forward(r3.data(), Cout, T_pool, s, Sdim, w.norm2_fc_w, w.norm2_fc_b);
+    adain1d_forward(r3.data(), Cout, T_pool, s, Sdim, w.norm2_fc_w, w.norm2_fc_b, w.norm2_w, w.norm2_b);
     for (size_t i = 0; i < r3.size(); ++i) if (r3[i] < 0) r3[i] *= 0.2f;
     std::vector<float> r4((size_t)Cout * T_pool);
     conv1d_forward(r3.data(), Cout, T_pool, w.conv2_w, w.conv2_b, Cout, 3, 1, 1, 1, r4.data(), T_pool);
@@ -82,15 +92,16 @@ struct DecoderFrontWeights {
     const float * F0_conv_b = nullptr;  // [1]
     const float * N_conv_w  = nullptr;  // [1,1,3]
     const float * N_conv_b  = nullptr;  // [1]
-    const float * asr_res_w = nullptr;  // [64,512,1]
-    const float * asr_res_b = nullptr;  // [64]
-    DecAdainResBlk encode;              // 514 -> 1024, learned_sc
-    DecAdainResBlk decode[4];           // 1090->1024 (x3), 1090->512 upsample
+    const float * asr_res_w = nullptr;  // [asr_res_C, Cin_asr, 1]
+    const float * asr_res_b = nullptr;  // [asr_res_C]
+    int asr_res_C = 64;                 // asr_res output channels (64 for Kokoro and Paradee)
+    DecAdainResBlk encode;              // Kokoro: 514 -> 1024, learned_sc
+    DecAdainResBlk decode[4];           // Kokoro: 1090->1024 (x3), 1090->512 upsample
 };
 
 // Decoder.forward up to (not including) the generator.
-//   asr[512,T_asr] (T_asr=132), F0_curve[2*T_asr], N[2*T_asr], s[128]
-// Output: x_out [512, 2*T_asr] (== generator_in_0); also returns the
+//   asr[Cin_asr,T_asr], F0_curve[2*T_asr], N[2*T_asr], s[Sdim]
+// Output: x_out [decode[3].Cout, 2*T_asr] (== generator_in_0); also returns the
 // stride-2 conv outputs F0_down[T_asr], N_down[T_asr] (caller passes them,
 // together with the ORIGINAL F0_curve, into the generator).
 inline void decoder_front(
@@ -100,36 +111,37 @@ inline void decoder_front(
         std::vector<float> & x_out,
         std::vector<float> & F0_down,
         std::vector<float> & N_down) {
-    const int Tc = 2 * T_asr;  // 264
+    const int Tc = 2 * T_asr;
 
     F0_down.assign(T_asr, 0.0f);
     conv1d_forward(F0_curve, 1, Tc, W.F0_conv_w, W.F0_conv_b, 1, 3, 2, 1, 1, F0_down.data(), T_asr);
     N_down.assign(T_asr, 0.0f);
     conv1d_forward(N_in, 1, Tc, W.N_conv_w, W.N_conv_b, 1, 3, 2, 1, 1, N_down.data(), T_asr);
 
-    // x = cat([asr, F0, N], dim=channels) -> [514, T_asr]
+    // x = cat([asr, F0, N], dim=channels) -> [Cin_asr + 2, T_asr]
     std::vector<float> xcat((size_t)(Cin_asr + 2) * T_asr);
     std::memcpy(xcat.data(), asr, sizeof(float) * (size_t)Cin_asr * T_asr);
     std::memcpy(xcat.data() + (size_t)Cin_asr * T_asr, F0_down.data(), sizeof(float) * T_asr);
     std::memcpy(xcat.data() + (size_t)(Cin_asr + 1) * T_asr, N_down.data(), sizeof(float) * T_asr);
 
     std::vector<float> x; int T_x;
-    dec_adainresblk1d_forward(W.encode, xcat.data(), T_asr, s, x, T_x);   // encode 514->1024
+    dec_adainresblk1d_forward(W.encode, xcat.data(), T_asr, s, x, T_x);   // encode Cin_asr+2 -> W
 
-    std::vector<float> asr_res((size_t)64 * T_asr);                       // asr_res Conv1d k1 512->64
-    conv1d_forward(asr, Cin_asr, T_asr, W.asr_res_w, W.asr_res_b, 64, 1, 1, 0, 1, asr_res.data(), T_asr);
+    const int Cr = W.asr_res_C;
+    std::vector<float> asr_res((size_t)Cr * T_asr);                       // asr_res Conv1d k1 Cin_asr->Cr
+    conv1d_forward(asr, Cin_asr, T_asr, W.asr_res_w, W.asr_res_b, Cr, 1, 1, 0, 1, asr_res.data(), T_asr);
 
     bool res = true;
     for (int b = 0; b < 4; ++b) {
         std::vector<float> blk_in;
-        if (res) {  // cat([x, asr_res, F0, N]) -> 1024+64+1+1 = 1090
+        if (res) {  // cat([x, asr_res, F0, N]) -> Kokoro 1024+64+1+1 = 1090
             const int Cx = (int)(x.size() / T_x);
-            const int Cin_blk = Cx + 64 + 1 + 1;
+            const int Cin_blk = Cx + Cr + 1 + 1;
             blk_in.assign((size_t)Cin_blk * T_x, 0.0f);
             std::memcpy(blk_in.data(), x.data(), sizeof(float) * (size_t)Cx * T_x);
-            std::memcpy(blk_in.data() + (size_t)Cx * T_x, asr_res.data(), sizeof(float) * 64 * T_x);
-            std::memcpy(blk_in.data() + (size_t)(Cx + 64) * T_x, F0_down.data(), sizeof(float) * T_x);
-            std::memcpy(blk_in.data() + (size_t)(Cx + 65) * T_x, N_down.data(), sizeof(float) * T_x);
+            std::memcpy(blk_in.data() + (size_t)Cx * T_x, asr_res.data(), sizeof(float) * (size_t)Cr * T_x);
+            std::memcpy(blk_in.data() + (size_t)(Cx + Cr) * T_x, F0_down.data(), sizeof(float) * T_x);
+            std::memcpy(blk_in.data() + (size_t)(Cx + Cr + 1) * T_x, N_down.data(), sizeof(float) * T_x);
         } else {
             blk_in.assign(x.begin(), x.end());
         }
@@ -138,7 +150,7 @@ inline void decoder_front(
         x.swap(y); T_x = T_y;
         if (W.decode[b].upsample) res = false;  // decode3 upsamples -> res stops
     }
-    x_out.swap(x);  // [512, 264]
+    x_out.swap(x);  // [decode[3].Cout, 2*T_asr]
 }
 
 } // namespace eliza_kokoro

tools/kokoro/include/kokoro-decoder.h

@@ -5,6 +5,9 @@
 // Wires the validated decoder_front (kokoro-decoder-front.h) + Generator
 // (kokoro-generator.h) against the model's all-F32 ggml context. Replaces the
 // J2-ship placeholder spectrogram in kokoro_synthesize (#9588).
+//
+// Every channel width is read from the GGUF tensor shapes, so the same code
+// runs Kokoro-82M and width-scaled students of it (Paradee-8M).
 
 #pragma once
 
@@ -13,10 +16,40 @@
 #include <string>
 #include <vector>
 
+struct ggml_context;
+
 namespace eliza_kokoro {
 
 struct kokoro_model;
 
+struct DecoderConfig {
+    std::string ns = "kokoro";          // tensor namespace ("<ns>.decoder.*", "<ns>.gen.*")
+    bool instnorm_affine = false;       // AdaIN1d uses the trained InstanceNorm affine (normN.norm.*, adainN.j.norm.*)
+    bool canonical_source_stft = false; // see GeneratorConfig::canonical_source_stft
+    bool precise_source = false;        // see GeneratorConfig::precise_source
+    int up_rates[2] = { 10, 6 };        // generator upsample rates (kernels and widths come from the tensors)
+    int istft_hop = 5;                  // generator iSTFT hop (n_fft comes from conv_post)
+    std::vector<float> * har_source_out = nullptr;  // generator m_source output
+    const std::vector<float> * har_source_in = nullptr;  // parity hook: replace m_source's output
+    bool (*cancel)(void * user) = nullptr;
+    void * cancel_user = nullptr;
+    bool cancelled = false;             // out: set when cancel() stopped the forward
+};
+
+// Generic decoder over a loaded F32 context.
+//   asr_ct : [asr_dim, T_frame] channel-major
+//   F0, N  : [2*T_frame]
+//   style  : [Sdim] decoder style (Sdim read from the GGUF)
+// Returns false on error (err set) or cancellation (cfg.cancelled, err "cancelled").
+bool decoder_forward(
+        ggml_context * ctx,
+        DecoderConfig & cfg,
+        const float * asr_ct, int asr_dim, int T_frame,
+        const float * F0, const float * N,
+        const float * style,
+        std::vector<float> & audio,
+        std::string & err);
+
 // Run the full decoder. Inputs come from kokoro_predictor_forward:
 //   asr_ct   : [512, T_frame] channel-major (transpose of PredictorOut.asr [T,512])
 //   F0, N    : [2*T_frame]    (PredictorOut.F0_pred / N_pred — the up-2x curves)

tools/kokoro/include/kokoro-generator.h

@@ -2,15 +2,18 @@
 //
 // kokoro-generator.h — iSTFTNet Generator.forward (StyleTTS-2 decoder back-end).
 //
-// The generator turns the decoder body output `x` [512, 264], the style
-// vector `s` [128], and the (un-downsampled) F0 curve `f0_curve` [264] into
-// `audio` [79200] (24 kHz).
+// The generator turns the decoder body output `x` [Cin, T0], the style
+// vector `s` [Sdim], and the (un-downsampled) F0 curve `f0_curve` [T0] into
+// `audio` [T0 * prod(upsample_rates) * hop] (24 kHz).
+//
+// Kokoro-82M: Cin 512, stage channels 256/128, Sdim 128.
+// Paradee-8M: Cin 128, stage channels 64/32, Sdim 16.
 //
 // Weights are raw float pointers (PyTorch row-major, weight_norm-fused):
 //   Conv1d weight       [Cout, Cin, K]
 //   ConvTranspose1d wt  [Cin,  Cout, K]
 //   Linear weight       [out,  in]
-//   AdaIN1d fc.weight    [2C,   style_dim]   (style_dim = 128)
+//   AdaIN1d fc.weight    [2C,   style_dim]
 //   Snake alpha          [C]
 // The caller supplies them via GeneratorWeights so the function composes with
 // any weight-loading boundary (GGUF tensor lookup, raw .f32 fixtures, …).
@@ -28,43 +31,97 @@ struct GenSubBlockWeights {
     const float * conv1_b = nullptr;     // [C]
     const float * conv2_w = nullptr;     // [C, C, K]
     const float * conv2_b = nullptr;     // [C]
-    const float * adain1_fc_w = nullptr; // [2C, 128]
+    const float * adain1_fc_w = nullptr; // [2C, Sdim]
     const float * adain1_fc_b = nullptr; // [2C]
-    const float * adain2_fc_w = nullptr; // [2C, 128]
+    const float * adain2_fc_w = nullptr; // [2C, Sdim]
     const float * adain2_fc_b = nullptr; // [2C]
     const float * alpha1 = nullptr;      // [C]
     const float * alpha2 = nullptr;      // [C]
+    // Trained InstanceNorm affine inside AdaIN1d (students only; null keeps
+    // Kokoro-82M's identity affine).
+    const float * adain1_norm_w = nullptr;  // [C]
+    const float * adain1_norm_b = nullptr;  // [C]
+    const float * adain2_norm_w = nullptr;  // [C]
+    const float * adain2_norm_b = nullptr;  // [C]
 };
 
 struct GenAdaResBlockWeights {
     GenSubBlockWeights sub[3];
 };
 
 struct GeneratorWeights {
-    // m_source.l_linear: Linear(9 -> 1).
-    const float * l_linear_w = nullptr;  // [1, 9]
+    // m_source.l_linear: Linear(harmonics -> 1).
+    const float * l_linear_w = nullptr;  // [1, harmonics]
     const float * l_linear_b = nullptr;  // [1]
 
     // ups[0], ups[1]: ConvTranspose1d. weight [Cin, Cout, K], bias [Cout].
     const float * ups_w[2] = { nullptr, nullptr };
     const float * ups_b[2] = { nullptr, nullptr };
 
-    // noise_convs[0], noise_convs[1]: Conv1d. weight [Cout, 22, K], bias [Cout].
+    // noise_convs[0], noise_convs[1]: Conv1d. weight [Cout, n_fft+2, K], bias [Cout].
     const float * noise_convs_w[2] = { nullptr, nullptr };
     const float * noise_convs_b[2] = { nullptr, nullptr };
 
     // noise_res[0] (k=7), noise_res[1] (k=11): AdaINResBlock1.
     GenAdaResBlockWeights noise_res[2];
 
-    // resblocks[0..5]: AdaINResBlock1 (stage0: k=3,7,11 ch=256; stage1: ch=128).
+    // resblocks[0..5]: AdaINResBlock1 (3 kernels per stage, 2 stages).
     GenAdaResBlockWeights resblocks[6];
 
-    // conv_post: Conv1d(128 -> 22, k=7, pad=3). weight [22, 128, 7], bias [22].
+    // conv_post: Conv1d(ch[1] -> n_fft+2, k=7, pad=3).
     const float * conv_post_w = nullptr;
     const float * conv_post_b = nullptr;
 };
 
-// Generator.forward. audio is resized to T0 * 300 (== 79200 for T0=264).
+// Generator shape and behaviour. The defaults are Kokoro-82M v1.0.
+struct GeneratorConfig {
+    int Cin = 512;                    // input channels (upsample_initial_channel)
+    int ch[2] = { 256, 128 };         // channels after ups[0], ups[1]
+    int Sdim = 128;                   // style width
+    int up_rates[2] = { 10, 6 };
+    int up_kernels[2] = { 20, 12 };
+    int rb_kernels[3] = { 3, 7, 11 };
+    int dil[3] = { 1, 3, 5 };
+    int n_fft = 20;
+    int hop = 5;
+    int harmonics = 9;                // harmonic_num + 1
+    float sine_amp = 0.1f;
+    float voiced_threshold = 10.0f;
+    float sample_rate = 24000.0f;
+
+    // Source STFT convention. false (Kokoro-82M as shipped): a direct double
+    // DFT, whose rounding noise decides the sign of near-zero imaginary parts
+    // and therefore the phase of near-empty bins. true (Paradee): imaginary
+    // parts and magnitudes below the DFT's rounding floor are set to exactly
+    // zero, so the DC/Nyquist bins of a real signal and the empty bins of a
+    // constant (unvoiced, noise-free) source get torch.stft's phase (0, or pi
+    // for a negative real part) instead of a sign picked by rounding noise.
+    bool canonical_source_stft = false;
+    // Harmonic source precision. false (Kokoro-82M as shipped): rad/phase
+    // pass through float between the SineGen steps. true (Paradee): every
+    // SineGen step runs in double and only sine_waves is rounded to float.
+    bool precise_source = false;
+    bool instnorm_affine = false;     // documents that the AdaIN1d norm_* pointers are set
+
+    // Optional outputs / hooks.
+    std::vector<float> * har_source_out = nullptr;   // m_source output [T0 * upsample_scale]
+    const std::vector<float> * har_source_in = nullptr;  // parity hook: use this source instead of m_source
+    bool (*cancel)(void * user) = nullptr;           // checked between generator stages
+    void * cancel_user = nullptr;
+};
+
+// Generator.forward with an explicit configuration. Returns false only when
+// `cfg.cancel` asked it to stop (audio is then cleared).
+bool generator_forward(
+        const float * x_in,      // [cfg.Cin, T0] channel-major
+        int T0,                  // input time (== 2 * predictor T_frame)
+        const float * s,         // [cfg.Sdim]
+        const float * f0_curve,  // [T0]
+        const GeneratorWeights & w,
+        const GeneratorConfig & cfg,
+        std::vector<float> & audio);
+
+// Kokoro-82M Generator.forward. audio is resized to T0 * 300 (== 79200 for T0=264).
 void kokoro_generator_forward(
         const float * x_in,      // [512, T0] channel-major
         int T0,                  // input time (== 2 * predictor T_frame)