Integrate reviewed native speech and ABI 18 on current main

CMakeLists.txt

@@ -115,6 +115,12 @@ option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when availab
 option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})
 option(LLAMA_TESTS_INSTALL "llama: install tests" ON)
 
+option(LLAMA_BUILD_KOKORO "Build native Kokoro speech" ON)
+option(LLAMA_BUILD_PARADEE "Build native Paradee speech" ON)
+option(LLAMA_BUILD_MISAKI_G2P "Build native misaki G2P" ON)
+option(LLAMA_BUILD_OMNIVOICE "Build native OmniVoice and fused inference" ON)
+option(OMNIVOICE_SHARED "Build libomnivoice for ABI consumers" OFF)
+
 # 3rd party libs
 option(LLAMA_OPENSSL    "llama: use openssl to support HTTPS" ON)
 option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
@@ -232,6 +238,21 @@ if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
     add_subdirectory(tools/mtmd)
 endif()
 
+if (NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
+    if (LLAMA_BUILD_KOKORO)
+        add_subdirectory(tools/kokoro)
+    endif()
+    if (LLAMA_BUILD_MISAKI_G2P)
+        add_subdirectory(tools/misaki-g2p)
+    endif()
+    if (LLAMA_BUILD_PARADEE AND LLAMA_BUILD_KOKORO)
+        add_subdirectory(tools/paradee)
+    endif()
+    if (LLAMA_BUILD_OMNIVOICE)
+        add_subdirectory(tools/omnivoice)
+    endif()
+endif()
+
 #
 # install
 #

tools/CMakeLists.txt

@@ -30,6 +30,18 @@ else()
     add_subdirectory(parser)
     add_subdirectory(tts)
     add_subdirectory(mtmd)
+    if (LLAMA_BUILD_KOKORO)
+        add_subdirectory(kokoro)
+    endif()
+    if (LLAMA_BUILD_MISAKI_G2P)
+        add_subdirectory(misaki-g2p)
+    endif()
+    if (LLAMA_BUILD_PARADEE AND LLAMA_BUILD_KOKORO)
+        add_subdirectory(paradee)
+    endif()
+    if (LLAMA_BUILD_OMNIVOICE)
+        add_subdirectory(omnivoice)
+    endif()
     if (GGML_RPC)
         add_subdirectory(rpc)
     endif()

tools/kokoro/CMakeLists.txt

@@ -19,6 +19,7 @@ set(KOKORO_PUBLIC_HEADERS
     include/kokoro-server-mount.h
     include/kokoro-tensor-names.h
     include/kokoro-layers.h
+    include/kokoro-model-internal.h
     include/kokoro-predictor.h)
 
 set(KOKORO_CORE_SOURCES
@@ -32,6 +33,8 @@ set(KOKORO_CORE_SOURCES
 add_library(kokoro_lib STATIC
     ${KOKORO_CORE_SOURCES}
     ${KOKORO_PUBLIC_HEADERS})
+# The fused inference shared library links this static archive.
+set_target_properties(kokoro_lib PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 target_include_directories(kokoro_lib PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -52,11 +55,9 @@ target_link_libraries(kokoro_lib PUBLIC ggml)
 if(TARGET ggml-base)
     target_link_libraries(kokoro_lib PUBLIC ggml-base)
 endif()
-if(TARGET ggml-cpu)
-    target_link_libraries(kokoro_lib PUBLIC ggml-cpu)
-endif()
 
 target_compile_features(kokoro_lib PUBLIC cxx_std_17)
+target_compile_definitions(kokoro_lib PRIVATE KOKORO_USE_GGML_BACKEND)
 
 # kokoro-layers.h routes the Conv1d/ConvTranspose1d/Linear/LSTM hot loops
 # through the Accelerate BLAS (im2col + sgemm/sgemv) on Apple platforms —
@@ -70,12 +71,18 @@ else()
     find_package(Threads REQUIRED)
     target_link_libraries(kokoro_lib PUBLIC Threads::Threads)
 endif()
+if(ANDROID)
+    target_link_libraries(kokoro_lib PUBLIC log)
+endif()
 
 # Real G2P via libespeak-ng. When present, kokoro_phonemize() drives
 # espeak_TextToPhonemes() (en-us IPA) and maps codepoints to Kokoro vocab ids,
-# reproducing the reference token sequence. When absent, the build falls back
-# to the degraded ASCII grapheme mapping and the TS layer must supply IPA.
-# Override the search with -DKOKORO_ESPEAK_ROOT=<prefix> (e.g. Homebrew).
+# reproducing the reference token sequence. When absent, the text path falls
+# back to the degraded ASCII grapheme mapping — so an espeak-less build reports
+# `eliza_inference_kokoro_g2p_kind() == ELIZA_KOKORO_G2P_ASCII` and the TS voice
+# layer supplies espeak-ng (WASM) IPA through
+# `eliza_inference_kokoro_synthesize_ipa()` (ABI v14, #11776) to stay
+# intelligible. Override the search with -DKOKORO_ESPEAK_ROOT=<prefix> (Homebrew).
 option(KOKORO_ENABLE_ESPEAK "Link libespeak-ng for real Kokoro G2P" ON)
 if(KOKORO_ENABLE_ESPEAK)
     find_path(ESPEAK_NG_INCLUDE_DIR espeak-ng/speak_lib.h
@@ -90,7 +97,7 @@ if(KOKORO_ENABLE_ESPEAK)
         target_compile_definitions(kokoro_lib PRIVATE KOKORO_USE_ESPEAK)
         message(STATUS "Kokoro G2P: libespeak-ng found (${ESPEAK_NG_LIBRARY}) — real IPA path enabled")
     else()
-        message(STATUS "Kokoro G2P: libespeak-ng not found — falling back to ASCII grapheme mapping (TS layer must supply IPA)")
+        message(STATUS "Kokoro G2P: libespeak-ng not found — ASCII grapheme fallback; g2p_kind() reports ASCII so the TS layer feeds IPA via kokoro_synthesize_ipa (#11776)")
     endif()
 endif()
 

tools/kokoro/include/kokoro-decoder-front.h

@@ -1,5 +1,9 @@
 // SPDX-License-Identifier: MIT
 // kokoro-decoder-front.h — Decoder.forward up to the generator (validated port, #9588).
+//
+// Channel widths come from the weights struct (filled from the GGUF tensor
+// shapes by kokoro-decoder.cpp), so width-scaled students of Kokoro-82M
+// (Paradee-8M: AdaIN width 256, style 16) share this code.
 #pragma once
 #include <cmath>
 #include <cstring>
@@ -24,6 +28,12 @@ struct DecAdainResBlk {
     const float * conv1x1_b  = nullptr;  // [Cout] (null — conv1x1 bias=False)
     const float * pool_w     = nullptr;  // [Cin, 1, 3] (upsample only)
     const float * pool_b     = nullptr;  // [Cin] (upsample only)
+    // Trained InstanceNorm affine inside AdaIN1d (students only; null keeps
+    // Kokoro-82M's identity affine).
+    const float * norm1_w    = nullptr;  // [Cin]
+    const float * norm1_b    = nullptr;  // [Cin]
+    const float * norm2_w    = nullptr;  // [Cout]
+    const float * norm2_b    = nullptr;  // [Cout]
 };
 
 // AdainResBlk1d (decode-block flavor: leaky_relu 0.2; pool/shortcut;
@@ -35,7 +45,7 @@ inline void dec_adainresblk1d_forward(
 
     // residual branch: norm1 -> leaky_relu(0.2) -> [pool] -> conv1 -> norm2 -> leaky_relu -> conv2
     std::vector<float> r(x, x + (size_t)Cin * T_in);
-    adain1d_forward(r.data(), Cin, T_in, s, Sdim, w.norm1_fc_w, w.norm1_fc_b);
+    adain1d_forward(r.data(), Cin, T_in, s, Sdim, w.norm1_fc_w, w.norm1_fc_b, w.norm1_w, w.norm1_b);
     for (size_t i = 0; i < r.size(); ++i) if (r[i] < 0) r[i] *= 0.2f;
 
     int T_pool = T_in;
@@ -47,7 +57,7 @@ inline void dec_adainresblk1d_forward(
     }
     std::vector<float> r3((size_t)Cout * T_pool);
     conv1d_forward(r.data(), Cin, T_pool, w.conv1_w, w.conv1_b, Cout, 3, 1, 1, 1, r3.data(), T_pool);
-    adain1d_forward(r3.data(), Cout, T_pool, s, Sdim, w.norm2_fc_w, w.norm2_fc_b);
+    adain1d_forward(r3.data(), Cout, T_pool, s, Sdim, w.norm2_fc_w, w.norm2_fc_b, w.norm2_w, w.norm2_b);
     for (size_t i = 0; i < r3.size(); ++i) if (r3[i] < 0) r3[i] *= 0.2f;
     std::vector<float> r4((size_t)Cout * T_pool);
     conv1d_forward(r3.data(), Cout, T_pool, w.conv2_w, w.conv2_b, Cout, 3, 1, 1, 1, r4.data(), T_pool);
@@ -82,15 +92,16 @@ struct DecoderFrontWeights {
     const float * F0_conv_b = nullptr;  // [1]
     const float * N_conv_w  = nullptr;  // [1,1,3]
     const float * N_conv_b  = nullptr;  // [1]
-    const float * asr_res_w = nullptr;  // [64,512,1]
-    const float * asr_res_b = nullptr;  // [64]
-    DecAdainResBlk encode;              // 514 -> 1024, learned_sc
-    DecAdainResBlk decode[4];           // 1090->1024 (x3), 1090->512 upsample
+    const float * asr_res_w = nullptr;  // [asr_res_C, Cin_asr, 1]
+    const float * asr_res_b = nullptr;  // [asr_res_C]
+    int asr_res_C = 64;                 // asr_res output channels (64 for Kokoro and Paradee)
+    DecAdainResBlk encode;              // Kokoro: 514 -> 1024, learned_sc
+    DecAdainResBlk decode[4];           // Kokoro: 1090->1024 (x3), 1090->512 upsample
 };
 
 // Decoder.forward up to (not including) the generator.
-//   asr[512,T_asr] (T_asr=132), F0_curve[2*T_asr], N[2*T_asr], s[128]
-// Output: x_out [512, 2*T_asr] (== generator_in_0); also returns the
+//   asr[Cin_asr,T_asr], F0_curve[2*T_asr], N[2*T_asr], s[Sdim]
+// Output: x_out [decode[3].Cout, 2*T_asr] (== generator_in_0); also returns the
 // stride-2 conv outputs F0_down[T_asr], N_down[T_asr] (caller passes them,
 // together with the ORIGINAL F0_curve, into the generator).
 inline void decoder_front(
@@ -100,36 +111,37 @@ inline void decoder_front(
         std::vector<float> & x_out,
         std::vector<float> & F0_down,
         std::vector<float> & N_down) {
-    const int Tc = 2 * T_asr;  // 264
+    const int Tc = 2 * T_asr;
 
     F0_down.assign(T_asr, 0.0f);
     conv1d_forward(F0_curve, 1, Tc, W.F0_conv_w, W.F0_conv_b, 1, 3, 2, 1, 1, F0_down.data(), T_asr);
     N_down.assign(T_asr, 0.0f);
     conv1d_forward(N_in, 1, Tc, W.N_conv_w, W.N_conv_b, 1, 3, 2, 1, 1, N_down.data(), T_asr);
 
-    // x = cat([asr, F0, N], dim=channels) -> [514, T_asr]
+    // x = cat([asr, F0, N], dim=channels) -> [Cin_asr + 2, T_asr]
     std::vector<float> xcat((size_t)(Cin_asr + 2) * T_asr);
     std::memcpy(xcat.data(), asr, sizeof(float) * (size_t)Cin_asr * T_asr);
     std::memcpy(xcat.data() + (size_t)Cin_asr * T_asr, F0_down.data(), sizeof(float) * T_asr);
     std::memcpy(xcat.data() + (size_t)(Cin_asr + 1) * T_asr, N_down.data(), sizeof(float) * T_asr);
 
     std::vector<float> x; int T_x;
-    dec_adainresblk1d_forward(W.encode, xcat.data(), T_asr, s, x, T_x);   // encode 514->1024
+    dec_adainresblk1d_forward(W.encode, xcat.data(), T_asr, s, x, T_x);   // encode Cin_asr+2 -> W
 
-    std::vector<float> asr_res((size_t)64 * T_asr);                       // asr_res Conv1d k1 512->64
-    conv1d_forward(asr, Cin_asr, T_asr, W.asr_res_w, W.asr_res_b, 64, 1, 1, 0, 1, asr_res.data(), T_asr);
+    const int Cr = W.asr_res_C;
+    std::vector<float> asr_res((size_t)Cr * T_asr);                       // asr_res Conv1d k1 Cin_asr->Cr
+    conv1d_forward(asr, Cin_asr, T_asr, W.asr_res_w, W.asr_res_b, Cr, 1, 1, 0, 1, asr_res.data(), T_asr);
 
     bool res = true;
     for (int b = 0; b < 4; ++b) {
         std::vector<float> blk_in;
-        if (res) {  // cat([x, asr_res, F0, N]) -> 1024+64+1+1 = 1090
+        if (res) {  // cat([x, asr_res, F0, N]) -> Kokoro 1024+64+1+1 = 1090
             const int Cx = (int)(x.size() / T_x);
-            const int Cin_blk = Cx + 64 + 1 + 1;
+            const int Cin_blk = Cx + Cr + 1 + 1;
             blk_in.assign((size_t)Cin_blk * T_x, 0.0f);
             std::memcpy(blk_in.data(), x.data(), sizeof(float) * (size_t)Cx * T_x);
-            std::memcpy(blk_in.data() + (size_t)Cx * T_x, asr_res.data(), sizeof(float) * 64 * T_x);
-            std::memcpy(blk_in.data() + (size_t)(Cx + 64) * T_x, F0_down.data(), sizeof(float) * T_x);
-            std::memcpy(blk_in.data() + (size_t)(Cx + 65) * T_x, N_down.data(), sizeof(float) * T_x);
+            std::memcpy(blk_in.data() + (size_t)Cx * T_x, asr_res.data(), sizeof(float) * (size_t)Cr * T_x);
+            std::memcpy(blk_in.data() + (size_t)(Cx + Cr) * T_x, F0_down.data(), sizeof(float) * T_x);
+            std::memcpy(blk_in.data() + (size_t)(Cx + Cr + 1) * T_x, N_down.data(), sizeof(float) * T_x);
         } else {
             blk_in.assign(x.begin(), x.end());
         }
@@ -138,7 +150,7 @@ inline void decoder_front(
         x.swap(y); T_x = T_y;
         if (W.decode[b].upsample) res = false;  // decode3 upsamples -> res stops
     }
-    x_out.swap(x);  // [512, 264]
+    x_out.swap(x);  // [decode[3].Cout, 2*T_asr]
 }
 
 } // namespace eliza_kokoro

tools/kokoro/include/kokoro-decoder.h

@@ -5,27 +5,67 @@
 // Wires the validated decoder_front (kokoro-decoder-front.h) + Generator
 // (kokoro-generator.h) against the model's all-F32 ggml context. Replaces the
 // J2-ship placeholder spectrogram in kokoro_synthesize (#9588).
+//
+// Every channel width is read from the GGUF tensor shapes, so the same code
+// runs Kokoro-82M and width-scaled students of it (Paradee-8M).
 
 #pragma once
 
+#include "kokoro.h"
+
 #include <string>
 #include <vector>
 
+struct ggml_context;
+
 namespace eliza_kokoro {
 
 struct kokoro_model;
 
+struct DecoderConfig {
+    std::string ns = "kokoro";          // tensor namespace ("<ns>.decoder.*", "<ns>.gen.*")
+    bool instnorm_affine = false;       // AdaIN1d uses the trained InstanceNorm affine (normN.norm.*, adainN.j.norm.*)
+    bool canonical_source_stft = false; // see GeneratorConfig::canonical_source_stft
+    bool precise_source = false;        // see GeneratorConfig::precise_source
+    int up_rates[2] = { 10, 6 };        // generator upsample rates (kernels and widths come from the tensors)
+    int istft_hop = 5;                  // generator iSTFT hop (n_fft comes from conv_post)
+    std::vector<float> * har_source_out = nullptr;  // generator m_source output
+    const std::vector<float> * har_source_in = nullptr;  // parity hook: replace m_source's output
+    bool (*cancel)(void * user) = nullptr;
+    void * cancel_user = nullptr;
+    bool cancelled = false;             // out: set when cancel() stopped the forward
+};
+
+// Generic decoder over a loaded F32 context.
+//   asr_ct : [asr_dim, T_frame] channel-major
+//   F0, N  : [2*T_frame]
+//   style  : [Sdim] decoder style (Sdim read from the GGUF)
+// Returns false on error (err set) or cancellation (cfg.cancelled, err "cancelled").
+bool decoder_forward(
+        ggml_context * ctx,
+        DecoderConfig & cfg,
+        const float * asr_ct, int asr_dim, int T_frame,
+        const float * F0, const float * N,
+        const float * style,
+        std::vector<float> & audio,
+        std::string & err);
+
 // Run the full decoder. Inputs come from kokoro_predictor_forward:
 //   asr_ct   : [512, T_frame] channel-major (transpose of PredictorOut.asr [T,512])
 //   F0, N    : [2*T_frame]    (PredictorOut.F0_pred / N_pred — the up-2x curves)
 //   ref_s_dec: [128]          decoder-half style (ref_s[:128])
 // Output: audio (24 kHz mono), resized to (2*T_frame)*300.
+// `cancel` (optional) is polled between the decoder front and the generator;
+// when it returns true the call stops early, returns false with err
+// "cancelled", and leaves `audio` empty.
 bool kokoro_decoder_forward(
         const kokoro_model * model,
         const float * asr_ct, int T_frame,
         const float * F0, const float * N,
         const float * ref_s_dec,
         std::vector<float> & audio,
-        std::string & err);
+        std::string & err,
+        kokoro_cancel_callback cancel = nullptr,
+        void * cancel_user_data = nullptr);
 
 } // namespace eliza_kokoro