diff --git a/CMakeLists.txt b/CMakeLists.txt index 60a4503d4..01473219d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,6 +131,16 @@ option(OMNIVOICE_SHARED "Build libomnivoice for ABI consumers" OFF) # Kokoro is a first-class voice surface in the eliza-1 stack. option(LLAMA_BUILD_KOKORO "Build the Kokoro-82M TTS subtree (tools/kokoro/)" ON) +# Paradee-8M v1.0 (single-voice distillation of Kokoro-82M, Apache-2.0). +# Produces paradee_lib + paradee-tts. Runs on kokoro_lib's layers, so it is +# built only together with LLAMA_BUILD_KOKORO. Default ON: Paradee is the +# shipped mobile voice. +option(LLAMA_BUILD_PARADEE "Build the Paradee-8M TTS subtree (tools/paradee/)" ON) + +# Native misaki grapheme-to-phoneme (Paradee/Kokoro phoneme spelling). The +# subtree lives in tools/misaki-g2p/ and is added when present. +option(LLAMA_BUILD_MISAKI_G2P "Build the misaki G2P subtree (tools/misaki-g2p/)" ON) + # W3-3 deprecation: ELIZA_FUSE_OMNIVOICE=ON is the pre-W3-3 flag for the # fork-root `omnivoice/` graft. The graft is gone (see the deprecation # block below); turn the flag into a redirect now so the option ON state @@ -281,6 +291,27 @@ if (LLAMA_BUILD_KOKORO AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)) add_subdirectory(tools/kokoro) endif() +# misaki G2P and Paradee — embed-as-library hooks (mirror kokoro above). Both +# precede tools/omnivoice so the fused `elizainference` target can fold them +# in with `if(TARGET ...)`. +if (LLAMA_BUILD_MISAKI_G2P AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)) + if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/misaki-g2p/CMakeLists.txt") + add_subdirectory(tools/misaki-g2p) + else() + message(STATUS "LLAMA_BUILD_MISAKI_G2P: tools/misaki-g2p/ is not in this checkout; skipped") + endif() +endif() +if (LLAMA_BUILD_PARADEE AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)) + # Paradee runs on the tools/kokoro layers. A build that turns Kokoro off + # (backend-only CI, for example) skips Paradee instead of failing; the + # fused library then reports paradee_supported() == 0. + if (LLAMA_BUILD_KOKORO) + add_subdirectory(tools/paradee) + else() + message(STATUS "LLAMA_BUILD_PARADEE: skipped because LLAMA_BUILD_KOKORO=OFF") + endif() +endif() + if (LLAMA_BUILD_OMNIVOICE) if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/omnivoice/CMakeLists.txt") message(FATAL_ERROR diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3a194f1c3..08d148989 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -41,6 +41,26 @@ else() # this ordering for the embed-as-library (LLAMA_BUILD_TOOLS=OFF) case. add_subdirectory(kokoro) endif() + if (LLAMA_BUILD_MISAKI_G2P) + # Native misaki G2P (the phoneme spelling Paradee and Kokoro were + # trained on). Added when the subtree is present. + if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/misaki-g2p/CMakeLists.txt") + add_subdirectory(misaki-g2p) + else() + message(STATUS "LLAMA_BUILD_MISAKI_G2P: tools/misaki-g2p/ is not in this checkout; skipped") + endif() + endif() + if (LLAMA_BUILD_PARADEE) + # Paradee-8M (width-scaled Kokoro student, single voice). Needs + # kokoro_lib; like kokoro it precedes omnivoice so the fused + # `elizainference` target can fold paradee_lib in. + # A build with Kokoro off skips Paradee rather than failing. + if (LLAMA_BUILD_KOKORO) + add_subdirectory(paradee) + else() + message(STATUS "LLAMA_BUILD_PARADEE: skipped because LLAMA_BUILD_KOKORO=OFF") + endif() + endif() if (LLAMA_BUILD_OMNIVOICE) # The merged omnivoice subtree lives under tools/omnivoice/. The # top-level CMakeLists.txt has a parallel add_subdirectory hook for diff --git a/tools/kokoro/include/kokoro-decoder-front.h b/tools/kokoro/include/kokoro-decoder-front.h index 4a89770c5..11244bba8 100644 --- a/tools/kokoro/include/kokoro-decoder-front.h +++ b/tools/kokoro/include/kokoro-decoder-front.h @@ -1,5 +1,9 @@ // SPDX-License-Identifier: MIT // kokoro-decoder-front.h — Decoder.forward up to the generator (validated port, #9588). +// +// Channel widths come from the weights struct (filled from the GGUF tensor +// shapes by kokoro-decoder.cpp), so width-scaled students of Kokoro-82M +// (Paradee-8M: AdaIN width 256, style 16) share this code. #pragma once #include #include @@ -24,6 +28,12 @@ struct DecAdainResBlk { const float * conv1x1_b = nullptr; // [Cout] (null — conv1x1 bias=False) const float * pool_w = nullptr; // [Cin, 1, 3] (upsample only) const float * pool_b = nullptr; // [Cin] (upsample only) + // Trained InstanceNorm affine inside AdaIN1d (students only; null keeps + // Kokoro-82M's identity affine). + const float * norm1_w = nullptr; // [Cin] + const float * norm1_b = nullptr; // [Cin] + const float * norm2_w = nullptr; // [Cout] + const float * norm2_b = nullptr; // [Cout] }; // AdainResBlk1d (decode-block flavor: leaky_relu 0.2; pool/shortcut; @@ -35,7 +45,7 @@ inline void dec_adainresblk1d_forward( // residual branch: norm1 -> leaky_relu(0.2) -> [pool] -> conv1 -> norm2 -> leaky_relu -> conv2 std::vector r(x, x + (size_t)Cin * T_in); - adain1d_forward(r.data(), Cin, T_in, s, Sdim, w.norm1_fc_w, w.norm1_fc_b); + adain1d_forward(r.data(), Cin, T_in, s, Sdim, w.norm1_fc_w, w.norm1_fc_b, w.norm1_w, w.norm1_b); for (size_t i = 0; i < r.size(); ++i) if (r[i] < 0) r[i] *= 0.2f; int T_pool = T_in; @@ -47,7 +57,7 @@ inline void dec_adainresblk1d_forward( } std::vector r3((size_t)Cout * T_pool); conv1d_forward(r.data(), Cin, T_pool, w.conv1_w, w.conv1_b, Cout, 3, 1, 1, 1, r3.data(), T_pool); - adain1d_forward(r3.data(), Cout, T_pool, s, Sdim, w.norm2_fc_w, w.norm2_fc_b); + adain1d_forward(r3.data(), Cout, T_pool, s, Sdim, w.norm2_fc_w, w.norm2_fc_b, w.norm2_w, w.norm2_b); for (size_t i = 0; i < r3.size(); ++i) if (r3[i] < 0) r3[i] *= 0.2f; std::vector r4((size_t)Cout * T_pool); conv1d_forward(r3.data(), Cout, T_pool, w.conv2_w, w.conv2_b, Cout, 3, 1, 1, 1, r4.data(), T_pool); @@ -82,15 +92,16 @@ struct DecoderFrontWeights { const float * F0_conv_b = nullptr; // [1] const float * N_conv_w = nullptr; // [1,1,3] const float * N_conv_b = nullptr; // [1] - const float * asr_res_w = nullptr; // [64,512,1] - const float * asr_res_b = nullptr; // [64] - DecAdainResBlk encode; // 514 -> 1024, learned_sc - DecAdainResBlk decode[4]; // 1090->1024 (x3), 1090->512 upsample + const float * asr_res_w = nullptr; // [asr_res_C, Cin_asr, 1] + const float * asr_res_b = nullptr; // [asr_res_C] + int asr_res_C = 64; // asr_res output channels (64 for Kokoro and Paradee) + DecAdainResBlk encode; // Kokoro: 514 -> 1024, learned_sc + DecAdainResBlk decode[4]; // Kokoro: 1090->1024 (x3), 1090->512 upsample }; // Decoder.forward up to (not including) the generator. -// asr[512,T_asr] (T_asr=132), F0_curve[2*T_asr], N[2*T_asr], s[128] -// Output: x_out [512, 2*T_asr] (== generator_in_0); also returns the +// asr[Cin_asr,T_asr], F0_curve[2*T_asr], N[2*T_asr], s[Sdim] +// Output: x_out [decode[3].Cout, 2*T_asr] (== generator_in_0); also returns the // stride-2 conv outputs F0_down[T_asr], N_down[T_asr] (caller passes them, // together with the ORIGINAL F0_curve, into the generator). inline void decoder_front( @@ -100,36 +111,37 @@ inline void decoder_front( std::vector & x_out, std::vector & F0_down, std::vector & N_down) { - const int Tc = 2 * T_asr; // 264 + const int Tc = 2 * T_asr; F0_down.assign(T_asr, 0.0f); conv1d_forward(F0_curve, 1, Tc, W.F0_conv_w, W.F0_conv_b, 1, 3, 2, 1, 1, F0_down.data(), T_asr); N_down.assign(T_asr, 0.0f); conv1d_forward(N_in, 1, Tc, W.N_conv_w, W.N_conv_b, 1, 3, 2, 1, 1, N_down.data(), T_asr); - // x = cat([asr, F0, N], dim=channels) -> [514, T_asr] + // x = cat([asr, F0, N], dim=channels) -> [Cin_asr + 2, T_asr] std::vector xcat((size_t)(Cin_asr + 2) * T_asr); std::memcpy(xcat.data(), asr, sizeof(float) * (size_t)Cin_asr * T_asr); std::memcpy(xcat.data() + (size_t)Cin_asr * T_asr, F0_down.data(), sizeof(float) * T_asr); std::memcpy(xcat.data() + (size_t)(Cin_asr + 1) * T_asr, N_down.data(), sizeof(float) * T_asr); std::vector x; int T_x; - dec_adainresblk1d_forward(W.encode, xcat.data(), T_asr, s, x, T_x); // encode 514->1024 + dec_adainresblk1d_forward(W.encode, xcat.data(), T_asr, s, x, T_x); // encode Cin_asr+2 -> W - std::vector asr_res((size_t)64 * T_asr); // asr_res Conv1d k1 512->64 - conv1d_forward(asr, Cin_asr, T_asr, W.asr_res_w, W.asr_res_b, 64, 1, 1, 0, 1, asr_res.data(), T_asr); + const int Cr = W.asr_res_C; + std::vector asr_res((size_t)Cr * T_asr); // asr_res Conv1d k1 Cin_asr->Cr + conv1d_forward(asr, Cin_asr, T_asr, W.asr_res_w, W.asr_res_b, Cr, 1, 1, 0, 1, asr_res.data(), T_asr); bool res = true; for (int b = 0; b < 4; ++b) { std::vector blk_in; - if (res) { // cat([x, asr_res, F0, N]) -> 1024+64+1+1 = 1090 + if (res) { // cat([x, asr_res, F0, N]) -> Kokoro 1024+64+1+1 = 1090 const int Cx = (int)(x.size() / T_x); - const int Cin_blk = Cx + 64 + 1 + 1; + const int Cin_blk = Cx + Cr + 1 + 1; blk_in.assign((size_t)Cin_blk * T_x, 0.0f); std::memcpy(blk_in.data(), x.data(), sizeof(float) * (size_t)Cx * T_x); - std::memcpy(blk_in.data() + (size_t)Cx * T_x, asr_res.data(), sizeof(float) * 64 * T_x); - std::memcpy(blk_in.data() + (size_t)(Cx + 64) * T_x, F0_down.data(), sizeof(float) * T_x); - std::memcpy(blk_in.data() + (size_t)(Cx + 65) * T_x, N_down.data(), sizeof(float) * T_x); + std::memcpy(blk_in.data() + (size_t)Cx * T_x, asr_res.data(), sizeof(float) * (size_t)Cr * T_x); + std::memcpy(blk_in.data() + (size_t)(Cx + Cr) * T_x, F0_down.data(), sizeof(float) * T_x); + std::memcpy(blk_in.data() + (size_t)(Cx + Cr + 1) * T_x, N_down.data(), sizeof(float) * T_x); } else { blk_in.assign(x.begin(), x.end()); } @@ -138,7 +150,7 @@ inline void decoder_front( x.swap(y); T_x = T_y; if (W.decode[b].upsample) res = false; // decode3 upsamples -> res stops } - x_out.swap(x); // [512, 264] + x_out.swap(x); // [decode[3].Cout, 2*T_asr] } } // namespace eliza_kokoro \ No newline at end of file diff --git a/tools/kokoro/include/kokoro-decoder.h b/tools/kokoro/include/kokoro-decoder.h index 7920b5199..312e13ed5 100644 --- a/tools/kokoro/include/kokoro-decoder.h +++ b/tools/kokoro/include/kokoro-decoder.h @@ -5,6 +5,9 @@ // Wires the validated decoder_front (kokoro-decoder-front.h) + Generator // (kokoro-generator.h) against the model's all-F32 ggml context. Replaces the // J2-ship placeholder spectrogram in kokoro_synthesize (#9588). +// +// Every channel width is read from the GGUF tensor shapes, so the same code +// runs Kokoro-82M and width-scaled students of it (Paradee-8M). #pragma once @@ -13,10 +16,40 @@ #include #include +struct ggml_context; + namespace eliza_kokoro { struct kokoro_model; +struct DecoderConfig { + std::string ns = "kokoro"; // tensor namespace (".decoder.*", ".gen.*") + bool instnorm_affine = false; // AdaIN1d uses the trained InstanceNorm affine (normN.norm.*, adainN.j.norm.*) + bool canonical_source_stft = false; // see GeneratorConfig::canonical_source_stft + bool precise_source = false; // see GeneratorConfig::precise_source + int up_rates[2] = { 10, 6 }; // generator upsample rates (kernels and widths come from the tensors) + int istft_hop = 5; // generator iSTFT hop (n_fft comes from conv_post) + std::vector * har_source_out = nullptr; // generator m_source output + const std::vector * har_source_in = nullptr; // parity hook: replace m_source's output + bool (*cancel)(void * user) = nullptr; + void * cancel_user = nullptr; + bool cancelled = false; // out: set when cancel() stopped the forward +}; + +// Generic decoder over a loaded F32 context. +// asr_ct : [asr_dim, T_frame] channel-major +// F0, N : [2*T_frame] +// style : [Sdim] decoder style (Sdim read from the GGUF) +// Returns false on error (err set) or cancellation (cfg.cancelled, err "cancelled"). +bool decoder_forward( + ggml_context * ctx, + DecoderConfig & cfg, + const float * asr_ct, int asr_dim, int T_frame, + const float * F0, const float * N, + const float * style, + std::vector & audio, + std::string & err); + // Run the full decoder. Inputs come from kokoro_predictor_forward: // asr_ct : [512, T_frame] channel-major (transpose of PredictorOut.asr [T,512]) // F0, N : [2*T_frame] (PredictorOut.F0_pred / N_pred — the up-2x curves) diff --git a/tools/kokoro/include/kokoro-generator.h b/tools/kokoro/include/kokoro-generator.h index d1d88f696..86a733b71 100644 --- a/tools/kokoro/include/kokoro-generator.h +++ b/tools/kokoro/include/kokoro-generator.h @@ -2,15 +2,18 @@ // // kokoro-generator.h — iSTFTNet Generator.forward (StyleTTS-2 decoder back-end). // -// The generator turns the decoder body output `x` [512, 264], the style -// vector `s` [128], and the (un-downsampled) F0 curve `f0_curve` [264] into -// `audio` [79200] (24 kHz). +// The generator turns the decoder body output `x` [Cin, T0], the style +// vector `s` [Sdim], and the (un-downsampled) F0 curve `f0_curve` [T0] into +// `audio` [T0 * prod(upsample_rates) * hop] (24 kHz). +// +// Kokoro-82M: Cin 512, stage channels 256/128, Sdim 128. +// Paradee-8M: Cin 128, stage channels 64/32, Sdim 16. // // Weights are raw float pointers (PyTorch row-major, weight_norm-fused): // Conv1d weight [Cout, Cin, K] // ConvTranspose1d wt [Cin, Cout, K] // Linear weight [out, in] -// AdaIN1d fc.weight [2C, style_dim] (style_dim = 128) +// AdaIN1d fc.weight [2C, style_dim] // Snake alpha [C] // The caller supplies them via GeneratorWeights so the function composes with // any weight-loading boundary (GGUF tensor lookup, raw .f32 fixtures, …). @@ -28,12 +31,18 @@ struct GenSubBlockWeights { const float * conv1_b = nullptr; // [C] const float * conv2_w = nullptr; // [C, C, K] const float * conv2_b = nullptr; // [C] - const float * adain1_fc_w = nullptr; // [2C, 128] + const float * adain1_fc_w = nullptr; // [2C, Sdim] const float * adain1_fc_b = nullptr; // [2C] - const float * adain2_fc_w = nullptr; // [2C, 128] + const float * adain2_fc_w = nullptr; // [2C, Sdim] const float * adain2_fc_b = nullptr; // [2C] const float * alpha1 = nullptr; // [C] const float * alpha2 = nullptr; // [C] + // Trained InstanceNorm affine inside AdaIN1d (students only; null keeps + // Kokoro-82M's identity affine). + const float * adain1_norm_w = nullptr; // [C] + const float * adain1_norm_b = nullptr; // [C] + const float * adain2_norm_w = nullptr; // [C] + const float * adain2_norm_b = nullptr; // [C] }; struct GenAdaResBlockWeights { @@ -41,30 +50,78 @@ struct GenAdaResBlockWeights { }; struct GeneratorWeights { - // m_source.l_linear: Linear(9 -> 1). - const float * l_linear_w = nullptr; // [1, 9] + // m_source.l_linear: Linear(harmonics -> 1). + const float * l_linear_w = nullptr; // [1, harmonics] const float * l_linear_b = nullptr; // [1] // ups[0], ups[1]: ConvTranspose1d. weight [Cin, Cout, K], bias [Cout]. const float * ups_w[2] = { nullptr, nullptr }; const float * ups_b[2] = { nullptr, nullptr }; - // noise_convs[0], noise_convs[1]: Conv1d. weight [Cout, 22, K], bias [Cout]. + // noise_convs[0], noise_convs[1]: Conv1d. weight [Cout, n_fft+2, K], bias [Cout]. const float * noise_convs_w[2] = { nullptr, nullptr }; const float * noise_convs_b[2] = { nullptr, nullptr }; // noise_res[0] (k=7), noise_res[1] (k=11): AdaINResBlock1. GenAdaResBlockWeights noise_res[2]; - // resblocks[0..5]: AdaINResBlock1 (stage0: k=3,7,11 ch=256; stage1: ch=128). + // resblocks[0..5]: AdaINResBlock1 (3 kernels per stage, 2 stages). GenAdaResBlockWeights resblocks[6]; - // conv_post: Conv1d(128 -> 22, k=7, pad=3). weight [22, 128, 7], bias [22]. + // conv_post: Conv1d(ch[1] -> n_fft+2, k=7, pad=3). const float * conv_post_w = nullptr; const float * conv_post_b = nullptr; }; -// Generator.forward. audio is resized to T0 * 300 (== 79200 for T0=264). +// Generator shape and behaviour. The defaults are Kokoro-82M v1.0. +struct GeneratorConfig { + int Cin = 512; // input channels (upsample_initial_channel) + int ch[2] = { 256, 128 }; // channels after ups[0], ups[1] + int Sdim = 128; // style width + int up_rates[2] = { 10, 6 }; + int up_kernels[2] = { 20, 12 }; + int rb_kernels[3] = { 3, 7, 11 }; + int dil[3] = { 1, 3, 5 }; + int n_fft = 20; + int hop = 5; + int harmonics = 9; // harmonic_num + 1 + float sine_amp = 0.1f; + float voiced_threshold = 10.0f; + float sample_rate = 24000.0f; + + // Source STFT convention. false (Kokoro-82M as shipped): a direct double + // DFT, whose rounding noise decides the sign of near-zero imaginary parts + // and therefore the phase of near-empty bins. true (Paradee): imaginary + // parts and magnitudes below the DFT's rounding floor are set to exactly + // zero, so the DC/Nyquist bins of a real signal and the empty bins of a + // constant (unvoiced, noise-free) source get torch.stft's phase (0, or pi + // for a negative real part) instead of a sign picked by rounding noise. + bool canonical_source_stft = false; + // Harmonic source precision. false (Kokoro-82M as shipped): rad/phase + // pass through float between the SineGen steps. true (Paradee): every + // SineGen step runs in double and only sine_waves is rounded to float. + bool precise_source = false; + bool instnorm_affine = false; // documents that the AdaIN1d norm_* pointers are set + + // Optional outputs / hooks. + std::vector * har_source_out = nullptr; // m_source output [T0 * upsample_scale] + const std::vector * har_source_in = nullptr; // parity hook: use this source instead of m_source + bool (*cancel)(void * user) = nullptr; // checked between generator stages + void * cancel_user = nullptr; +}; + +// Generator.forward with an explicit configuration. Returns false only when +// `cfg.cancel` asked it to stop (audio is then cleared). +bool generator_forward( + const float * x_in, // [cfg.Cin, T0] channel-major + int T0, // input time (== 2 * predictor T_frame) + const float * s, // [cfg.Sdim] + const float * f0_curve, // [T0] + const GeneratorWeights & w, + const GeneratorConfig & cfg, + std::vector & audio); + +// Kokoro-82M Generator.forward. audio is resized to T0 * 300 (== 79200 for T0=264). void kokoro_generator_forward( const float * x_in, // [512, T0] channel-major int T0, // input time (== 2 * predictor T_frame) diff --git a/tools/kokoro/include/kokoro-layers.h b/tools/kokoro/include/kokoro-layers.h index b834c128c..398b560bf 100644 --- a/tools/kokoro/include/kokoro-layers.h +++ b/tools/kokoro/include/kokoro-layers.h @@ -384,6 +384,154 @@ inline void kokoro_axpy_f32(float a, const float * x, float * y, int n) { } // namespace detail #endif // KOKORO_USE_PORTABLE_FAST +// ================================================================= +// Opt-in layer strategies. Kokoro-82M never sets them, so its arithmetic is +// unchanged; Paradee enables them for its synthesis calls. +// tap_gemm_conv: stride-1 Conv1d on Accelerate as one SGEMM per kernel +// tap over a zero-padded input, instead of im2col (whose +// column matrix is Cin*K*T floats, ~70 MB for a 32-channel, +// K=11, 10 s generator stage). On the portable path, +// register-tiled Conv1d / ConvTranspose1d kernels instead of +// per-tap AXPY loops. +// fast_snake: Snake1D with a polynomial sin^2 (|error| < 5e-7) that +// vectorises, instead of one libm sin call per element. +// ================================================================= +namespace detail { +struct KokoroLayerOptions { + bool tap_gemm_conv = false; + bool fast_snake = false; +}; +inline KokoroLayerOptions & kokoro_layer_options() { + static thread_local KokoroLayerOptions options; + return options; +} + +class KokoroLayerOptionsScope { +public: + explicit KokoroLayerOptionsScope(const KokoroLayerOptions & o) : previous_(kokoro_layer_options()) { + kokoro_layer_options() = o; + } + ~KokoroLayerOptionsScope() { kokoro_layer_options() = previous_; } + KokoroLayerOptionsScope(const KokoroLayerOptionsScope &) = delete; + KokoroLayerOptionsScope & operator=(const KokoroLayerOptionsScope &) = delete; +private: + KokoroLayerOptions previous_; +}; + +// sin(y)^2 for float y: reduction by pi in double (sin^2 has period pi; a +// float reduction loses ~1e-7 per 100 rad), then an odd degree-11 +// polynomial for sin on [-pi/2, pi/2] in float. +inline float fast_sin2(float y) { + const double n = std::nearbyint((double) y * 0.318309886183790671538); + const float r = (float) ((double) y - n * 3.14159265358979323846); + const float r2 = r * r; + float p = -2.50521083854417187751e-08f; + p = p * r2 + 2.75573192239858906526e-06f; + p = p * r2 - 1.98412698412698412526e-04f; + p = p * r2 + 8.33333333333333321769e-03f; + p = p * r2 - 1.66666666666666657415e-01f; + const float sn = r + r * r2 * p; + return sn * sn; +} + +#if defined(KOKORO_USE_PORTABLE_FAST) && defined(KOKORO_USE_NEON) +#define KOKORO_HAVE_TILED_KERNELS 1 +// Register-tiled NEON kernels for the portable path (tap_gemm_conv). A tile +// is 4 output rows x 16 time steps held in 16 q-register accumulators across +// the whole reduction, with the 4 weights of each (ci, k) packed into one +// vector, so each 16-float input load feeds 64 FMAs (the AXPY path does one +// load, one FMA and one store per 4 MACs). Tiles run on the layer pool. +constexpr int KT_MR = 4; +constexpr int KT_NR = 16; + +// a: KT_MR-wide packed A columns (a[p*4 + r]), b: B rows (b + p*ldb), Kd steps. +inline void kt_micro(const float * a, const float * b, size_t ldb, int Kd, float32x4_t acc[KT_MR][4]) { + for (int p = 0; p < Kd; ++p) { + const float * bp = b + (size_t) p * ldb; + const float32x4_t x0 = vld1q_f32(bp), x1 = vld1q_f32(bp + 4), x2 = vld1q_f32(bp + 8), x3 = vld1q_f32(bp + 12); + const float32x4_t w = vld1q_f32(a + (size_t) p * KT_MR); +#define KT_ROW(r) \ + acc[r][0] = vfmaq_laneq_f32(acc[r][0], x0, w, r); acc[r][1] = vfmaq_laneq_f32(acc[r][1], x1, w, r); \ + acc[r][2] = vfmaq_laneq_f32(acc[r][2], x2, w, r); acc[r][3] = vfmaq_laneq_f32(acc[r][3], x3, w, r); + KT_ROW(0) KT_ROW(1) KT_ROW(2) KT_ROW(3) +#undef KT_ROW + } +} + +inline void kt_store(float32x4_t acc[KT_MR][4], float * c, size_t ldc, int mr, int nr) { + for (int r = 0; r < mr; ++r) { + float tmp[KT_NR]; + vst1q_f32(tmp, acc[r][0]); vst1q_f32(tmp + 4, acc[r][1]); + vst1q_f32(tmp + 8, acc[r][2]); vst1q_f32(tmp + 12, acc[r][3]); + std::memcpy(c + (size_t) r * ldc, tmp, sizeof(float) * (size_t) nr); + } +} + +// y[Cout, T_out] = b + sum_{ci,k} W[co, ci, k] * xp[ci, to + k*dil] +// xp: [Cin, ldx] zero-padded input, readable up to to + (K-1)*dil + KT_NR. +inline void conv1d_s1_tiled(const float * xp, int ldx, int Cin, + const float * W, const float * b, int Cout, int K, int dil, + float * y, int T_out) { + const int n_mb = (Cout + KT_MR - 1) / KT_MR; + const int n_nb = (T_out + KT_NR - 1) / KT_NR; + // Pack weights per row block: wp[mb][ci][k][r]. + std::vector wp((size_t) n_mb * Cin * K * KT_MR, 0.0f); + for (int co = 0; co < Cout; ++co) + for (int ci = 0; ci < Cin; ++ci) + for (int k = 0; k < K; ++k) + wp[(((size_t) (co / KT_MR) * Cin + ci) * K + k) * KT_MR + co % KT_MR] = + W[((size_t) co * Cin + ci) * K + k]; + const size_t total_macs = (size_t) Cout * T_out * Cin * K; + kokoro_parallel_for(total_macs, n_mb * n_nb, [&](int tile) { + const int mb = tile / n_nb; + const int co0 = mb * KT_MR; + const int t0 = (tile % n_nb) * KT_NR; + const int mr = std::min(KT_MR, Cout - co0); + const int nr = std::min(KT_NR, T_out - t0); + float32x4_t acc[KT_MR][4]; + for (int r = 0; r < KT_MR; ++r) { + const float32x4_t bv = vdupq_n_f32((b && r < mr) ? b[co0 + r] : 0.0f); + acc[r][0] = acc[r][1] = acc[r][2] = acc[r][3] = bv; + } + const float * wb = wp.data() + (size_t) mb * Cin * K * KT_MR; + for (int ci = 0; ci < Cin; ++ci) { + const float * xr = xp + (size_t) ci * ldx + t0; + if (dil == 1) { + // taps k are consecutive rows of a Hankel view: ldb = 1 + kt_micro(wb + (size_t) ci * K * KT_MR, xr, 1, K, acc); + } else { + kt_micro(wb + (size_t) ci * K * KT_MR, xr, (size_t) dil, K, acc); + } + } + kt_store(acc, y + (size_t) co0 * T_out + t0, (size_t) T_out, mr, nr); + }); +} + +// C[M, N] = A[M, Kd] (row-major, lda) * B[Kd, N] (row-major, ldb, readable +// to N rounded up to KT_NR). +inline void sgemm_tiled(int M, int N, int Kd, const float * A, int lda, const float * B, int ldb, + float * C, int ldc) { + const int n_mb = (M + KT_MR - 1) / KT_MR; + const int n_nb = (N + KT_NR - 1) / KT_NR; + std::vector ap((size_t) n_mb * Kd * KT_MR, 0.0f); + for (int i = 0; i < M; ++i) + for (int p = 0; p < Kd; ++p) + ap[((size_t) (i / KT_MR) * Kd + p) * KT_MR + i % KT_MR] = A[(size_t) i * lda + p]; + kokoro_parallel_for((size_t) M * N * Kd, n_mb * n_nb, [&](int tile) { + const int mb = tile / n_nb; + const int i0 = mb * KT_MR; + const int j0 = (tile % n_nb) * KT_NR; + const int mr = std::min(KT_MR, M - i0); + const int nr = std::min(KT_NR, N - j0); + float32x4_t acc[KT_MR][4]; + for (int r = 0; r < KT_MR; ++r) acc[r][0] = acc[r][1] = acc[r][2] = acc[r][3] = vdupq_n_f32(0.0f); + kt_micro(ap.data() + (size_t) mb * Kd * KT_MR, B + j0, (size_t) ldb, Kd, acc); + kt_store(acc, C + (size_t) i0 * ldc + j0, (size_t) ldc, mr, nr); + }); +} +#endif +} // namespace detail + // ================================================================= // Tensor (lightweight host tensor — view or owned) // ================================================================= @@ -452,6 +600,33 @@ inline void conv1d_forward( } #endif #if defined(KOKORO_USE_ACCELERATE) + if (detail::kokoro_layer_options().tap_gemm_conv && stride == 1 && K > 1) { + // y = sum_k W[:, :, k] · x_pad[:, k*dil : k*dil + T_out] + const int Tp = T + 2 * pad; + std::vector xp((size_t) Cin * Tp, 0.0f); + for (int ci = 0; ci < Cin; ++ci) { + std::memcpy(xp.data() + (size_t) ci * Tp + pad, x + (size_t) ci * T, sizeof(float) * (size_t) T); + } + std::vector wk((size_t) K * Cout * Cin); + for (int co = 0; co < Cout; ++co) + for (int ci = 0; ci < Cin; ++ci) + for (int k = 0; k < K; ++k) + wk[((size_t) k * Cout + co) * Cin + ci] = W[((size_t) co * Cin + ci) * K + k]; + if (b) { + for (int co = 0; co < Cout; ++co) { + float * yc = y + (size_t) co * T_out; + for (int to = 0; to < T_out; ++to) yc[to] = b[co]; + } + } + for (int k = 0; k < K; ++k) { + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + Cout, T_out, Cin, + 1.0f, wk.data() + (size_t) k * Cout * Cin, Cin, + xp.data() + (size_t) k * dilation, Tp, + (b || k > 0) ? 1.0f : 0.0f, y, T_out); + } + return; + } // im2col + one SGEMM. The PyTorch weight layout [Cout, Cin, K] is already // a row-major [Cout, Cin*K] GEMM operand; the column matrix is // [Cin*K, T_out] with zero padding baked in, so @@ -491,6 +666,40 @@ inline void conv1d_forward( 1.0f, W, Cin * K, cols.data(), T_out, beta, y, T_out); #elif defined(KOKORO_USE_PORTABLE_FAST) +#if defined(KOKORO_HAVE_TILED_KERNELS) + if (detail::kokoro_layer_options().tap_gemm_conv && stride == 1) { + const int ldx = T + 2 * pad + detail::KT_NR; + std::vector xp((size_t) Cin * ldx, 0.0f); + for (int ci = 0; ci < Cin; ++ci) { + std::memcpy(xp.data() + (size_t) ci * ldx + pad, x + (size_t) ci * T, sizeof(float) * (size_t) T); + } + detail::conv1d_s1_tiled(xp.data(), ldx, Cin, W, b, Cout, K, dilation, y, T_out); + return; + } + if (detail::kokoro_layer_options().tap_gemm_conv && stride > 1) { + // Strided: im2col [Cin*K, T_out] (small for these shapes) + tiled GEMM. + const int ldc = T_out + detail::KT_NR; + std::vector cols((size_t) Cin * K * ldc, 0.0f); + for (int ci = 0; ci < Cin; ++ci) { + const float * xi = x + (size_t) ci * T; + for (int k = 0; k < K; ++k) { + float * col = cols.data() + ((size_t) ci * K + k) * ldc; + for (int to = 0; to < T_out; ++to) { + const int ti = to * stride + k * dilation - pad; + if (ti >= 0 && ti < T) col[to] = xi[ti]; + } + } + } + detail::sgemm_tiled(Cout, T_out, Cin * K, W, Cin * K, cols.data(), ldc, y, T_out); + if (b) { + for (int co = 0; co < Cout; ++co) { + float * yc = y + (size_t) co * T_out; + for (int to = 0; to < T_out; ++to) yc[to] += b[co]; + } + } + return; + } +#endif // Parallel over output channels (each thread owns disjoint y rows), and // per (ci, k) tap a branch-free AXPY over the valid `to` range — the // per-element bounds check of the reference loop is hoisted into the @@ -640,6 +849,32 @@ inline void convtranspose1d_forward( } } #elif defined(KOKORO_USE_PORTABLE_FAST) +#if defined(KOKORO_HAVE_TILED_KERNELS) + if (detail::kokoro_layer_options().tap_gemm_conv) { + // tmp[Cout*K, T] = W^T x, then col2im (as the Accelerate path). + std::vector wt((size_t) Cout * K * Cin); + for (int ci = 0; ci < Cin; ++ci) + for (int j = 0; j < Cout * K; ++j) + wt[(size_t) j * Cin + ci] = W[(size_t) ci * Cout * K + j]; + const int ldb = T + detail::KT_NR; + std::vector xb((size_t) Cin * ldb, 0.0f); + for (int ci = 0; ci < Cin; ++ci) std::memcpy(xb.data() + (size_t) ci * ldb, x + (size_t) ci * T, sizeof(float) * (size_t) T); + std::vector tmp((size_t) Cout * K * T); + detail::sgemm_tiled(Cout * K, T, Cin, wt.data(), Cin, xb.data(), ldb, tmp.data(), T); + for (int co = 0; co < Cout; ++co) { + float * yc = y + (size_t) co * T_out; + for (int k = 0; k < K; ++k) { + const float * row = tmp.data() + ((size_t) co * K + k) * T; + const int t_off = k - pad; + for (int t_in = 0; t_in < T; ++t_in) { + const int to = t_in * stride + t_off; + if (to >= 0 && to < T_out) yc[to] += row[t_in]; + } + } + } + return; + } +#endif // Parallel over output channels (each thread scatters into disjoint y // rows — no races). Two branch-free AXPY shapes, valid ranges hoisted // out of the inner loop: @@ -888,7 +1123,9 @@ inline void bilstm_forward( inline void adain1d_forward( float * x, int C, int T, const float * s, int Sdim, - const float * fc_W /* [2C, Sdim] */, const float * fc_b /* [2C] */) { + const float * fc_W /* [2C, Sdim] */, const float * fc_b /* [2C] */, + const float * norm_W = nullptr /* [C], InstanceNorm affine weight */, + const float * norm_b = nullptr /* [C], InstanceNorm affine bias */) { std::vector h(2 * C); linear_forward(s, Sdim, fc_W, fc_b, 2 * C, h.data()); // gamma = h[0..C], beta = h[C..2C] @@ -905,7 +1142,11 @@ inline void adain1d_forward( // saved state_dict shows no norm.weight key under norm1/norm2 of // AdainResBlk1d, only fc.weight/fc.bias. The default-affine // weight=1/bias=0 is correct then. - instance_norm1d_forward(x, C, T, 1e-5f, nullptr, nullptr); + // + // Models trained with the affine parameters live (Paradee's students) + // pass them as norm_W / norm_b; Kokoro-82M passes nullptr and keeps the + // identity affine, so its arithmetic is unchanged. + instance_norm1d_forward(x, C, T, 1e-5f, norm_W, norm_b); for (int c = 0; c < C; ++c) { const float gamma = h[c]; const float beta = h[C + c]; @@ -992,6 +1233,15 @@ struct AdainResBlk1dWeights { // Snake1D activation: x = x + (1/a) * sin(a*x)^2 // ================================================================= inline void snake1d_forward(float * x, int C, int T, const float * a /* [C] */) { + if (detail::kokoro_layer_options().fast_snake) { + for (int c = 0; c < C; ++c) { + const float av = std::abs(a[c]) > 1e-8f ? a[c] : 1e-8f; + const float inv_a = 1.0f / av; + float * xc = x + (size_t)c * T; + for (int t = 0; t < T; ++t) xc[t] = xc[t] + inv_a * detail::fast_sin2(av * xc[t]); + } + return; + } for (int c = 0; c < C; ++c) { const float av = std::abs(a[c]) > 1e-8f ? a[c] : 1e-8f; const float inv_a = 1.0f / av; diff --git a/tools/kokoro/include/kokoro-predictor.h b/tools/kokoro/include/kokoro-predictor.h index efea20d07..f28fce604 100644 --- a/tools/kokoro/include/kokoro-predictor.h +++ b/tools/kokoro/include/kokoro-predictor.h @@ -9,14 +9,21 @@ // numerically equivalent activations to the upstream Python reference. // // Stages (all forwarded by `kokoro_predictor_forward`): -// 1. Albert BERT (12 layers, num_hidden_groups=1) over input phoneme ids. -// 2. bert_encoder Linear 768→512 → d_en [T, 512]. +// 1. Albert BERT (n_layer, num_hidden_groups=1) over input phoneme ids. +// 2. bert_encoder Linear d_model→hid → d_en [T, hid]. // 3. predictor.text_encoder (DurationEncoder): 3 × (BiLSTM + AdaLayerNorm), -// conditioned on ref_s[:, 128:] (predictor-half style). -// Output `d` has shape [T, 512+128=640] (concat of LSTM output + style). -// 4. predictor.lstm BiLSTM over `d` → x [T, 512]. -// 5. duration_proj Linear (512→50) → duration logits → sigmoid → sum/speed +// conditioned on the predictor style. +// Output `d` has shape [T, hid+sty] (concat of LSTM output + style). +// 4. predictor.lstm BiLSTM over `d` → x [T, hid]. +// 5. duration_proj Linear (hid→max_dur) → duration logits → sigmoid → sum/speed // → per-phoneme float duration. round.clamp(min=1).long() → pred_dur. +// +// Widths come from the GGUF tensor shapes, so the same code runs Kokoro-82M +// (hid 512, style 128) and width-scaled students of it such as Paradee-8M +// (hid 192, style 32). `predictor_forward` takes a PredictorConfig for the +// student-only parts (tensor namespace, asr projection MLP, InstanceNorm +// affine, torch rounding, cancellation); `kokoro_predictor_forward` is the +// Kokoro-82M entry and keeps its arithmetic unchanged. #pragma once @@ -26,6 +33,8 @@ #include #include +struct ggml_context; + namespace eliza_kokoro { // Forward declaration — kokoro_model holds the actual tensor pointers. @@ -36,22 +45,69 @@ struct kokoro_model; struct PredictorOut { int T_phon = 0; // padded input length (BOS+phonemes+EOS) int T_frame = 0; // total frames after length-regulate (sum of pred_dur) + int hid = 0; // predictor hidden width (Kokoro 512) + int sty = 0; // predictor style width (Kokoro 128) + int asr_dim = 0; // width of `asr` rows (Kokoro 512 = hid) - std::vector d; // [T_phon, 640] = DurationEncoder output (LSTM cat style) - std::vector x_lstm; // [T_phon, 512] = predictor.lstm output (forward+reverse concat) + std::vector bert; // [T_phon, d_model] (only when PredictorConfig::keep_bert) + std::vector d; // [T_phon, hid+sty] = DurationEncoder output (LSTM cat style) + std::vector x_lstm; // [T_phon, hid] = predictor.lstm output (forward+reverse concat) std::vector duration; // [T_phon] = sigmoid+sum (pre-round, post-speed) std::vector pred_dur; // [T_phon] = round.clamp(1).long() // After length-regulate: - std::vector en; // [T_frame, 640] = d transposed and matmul'd with pred_aln_trg - std::vector t_en; // [T_phon, 512] = text_encoder forward (separate from BERT branch) - std::vector asr; // [T_frame, 512] = t_en @ pred_aln_trg + std::vector en; // [T_frame, hid+sty] = d transposed and matmul'd with pred_aln_trg + std::vector t_en; // [T_phon, hid] = text_encoder forward (separate from BERT branch) + std::vector asr; // [T_frame, asr_dim] = (asr_proj(t_en) or t_en) @ pred_aln_trg // F0 / N predictions (run via predictor.F0Ntrain on `en`). - std::vector F0_pred; // [T_frame] - std::vector N_pred; // [T_frame] + std::vector F0_pred; // [2*T_frame] + std::vector N_pred; // [2*T_frame] + + bool cancelled = false; // set when PredictorConfig::cancel stopped the forward +}; + +// Cancellation hook: return true to stop. Checked between stages. +using predictor_cancel_fn = bool (*)(void * user); + +struct PredictorConfig { + // Tensor namespace: ".bert.*", ".predictor.*", ... ("kokoro" for + // Kokoro-82M, "paradee" for Paradee-8M). + std::string ns = "kokoro"; + + // Albert text branch (from the GGUF hparams). + int text_n_layer = 12; + int text_n_head = 12; + int text_d_model = 768; + int text_d_ff = 2048; + + int n_layer = 3; // DurationEncoder / TextEncoder depth + int max_dur = 50; // duration_proj width + int text_kernel = 5; // TextEncoder conv kernel + + // Predictor style vector, [sty] floats (sty read from the GGUF). + const float * style = nullptr; + + // Student-only behaviour; all false reproduces Kokoro-82M exactly. + bool torch_round = false; // round half to even like torch.round (Kokoro: std::round) + bool instnorm_affine = false; // AdaIN1d applies the trained InstanceNorm affine (.normN.norm.*) + bool asr_proj_mlp = false; // asr = Linear(.asr_proj.1)(GELU(Linear(.asr_proj.0)(t_en))) + bool keep_bert = false; // keep the BERT output in PredictorOut::bert (stage dumps) + + predictor_cancel_fn cancel = nullptr; + void * cancel_user = nullptr; }; +// Generic predictor forward over a loaded F32 ggml context. Returns false on +// error (`err_out` set) or cancellation (`out.cancelled`, err_out "cancelled"). +bool predictor_forward( + ggml_context * ctx, + const PredictorConfig & cfg, + const std::vector & phoneme_ids, + float speed, + PredictorOut & out, + std::string & err_out); + // Run the predictor + text_encoder + length-regulation. `phoneme_ids` is the // raw phoneme sequence (BOS+phon+EOS); `ref_s` is the 256-dim style vector // (split internally into predictor + decoder halves). diff --git a/tools/kokoro/src/kokoro-decoder.cpp b/tools/kokoro/src/kokoro-decoder.cpp index 07deda551..eaec858b0 100644 --- a/tools/kokoro/src/kokoro-decoder.cpp +++ b/tools/kokoro/src/kokoro-decoder.cpp @@ -4,10 +4,14 @@ // StyleTTS-2 / iSTFTNet decoder, reading weights from the model's all-F32 // ggml context (dequantized at load). Validated against the PyTorch reference // stage-by-stage (#9588). +// +// Channel widths, kernel sizes and the style width are read from the tensor +// shapes (ggml ne[] is the reversed PyTorch shape), so Kokoro-82M and its +// width-scaled students share this file. #include "kokoro-decoder.h" #include "kokoro-decoder-front.h" // DecoderFrontWeights, DecAdainResBlk, decoder_front -#include "kokoro-generator.h" // GeneratorWeights, kokoro_generator_forward +#include "kokoro-generator.h" // GeneratorWeights, generator_forward #include "kokoro-model-internal.h" #include "kokoro-profile.h" @@ -21,16 +25,27 @@ namespace { struct Lk { ggml_context * ctx; - const float * get(const std::string & name) const { - ggml_tensor * t = kokoro_find_tensor_compat(ctx, name); + std::string ns; + std::string full(const std::string & rel) const { return ns + rel; } + ggml_tensor * tensor(const std::string & rel) const { + return kokoro_find_tensor_compat(ctx, full(rel)); + } + const float * get(const std::string & rel) const { + ggml_tensor * t = tensor(rel); return t ? (const float *) t->data : nullptr; } }; -// Fill an AdainResBlk1d (decode flavor) from a tensor-name prefix. -void fill_dec_block(const Lk & L, DecAdainResBlk & b, const std::string & pfx, - int Cin, int Cout, bool upsample) { - b.Cin = Cin; b.Cout = Cout; b.Sdim = 128; +// Fill an AdainResBlk1d (decode flavor) from a tensor-name prefix. Cin/Cout +// come from conv1.weight [Cout, Cin, 3] and Sdim from norm1.fc.weight +// [2*Cin, Sdim]. +bool fill_dec_block(const Lk & L, DecAdainResBlk & b, const std::string & pfx, + bool upsample, bool affine) { + ggml_tensor * c1 = L.tensor(pfx + ".conv1.weight"); + ggml_tensor * n1 = L.tensor(pfx + ".norm1.fc.weight"); + if (!c1 || !n1) return false; + const int Cin = (int) c1->ne[1], Cout = (int) c1->ne[2]; + b.Cin = Cin; b.Cout = Cout; b.Sdim = (int) n1->ne[0]; b.upsample = upsample; b.learned_sc = (Cin != Cout); b.norm1_fc_w = L.get(pfx + ".norm1.fc.weight"); @@ -45,10 +60,18 @@ void fill_dec_block(const Lk & L, DecAdainResBlk & b, const std::string & pfx, b.conv1x1_b = nullptr; // conv1x1 bias=False upstream b.pool_w = upsample ? L.get(pfx + ".pool.weight") : nullptr; b.pool_b = upsample ? L.get(pfx + ".pool.bias") : nullptr; + if (affine) { + b.norm1_w = L.get(pfx + ".norm1.norm.weight"); + b.norm1_b = L.get(pfx + ".norm1.norm.bias"); + b.norm2_w = L.get(pfx + ".norm2.norm.weight"); + b.norm2_b = L.get(pfx + ".norm2.norm.bias"); + if (!b.norm1_w || !b.norm1_b || !b.norm2_w || !b.norm2_b) return false; + } + return true; } // Fill an AdaINResBlock1 (generator flavor: 3 sub-blocks, Snake1D) from a prefix. -void fill_gen_block(const Lk & L, GenAdaResBlockWeights & b, const std::string & pfx) { +bool fill_gen_block(const Lk & L, GenAdaResBlockWeights & b, const std::string & pfx, bool affine) { for (int j = 0; j < 3; ++j) { const std::string js = std::to_string(j); GenSubBlockWeights & s = b.sub[j]; @@ -62,67 +85,116 @@ void fill_gen_block(const Lk & L, GenAdaResBlockWeights & b, const std::string & s.adain2_fc_b = L.get(pfx + ".adain2." + js + ".fc.bias"); s.alpha1 = L.get(pfx + ".alpha1." + js); s.alpha2 = L.get(pfx + ".alpha2." + js); + if (affine) { + s.adain1_norm_w = L.get(pfx + ".adain1." + js + ".norm.weight"); + s.adain1_norm_b = L.get(pfx + ".adain1." + js + ".norm.bias"); + s.adain2_norm_w = L.get(pfx + ".adain2." + js + ".norm.weight"); + s.adain2_norm_b = L.get(pfx + ".adain2." + js + ".norm.bias"); + if (!s.adain1_norm_w || !s.adain1_norm_b || !s.adain2_norm_w || !s.adain2_norm_b) return false; + } } + return true; } } // namespace -bool kokoro_decoder_forward( - const kokoro_model * model, - const float * asr_ct, int T_frame, +bool decoder_forward( + ggml_context * ctx, + DecoderConfig & cfg, + const float * asr_ct, int asr_dim, int T_frame, const float * F0, const float * N, - const float * ref_s_dec, + const float * style, std::vector & audio, - std::string & err, - kokoro_cancel_callback cancel, - void * cancel_user_data) { + std::string & err) { audio.clear(); - if (!model) { err = "null model"; return false; } - if (T_frame <= 0) { err = "non-positive T_frame"; return false; } - - ggml_context * ctx = kokoro_model_ggml_ctx(model); + cfg.cancelled = false; if (!ctx) { err = "null model context"; return false; } - Lk L{ctx}; + if (T_frame <= 0) { err = "non-positive T_frame"; return false; } + Lk L{ctx, cfg.ns}; + const bool affine = cfg.instnorm_affine; // --- decoder_front weights --- DecoderFrontWeights W; - W.F0_conv_w = L.get("kokoro.decoder.F0_conv.weight"); - W.F0_conv_b = L.get("kokoro.decoder.F0_conv.bias"); - W.N_conv_w = L.get("kokoro.decoder.N_conv.weight"); - W.N_conv_b = L.get("kokoro.decoder.N_conv.bias"); - W.asr_res_w = L.get("kokoro.decoder.asr_res.weight"); - W.asr_res_b = L.get("kokoro.decoder.asr_res.bias"); - fill_dec_block(L, W.encode, "kokoro.decoder.encode", 514, 1024, /*upsample*/false); - fill_dec_block(L, W.decode[0], "kokoro.decoder.decode.0", 1090, 1024, false); - fill_dec_block(L, W.decode[1], "kokoro.decoder.decode.1", 1090, 1024, false); - fill_dec_block(L, W.decode[2], "kokoro.decoder.decode.2", 1090, 1024, false); - fill_dec_block(L, W.decode[3], "kokoro.decoder.decode.3", 1090, 512, /*upsample*/true); - - if (!W.F0_conv_w || !W.asr_res_w || !W.encode.conv1_w || !W.decode[3].pool_w) { - err = "missing decoder weights (is the GGUF a full Kokoro model?)"; + W.F0_conv_w = L.get(".decoder.F0_conv.weight"); + W.F0_conv_b = L.get(".decoder.F0_conv.bias"); + W.N_conv_w = L.get(".decoder.N_conv.weight"); + W.N_conv_b = L.get(".decoder.N_conv.bias"); + W.asr_res_w = L.get(".decoder.asr_res.weight"); + W.asr_res_b = L.get(".decoder.asr_res.bias"); + ggml_tensor * asr_res_t = L.tensor(".decoder.asr_res.weight"); // [Cr, asr_dim, 1] + bool ok = asr_res_t != nullptr; + if (ok) W.asr_res_C = (int) asr_res_t->ne[2]; + ok = ok && fill_dec_block(L, W.encode, ".decoder.encode", /*upsample*/false, affine); + for (int b = 0; b < 4 && ok; ++b) { + ok = fill_dec_block(L, W.decode[b], ".decoder.decode." + std::to_string(b), + /*upsample*/ b == 3, affine); + } + if (!ok || !W.F0_conv_w || !W.asr_res_w || !W.encode.conv1_w || !W.decode[3].pool_w) { + err = "missing decoder weights (is the GGUF a full " + cfg.ns + " model?)"; + return false; + } + if (W.encode.Cin != asr_dim + 2 || (int) asr_res_t->ne[1] != asr_dim) { + err = "decoder input width " + std::to_string(W.encode.Cin - 2) + + " != asr width " + std::to_string(asr_dim); return false; } // --- generator weights --- GeneratorWeights G; - G.l_linear_w = L.get("kokoro.gen.m_source.l_linear.weight"); - G.l_linear_b = L.get("kokoro.gen.m_source.l_linear.bias"); - for (int i = 0; i < 2; ++i) { + G.l_linear_w = L.get(".gen.m_source.l_linear.weight"); + G.l_linear_b = L.get(".gen.m_source.l_linear.bias"); + for (int i = 0; i < 2 && ok; ++i) { const std::string is = std::to_string(i); - G.ups_w[i] = L.get("kokoro.gen.ups." + is + ".weight"); - G.ups_b[i] = L.get("kokoro.gen.ups." + is + ".bias"); - G.noise_convs_w[i] = L.get("kokoro.gen.noise_convs." + is + ".weight"); - G.noise_convs_b[i] = L.get("kokoro.gen.noise_convs." + is + ".bias"); - fill_gen_block(L, G.noise_res[i], "kokoro.gen.noise_res." + is); + G.ups_w[i] = L.get(".gen.ups." + is + ".weight"); + G.ups_b[i] = L.get(".gen.ups." + is + ".bias"); + G.noise_convs_w[i] = L.get(".gen.noise_convs." + is + ".weight"); + G.noise_convs_b[i] = L.get(".gen.noise_convs." + is + ".bias"); + ok = fill_gen_block(L, G.noise_res[i], ".gen.noise_res." + is, affine); } - for (int i = 0; i < 6; ++i) { - fill_gen_block(L, G.resblocks[i], "kokoro.gen.resblocks." + std::to_string(i)); + for (int i = 0; i < 6 && ok; ++i) { + ok = fill_gen_block(L, G.resblocks[i], ".gen.resblocks." + std::to_string(i), affine); + } + G.conv_post_w = L.get(".gen.conv_post.weight"); + G.conv_post_b = L.get(".gen.conv_post.bias"); + + ggml_tensor * ups0 = L.tensor(".gen.ups.0.weight"); // [Cin, ch0, K0] + ggml_tensor * ups1 = L.tensor(".gen.ups.1.weight"); // [ch0, ch1, K1] + ggml_tensor * post = L.tensor(".gen.conv_post.weight"); // [n_fft+2, ch1, 7] + ggml_tensor * lin = L.tensor(".gen.m_source.l_linear.weight"); // [1, harmonics] + if (!ok || !G.l_linear_w || !G.ups_w[0] || !G.conv_post_w || !G.resblocks[5].sub[2].conv2_w || + !ups0 || !ups1 || !post || !lin) { + err = "missing generator weights (is the GGUF a full " + cfg.ns + " model?)"; + return false; } - G.conv_post_w = L.get("kokoro.gen.conv_post.weight"); - G.conv_post_b = L.get("kokoro.gen.conv_post.bias"); - if (!G.l_linear_w || !G.ups_w[0] || !G.conv_post_w || !G.resblocks[5].sub[2].conv2_w) { - err = "missing generator weights (is the GGUF a full Kokoro model?)"; + GeneratorConfig gc; + gc.Cin = (int) ups0->ne[2]; + gc.ch[0] = (int) ups0->ne[1]; + gc.ch[1] = (int) ups1->ne[1]; + gc.up_kernels[0] = (int) ups0->ne[0]; + gc.up_kernels[1] = (int) ups1->ne[0]; + gc.up_rates[0] = cfg.up_rates[0]; + gc.up_rates[1] = cfg.up_rates[1]; + for (int j = 0; j < 3; ++j) { + ggml_tensor * rb = L.tensor(".gen.resblocks." + std::to_string(j) + ".convs1.0.weight"); + if (!rb) { err = "missing generator resblock weights"; return false; } + gc.rb_kernels[j] = (int) rb->ne[0]; + } + gc.n_fft = (int) post->ne[2] - 2; + gc.hop = cfg.istft_hop; + gc.harmonics = (int) lin->ne[0]; + ggml_tensor * gen_fc = L.tensor(".gen.resblocks.0.adain1.0.fc.weight"); // [2C, Sdim] + gc.Sdim = gen_fc ? (int) gen_fc->ne[0] : W.encode.Sdim; + gc.canonical_source_stft = cfg.canonical_source_stft; + gc.precise_source = cfg.precise_source; + gc.instnorm_affine = affine; + gc.har_source_out = cfg.har_source_out; + gc.har_source_in = cfg.har_source_in; + gc.cancel = cfg.cancel; + gc.cancel_user = cfg.cancel_user; + if (W.decode[3].Cout != gc.Cin || (int) ups1->ne[2] != gc.ch[0] || (int) post->ne[1] != gc.ch[1] || + gc.Sdim != W.encode.Sdim || gc.n_fft <= 0) { + err = "inconsistent decoder/generator tensor shapes"; return false; } @@ -130,24 +202,50 @@ bool kokoro_decoder_forward( std::vector x, F0_down, N_down; { kokoro_phase_timer t("decoder front"); - decoder_front(W, asr_ct, /*Cin_asr*/512, T_frame, F0, N, ref_s_dec, x, F0_down, N_down); + decoder_front(W, asr_ct, asr_dim, T_frame, F0, N, style, x, F0_down, N_down); + } + if (cfg.cancel && cfg.cancel(cfg.cancel_user)) { + cfg.cancelled = true; + err = "cancelled"; + return false; } const int T0 = 2 * T_frame; // decoder_front upsamples T_frame -> 2*T_frame - if ((int) (x.size() / 512) != T0) { + if ((int) (x.size() / (size_t) gc.Cin) != T0) { err = "decoder_front output width mismatch"; return false; } - // The generator is the most expensive stage; skip it when cancelled. - if (cancel && cancel(cancel_user_data)) { - err = "cancelled"; - return false; - } { kokoro_phase_timer t("generator (iSTFTNet)"); - kokoro_generator_forward(x.data(), T0, ref_s_dec, F0, G, audio); + if (!generator_forward(x.data(), T0, style, F0, G, gc, audio)) { + cfg.cancelled = true; + err = "cancelled"; + return false; + } } return true; } +bool kokoro_decoder_forward( + const kokoro_model * model, + const float * asr_ct, int T_frame, + const float * F0, const float * N, + const float * ref_s_dec, + std::vector & audio, + std::string & err, + kokoro_cancel_callback cancel, + void * cancel_user_data) { + audio.clear(); + if (!model) { err = "null model"; return false; } + DecoderConfig cfg; // Kokoro-82M: namespace "kokoro", identity InstanceNorm affine, shipped source STFT + ggml_context * ctx = kokoro_model_ggml_ctx(model); + // asr width from asr_res.weight [64, asr_dim, 1] (512 for Kokoro-82M). + ggml_tensor * asr_res = kokoro_find_tensor_compat(ctx, "kokoro.decoder.asr_res.weight"); + if (!asr_res) { err = "missing decoder weights (is the GGUF a full kokoro model?)"; return false; } + cfg.cancel = cancel; + cfg.cancel_user = cancel_user_data; + return decoder_forward(ctx, cfg, asr_ct, (int) asr_res->ne[1], T_frame, + F0, N, ref_s_dec, audio, err); +} + } // namespace eliza_kokoro diff --git a/tools/kokoro/src/kokoro-generator.cpp b/tools/kokoro/src/kokoro-generator.cpp index 97d93d457..866542bfa 100644 --- a/tools/kokoro/src/kokoro-generator.cpp +++ b/tools/kokoro/src/kokoro-generator.cpp @@ -15,6 +15,10 @@ // - spec = exp(x[:11]), phase = sin(x[11:22]), // - inverse STFT (center=True) -> audio. // +// Widths, rates and the source convention come from GeneratorConfig; the +// defaults (and kokoro_generator_forward) are Kokoro v1.0. Paradee-8M runs the +// same code at Cin 128, stage channels 64/32, style 16. +// // Config (kokoro v1.0 istftnet): // style_dim=128, upsample_initial_channel=512, // upsample_rates=[10,6], upsample_kernel_sizes=[20,12], @@ -113,9 +117,53 @@ static void interp_linear(const float * x, int C, int T_in, int T_out, float * y // window. Output: mag[F, n_frames], phase[F, n_frames], F=n_fft/2+1, // n_frames = N/hop + 1. // ---------------------------------------------------------------------------- +// Canonical source STFT (GeneratorConfig::canonical_source_stft): the same +// centred DFT with a double-precision window and basis tables, then values +// at or below the frame's rounding floor (1e-12 * sum|x*w|) set to exactly 0 +// and torch.angle's convention for the phase (pi for a negative real value). +// A float-rounded window would leave ~1e-8 residue in the empty bins of a +// constant frame, above any useful floor, so the window here stays double. +static void stft_center_canonical(const std::vector & padded, int n_fft, int hop, int win, + int n_frames, std::vector & mag, std::vector & phase) { + const int F = n_fft / 2 + 1; + std::vector w((size_t) win), cs((size_t) F * win), sn((size_t) F * win); + for (int k = 0; k < win; ++k) w[(size_t) k] = 0.5 - 0.5 * std::cos(2.0 * K_PI * (double) k / (double) win); + for (int f = 0; f < F; ++f) { + for (int k = 0; k < win; ++k) { + const double ang = 2.0 * K_PI * (double) f * (double) k / (double) n_fft; + cs[(size_t) f * win + k] = std::cos(ang) * w[(size_t) k]; + sn[(size_t) f * win + k] = -std::sin(ang) * w[(size_t) k]; + } + } + std::vector x((size_t) win); + for (int t = 0; t < n_frames; ++t) { + const int off = t * hop; + double floor_abs = 0.0; + for (int k = 0; k < win; ++k) { + x[(size_t) k] = (double) padded[(size_t) (off + k)]; + floor_abs += std::fabs(x[(size_t) k] * w[(size_t) k]); + } + floor_abs *= 1e-12; + for (int f = 0; f < F; ++f) { + double re = 0.0, im = 0.0; + const double * c = cs.data() + (size_t) f * win; + const double * s = sn.data() + (size_t) f * win; + for (int k = 0; k < win; ++k) { + re += x[(size_t) k] * c[k]; + im += x[(size_t) k] * s[k]; + } + if (std::fabs(re) <= floor_abs) re = 0.0; + if (std::fabs(im) <= floor_abs) im = 0.0; + mag[(size_t) f * n_frames + t] = (float) std::sqrt(re * re + im * im); + phase[(size_t) f * n_frames + t] = + (im == 0.0 && re < 0.0) ? (float) K_PI : (float) std::atan2(im, re); + } + } +} + static void stft_center(const float * sig, int N, int n_fft, int hop, int win, std::vector & mag, std::vector & phase, - int & F, int & n_frames) { + int & F, int & n_frames, bool canonical) { F = n_fft / 2 + 1; const int pad = n_fft / 2; const int Np = N + 2 * pad; @@ -130,6 +178,10 @@ static void stft_center(const float * sig, int N, int n_fft, int hop, int win, const std::vector window = hann_periodic(win); mag.assign((size_t) F * n_frames, 0.0f); phase.assign((size_t) F * n_frames, 0.0f); + if (canonical) { + stft_center_canonical(padded, n_fft, hop, win, n_frames, mag, phase); + return; + } for (int t = 0; t < n_frames; ++t) { const int off = t * hop; for (int f = 0; f < F; ++f) { @@ -141,8 +193,10 @@ static void stft_center(const float * sig, int N, int n_fft, int hop, int win, re += v * std::cos(ang); im += v * std::sin(ang); } - mag[(size_t) f * n_frames + t] = (float) std::sqrt(re * re + im * im); - phase[(size_t) f * n_frames + t] = (float) std::atan2(im, re); + { + mag[(size_t) f * n_frames + t] = (float) std::sqrt(re * re + im * im); + phase[(size_t) f * n_frames + t] = (float) std::atan2(im, re); + } } } } @@ -163,17 +217,32 @@ static void istft_center(const float * mag, const float * phase, const std::vector window = hann_periodic(win); std::vector frame((size_t) n_fft); + // Basis tables and per-frame (re, im): the same double values the + // straightforward triple loop computes, evaluated once instead of n_fft + // times, so the sums (and their order) are unchanged. + std::vector cos_t((size_t) F * n_fft), sin_t((size_t) F * n_fft); + for (int f = 0; f < F; ++f) { + for (int n = 0; n < n_fft; ++n) { + const double ang = 2.0 * K_PI * (double) f * (double) n / (double) n_fft; + cos_t[(size_t) f * n_fft + n] = std::cos(ang); + sin_t[(size_t) f * n_fft + n] = std::sin(ang); + } + } + std::vector re_f((size_t) F), im_f((size_t) F); for (int t = 0; t < n_frames; ++t) { + for (int f = 0; f < F; ++f) { + const double m = mag[(size_t) f * n_frames + t]; + const double p = phase[(size_t) f * n_frames + t]; + re_f[(size_t) f] = m * std::cos(p); + im_f[(size_t) f] = m * std::sin(p); + } // irfft of the hermitian spectrum -> n_fft real samples. for (int n = 0; n < n_fft; ++n) { double s = 0.0; for (int f = 0; f < F; ++f) { - const double m = mag[(size_t) f * n_frames + t]; - const double p = phase[(size_t) f * n_frames + t]; - const double re = m * std::cos(p); - const double im = m * std::sin(p); - const double ang = 2.0 * K_PI * (double) f * (double) n / (double) n_fft; - double term = re * std::cos(ang) - im * std::sin(ang); + const double re = re_f[(size_t) f]; + const double im = im_f[(size_t) f]; + double term = re * cos_t[(size_t) f * n_fft + n] - im * sin_t[(size_t) f * n_fft + n]; if (f != 0 && !(n_fft % 2 == 0 && f == F - 1)) term *= 2.0; s += term; } @@ -203,6 +272,9 @@ static void istft_center(const float * mag, const float * phase, // ---------------------------------------------------------------------------- static int get_padding(int k, int d) { return (k * d - d) / 2; } +// Length after F.interpolate(scale_factor = 1 / upsample_scale). +static int Lds_of(int L, int upsample_scale) { return L / upsample_scale; } + static void adain_resblock1(const GenAdaResBlockWeights & w, float * x, int C, int T, const float * s, int Sdim, int K, const int dil[3]) { @@ -210,12 +282,14 @@ static void adain_resblock1(const GenAdaResBlockWeights & w, for (int i = 0; i < 3; ++i) { const GenSubBlockWeights & sb = w.sub[i]; std::memcpy(xt.data(), x, sizeof(float) * (size_t) C * T); - adain1d_forward(xt.data(), C, T, s, Sdim, sb.adain1_fc_w, sb.adain1_fc_b); + adain1d_forward(xt.data(), C, T, s, Sdim, sb.adain1_fc_w, sb.adain1_fc_b, + sb.adain1_norm_w, sb.adain1_norm_b); snake1d_forward(xt.data(), C, T, sb.alpha1); const int d1 = dil[i]; conv1d_forward(xt.data(), C, T, sb.conv1_w, sb.conv1_b, C, K, 1, get_padding(K, d1), d1, y1.data(), T); - adain1d_forward(y1.data(), C, T, s, Sdim, sb.adain2_fc_w, sb.adain2_fc_b); + adain1d_forward(y1.data(), C, T, s, Sdim, sb.adain2_fc_w, sb.adain2_fc_b, + sb.adain2_norm_w, sb.adain2_norm_b); snake1d_forward(y1.data(), C, T, sb.alpha2); conv1d_forward(y1.data(), C, T, sb.conv2_w, sb.conv2_b, C, K, 1, get_padding(K, 1), 1, y2.data(), T); @@ -226,25 +300,31 @@ static void adain_resblock1(const GenAdaResBlockWeights & w, } // namespace // ============================================================================ -// kokoro_generator_forward +// generator_forward // ============================================================================ -void kokoro_generator_forward( - const float * x_in, // [512, T0] +bool generator_forward( + const float * x_in, // [cfg.Cin, T0] int T0, // input time (== 2 * predictor T_frame) - const float * s, // [128] + const float * s, // [cfg.Sdim] const float * f0_curve, // [T0] const GeneratorWeights & w, - std::vector & audio /* out [T0 * 300] */) { - - const int Sdim = 128; - const int up0 = 10, up1 = 6; // upsample rates - const int hop = 5; - const int n_fft = 20, win = 20; - const int upsample_scale = up0 * up1 * hop; // 300 - const int dim = 9; // harmonic_num (8) + fundamental - const float sine_amp = 0.1f; - const float voiced_threshold = 10.0f; - const float sr = 24000.0f; + const GeneratorConfig & cfg, + std::vector & audio /* out [T0 * upsample_scale] */) { + + const int Sdim = cfg.Sdim; + const int up0 = cfg.up_rates[0], up1 = cfg.up_rates[1]; // upsample rates + const int hop = cfg.hop; + const int n_fft = cfg.n_fft, win = cfg.n_fft; + const int upsample_scale = up0 * up1 * hop; // Kokoro: 300 + const int dim = cfg.harmonics; // harmonic_num + fundamental (Kokoro: 9) + const float sine_amp = cfg.sine_amp; + const float voiced_threshold = cfg.voiced_threshold; + const float sr = cfg.sample_rate; + const auto cancelled = [&]() { + if (!cfg.cancel || !cfg.cancel(cfg.cancel_user)) return false; + audio.clear(); + return true; + }; // Phase checkpoints (KOKORO_PROFILE=1): logs the delta since the previous // mark, attributing generator time to source/stft, the two upsample @@ -267,45 +347,83 @@ void kokoro_generator_forward( for (int r = 0; r < upsample_scale; ++r) f0_up[(size_t) t * upsample_scale + r] = f0_curve[t]; - // fn = f0 * [1..9]; rad = (fn / sr) % 1 ; [dim, L] channel-major. - std::vector rad((size_t) dim * L); - for (int h = 0; h < dim; ++h) { - const float mul = (float) (h + 1); - for (int t = 0; t < L; ++t) { - float v = f0_up[(size_t) t] * mul / sr; - v = v - std::floor(v); // % 1 - rad[(size_t) h * L + t] = v; + std::vector sine_waves((size_t) dim * L); + if (!cfg.precise_source) { + // fn = f0 * [1..9]; rad = (fn / sr) % 1 ; [dim, L] channel-major. + std::vector rad((size_t) dim * L); + for (int h = 0; h < dim; ++h) { + const float mul = (float) (h + 1); + for (int t = 0; t < L; ++t) { + float v = f0_up[(size_t) t] * mul / sr; + v = v - std::floor(v); // % 1 + rad[(size_t) h * L + t] = v; + } } - } - // Deterministic: rad[:,0,:] += 0 (no random initial phase). - // _f02sine: downsample rad by 1/upsample_scale (linear), cumsum*2pi, - // upsample by *upsample_scale (with phase scaled by upsample_scale), sin. - const int Lds = L / upsample_scale; // 264 - std::vector rad_ds((size_t) dim * Lds); - interp_linear(rad.data(), dim, L, Lds, rad_ds.data()); - // cumsum over time per channel, * 2*pi. - std::vector phase_ds((size_t) dim * Lds); - for (int h = 0; h < dim; ++h) { - double cum = 0.0; - for (int t = 0; t < Lds; ++t) { - cum += (double) rad_ds[(size_t) h * Lds + t]; - phase_ds[(size_t) h * Lds + t] = (float) (cum * 2.0 * K_PI); + // Deterministic: rad[:,0,:] += 0 (no random initial phase). + // _f02sine: downsample rad by 1/upsample_scale (linear), cumsum*2pi, + // upsample by *upsample_scale (with phase scaled by upsample_scale), sin. + const int Lds = L / upsample_scale; // 264 + std::vector rad_ds((size_t) dim * Lds); + interp_linear(rad.data(), dim, L, Lds, rad_ds.data()); + // cumsum over time per channel, * 2*pi. + std::vector phase_ds((size_t) dim * Lds); + for (int h = 0; h < dim; ++h) { + double cum = 0.0; + for (int t = 0; t < Lds; ++t) { + cum += (double) rad_ds[(size_t) h * Lds + t]; + phase_ds[(size_t) h * Lds + t] = (float) (cum * 2.0 * K_PI); + } } - } - // F.interpolate(phase * upsample_scale, scale=upsample_scale, linear). - std::vector phase_scaled((size_t) dim * Lds); - for (size_t i = 0; i < (size_t) dim * Lds; ++i) - phase_scaled[i] = phase_ds[i] * (float) upsample_scale; - std::vector phase_up((size_t) dim * L); - interp_linear(phase_scaled.data(), dim, Lds, L, phase_up.data()); - // sines = sin(phase) * sine_amp ; voiced mask uv = (f0 > thr). - // sine_waves = sines * uv (+ noise = 0). sine_merge = tanh(l_linear(sine_waves)). - std::vector sine_waves((size_t) dim * L); - for (int h = 0; h < dim; ++h) { - for (int t = 0; t < L; ++t) { - const float uv = (f0_up[(size_t) t] > voiced_threshold) ? 1.0f : 0.0f; - sine_waves[(size_t) h * L + t] = - std::sin(phase_up[(size_t) h * L + t]) * sine_amp * uv; + // F.interpolate(phase * upsample_scale, scale=upsample_scale, linear). + std::vector phase_scaled((size_t) dim * Lds); + for (size_t i = 0; i < (size_t) dim * Lds; ++i) + phase_scaled[i] = phase_ds[i] * (float) upsample_scale; + std::vector phase_up((size_t) dim * L); + interp_linear(phase_scaled.data(), dim, Lds, L, phase_up.data()); + // sines = sin(phase) * sine_amp ; voiced mask uv = (f0 > thr). + // sine_waves = sines * uv (+ noise = 0). sine_merge = tanh(l_linear(sine_waves)). + for (int h = 0; h < dim; ++h) { + for (int t = 0; t < L; ++t) { + const float uv = (f0_up[(size_t) t] > voiced_threshold) ? 1.0f : 0.0f; + sine_waves[(size_t) h * L + t] = + std::sin(phase_up[(size_t) h * L + t]) * sine_amp * uv; + } + } + } else { + // The same SineGen._f02sine steps in double, so the phase of the top + // harmonic (~1e5 rad after a few seconds) keeps sub-microradian + // precision; values are rounded to float only as sine_waves. + const double scale_dn = (double) L / (double) Lds_of(L, upsample_scale); + const int Lds = Lds_of(L, upsample_scale); + const double scale_up = (double) Lds / (double) L; + std::vector phase_ds((size_t) Lds); + for (int h = 0; h < dim; ++h) { + const double mul = (double) (h + 1); + auto rad_at = [&](int t) { + double v = (double) f0_up[(size_t) t] * mul / (double) sr; + return v - std::floor(v); // % 1 + }; + double cum = 0.0; + for (int o = 0; o < Lds; ++o) { + double src = ((double) o + 0.5) * scale_dn - 0.5; + if (src < 0.0) src = 0.0; + int s0 = std::min((int) src, L - 1); + const int s1 = s0 < L - 1 ? s0 + 1 : s0; + const double frac = std::min(std::max(src - (double) s0, 0.0), 1.0); + const double r0 = rad_at(s0), r1 = rad_at(s1); + cum += r0 * (1.0 - frac) + r1 * frac; + phase_ds[(size_t) o] = cum * 2.0 * K_PI * (double) upsample_scale; + } + for (int t = 0; t < L; ++t) { + double src = ((double) t + 0.5) * scale_up - 0.5; + if (src < 0.0) src = 0.0; + int s0 = std::min((int) src, Lds - 1); + const int s1 = s0 < Lds - 1 ? s0 + 1 : s0; + const double frac = std::min(std::max(src - (double) s0, 0.0), 1.0); + const double ph = phase_ds[(size_t) s0] * (1.0 - frac) + phase_ds[(size_t) s1] * frac; + const double uv = ((double) f0_up[(size_t) t] > (double) voiced_threshold) ? 1.0 : 0.0; + sine_waves[(size_t) h * L + t] = (float) (std::sin(ph) * (double) sine_amp * uv); + } } } // l_linear: Linear(9 -> 1). weight [1, 9], bias [1]. har_source[L]. @@ -316,14 +434,20 @@ void kokoro_generator_forward( acc += (double) w.l_linear_w[h] * (double) sine_waves[(size_t) h * L + t]; har_source[(size_t) t] = std::tanh((float) acc); } + if (cfg.har_source_in && cfg.har_source_in->size() == har_source.size()) { + har_source = *cfg.har_source_in; + } DBG("har_source", har_source.data(), har_source.size()); + if (cfg.har_source_out) *cfg.har_source_out = har_source; + if (cancelled()) return false; // ------------------------------------------------------------------ // 2. STFT of har_source -> har = cat(mag, phase) [22, n_frames]. // ------------------------------------------------------------------ std::vector hmag, hphase; int F = 0, n_frames = 0; - stft_center(har_source.data(), L, n_fft, hop, win, hmag, hphase, F, n_frames); + stft_center(har_source.data(), L, n_fft, hop, win, hmag, hphase, F, n_frames, + cfg.canonical_source_stft); const int Hc = 2 * F; // 22 std::vector har((size_t) Hc * n_frames); std::memcpy(har.data(), hmag.data(), sizeof(float) * (size_t) F * n_frames); @@ -343,15 +467,16 @@ void kokoro_generator_forward( // 3. Upsample stages. // ------------------------------------------------------------------ const int ups_rate[2] = { up0, up1 }; - const int ups_k[2] = { 20, 12 }; - const int ch_after[2] = { 256, 128 }; // upsample_initial_channel >> (i+1) - const int dil135[3] = { 1, 3, 5 }; + const int ups_k[2] = { cfg.up_kernels[0], cfg.up_kernels[1] }; + const int ch_after[2] = { cfg.ch[0], cfg.ch[1] }; // upsample_initial_channel >> (i+1) + const int dil135[3] = { cfg.dil[0], cfg.dil[1], cfg.dil[2] }; - // current x: [512, 264] (copy, mutable). - int curC = 512, curT = T0; + // current x: [Cin, T0] (copy, mutable). + int curC = cfg.Cin, curT = T0; std::vector x(x_in, x_in + (size_t) curC * curT); for (int i = 0; i < 2; ++i) { + if (cancelled()) return false; const int u = ups_rate[i], k = ups_k[i]; const int outC = ch_after[i]; @@ -413,8 +538,9 @@ void kokoro_generator_forward( // resblocks: xs_sum = sum_j resblock[i*3+j](x, s) / 3. std::vector accum((size_t) outC * xT, 0.0f); - const int rb_k[3] = { 3, 7, 11 }; + const int rb_k[3] = { cfg.rb_kernels[0], cfg.rb_kernels[1], cfg.rb_kernels[2] }; for (int j = 0; j < 3; ++j) { + if (cancelled()) return false; std::vector rb(xs); // copy current x adain_resblock1(w.resblocks[i * 3 + j], rb.data(), outC, xT, s, Sdim, rb_k[j], dil135); @@ -431,6 +557,7 @@ void kokoro_generator_forward( // ------------------------------------------------------------------ // 4. conv_post -> spec/phase -> iSTFT. // ------------------------------------------------------------------ + if (cancelled()) return false; leaky_relu(x.data(), curC * curT, 0.01f); // F.leaky_relu (no slope arg) = default 0.01 const int cpC = n_fft + 2; // 22 std::vector cp((size_t) cpC * curT); @@ -449,6 +576,21 @@ void kokoro_generator_forward( } istft_center(spec.data(), phase.data(), n_fft, hop, win, curT, audio); prof_mark(" gen: post+istft"); + return true; +} + +// ============================================================================ +// kokoro_generator_forward (Kokoro-82M defaults) +// ============================================================================ +void kokoro_generator_forward( + const float * x_in, // [512, T0] + int T0, // input time (== 2 * predictor T_frame) + const float * s, // [128] + const float * f0_curve, // [T0] + const GeneratorWeights & w, + std::vector & audio /* out [T0 * 300] */) { + GeneratorConfig cfg; + generator_forward(x_in, T0, s, f0_curve, w, cfg, audio); } } // namespace eliza_kokoro diff --git a/tools/kokoro/src/kokoro-predictor.cpp b/tools/kokoro/src/kokoro-predictor.cpp index a32ded1a7..e23273f55 100644 --- a/tools/kokoro/src/kokoro-predictor.cpp +++ b/tools/kokoro/src/kokoro-predictor.cpp @@ -43,7 +43,17 @@ namespace { // Tensor lookup helper. struct TLookup { ggml_context * ctx = nullptr; - const float * get(const std::string & name, std::string * err = nullptr) const { + std::string ns = "kokoro"; + // Expand a namespace-relative name: ".bert.x" -> ".bert.x". Names that + // do not start with '.' are used as given. + std::string full(const std::string & name) const { + return (!name.empty() && name[0] == '.') ? ns + name : name; + } + ggml_tensor * tensor(const std::string & name) const { + return kokoro_find_tensor_compat(ctx, full(name)); + } + const float * get(const std::string & rel, std::string * err = nullptr) const { + const std::string name = full(rel); ggml_tensor * t = kokoro_find_tensor_compat(ctx, name); if (!t) { if (err) *err = "missing tensor '" + name + "'"; @@ -51,12 +61,12 @@ struct TLookup { } return (const float *) t->data; } - const float * get_or_null(const std::string & name) const { - ggml_tensor * t = kokoro_find_tensor_compat(ctx, name); + const float * get_or_null(const std::string & rel) const { + ggml_tensor * t = kokoro_find_tensor_compat(ctx, full(rel)); return t ? (const float *) t->data : nullptr; } - bool has(const std::string & name) const { - return kokoro_find_tensor_compat(ctx, name) != nullptr; + bool has(const std::string & rel) const { + return kokoro_find_tensor_compat(ctx, full(rel)) != nullptr; } }; @@ -96,19 +106,22 @@ static void bert_forward( std::vector & out /* [N, d_model] */, std::string & err) { const int N = (int) ids.size(); - const int d_embd = 128; + // Factorized ALBERT embedding width (128 for Kokoro-82M and Paradee-8M), + // read from the token embedding [vocab, d_embd] (ggml ne[0] = d_embd). + ggml_tensor * tok_t = T.tensor(".bert.token_embd.weight"); + const int d_embd = tok_t ? (int) tok_t->ne[0] : 128; const int head_dim = d_model / n_head; - const float * tok_embd = T.get("kokoro.bert.token_embd.weight", &err); // [vocab, 128] - const float * pos_embd = T.get("kokoro.bert.position_embd.weight", &err); // [maxpos, 128] - const float * tt_embd = T.get("kokoro.bert.tok_type_embd.weight", &err); // [2, 128] - const float * ln_w = T.get("kokoro.bert.embd_ln.weight", &err); - const float * ln_b = T.get("kokoro.bert.embd_ln.bias", &err); - const float * proj_w = T.get("kokoro.bert.embd_proj.weight", &err); // [768, 128] - const float * proj_b = T.get("kokoro.bert.embd_proj.bias", &err); + const float * tok_embd = T.get(".bert.token_embd.weight", &err); // [vocab, 128] + const float * pos_embd = T.get(".bert.position_embd.weight", &err); // [maxpos, 128] + const float * tt_embd = T.get(".bert.tok_type_embd.weight", &err); // [2, 128] + const float * ln_w = T.get(".bert.embd_ln.weight", &err); + const float * ln_b = T.get(".bert.embd_ln.bias", &err); + const float * proj_w = T.get(".bert.embd_proj.weight", &err); // [768, 128] + const float * proj_b = T.get(".bert.embd_proj.bias", &err); if (!tok_embd || !pos_embd || !tt_embd || !ln_w || !ln_b || !proj_w || !proj_b) return; - // Step 1+2+3: embed → LN(128) → project to 768. + // Step 1+2+3: embed → LN(d_embd) → project to d_model. std::vector h(N * d_embd, 0.0f); for (int t = 0; t < N; ++t) { const int id = ids[t]; @@ -128,22 +141,22 @@ static void bert_forward( } // Step 4: 12 layers — shared params (Albert num_hidden_groups=1). - const float * attn_q_w = T.get("kokoro.bert.layer.attn_q.weight"); - const float * attn_q_b = T.get("kokoro.bert.layer.attn_q.bias"); - const float * attn_k_w = T.get("kokoro.bert.layer.attn_k.weight"); - const float * attn_k_b = T.get("kokoro.bert.layer.attn_k.bias"); - const float * attn_v_w = T.get("kokoro.bert.layer.attn_v.weight"); - const float * attn_v_b = T.get("kokoro.bert.layer.attn_v.bias"); - const float * attn_o_w = T.get("kokoro.bert.layer.attn_o.weight"); - const float * attn_o_b = T.get("kokoro.bert.layer.attn_o.bias"); - const float * attn_ln_w = T.get("kokoro.bert.layer.attn_ln.weight"); - const float * attn_ln_b = T.get("kokoro.bert.layer.attn_ln.bias"); - const float * full_ln_w = T.get("kokoro.bert.layer.full_ln.weight"); - const float * full_ln_b = T.get("kokoro.bert.layer.full_ln.bias"); - const float * ffn_w = T.get("kokoro.bert.layer.ffn.weight"); // [2048, 768] - const float * ffn_b = T.get("kokoro.bert.layer.ffn.bias"); - const float * ffn_out_w = T.get("kokoro.bert.layer.ffn_out.weight"); // [768, 2048] - const float * ffn_out_b = T.get("kokoro.bert.layer.ffn_out.bias"); + const float * attn_q_w = T.get(".bert.layer.attn_q.weight"); + const float * attn_q_b = T.get(".bert.layer.attn_q.bias"); + const float * attn_k_w = T.get(".bert.layer.attn_k.weight"); + const float * attn_k_b = T.get(".bert.layer.attn_k.bias"); + const float * attn_v_w = T.get(".bert.layer.attn_v.weight"); + const float * attn_v_b = T.get(".bert.layer.attn_v.bias"); + const float * attn_o_w = T.get(".bert.layer.attn_o.weight"); + const float * attn_o_b = T.get(".bert.layer.attn_o.bias"); + const float * attn_ln_w = T.get(".bert.layer.attn_ln.weight"); + const float * attn_ln_b = T.get(".bert.layer.attn_ln.bias"); + const float * full_ln_w = T.get(".bert.layer.full_ln.weight"); + const float * full_ln_b = T.get(".bert.layer.full_ln.bias"); + const float * ffn_w = T.get(".bert.layer.ffn.weight"); // [2048, 768] + const float * ffn_b = T.get(".bert.layer.ffn.bias"); + const float * ffn_out_w = T.get(".bert.layer.ffn_out.weight"); // [768, 2048] + const float * ffn_out_b = T.get(".bert.layer.ffn_out.bias"); std::vector q(N * d_model), k(N * d_model), v(N * d_model); std::vector ctx(N * d_model); @@ -275,7 +288,7 @@ static void duration_encoder_forward( for (int il = 0; il < n_layer; ++il) { // BiLSTM over x [T, hid+sty] → y [T, 2*half=hid]. - const std::string pfx = "kokoro.predictor.de.lstm" + std::to_string(il) + "."; + const std::string pfx = ".predictor.de.lstm" + std::to_string(il) + "."; lw.W_ih = TL.get(pfx + "weight_ih_l0"); lw.W_hh = TL.get(pfx + "weight_hh_l0"); lw.b_ih = TL.get(pfx + "bias_ih_l0"); @@ -288,7 +301,7 @@ static void duration_encoder_forward( bilstm_forward(x.data(), T_phon, d_hid + sty_dim, half, lw, y.data()); // AdaLayerNorm(style_dim, channels=d_hid). fc(s) → 2*d_hid; chunk → gamma,beta. - const std::string apfx = "kokoro.predictor.de.adaln" + std::to_string(il) + ".fc."; + const std::string apfx = ".predictor.de.adaln" + std::to_string(il) + ".fc."; const float * fc_w = TL.get(apfx + "weight"); const float * fc_b = TL.get(apfx + "bias"); @@ -320,14 +333,14 @@ static void predictor_main_lstm_forward( const float * d /* [T_phon, 640] */, int T_phon, int d_hid, int sty_dim, std::vector & out /* [T_phon, 512] */) { LSTMWeights lw; - lw.W_ih = TL.get("kokoro.predictor.lstm.weight_ih_l0"); - lw.W_hh = TL.get("kokoro.predictor.lstm.weight_hh_l0"); - lw.b_ih = TL.get("kokoro.predictor.lstm.bias_ih_l0"); - lw.b_hh = TL.get("kokoro.predictor.lstm.bias_hh_l0"); - lw.W_ih_r = TL.get("kokoro.predictor.lstm.weight_ih_l0_r"); - lw.W_hh_r = TL.get("kokoro.predictor.lstm.weight_hh_l0_r"); - lw.b_ih_r = TL.get("kokoro.predictor.lstm.bias_ih_l0_r"); - lw.b_hh_r = TL.get("kokoro.predictor.lstm.bias_hh_l0_r"); + lw.W_ih = TL.get(".predictor.lstm.weight_ih_l0"); + lw.W_hh = TL.get(".predictor.lstm.weight_hh_l0"); + lw.b_ih = TL.get(".predictor.lstm.bias_ih_l0"); + lw.b_hh = TL.get(".predictor.lstm.bias_hh_l0"); + lw.W_ih_r = TL.get(".predictor.lstm.weight_ih_l0_r"); + lw.W_hh_r = TL.get(".predictor.lstm.weight_hh_l0_r"); + lw.b_ih_r = TL.get(".predictor.lstm.bias_ih_l0_r"); + lw.b_hh_r = TL.get(".predictor.lstm.bias_hh_l0_r"); out.assign(T_phon * d_hid, 0.0f); bilstm_forward(d, T_phon, d_hid + sty_dim, d_hid / 2, lw, out.data()); } @@ -339,10 +352,11 @@ static void predictor_main_lstm_forward( static void duration_predictor_forward( const TLookup & TL, const float * x_lstm, int T_phon, int d_hid, int max_dur, float speed, + bool torch_round, std::vector & duration /* [T_phon] */, std::vector & pred_dur /* [T_phon] */) { - const float * w = TL.get("kokoro.predictor.duration_proj.weight"); - const float * b = TL.get("kokoro.predictor.duration_proj.bias"); + const float * w = TL.get(".predictor.duration_proj.weight"); + const float * b = TL.get(".predictor.duration_proj.bias"); duration.assign(T_phon, 0.0f); pred_dur.assign(T_phon, 1); std::vector tmp(max_dur); @@ -355,7 +369,10 @@ static void duration_predictor_forward( } const float dur_f = (float)(sum / speed); duration[t] = dur_f; - const int32_t d_round = (int32_t) std::round(dur_f); + // torch.round rounds half to even; Kokoro-82M keeps std::round + // (half away from zero) so its durations are unchanged. + const int32_t d_round = torch_round ? (int32_t) std::nearbyint(dur_f) + : (int32_t) std::round(dur_f); pred_dur[t] = std::max(1, d_round); } } @@ -400,11 +417,10 @@ static void length_regulate( // ================================================================= static void text_encoder_forward( const TLookup & TL, - const std::vector & ids, int n_cnn, int hid, + const std::vector & ids, int n_cnn, int hid, int K, std::vector & out /* [T, hid] */) { const int T = (int) ids.size(); - const int K = 5; - const float * embd = TL.get("kokoro.text_encoder.embd.weight"); // [vocab, hid] + const float * embd = TL.get(".text_encoder.embd.weight"); // [vocab, hid] if (!embd) return; // Step 1: embedding → [T, hid]. @@ -421,7 +437,7 @@ static void text_encoder_forward( std::vector tmp(hid * T); for (int il = 0; il < n_cnn; ++il) { - const std::string cpfx = "kokoro.text_encoder.cnn." + std::to_string(il) + "."; + const std::string cpfx = ".text_encoder.cnn." + std::to_string(il) + "."; const float * cw = TL.get(cpfx + "weight"); const float * cb = TL.get(cpfx + "bias"); conv1d_forward(x_ct.data(), hid, T, cw, cb, hid, K, @@ -432,8 +448,8 @@ static void text_encoder_forward( // LayerNorm: gamma/beta over channel dim. The Python LayerNorm // transposes (1,-1) before F.layer_norm — that swaps channels onto // the last dim. So we normalize across channels for each time step. - const float * gamma = TL.get(std::string("kokoro.text_encoder.ln.") + std::to_string(il) + ".gamma"); - const float * beta = TL.get(std::string("kokoro.text_encoder.ln.") + std::to_string(il) + ".beta"); + const float * gamma = TL.get(std::string(".text_encoder.ln.") + std::to_string(il) + ".gamma"); + const float * beta = TL.get(std::string(".text_encoder.ln.") + std::to_string(il) + ".beta"); for (int t = 0; t < T; ++t) { double sum = 0, sumsq = 0; for (int c = 0; c < hid; ++c) { @@ -462,14 +478,14 @@ static void text_encoder_forward( // BiLSTM. LSTMWeights lw; - lw.W_ih = TL.get("kokoro.text_encoder.lstm.weight_ih_l0"); - lw.W_hh = TL.get("kokoro.text_encoder.lstm.weight_hh_l0"); - lw.b_ih = TL.get("kokoro.text_encoder.lstm.bias_ih_l0"); - lw.b_hh = TL.get("kokoro.text_encoder.lstm.bias_hh_l0"); - lw.W_ih_r = TL.get("kokoro.text_encoder.lstm.weight_ih_l0_r"); - lw.W_hh_r = TL.get("kokoro.text_encoder.lstm.weight_hh_l0_r"); - lw.b_ih_r = TL.get("kokoro.text_encoder.lstm.bias_ih_l0_r"); - lw.b_hh_r = TL.get("kokoro.text_encoder.lstm.bias_hh_l0_r"); + lw.W_ih = TL.get(".text_encoder.lstm.weight_ih_l0"); + lw.W_hh = TL.get(".text_encoder.lstm.weight_hh_l0"); + lw.b_ih = TL.get(".text_encoder.lstm.bias_ih_l0"); + lw.b_hh = TL.get(".text_encoder.lstm.bias_hh_l0"); + lw.W_ih_r = TL.get(".text_encoder.lstm.weight_ih_l0_r"); + lw.W_hh_r = TL.get(".text_encoder.lstm.weight_hh_l0_r"); + lw.b_ih_r = TL.get(".text_encoder.lstm.bias_ih_l0_r"); + lw.b_hh_r = TL.get(".text_encoder.lstm.bias_hh_l0_r"); out.assign(T * hid, 0.0f); bilstm_forward(x_th2.data(), T, hid, hid / 2, lw, out.data()); } @@ -500,7 +516,7 @@ static void text_encoder_forward( static void adainresblk1d_forward( const TLookup & TL, const std::string & pfx, const float * x, int Cin, int T_in, int Sdim, const float * s, - int Cout, bool upsample, bool learned_sc, + int Cout, bool upsample, bool learned_sc, bool instnorm_affine, std::vector & y, int & T_out) { const float * n1_w = TL.get(pfx + ".norm1.fc.weight"); @@ -515,11 +531,16 @@ static void adainresblk1d_forward( const float * c1x1_b = learned_sc ? TL.get_or_null(pfx + ".conv1x1.bias") : nullptr; const float * pool_w = upsample ? TL.get(pfx + ".pool.weight") : nullptr; const float * pool_b = upsample ? TL.get(pfx + ".pool.bias") : nullptr; + // Trained InstanceNorm affine (students); absent/unused for Kokoro-82M. + const float * n1_nw = instnorm_affine ? TL.get(pfx + ".norm1.norm.weight") : nullptr; + const float * n1_nb = instnorm_affine ? TL.get(pfx + ".norm1.norm.bias") : nullptr; + const float * n2_nw = instnorm_affine ? TL.get(pfx + ".norm2.norm.weight") : nullptr; + const float * n2_nb = instnorm_affine ? TL.get(pfx + ".norm2.norm.bias") : nullptr; // === Residual branch === // norm1 (AdaIN1d on Cin channels) std::vector r(x, x + (size_t)Cin * T_in); // copy - adain1d_forward(r.data(), Cin, T_in, s, Sdim, n1_w, n1_b); + adain1d_forward(r.data(), Cin, T_in, s, Sdim, n1_w, n1_b, n1_nw, n1_nb); // leaky relu for (size_t i = 0; i < r.size(); ++i) if (r[i] < 0) r[i] *= 0.2f; @@ -540,7 +561,7 @@ static void adainresblk1d_forward( /*stride*/1, /*pad*/1, /*dil*/1, r3.data(), T_after_pool); // norm2 (AdaIN1d on Cout channels) - adain1d_forward(r3.data(), Cout, T_after_pool, s, Sdim, n2_w, n2_b); + adain1d_forward(r3.data(), Cout, T_after_pool, s, Sdim, n2_w, n2_b, n2_nw, n2_nb); // leaky relu for (size_t i = 0; i < r3.size(); ++i) if (r3[i] < 0) r3[i] *= 0.2f; @@ -600,17 +621,17 @@ static void adainresblk1d_forward( static void f0n_train_forward( const TLookup & TL, const float * en /* [T_frame, 640] */, int T_frame, int d_hid, int sty_dim, - const float * s /* [sty_dim] = 128 */, + const float * s /* [sty_dim] */, bool instnorm_affine, std::vector & F0_pred, std::vector & N_pred) { LSTMWeights lw; - lw.W_ih = TL.get("kokoro.predictor.shared.weight_ih_l0"); - lw.W_hh = TL.get("kokoro.predictor.shared.weight_hh_l0"); - lw.b_ih = TL.get("kokoro.predictor.shared.bias_ih_l0"); - lw.b_hh = TL.get("kokoro.predictor.shared.bias_hh_l0"); - lw.W_ih_r = TL.get("kokoro.predictor.shared.weight_ih_l0_r"); - lw.W_hh_r = TL.get("kokoro.predictor.shared.weight_hh_l0_r"); - lw.b_ih_r = TL.get("kokoro.predictor.shared.bias_ih_l0_r"); - lw.b_hh_r = TL.get("kokoro.predictor.shared.bias_hh_l0_r"); + lw.W_ih = TL.get(".predictor.shared.weight_ih_l0"); + lw.W_hh = TL.get(".predictor.shared.weight_hh_l0"); + lw.b_ih = TL.get(".predictor.shared.bias_ih_l0"); + lw.b_hh = TL.get(".predictor.shared.bias_hh_l0"); + lw.W_ih_r = TL.get(".predictor.shared.weight_ih_l0_r"); + lw.W_hh_r = TL.get(".predictor.shared.weight_hh_l0_r"); + lw.b_ih_r = TL.get(".predictor.shared.bias_ih_l0_r"); + lw.b_hh_r = TL.get(".predictor.shared.bias_hh_l0_r"); std::vector x_shared(T_frame * d_hid); bilstm_forward(en, T_frame, d_hid + sty_dim, d_hid / 2, lw, x_shared.data()); @@ -631,19 +652,19 @@ static void f0n_train_forward( // F0.1: Cin=d_hid, Cout=d_hid/2, upsample=True, learned_sc=True // F0.2: Cin=d_hid/2, Cout=d_hid/2, upsample=False, learned_sc=False std::vector tmp; int T_after = T_frame; - adainresblk1d_forward(TL, "kokoro.predictor.F0.0", F0.data(), d_hid, T_after, sty_dim, s, - d_hid, false, false, tmp, T_after); + adainresblk1d_forward(TL, ".predictor.F0.0", F0.data(), d_hid, T_after, sty_dim, s, + d_hid, false, false, instnorm_affine, tmp, T_after); F0 = std::move(tmp); - adainresblk1d_forward(TL, "kokoro.predictor.F0.1", F0.data(), d_hid, T_after, sty_dim, s, - d_hid / 2, true, true, tmp, T_after); + adainresblk1d_forward(TL, ".predictor.F0.1", F0.data(), d_hid, T_after, sty_dim, s, + d_hid / 2, true, true, instnorm_affine, tmp, T_after); F0 = std::move(tmp); - adainresblk1d_forward(TL, "kokoro.predictor.F0.2", F0.data(), d_hid / 2, T_after, sty_dim, s, - d_hid / 2, false, false, tmp, T_after); + adainresblk1d_forward(TL, ".predictor.F0.2", F0.data(), d_hid / 2, T_after, sty_dim, s, + d_hid / 2, false, false, instnorm_affine, tmp, T_after); F0 = std::move(tmp); // F0_proj: Conv1d(d_hid/2, 1, kernel=1) → [1, T_after]. { - const float * w = TL.get("kokoro.predictor.F0_proj.weight"); - const float * b = TL.get("kokoro.predictor.F0_proj.bias"); + const float * w = TL.get(".predictor.F0_proj.weight"); + const float * b = TL.get(".predictor.F0_proj.bias"); F0_pred.assign(T_after, 0.0f); std::vector proj(1 * T_after); conv1d_forward(F0.data(), d_hid / 2, T_after, w, b, 1, 1, 1, 0, 1, proj.data(), T_after); @@ -652,100 +673,209 @@ static void f0n_train_forward( // N chain (mirror). int T_after_N = T_frame; - adainresblk1d_forward(TL, "kokoro.predictor.N.0", N.data(), d_hid, T_after_N, sty_dim, s, - d_hid, false, false, tmp, T_after_N); + adainresblk1d_forward(TL, ".predictor.N.0", N.data(), d_hid, T_after_N, sty_dim, s, + d_hid, false, false, instnorm_affine, tmp, T_after_N); N = std::move(tmp); - adainresblk1d_forward(TL, "kokoro.predictor.N.1", N.data(), d_hid, T_after_N, sty_dim, s, - d_hid / 2, true, true, tmp, T_after_N); + adainresblk1d_forward(TL, ".predictor.N.1", N.data(), d_hid, T_after_N, sty_dim, s, + d_hid / 2, true, true, instnorm_affine, tmp, T_after_N); N = std::move(tmp); - adainresblk1d_forward(TL, "kokoro.predictor.N.2", N.data(), d_hid / 2, T_after_N, sty_dim, s, - d_hid / 2, false, false, tmp, T_after_N); + adainresblk1d_forward(TL, ".predictor.N.2", N.data(), d_hid / 2, T_after_N, sty_dim, s, + d_hid / 2, false, false, instnorm_affine, tmp, T_after_N); N = std::move(tmp); { - const float * w = TL.get("kokoro.predictor.N_proj.weight"); - const float * b = TL.get("kokoro.predictor.N_proj.bias"); + const float * w = TL.get(".predictor.N_proj.weight"); + const float * b = TL.get(".predictor.N_proj.bias"); std::vector proj(1 * T_after_N); conv1d_forward(N.data(), d_hid / 2, T_after_N, w, b, 1, 1, 1, 0, 1, proj.data(), T_after_N); N_pred = std::move(proj); } } + +// ================================================================= +// asr projection MLP (Paradee TextStudent preset "+mlp"): +// asr_tok = Linear(asr_dim, asr_dim)(GELU(Linear(hid, asr_dim)(t_en))) +// nn.GELU() is the exact (erf) GELU, not ALBERT's tanh approximation. +// ================================================================= +static bool asr_proj_mlp_forward( + const TLookup & TL, const float * t_en, int T_phon, int hid, + std::vector & out, int & asr_dim, std::string & err) { + ggml_tensor * w0t = TL.tensor(".asr_proj.0.weight"); // [asr_dim, hid] + ggml_tensor * w1t = TL.tensor(".asr_proj.1.weight"); // [asr_dim, asr_dim] + const float * b0 = TL.get(".asr_proj.0.bias"); + const float * b1 = TL.get(".asr_proj.1.bias"); + if (!w0t || !w1t || !b0 || !b1) { err = "missing asr_proj tensors"; return false; } + if ((int) w0t->ne[0] != hid || w1t->ne[0] != w0t->ne[1] || w1t->ne[1] != w0t->ne[1]) { + err = "asr_proj tensor shapes do not match the predictor width"; + return false; + } + asr_dim = (int) w0t->ne[1]; + const float * w0 = (const float *) w0t->data; + const float * w1 = (const float *) w1t->data; + out.assign((size_t) T_phon * asr_dim, 0.0f); + std::vector h((size_t) asr_dim); + const float inv_sqrt2 = 0.70710678118654752440f; + for (int t = 0; t < T_phon; ++t) { + linear_forward(t_en + (size_t) t * hid, hid, w0, b0, asr_dim, h.data()); + for (int j = 0; j < asr_dim; ++j) { + const float v = h[(size_t) j]; + h[(size_t) j] = 0.5f * v * (1.0f + std::erf(v * inv_sqrt2)); + } + linear_forward(h.data(), asr_dim, w1, b1, asr_dim, out.data() + (size_t) t * asr_dim); + } + return true; +} + } // namespace // ================================================================= -// Public entry: full predictor forward. +// Generic entry: full predictor forward over a loaded F32 context. // ================================================================= -bool kokoro_predictor_forward( - const kokoro_model * model, +bool predictor_forward( + ggml_context * ctx, + const PredictorConfig & cfg, const std::vector & phoneme_ids, - const float * ref_s, float speed, + float speed, PredictorOut & out, std::string & err_out) { - if (!model) { err_out = "null model"; return false; } - const kokoro_hparams * hp = kokoro_get_hparams(model); - if (!hp) { err_out = "null hparams"; return false; } - - TLookup TL { kokoro_model_ggml_ctx(model) }; + if (!ctx) { err_out = "null model context"; return false; } + if (!cfg.style) { err_out = "null predictor style"; return false; } + if (phoneme_ids.empty()) { err_out = "empty phoneme ids"; return false; } + + TLookup TL { ctx, cfg.ns }; + const auto cancelled = [&]() { + if (!cfg.cancel || !cfg.cancel(cfg.cancel_user)) return false; + out.cancelled = true; + err_out = "cancelled"; + return true; + }; + + // Widths from the tensor shapes (ggml ne[] is the reversed PyTorch shape): + // bert_encoder.weight [hid, d_model] -> ne[1] = hid + // predictor.de.adaln0.fc.weight [2*hid, sty] -> ne[0] = sty + ggml_tensor * be_t = TL.tensor(".bert_encoder.weight"); + ggml_tensor * adaln_t = TL.tensor(".predictor.de.adaln0.fc.weight"); + ggml_tensor * dur_t = TL.tensor(".predictor.duration_proj.weight"); + if (!be_t || !adaln_t || !dur_t) { err_out = "missing predictor tensors"; return false; } + const int hid = (int) be_t->ne[1]; + const int sty = (int) adaln_t->ne[0]; + if ((int) be_t->ne[0] != cfg.text_d_model || (int) adaln_t->ne[1] != 2 * hid || + (int) dur_t->ne[0] != hid || (int) dur_t->ne[1] != cfg.max_dur) { + err_out = "predictor tensor shapes do not match the hparams"; + return false; + } const int T_phon = (int) phoneme_ids.size(); out.T_phon = T_phon; - const int d_model = hp->text_d_model; - const int d_ff = hp->text_d_ff; - const int n_layer = hp->text_n_layer; - const int n_head = hp->text_n_head; - const int hid = 512; // hidden_dim from config.json - const int sty = 128; // per-half style_dim - const int max_dur = 50; - const int pred_nl = 3; + out.hid = hid; + out.sty = sty; + out.cancelled = false; + const int d_model = cfg.text_d_model; + const int d_ff = cfg.text_d_ff; + const int n_layer = cfg.text_n_layer; + const int n_head = cfg.text_n_head; + const int max_dur = cfg.max_dur; + const int pred_nl = cfg.n_layer; + const float * style = cfg.style; // 1. BERT. std::vector bert_out; bert_forward(TL, phoneme_ids, d_model, d_ff, n_layer, n_head, bert_out, err_out); if (!err_out.empty()) return false; + if (cancelled()) return false; - // 2. bert_encoder Linear 768→512 → d_en [T, 512] row-major. - const float * be_w = TL.get("kokoro.bert_encoder.weight"); - const float * be_b = TL.get("kokoro.bert_encoder.bias"); + // 2. bert_encoder Linear d_model→hid → d_en [T, hid] row-major. + const float * be_w = TL.get(".bert_encoder.weight"); + const float * be_b = TL.get(".bert_encoder.bias"); if (!be_w || !be_b) { err_out = "missing bert_encoder weights"; return false; } std::vector d_en((size_t)T_phon * hid); for (int t = 0; t < T_phon; ++t) { linear_forward(bert_out.data() + (size_t)t * d_model, d_model, be_w, be_b, hid, d_en.data() + (size_t)t * hid); } + if (cfg.keep_bert) out.bert = std::move(bert_out); // 3. DurationEncoder. - duration_encoder_forward(TL, d_en.data(), T_phon, hid, sty, pred_nl, - ref_s + hid /* predictor-half = ref_s[128:] */, out.d); + duration_encoder_forward(TL, d_en.data(), T_phon, hid, sty, pred_nl, style, out.d); // 4. predictor.lstm. predictor_main_lstm_forward(TL, out.d.data(), T_phon, hid, sty, out.x_lstm); // 5. duration_proj → sigmoid → sum → /speed → pred_dur. duration_predictor_forward(TL, out.x_lstm.data(), T_phon, hid, max_dur, speed, - out.duration, out.pred_dur); + cfg.torch_round, out.duration, out.pred_dur); + if (cancelled()) return false; - // 6. Length regulate `d` → en [T_frame, 640]. + // 6. Length regulate `d` → en [T_frame, hid+sty]. length_regulate(out.d.data(), T_phon, hid + sty, out.pred_dur, out.en, out.T_frame); // 7. text_encoder forward (separate from BERT). - text_encoder_forward(TL, phoneme_ids, pred_nl, hid, out.t_en); + text_encoder_forward(TL, phoneme_ids, pred_nl, hid, cfg.text_kernel, out.t_en); - // 8. Length regulate t_en [T_phon, 512] → asr [T_frame, 512]. + // 8. Length regulate the asr features [T_phon, asr_dim] → asr [T_frame, asr_dim]. { + const float * asr_src = out.t_en.data(); + std::vector asr_tok; + int asr_dim = hid; + if (cfg.asr_proj_mlp) { + if (!asr_proj_mlp_forward(TL, out.t_en.data(), T_phon, hid, asr_tok, asr_dim, err_out)) { + return false; + } + asr_src = asr_tok.data(); + } + out.asr_dim = asr_dim; int T_frame_check = 0; - length_regulate(out.t_en.data(), T_phon, hid, out.pred_dur, out.asr, T_frame_check); + length_regulate(asr_src, T_phon, asr_dim, out.pred_dur, out.asr, T_frame_check); if (T_frame_check != out.T_frame) { err_out = "internal: T_frame mismatch between en and asr"; return false; } } + if (cancelled()) return false; // 9. F0/N predict. // The shared LSTM expects input [T_frame, hid+sty]. en is already that shape // (length-regulated d, which is hid+sty wide). f0n_train_forward(TL, out.en.data(), out.T_frame, hid, sty, - ref_s + hid /* predictor-half */, out.F0_pred, out.N_pred); + style, cfg.instnorm_affine, out.F0_pred, out.N_pred); + if (cancelled()) return false; return true; } +// ================================================================= +// Kokoro-82M entry. +// ================================================================= +bool kokoro_predictor_forward( + const kokoro_model * model, + const std::vector & phoneme_ids, + const float * ref_s, float speed, + PredictorOut & out, std::string & err_out) { + if (!model) { err_out = "null model"; return false; } + const kokoro_hparams * hp = kokoro_get_hparams(model); + if (!hp) { err_out = "null hparams"; return false; } + if (!ref_s) { err_out = "null ref_s"; return false; } + + ggml_context * ctx = kokoro_model_ggml_ctx(model); + TLookup TL { ctx, "kokoro" }; + ggml_tensor * be_t = TL.tensor(".bert_encoder.weight"); + if (!be_t) { err_out = "missing bert_encoder weights"; return false; } + + PredictorConfig cfg; + cfg.ns = "kokoro"; + cfg.text_n_layer = hp->text_n_layer; + cfg.text_n_head = hp->text_n_head; + cfg.text_d_model = hp->text_d_model; + cfg.text_d_ff = hp->text_d_ff; + cfg.n_layer = 3; + cfg.max_dur = 50; + cfg.text_kernel = 5; + // KNOWN ISSUE, kept bit-for-bit here: the predictor style is read at + // ref_s + hid (hid = 512 for Kokoro-82M), not at ref_s + 128 where + // kokoro/model.py takes ref_s[:, 128:]. That is two rows further into the + // voice pack, and past its end for the longest inputs. Fixing it changes + // Kokoro's audio, so it is a separate change with its own parity run. + cfg.style = ref_s + (int) be_t->ne[1]; + return predictor_forward(ctx, cfg, phoneme_ids, speed, out, err_out); +} + } // namespace eliza_kokoro diff --git a/tools/kokoro/src/kokoro.cpp b/tools/kokoro/src/kokoro.cpp index 4618698bf..2f8c1c1d3 100644 --- a/tools/kokoro/src/kokoro.cpp +++ b/tools/kokoro/src/kokoro.cpp @@ -677,16 +677,17 @@ static kokoro_status kokoro_synthesize_from_input_ids( } if (cancelled()) return KOKORO_E_CANCELLED; const int T = pred.T_frame; - if (T <= 0 || (int) pred.asr.size() != T * 512) { + const int A = pred.asr_dim; // 512 for Kokoro-82M (the text encoder width) + if (T <= 0 || A <= 0 || pred.asr.size() != (size_t) T * (size_t) A) { err_out = "predictor produced empty/invalid asr (T=" + std::to_string(T) + ")"; return KOKORO_E_RUNTIME; } - // Transpose asr [T, 512] (T-major) → [512, T] (channel-major). - std::vector asr_ct((size_t) 512 * (size_t) T); + // Transpose asr [T, A] (T-major) → [A, T] (channel-major). + std::vector asr_ct((size_t) A * (size_t) T); for (int t = 0; t < T; ++t) { - const float * row = pred.asr.data() + (size_t) t * 512; - for (int c = 0; c < 512; ++c) { + const float * row = pred.asr.data() + (size_t) t * A; + for (int c = 0; c < A; ++c) { asr_ct[(size_t) c * (size_t) T + t] = row[c]; } } diff --git a/tools/kokoro/tests/CMakeLists.txt b/tools/kokoro/tests/CMakeLists.txt index a7725c941..ce1fd13e4 100644 --- a/tools/kokoro/tests/CMakeLists.txt +++ b/tools/kokoro/tests/CMakeLists.txt @@ -39,6 +39,12 @@ add_executable(test-kokoro-backend test_kokoro_backend.cpp) target_link_libraries(test-kokoro-backend PRIVATE kokoro_lib) add_test(NAME test-kokoro-backend COMMAND test-kokoro-backend) +# Opt-in layer strategies (KokoroLayerOptions) against the default kernels. +# Kokoro-82M never enables them; Paradee-8M does. +add_executable(test-kokoro-layer-options test_kokoro_layer_options.cpp) +target_link_libraries(test-kokoro-layer-options PRIVATE kokoro_lib) +add_test(NAME test-kokoro-layer-options COMMAND test-kokoro-layer-options) + # Synthesis cancellation. Weight-free checks always run; set KOKORO_TEST_MODEL # and KOKORO_TEST_VOICE to also cancel a real synthesis at every stage boundary. add_executable(test-kokoro-cancel test_kokoro_cancel.cpp) diff --git a/tools/kokoro/tests/test_kokoro_layer_options.cpp b/tools/kokoro/tests/test_kokoro_layer_options.cpp new file mode 100644 index 000000000..7fdd8e520 --- /dev/null +++ b/tools/kokoro/tests/test_kokoro_layer_options.cpp @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +// +// test-kokoro-layer-options — the opt-in layer strategies in kokoro-layers.h +// (KokoroLayerOptions: tap_gemm_conv, fast_snake) against the default +// kernels. Kokoro-82M never enables them; Paradee-8M does. Checks that: +// - Conv1d (stride 1, dilated, strided) and ConvTranspose1d give the same +// result as the default path within float rounding, +// - the polynomial sin^2 in fast Snake1D is within 5e-7 of libm, +// - the options are scoped (restored when the scope ends). + +#include "kokoro-layers.h" + +#include +#include +#include +#include + +using namespace eliza_kokoro; + +static int failures = 0; +#define CHECK(cond, ...) do { if (!(cond)) { std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + std::fprintf(stderr, __VA_ARGS__); std::fprintf(stderr, "\n"); ++failures; } } while (0) + +static std::vector randn(size_t n, std::mt19937 & g, float scale) { + std::normal_distribution d(0.0f, scale); + std::vector v(n); + for (auto & x : v) x = d(g); + return v; +} + +static double rel_err(const std::vector & a, const std::vector & b) { + double num = 0, den = 0; + for (size_t i = 0; i < a.size(); ++i) { + num = std::max(num, (double) std::fabs(a[i] - b[i])); + den = std::max(den, (double) std::fabs(a[i])); + } + return num / std::max(den, 1e-30); +} + +static detail::KokoroLayerOptions opts(bool on) { + detail::KokoroLayerOptions o; + o.tap_gemm_conv = on; + o.fast_snake = on; + return o; +} + +static void check_conv(std::mt19937 & g, int Cin, int Cout, int K, int T, int stride, int pad, int dil) { + const int T_out = (T + 2 * pad - dil * (K - 1) - 1) / stride + 1; + auto x = randn((size_t) Cin * T, g, 1.0f), W = randn((size_t) Cout * Cin * K, g, 0.1f), b = randn(Cout, g, 0.5f); + std::vector y0((size_t) Cout * T_out), y1((size_t) Cout * T_out); + conv1d_forward(x.data(), Cin, T, W.data(), b.data(), Cout, K, stride, pad, dil, y0.data(), T_out); + { + detail::KokoroLayerOptionsScope s(opts(true)); + conv1d_forward(x.data(), Cin, T, W.data(), b.data(), Cout, K, stride, pad, dil, y1.data(), T_out); + } + const double e = rel_err(y0, y1); + std::printf("conv1d Cin=%d Cout=%d K=%d T=%d stride=%d dil=%d: rel err %.3g\n", Cin, Cout, K, T, stride, dil, e); + CHECK(e < 1e-5, "conv1d mismatch %.3g", e); +} + +static void check_convtranspose(std::mt19937 & g, int Cin, int Cout, int K, int T, int stride) { + const int pad = (K - stride) / 2; + const int T_out = convtranspose1d_out_len(T, K, stride, pad, 0); + auto x = randn((size_t) Cin * T, g, 1.0f), W = randn((size_t) Cin * Cout * K, g, 0.1f), b = randn(Cout, g, 0.5f); + std::vector y0((size_t) Cout * T_out), y1((size_t) Cout * T_out); + convtranspose1d_forward(x.data(), Cin, T, W.data(), b.data(), Cout, K, stride, pad, 0, y0.data(), T_out); + { + detail::KokoroLayerOptionsScope s(opts(true)); + convtranspose1d_forward(x.data(), Cin, T, W.data(), b.data(), Cout, K, stride, pad, 0, y1.data(), T_out); + } + const double e = rel_err(y0, y1); + std::printf("convtranspose1d Cin=%d Cout=%d K=%d T=%d stride=%d: rel err %.3g\n", Cin, Cout, K, T, stride, e); + CHECK(e < 1e-5, "convtranspose1d mismatch %.3g", e); +} + +int main() { + std::mt19937 g(1234); + check_conv(g, 32, 32, 11, 1001, 1, 5, 1); + check_conv(g, 32, 32, 7, 997, 1, 15, 5); // dilated (get_padding(7, 5) = 15) + check_conv(g, 514, 256, 3, 47, 1, 1, 1); + check_conv(g, 30, 7, 3, 33, 1, 1, 1); // ragged row / time tiles + check_conv(g, 22, 64, 12, 601, 6, 3, 1); // strided (noise_convs[0]) + check_convtranspose(g, 128, 64, 20, 40, 10); + check_convtranspose(g, 64, 32, 12, 401, 6); + + // fast sin^2 + double worst = 0; + for (int i = -200000; i <= 200000; ++i) { + const float y = (float) i * 1e-3f; + const double ref = std::sin((double) y) * std::sin((double) y); + worst = std::max(worst, std::fabs((double) detail::fast_sin2(y) - ref)); + } + std::printf("fast_sin2 on [-200, 200]: max |err| %.3g\n", worst); + CHECK(worst < 5e-7, "fast_sin2 error %.3g", worst); + + // Snake with the option on matches libm within that bound. + { + auto x = randn(4 * 1000, g, 3.0f), a = randn(4, g, 1.0f); + std::vector x1 = x; + snake1d_forward(x.data(), 4, 1000, a.data()); + { + detail::KokoroLayerOptionsScope s(opts(true)); + snake1d_forward(x1.data(), 4, 1000, a.data()); + } + double e = 0; + for (int c = 0; c < 4; ++c) + for (int t = 0; t < 1000; ++t) + e = std::max(e, std::fabs((double) x[c * 1000 + t] - x1[c * 1000 + t]) * std::fabs(a[c])); + CHECK(e < 1e-6, "snake mismatch %.3g", e); + } + + // Scoped: the options are off again outside the scope. + { + detail::KokoroLayerOptionsScope s(opts(true)); + CHECK(detail::kokoro_layer_options().fast_snake, "scope did not set the option"); + } + CHECK(!detail::kokoro_layer_options().fast_snake && !detail::kokoro_layer_options().tap_gemm_conv, + "options leaked out of the scope"); + + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("test-kokoro-layer-options: OK\n"); + return 0; +} diff --git a/tools/omnivoice/include/eliza-inference-ffi.h b/tools/omnivoice/include/eliza-inference-ffi.h index b7b31a79d..a6e9ce9c7 100644 --- a/tools/omnivoice/include/eliza-inference-ffi.h +++ b/tools/omnivoice/include/eliza-inference-ffi.h @@ -627,7 +627,8 @@ int eliza_inference_paradee_phonemize( /* Text -> PCM. Native misaki G2P, then the phoneme string is split into * <= 510-id chunks (sentence ends, then clause punctuation, then spaces), * each chunk is padded and synthesized, and the PCM is concatenated. `speed` - * scales the speaking rate (1.0 = native; <= 0 or non-finite means 1.0). + * scales the speaking rate (1.0 = native; <= 0 or non-finite means 1.0; + * other values are clamped to [0.25, 4]). * On success `*out_pcm` is a malloc-owned buffer of `*out_samples` floats at * `paradee_sample_rate` (NULL only when `*out_samples == 0`); release it with * `eliza_inference_free_pcm`. Returns ELIZA_OK, ELIZA_ERR_CANCELLED after diff --git a/tools/omnivoice/src/eliza-inference-ffi.cpp b/tools/omnivoice/src/eliza-inference-ffi.cpp index f19bb1e26..bd44a856d 100644 --- a/tools/omnivoice/src/eliza-inference-ffi.cpp +++ b/tools/omnivoice/src/eliza-inference-ffi.cpp @@ -2272,8 +2272,13 @@ static int eliza_paradee_chunk( { ",", ";", ":", "\xE2\x80\x94" /* — */ }, { " " }, }; + /* Unpadded and unlimited: chunks are padded below, and the ABI drops + * characters outside the vocab (paradee_phonemes_to_ids would add pads, + * reject more than 510 symbols and reject unknown symbols). */ std::vector ids; - if (eliza_paradee::paradee_phonemes_to_ids(model, phonemes, ids, err) != eliza_paradee::PARADEE_OK) { + size_t n_dropped = 0; + if (eliza_paradee::paradee_phonemes_to_symbol_ids_lenient(model, phonemes, ids, n_dropped, err) != + eliza_paradee::PARADEE_OK) { return ELIZA_ERR_INVALID_ARG; } if (ids.empty()) return ELIZA_OK; @@ -2331,6 +2336,7 @@ static int eliza_paradee_synthesize_owned( return ELIZA_ERR_INVALID_ARG; } if (!(speed > 0.0f) || !std::isfinite(speed)) speed = 1.0f; + speed = std::min(std::max(speed, eliza_paradee::PARADEE_MIN_SPEED), eliza_paradee::PARADEE_MAX_SPEED); const std::string in(input, input_len ? input_len : std::strlen(input)); std::lock_guard lock(ctx->paradee_mutex); diff --git a/tools/omnivoice/tests/ffi-paradee-asr.cpp b/tools/omnivoice/tests/ffi-paradee-asr.cpp index 33b85834e..804d95db8 100644 --- a/tools/omnivoice/tests/ffi-paradee-asr.cpp +++ b/tools/omnivoice/tests/ffi-paradee-asr.cpp @@ -172,6 +172,13 @@ static void test_paradee_load_and_synth() { CHECK(pcm != nullptr && pcm[0] == (float) (43 + ('h' - 'a')) / 1000.0f); eliza_inference_free_pcm(pcm); } + // Out-of-range speeds are clamped to the library's range, not rejected. + for (float speed : { 0.01f, 100.0f }) { + float * pcm = nullptr; + CHECK_RC(eliza_inference_paradee_synthesize_alloc(ctx, "hi", 2, speed, &pcm, &n, &err), ELIZA_OK); + CHECK_RC((int) n, 2 * FAKE_PARADEE_SAMPLES_PER_ID); + eliza_inference_free_pcm(pcm); + } CHECK_RC(eliza_inference_paradee_synthesize_alloc(ctx, "x", 1, 1.0f, nullptr, &n, &err), ELIZA_ERR_INVALID_ARG); clear_err(&err); diff --git a/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp b/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp index e7b675466..a41858b5f 100644 --- a/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp +++ b/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp @@ -54,11 +54,13 @@ int paradee_sample_rate(const paradee_model * model) noexcept { return model ? 24000 : -1; } -paradee_status paradee_phonemes_to_ids( +paradee_status paradee_phonemes_to_symbol_ids_lenient( const paradee_model * model, const std::string & s, std::vector & ids, + size_t & n_dropped, std::string & err_out) noexcept { + n_dropped = 0; if (!model) { err_out = "fake: null model"; return PARADEE_E_INVALID_ARG; } ids.clear(); for (size_t i = 0; i < s.size();) { @@ -76,7 +78,7 @@ paradee_status paradee_phonemes_to_ids( else if (cp == "\xE2\x80\x94") ids.push_back(9); else if (cp == "\xE2\x80\xA6") ids.push_back(10); else if (cp == " ") ids.push_back(16); - else if (cp == "#") continue; + else if (cp == "#") { ++n_dropped; continue; } else if (len == 1 && c >= 'a' && c <= 'z') ids.push_back(43 + (c - 'a')); else ids.push_back(100); } @@ -95,7 +97,10 @@ paradee_status paradee_synthesize( err_out = "fake: bad input_ids (size " + std::to_string(input_ids.size()) + ")"; return PARADEE_E_INVALID_ARG; } - if (!(opts.speed > 0.0f)) { err_out = "fake: bad speed"; return PARADEE_E_INVALID_ARG; } + if (!(opts.speed >= PARADEE_MIN_SPEED && opts.speed <= PARADEE_MAX_SPEED)) { + err_out = "fake: bad speed"; + return PARADEE_E_INVALID_ARG; + } g_state.synth_calls.fetch_add(1); { std::lock_guard lock(g_state.sizes_mutex); diff --git a/tools/omnivoice/tests/paradee-contract/paradee.h b/tools/omnivoice/tests/paradee-contract/paradee.h index f7e795b7e..2181dbfd6 100644 --- a/tools/omnivoice/tests/paradee-contract/paradee.h +++ b/tools/omnivoice/tests/paradee-contract/paradee.h @@ -10,6 +10,7 @@ // header changes a signature used by the bridge, change this copy too. #pragma once +#include #include #include #include @@ -19,6 +20,8 @@ namespace eliza_paradee { // Model input limit, including the 0 pad token at each end. constexpr int PARADEE_MAX_INPUT_IDS = 512; +constexpr float PARADEE_MIN_SPEED = 0.25f; +constexpr float PARADEE_MAX_SPEED = 4.0f; struct paradee_model; struct paradee_model_deleter { void operator()(paradee_model *) const noexcept; }; @@ -30,12 +33,14 @@ struct paradee_audio { }; enum paradee_status { - PARADEE_OK = 0, - PARADEE_E_INVALID_ARG = 1, - PARADEE_E_LOAD_FAIL = 2, - PARADEE_E_OOM = 3, - PARADEE_E_CANCELLED = 4, - PARADEE_E_RUNTIME = 5, + PARADEE_OK = 0, + PARADEE_E_INVALID_ARG = 1, + PARADEE_E_UNKNOWN_SYMBOL = 2, + PARADEE_E_TOO_LONG = 3, + PARADEE_E_RUNTIME = 4, + PARADEE_CANCELLED = 5, + PARADEE_E_OOM = 6, + PARADEE_E_CANCELLED = PARADEE_CANCELLED, }; // Polled by the synthesis graph between stages. Returning true stops the @@ -52,12 +57,14 @@ paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string int paradee_sample_rate(const paradee_model * model) noexcept; -// Map a misaki phoneme string to model vocab ids (no pad tokens added). -// Characters outside the vocab are dropped, as kokoro/misaki do. -paradee_status paradee_phonemes_to_ids( +// Map a misaki phoneme string to model vocab ids (no pad tokens added, no +// length limit). Characters outside the vocab are dropped and counted, as +// kokoro/misaki do; empty input gives no ids and PARADEE_OK. +paradee_status paradee_phonemes_to_symbol_ids_lenient( const paradee_model * model, const std::string & misaki_phonemes, std::vector & ids_out, + size_t & n_dropped, std::string & err_out) noexcept; // Run one forward pass. `input_ids` is the full model input including the diff --git a/tools/paradee/CMakeLists.txt b/tools/paradee/CMakeLists.txt new file mode 100644 index 000000000..0cf849b45 --- /dev/null +++ b/tools/paradee/CMakeLists.txt @@ -0,0 +1,47 @@ +# tools/paradee/CMakeLists.txt +# +# Paradee-8M v1.0 TTS (sahilmahendrakar/Paradee-8M-v1.0, Apache-2.0): a +# width-scaled, single-voice distillation of Kokoro-82M. Built when +# LLAMA_BUILD_PARADEE=ON (see the top-level CMakeLists.txt). +# +# Produces: +# paradee_lib static library (loader, synthesis, phase-locking filter). +# Reuses kokoro_lib's StyleTTS-2 / iSTFTNet layers. +# paradee-tts CLI: misaki phonemes -> 24 kHz WAV (host only). +# paradee-stage-dump writes every stage for the PyTorch parity gate. + +if (NOT TARGET kokoro_lib) + message(FATAL_ERROR "LLAMA_BUILD_PARADEE=ON needs kokoro_lib (LLAMA_BUILD_KOKORO=ON): " + "Paradee runs on the tools/kokoro layers.") +endif() + +add_library(paradee_lib STATIC + src/paradee.cpp + src/paradee-phase-lock.cpp + include/paradee.h) +set_target_properties(paradee_lib PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_include_directories(paradee_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_link_libraries(paradee_lib PUBLIC kokoro_lib ggml) +if (TARGET ggml-base) + target_link_libraries(paradee_lib PUBLIC ggml-base) +endif() +target_compile_features(paradee_lib PUBLIC cxx_std_17) +# Same layer configuration as kokoro_lib, so the inline kokoro-layers.h +# definitions agree across the two archives. +target_compile_definitions(paradee_lib PRIVATE KOKORO_USE_GGML_BACKEND) + +if (NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") + include(GNUInstallDirs) + add_executable(paradee-tts tools/paradee-tts.cpp) + target_link_libraries(paradee-tts PRIVATE paradee_lib) + install(TARGETS paradee-tts + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + BUNDLE DESTINATION ${CMAKE_INSTALL_BINDIR}) + + add_executable(paradee-stage-dump tools/paradee-stage-dump.cpp) + target_link_libraries(paradee-stage-dump PRIVATE paradee_lib) +endif() + +if (LLAMA_BUILD_TESTS) + add_subdirectory(tests) +endif() diff --git a/tools/paradee/README.md b/tools/paradee/README.md new file mode 100644 index 000000000..67f3b6ac6 --- /dev/null +++ b/tools/paradee/README.md @@ -0,0 +1,125 @@ +# tools/paradee: Paradee-8M v1.0 TTS + +Paradee-8M v1.0 ([sahilmahendrakar/Paradee-8M-v1.0](https://huggingface.co/sahilmahendrakar/Paradee-8M-v1.0), Apache-2.0) +is Kokoro-82M's StyleTTS-2 + iSTFTNet code at smaller widths. It is distilled to one voice (`af_heart`), has +8.07M parameters and outputs 24 kHz audio. Its input is misaki phoneme ids with a `0` pad at each end, at most +512 ids, plus a speed value. + +Upstream ships the model as an ONNX graph and as PyTorch weights. This directory runs the PyTorch weights +natively, with no ONNX: + +- `convert_paradee_pt_to_gguf.py` writes a GGUF (`general.architecture = paradee`) from the pinned + `pytorch/text_side.pt` and `pytorch/decoder.pt`. +- `paradee_lib` runs it on the `tools/kokoro` layers. Those layers now read their widths from the tensor + shapes, and Kokoro-82M's output is unchanged bit for bit. +- `src/paradee-phase-lock.cpp` is a native implementation of the model's phase-locking filter. + +## Pipeline + +| Stage | Kokoro-82M | Paradee-8M | +|---|---|---| +| ALBERT | 768 wide, 12 heads, 12 layers | 256 wide, 4 heads, 6 layers (one shared group) | +| predictor hidden / style | 512 / 128 (a voice row) | 192 / 32 (learned constant `paradee.style.text`) | +| text encoder → decoder `asr` | 512, used directly | 192 → `asr_proj` (Linear 192→512, exact GELU, Linear 512→512) | +| decoder AdaIN width / style | 1024 / 128 | 256 / 16 (learned constant `paradee.style.decoder`) | +| generator channels | 512 → 256 → 128 | 128 → 64 → 32 | +| AdaIN1d InstanceNorm affine | identity (untrained) | trained, applied | +| durations | `std::round` | `torch.round` (half to even) | +| after the generator | none | phase lock: STFT 1024/256, 2–8 kHz, voiced frames (F0 > 60 Hz), 33-frame smoothing | + +The harmonic source is deterministic, as in `tools/kokoro`: it has no random initial phase and no additive +noise. Paradee runs it in double precision. Its STFT sets values below the DFT rounding floor to exactly +zero, so the empty bins of an unvoiced, noise-free source have a defined phase. + +## API (`include/paradee.h`) + +```cpp +namespace eliza_paradee { + paradee_model_ptr paradee_load_model(const std::string & gguf, std::string & err); + int paradee_sample_rate(const paradee_model *); + paradee_status paradee_phonemes_to_ids(const paradee_model *, const std::string & misaki_utf8, + std::vector & ids, std::string & err); + paradee_status paradee_synthesize_ids(const paradee_model *, const int32_t * ids, size_t n, float speed, + bool (*cancel)(void *), void * user, + std::vector & pcm, std::string & err); +} +``` + +- `paradee_phonemes_to_ids` adds the pads. It rejects more than 510 symbols (`PARADEE_E_TOO_LONG`), any + symbol outside the vocabulary (`PARADEE_E_UNKNOWN_SYMBOL`, with the code point named), and invalid UTF-8. + It never drops a symbol without saying so. +- `paradee_synthesize_ids` returns `PARADEE_CANCELLED`, with `pcm` empty, as soon as `cancel` returns true. + It checks `cancel` between stages and inside the generator and phase-lock loops. Calls on one model run + one at a time. +- `paradee_synthesize` takes the same arguments in two structs (`paradee_synth_options` and + `paradee_audio`). +- Diagnostics: + - `paradee_synthesize_ids_staged` and `paradee_decode_staged` return every stage. + - `paradee_phase_lock` runs the filter alone. + +## Build and use + +`LLAMA_BUILD_PARADEE` is ON by default. It needs `LLAMA_BUILD_KOKORO`. It builds `paradee_lib`, `paradee-tts` +and `paradee-stage-dump`. + +```sh +python tools/paradee/convert_paradee_pt_to_gguf.py --model-dir \ + --output paradee-8m-v1.0-f32.gguf --check-output +build/bin/paradee-tts --model paradee-8m-v1.0-f32.gguf --phonemes "həlˈO wˈɜɹld." --output hello.wav +``` + +The input is misaki's phoneme spelling, not text and not eSpeak IPA. `tools/misaki-g2p` provides the text +front end. + +## Pins (`pins.json`) + +- **Model:** HF revision `c7f83f86c04e84d0ae94cbe1aad7618e0544f80a`. The `v1.0` tag resolves to commit + `8f34b01e`, which differs only in `README.md`. +- **Files:** size and sha256 of the `.pt` files, `config.json`, `tokenizer.json`, `samples/sentences.txt` and + `samples/*_paradee.wav`. +- **Source:** github.com/sahilmahendrakar/paradee at `9c8b4d75`. The sha256 of each training script the + reference imports is pinned. +- **Reference environment:** torch 2.14.0, kokoro 0.9.4 and misaki 0.9.4, from `training/uv.lock`. +- **GGUF:** the expected sha256 of the F32 converter output. The converter is byte-reproducible. + `--check-output` enforces this sha256. +- **No F16 pin:** `--outtype f16` halves the file (16 MB) but fails the parity gate (end-to-end SNR + 16–23 dB), so it is not pinned or shipped. + +## Parity gate + +`tools/dump_paradee_reference.py` (run it in the reference environment) has four subcommands: + +| Subcommand | What it does | +|---|---| +| `phonemes` | Writes `tests/fixtures/sentences.misaki.json`: misaki 0.9.4 phonemes of `samples/sentences.txt`, from `en.G2P(trf=False, british=False, fallback=EspeakFallback)`. The fixture is committed. | +| `dump` | Runs the pinned PyTorch model on those phonemes and writes every stage: bert, d, dur, integer durations, F0, N, asr, decoder output, source, final waveform, and the float32 lock. | +| `lock-fixture` | Writes `tests/fixtures/phase_lock_synthetic.f32` for the self-contained lock test. | +| `report` | Log-mel distance to the published seeded samples, and WER with whisper.cpp. Report only. | + +The reference is deterministic and makes three documented changes, which the native graph also makes: + +- `SineGen` has `rand_ini = 0` and no noise. +- `SineGen` runs in float64. +- The source STFT is the canonical float64 STFT described above. + +```sh +python tools/paradee/tools/dump_paradee_reference.py dump --model-dir --paradee-src --out ref +cmake -B build -DLLAMA_BUILD_TESTS=ON -DPARADEE_GGUF=$PWD/paradee-8m-v1.0-f32.gguf -DPARADEE_REF_DIR=$PWD/ref +ctest --test-dir build -R paradee --output-on-failure +``` + +| Test | Needs | Gates | +|---|---|---| +| `test-paradee-phase-lock` | nothing (committed fixture) | Lock within 1e-4 of `phase_lock.lock()` evaluated in float64; no further from the shipped float32 `phase_lock.lock()` than float32 is from float64; pass-through, length and cancel behaviour | +| `test-paradee-api` | `PARADEE_GGUF` | ids, vocabulary and length errors, input validation, determinism, speed, cancellation at several points | +| `test-paradee-parity` | `PARADEE_GGUF`, `PARADEE_REF_DIR` | integer durations identical; bert/d/dur/F0/N/asr within 1e-3 × RMS; source (from the reference F0) and decoder output (from the reference asr/F0/N/source) within 1e-3 × RMS; lock alone within 1e-4 of float64 `phase_lock.lock()`; final waveform SNR ≥ 35 dB | + +The model-backed tests exit 77 (CTest SKIP) when the model or the reference is not configured. + +Each acoustic stage in the parity test gets the reference's inputs. The harmonic source integrates F0 +over the whole utterance, so F0's float32 rounding (about 1e-6 relative) would otherwise show up as a +source phase error and be charged to the wrong stage. The end-to-end SNR covers the whole chain. + +The shipped float32 `phase_lock.lock()` differs from its own float64 evaluation by up to about 1e-3, at +bins where the source is nearly silent. No independent implementation can match it within 1e-4. The gate +therefore uses the float64 evaluation and reports the float32 distance. diff --git a/tools/paradee/convert_paradee_pt_to_gguf.py b/tools/paradee/convert_paradee_pt_to_gguf.py new file mode 100644 index 000000000..08ed16500 --- /dev/null +++ b/tools/paradee/convert_paradee_pt_to_gguf.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +""" +convert_paradee_pt_to_gguf.py - convert the Paradee-8M v1.0 PyTorch weights +(sahilmahendrakar/Paradee-8M-v1.0, Apache-2.0) into the GGUF that +tools/paradee loads. + +Paradee is Kokoro-82M's own StyleTTS-2 + iSTFTNet code at smaller widths, +distilled to one voice (af_heart). It ships two checkpoints: + + pytorch/text_side.pt {'model': state_dict} TextStudent preset 's+mlp' + pytorch/decoder.pt {'model': state_dict} StudentDecoder preset 'A' + +Each checkpoint holds its own learned constant style vector (text 32-d, +decoder 16-d); there is no voice .bin. The model's published runtime graph is +ONNX; this converter reads the PyTorch weights instead so that inference runs +natively in this fork. + +Every input is checked against tools/paradee/pins.json (size + sha256) before +torch.load(weights_only=True) touches it. weight_norm pairs are folded with +torch._weight_norm, the same function PyTorch uses when it removes the +parametrisation, so the GGUF weights are bit-identical to the reference +module's effective weights. Every state-dict key must be consumed (only the +unused ALBERT pooler is skipped), and every tensor shape is checked against +the hparams written to the GGUF. + +The output is byte-reproducible: the same inputs and the same converter +produce the same file. pins.json records the expected sha256. The metadata +records the input pins and the git blob sha of this script (not the commit +sha, which would make the expected output depend on the commit that pins it). + +usage: + convert_paradee_pt_to_gguf.py --model-dir \ + --output paradee-8m-v1.0-f32.gguf [--outtype f32|f16] [--check-output] +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path + +import numpy as np + +_HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(_HERE.parent.parent / "gguf-py")) + +import gguf # noqa: E402 + +PINS_PATH = _HERE / "pins.json" +ARCH = "paradee" +NS = "paradee" + +# Architecture of Paradee-8M v1.0, read from the checkpoint shapes and from +# training/scripts/{student.py, student_decoder.py} at the pinned source +# commit. The converter checks every tensor against these values. +HPARAMS = { + # CustomAlbert (transformers AlbertConfig, num_hidden_groups=1). + "text.n_layer": 6, + "text.n_head": 4, + "text.d_model": 256, + "text.d_ff": 768, + "text.embd_dim": 128, + "text.vocab_size": 178, + "text.max_position": 512, + # ProsodyPredictor / TextEncoder. + "hidden_dim": 192, + "predictor.n_layer": 3, + "predictor.max_dur": 50, + "text_encoder.kernel": 5, + # Styles (learned constants, one per half). + "style.text_dim": 32, + "style.decoder_dim": 16, + # asr_proj MLP: Linear(192, 512) -> GELU -> Linear(512, 512). + "asr.dim": 512, + # StudentDecoder preset A + Kokoro iSTFTNet generator. + "decoder.adain_dim": 256, + "decoder.initial_ch": 128, + "decoder.asr_res_dim": 64, + "decoder.upsample_rates": [10, 6], + "decoder.upsample_kernels": [20, 12], + "decoder.resblock_kernels": [3, 7, 11], + "decoder.resblock_dilations": [1, 3, 5, 1, 3, 5, 1, 3, 5], + "decoder.istft_n_fft": 20, + "decoder.istft_hop": 5, + "decoder.harmonic_num": 8, + "decoder.sine_amp": 0.1, + "decoder.voiced_threshold": 10.0, + "audio.sample_rate": 24000, + # Phase-locking filter (training/scripts/phase_lock.py, export_paradee.py Lock(L=33)). + "lock.n_fft": 1024, + "lock.hop": 256, + "lock.band_lo_hz": 2000, + "lock.band_hi_hz": 8000, + "lock.smooth_frames": 33, + "lock.voiced_f0_hz": 60.0, + "lock.f0_hop": 300, + # Input contract (paradee/tts.py): [0] + ids + [0], at most 512 ids. + "input.pad_id": 0, + "input.max_ids": 512, +} + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def git_blob_sha1(path: Path) -> str: + data = path.read_bytes() + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + + +def check_pins(model_dir: Path, pins: dict) -> None: + for rel in ("pytorch/text_side.pt", "pytorch/decoder.pt", "config.json", "tokenizer.json"): + pin = pins["files"][rel] + path = model_dir / rel + if not path.is_file(): + raise SystemExit(f"missing input {path}") + size = path.stat().st_size + if size != pin["size"]: + raise SystemExit(f"{rel}: size {size} != pinned {pin['size']}") + digest = sha256_file(path) + if digest != pin["sha256"]: + raise SystemExit(f"{rel}: sha256 {digest} != pinned {pin['sha256']}") + + +def load_vocab(model_dir: Path) -> list[tuple[str, int]]: + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + tok = json.loads((model_dir / "tokenizer.json").read_text(encoding="utf-8")) + vocab = config["vocab"] + if config.get("sample_rate") != HPARAMS["audio.sample_rate"]: + raise SystemExit(f"config.json sample_rate {config.get('sample_rate')} != 24000") + if config.get("n_token") != HPARAMS["text.vocab_size"]: + raise SystemExit(f"config.json n_token {config.get('n_token')} != 178") + # tokenizer.json is the same vocabulary plus the '$' pad symbol at id 0. + tok_vocab = dict(tok["model"]["vocab"]) + if tok_vocab.pop("$", None) != 0 or tok_vocab != vocab: + raise SystemExit("tokenizer.json vocabulary differs from config.json") + ids = sorted(vocab.values()) + if len(set(ids)) != len(ids) or ids[0] < 1 or ids[-1] >= HPARAMS["text.vocab_size"]: + raise SystemExit("config.json vocabulary ids are not unique in 1..177") + for sym in vocab: + if len(sym) != 1: + raise SystemExit(f"vocabulary symbol {sym!r} is not one code point") + return sorted(vocab.items(), key=lambda kv: kv[1]) + + +def fold_weight_norm(sd: dict) -> dict: + """Fold weight_g/weight_v into weight with torch._weight_norm(v, g, 0).""" + import torch + + out = {} + pairs: dict[str, dict] = {} + for k, v in sd.items(): + if k.endswith(".weight_v"): + pairs.setdefault(k[: -len(".weight_v")], {})["v"] = v + elif k.endswith(".weight_g"): + pairs.setdefault(k[: -len(".weight_g")], {})["g"] = v + else: + out[k] = v + for base, p in pairs.items(): + if set(p) != {"v", "g"}: + raise SystemExit(f"incomplete weight_norm pair for {base}") + out[base + ".weight"] = torch._weight_norm(p["v"], p["g"], 0) + return out + + +def lstm_names(src: str, dst: str) -> list[tuple[str, str]]: + out = [] + for kind in ("weight_ih", "weight_hh", "bias_ih", "bias_hh"): + out.append((f"{src}.{kind}_l0", f"{dst}.{kind}_l0")) + out.append((f"{src}.{kind}_l0_reverse", f"{dst}.{kind}_l0_r")) + return out + + +def has_weight(sd: dict, base: str) -> bool: + return f"{base}.weight" in sd or f"{base}.weight_v" in sd + + +def adain_resblk_names(src: str, dst: str, sd: dict) -> list[tuple[str, str]]: + """AdainResBlk1d (kokoro istftnet): conv1/conv2, AdaIN1d norm1/norm2 (fc + + the trained InstanceNorm affine), optional conv1x1 (bias=False) and pool.""" + out = [] + for conv in ("conv1", "conv2"): + out += [(f"{src}.{conv}.weight", f"{dst}.{conv}.weight"), (f"{src}.{conv}.bias", f"{dst}.{conv}.bias")] + for norm in ("norm1", "norm2"): + for part in ("fc.weight", "fc.bias", "norm.weight", "norm.bias"): + out.append((f"{src}.{norm}.{part}", f"{dst}.{norm}.{part}")) + if has_weight(sd, f"{src}.conv1x1"): + out.append((f"{src}.conv1x1.weight", f"{dst}.conv1x1.weight")) + if has_weight(sd, f"{src}.pool"): + out += [(f"{src}.pool.weight", f"{dst}.pool.weight"), (f"{src}.pool.bias", f"{dst}.pool.bias")] + return out + + +def gen_resblock_names(src: str, dst: str) -> list[tuple[str, str]]: + """AdaINResBlock1 (Snake): convs1/convs2 x3, adain1/adain2 x3, alpha1/alpha2 x3.""" + out = [] + for j in range(3): + for branch in ("convs1", "convs2"): + out += [(f"{src}.{branch}.{j}.weight", f"{dst}.{branch}.{j}.weight"), + (f"{src}.{branch}.{j}.bias", f"{dst}.{branch}.{j}.bias")] + for branch in ("adain1", "adain2"): + for part in ("fc.weight", "fc.bias", "norm.weight", "norm.bias"): + out.append((f"{src}.{branch}.{j}.{part}", f"{dst}.{branch}.{j}.{part}")) + out += [(f"{src}.alpha1.{j}", f"{dst}.alpha1.{j}"), (f"{src}.alpha2.{j}", f"{dst}.alpha2.{j}")] + return out + + +def text_side_map(sd: dict) -> list[tuple[str, str]]: + m = [ + ("bert.embeddings.word_embeddings.weight", f"{NS}.bert.token_embd.weight"), + ("bert.embeddings.position_embeddings.weight", f"{NS}.bert.position_embd.weight"), + ("bert.embeddings.token_type_embeddings.weight", f"{NS}.bert.tok_type_embd.weight"), + ("bert.embeddings.LayerNorm.weight", f"{NS}.bert.embd_ln.weight"), + ("bert.embeddings.LayerNorm.bias", f"{NS}.bert.embd_ln.bias"), + ("bert.encoder.embedding_hidden_mapping_in.weight", f"{NS}.bert.embd_proj.weight"), + ("bert.encoder.embedding_hidden_mapping_in.bias", f"{NS}.bert.embd_proj.bias"), + ("bert_encoder.weight", f"{NS}.bert_encoder.weight"), + ("bert_encoder.bias", f"{NS}.bert_encoder.bias"), + ] + layer = "bert.encoder.albert_layer_groups.0.albert_layers.0." + for src, dst in (("attention.query", "attn_q"), ("attention.key", "attn_k"), ("attention.value", "attn_v"), + ("attention.dense", "attn_o"), ("attention.LayerNorm", "attn_ln"), + ("full_layer_layer_norm", "full_ln"), ("ffn", "ffn"), ("ffn_output", "ffn_out")): + m += [(f"{layer}{src}.weight", f"{NS}.bert.layer.{dst}.weight"), (f"{layer}{src}.bias", f"{NS}.bert.layer.{dst}.bias")] + for i in range(HPARAMS["predictor.n_layer"]): + m += lstm_names(f"predictor.text_encoder.lstms.{2 * i}", f"{NS}.predictor.de.lstm{i}") + m += [(f"predictor.text_encoder.lstms.{2 * i + 1}.fc.weight", f"{NS}.predictor.de.adaln{i}.fc.weight"), + (f"predictor.text_encoder.lstms.{2 * i + 1}.fc.bias", f"{NS}.predictor.de.adaln{i}.fc.bias")] + m += lstm_names("predictor.lstm", f"{NS}.predictor.lstm") + m += [("predictor.duration_proj.linear_layer.weight", f"{NS}.predictor.duration_proj.weight"), + ("predictor.duration_proj.linear_layer.bias", f"{NS}.predictor.duration_proj.bias")] + m += lstm_names("predictor.shared", f"{NS}.predictor.shared") + for chain in ("F0", "N"): + for i in range(3): + m += adain_resblk_names(f"predictor.{chain}.{i}", f"{NS}.predictor.{chain}.{i}", sd) + m += [(f"predictor.{chain}_proj.weight", f"{NS}.predictor.{chain}_proj.weight"), + (f"predictor.{chain}_proj.bias", f"{NS}.predictor.{chain}_proj.bias")] + m.append(("text_encoder.embedding.weight", f"{NS}.text_encoder.embd.weight")) + for i in range(HPARAMS["predictor.n_layer"]): + m += [(f"text_encoder.cnn.{i}.0.weight", f"{NS}.text_encoder.cnn.{i}.weight"), + (f"text_encoder.cnn.{i}.0.bias", f"{NS}.text_encoder.cnn.{i}.bias"), + (f"text_encoder.cnn.{i}.1.gamma", f"{NS}.text_encoder.ln.{i}.gamma"), + (f"text_encoder.cnn.{i}.1.beta", f"{NS}.text_encoder.ln.{i}.beta")] + m += lstm_names("text_encoder.lstm", f"{NS}.text_encoder.lstm") + m += [("asr_proj.0.weight", f"{NS}.asr_proj.0.weight"), ("asr_proj.0.bias", f"{NS}.asr_proj.0.bias"), + ("asr_proj.2.weight", f"{NS}.asr_proj.1.weight"), ("asr_proj.2.bias", f"{NS}.asr_proj.1.bias")] + m.append(("style", f"{NS}.style.text")) + return m + + +TEXT_SIDE_SKIPPED = ("bert.pooler.weight", "bert.pooler.bias") # ALBERT pooler: not used by TextStudent + + +def decoder_map(sd: dict) -> list[tuple[str, str]]: + m = adain_resblk_names("encode", f"{NS}.decoder.encode", sd) + for i in range(4): + m += adain_resblk_names(f"decode.{i}", f"{NS}.decoder.decode.{i}", sd) + for conv in ("F0_conv", "N_conv"): + m += [(f"{conv}.weight", f"{NS}.decoder.{conv}.weight"), (f"{conv}.bias", f"{NS}.decoder.{conv}.bias")] + m += [("asr_res.0.weight", f"{NS}.decoder.asr_res.weight"), ("asr_res.0.bias", f"{NS}.decoder.asr_res.bias")] + g = "generator" + m += [(f"{g}.m_source.l_linear.weight", f"{NS}.gen.m_source.l_linear.weight"), + (f"{g}.m_source.l_linear.bias", f"{NS}.gen.m_source.l_linear.bias")] + for i in range(2): + for part in ("weight", "bias"): + m.append((f"{g}.noise_convs.{i}.{part}", f"{NS}.gen.noise_convs.{i}.{part}")) + m.append((f"{g}.ups.{i}.{part}", f"{NS}.gen.ups.{i}.{part}")) + m += gen_resblock_names(f"{g}.noise_res.{i}", f"{NS}.gen.noise_res.{i}") + for i in range(6): + m += gen_resblock_names(f"{g}.resblocks.{i}", f"{NS}.gen.resblocks.{i}") + m += [(f"{g}.conv_post.weight", f"{NS}.gen.conv_post.weight"), (f"{g}.conv_post.bias", f"{NS}.gen.conv_post.bias")] + m.append(("style", f"{NS}.style.decoder")) + return m + + +def expected_shapes() -> dict[str, tuple[int, ...]]: + """Shapes (PyTorch order) of the tensors whose widths the C++ graph reads + from hparams. Everything else is checked for consistency at load time.""" + h = HPARAMS + d, e, ff, hid = h["text.d_model"], h["text.embd_dim"], h["text.d_ff"], h["hidden_dim"] + st, sd_, asr = h["style.text_dim"], h["style.decoder_dim"], h["asr.dim"] + W, uic = h["decoder.adain_dim"], h["decoder.initial_ch"] + return { + f"{NS}.bert.token_embd.weight": (h["text.vocab_size"], e), + f"{NS}.bert.position_embd.weight": (h["text.max_position"], e), + f"{NS}.bert.embd_proj.weight": (d, e), + f"{NS}.bert.layer.attn_q.weight": (d, d), + f"{NS}.bert.layer.ffn.weight": (ff, d), + f"{NS}.bert_encoder.weight": (hid, d), + f"{NS}.predictor.de.lstm0.weight_ih_l0": (2 * hid, hid + st), + f"{NS}.predictor.de.adaln0.fc.weight": (2 * hid, st), + f"{NS}.predictor.duration_proj.weight": (h["predictor.max_dur"], hid), + f"{NS}.predictor.F0.1.conv1.weight": (hid // 2, hid, 3), + f"{NS}.predictor.F0_proj.weight": (1, hid // 2, 1), + f"{NS}.text_encoder.embd.weight": (h["text.vocab_size"], hid), + f"{NS}.text_encoder.cnn.0.weight": (hid, hid, h["text_encoder.kernel"]), + f"{NS}.asr_proj.0.weight": (asr, hid), + f"{NS}.asr_proj.1.weight": (asr, asr), + f"{NS}.style.text": (st,), + f"{NS}.style.decoder": (sd_,), + f"{NS}.decoder.encode.conv1.weight": (W, asr + 2, 3), + f"{NS}.decoder.decode.0.conv1.weight": (W, W + h["decoder.asr_res_dim"] + 2, 3), + f"{NS}.decoder.decode.3.conv1.weight": (uic, W + h["decoder.asr_res_dim"] + 2, 3), + f"{NS}.decoder.asr_res.weight": (h["decoder.asr_res_dim"], asr, 1), + f"{NS}.decoder.encode.norm1.fc.weight": (2 * (asr + 2), sd_), + f"{NS}.gen.ups.0.weight": (uic, uic // 2, h["decoder.upsample_kernels"][0]), + f"{NS}.gen.ups.1.weight": (uic // 2, uic // 4, h["decoder.upsample_kernels"][1]), + f"{NS}.gen.noise_convs.0.weight": (uic // 2, h["decoder.istft_n_fft"] + 2, 2 * h["decoder.upsample_rates"][1]), + f"{NS}.gen.noise_convs.1.weight": (uic // 4, h["decoder.istft_n_fft"] + 2, 1), + f"{NS}.gen.conv_post.weight": (h["decoder.istft_n_fft"] + 2, uic // 4, 7), + f"{NS}.gen.m_source.l_linear.weight": (1, h["decoder.harmonic_num"] + 1), + } + + +def collect(sd_raw: dict, mapping: list[tuple[str, str]], skipped: tuple[str, ...], label: str) -> dict[str, np.ndarray]: + sd = fold_weight_norm(sd_raw) + out: dict[str, np.ndarray] = {} + used = set() + for src, dst in mapping: + if src not in sd: + raise SystemExit(f"{label}: missing tensor {src}") + t = sd[src].detach().cpu().contiguous() + if t.dtype.is_floating_point is False: + raise SystemExit(f"{label}: {src} is not floating point") + arr = t.float().numpy() + if dst.endswith(".style.text") or dst.endswith(".style.decoder"): + arr = arr.reshape(-1) + out[dst] = np.ascontiguousarray(arr, dtype=np.float32) + used.add(src) + leftover = sorted(set(sd) - used - set(skipped)) + if leftover: + raise SystemExit(f"{label}: unconsumed tensors: {leftover[:8]} ({len(leftover)} total)") + return out + + +def write_gguf(out_path: Path, tensors: dict[str, np.ndarray], vocab: list[tuple[str, int]], pins: dict, + outtype: str) -> None: + writer = gguf.GGUFWriter(str(out_path), arch=ARCH) + writer.add_string("general.name", "Paradee-8M v1.0") + writer.add_string("general.license", "apache-2.0") + writer.add_string("general.source.url", f"https://huggingface.co/{pins['model']['repo']}") + writer.add_string("general.base_model.0.name", "Kokoro-82M") + writer.add_string("general.description", + "Paradee-8M v1.0 single-voice (af_heart) TTS, distilled from Kokoro-82M. " + "StyleTTS-2 text side + iSTFTNet decoder + phase-locking filter.") + for key, val in HPARAMS.items(): + full = f"{NS}.{key}" + if isinstance(val, list): + writer.add_array(full, [int(x) for x in val]) + elif isinstance(val, float): + writer.add_float32(full, val) + else: + writer.add_int32(full, int(val)) + writer.add_string(f"{NS}.voice", "af_heart") + writer.add_string(f"{NS}.phoneme_format", "misaki") + writer.add_string(f"{NS}.input.pad_symbol", "$") + writer.add_array(f"{NS}.vocab.symbols", [s for s, _ in vocab]) + writer.add_array(f"{NS}.vocab.ids", [int(i) for _, i in vocab]) + + model = pins["model"] + writer.add_string(f"{NS}.pins.hf_repo", model["repo"]) + writer.add_string(f"{NS}.pins.hf_revision", model["revision"]) + writer.add_string(f"{NS}.pins.hf_tag", model["tag"]) + for rel in ("pytorch/text_side.pt", "pytorch/decoder.pt", "config.json", "tokenizer.json"): + writer.add_string(f"{NS}.pins.sha256.{rel}", pins["files"][rel]["sha256"]) + writer.add_string(f"{NS}.pins.source_repo", pins["source"]["repo"]) + writer.add_string(f"{NS}.pins.source_commit", pins["source"]["commit"]) + writer.add_string(f"{NS}.converter.git_blob_sha1", git_blob_sha1(Path(__file__))) + writer.add_string(f"{NS}.converter.outtype", outtype) + + for name in sorted(tensors): + arr = tensors[name] + if outtype == "f16" and arr.ndim >= 2 and not name.endswith((".style.text", ".style.decoder")): + arr = arr.astype(np.float16) + writer.add_tensor(name, arr) + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--model-dir", required=True, type=Path, + help="local snapshot of the pinned HF revision (pytorch/*.pt, config.json, tokenizer.json)") + ap.add_argument("--output", required=True, type=Path) + ap.add_argument("--outtype", choices=("f32", "f16"), default="f32", + help="f32 (default, the parity-gated file) or f16 for >=2-D weights " + "(experimental: fails test-paradee-parity, not pinned)") + ap.add_argument("--check-output", action="store_true", + help="fail unless the output sha256 equals the one pinned for this outtype") + args = ap.parse_args(argv) + + pins = json.loads(PINS_PATH.read_text(encoding="utf-8")) + check_pins(args.model_dir, pins) + vocab = load_vocab(args.model_dir) + + import torch + + text_sd = torch.load(args.model_dir / "pytorch/text_side.pt", map_location="cpu", weights_only=True) + dec_sd = torch.load(args.model_dir / "pytorch/decoder.pt", map_location="cpu", weights_only=True) + if set(text_sd) != {"model"} or set(dec_sd) != {"model"}: + raise SystemExit("expected {'model': state_dict} checkpoints") + text_sd, dec_sd = text_sd["model"], dec_sd["model"] + + tensors = collect(text_sd, text_side_map(text_sd), TEXT_SIDE_SKIPPED, "text_side.pt") + dec = collect(dec_sd, decoder_map(dec_sd), (), "decoder.pt") + overlap = set(tensors) & set(dec) + if overlap: + raise SystemExit(f"tensor name collision: {sorted(overlap)}") + tensors.update(dec) + + for name, shape in expected_shapes().items(): + if tuple(tensors[name].shape) != shape: + raise SystemExit(f"{name}: shape {tuple(tensors[name].shape)} != expected {shape}") + n_params = sum(int(a.size) for a in tensors.values()) + + args.output.parent.mkdir(parents=True, exist_ok=True) + write_gguf(args.output, tensors, vocab, pins, args.outtype) + digest = sha256_file(args.output) + print(f"wrote {args.output}: {len(tensors)} tensors, {n_params} parameters, " + f"{args.output.stat().st_size} bytes, sha256 {digest}") + + if args.check_output: + want = pins["gguf"][args.outtype]["sha256"] + if digest != want: + print(f"error: sha256 {digest} != pinned {want}", file=sys.stderr) + return 1 + print("output matches pins.json") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/paradee/include/paradee.h b/tools/paradee/include/paradee.h new file mode 100644 index 000000000..49b38d157 --- /dev/null +++ b/tools/paradee/include/paradee.h @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: MIT +// +// paradee.h — Paradee-8M v1.0 text-to-speech, native in the elizaOS llama.cpp +// fork (tools/paradee). +// +// Paradee (sahilmahendrakar/Paradee-8M-v1.0, Apache-2.0) is Kokoro-82M's +// StyleTTS-2 + iSTFTNet code at smaller widths, distilled to the single +// af_heart voice. 8.07M parameters, 24 kHz output. The upstream runtime is an +// ONNX graph; this port runs the PyTorch weights (converted by +// convert_paradee_pt_to_gguf.py) through the generalised tools/kokoro layers +// plus a native implementation of the model's phase-locking filter. No ONNX. +// +// misaki phonemes (UTF-8) +// -> paradee_phonemes_to_ids : [0] + vocab ids + [0], at most 512 ids +// -> text side : ALBERT (6 x 256) -> duration encoder -> durations +// (round(dur / speed), min 1) -> F0 / N curves; +// TextEncoder -> asr_proj MLP (192 -> 512) +// -> decoder : AdaIN res blocks (256) -> iSTFTNet generator (128 -> 64 -> 32) +// with a deterministic harmonic source (no random phase, +// no additive noise, as in tools/kokoro) +// -> phase lock : in voiced frames, the 2-8 kHz phase follows the +// harmonic source (STFT 1024 / hop 256, 33-frame smoothing) +// -> 24 kHz mono float PCM +// +// Input phonemes must be misaki's spelling (Kokoro's G2P); eSpeak IPA must be +// converted to misaki spelling first. + +#pragma once + +#include +#include +#include +#include +#include + +namespace eliza_paradee { + +struct paradee_model; +struct paradee_model_deleter { void operator()(paradee_model *) const noexcept; }; +using paradee_model_ptr = std::unique_ptr; + +// Input contract of Paradee-8M v1.0: at most 512 ids, the pad (0) at each end. +constexpr int PARADEE_MAX_INPUT_IDS = 512; +constexpr int PARADEE_PAD_ID = 0; + +// Accepted speed range. Durations scale with 1/speed, so a tiny speed would +// make the frame count (and every decoder buffer) arbitrarily large. +constexpr float PARADEE_MIN_SPEED = 0.25f; +constexpr float PARADEE_MAX_SPEED = 4.0f; + +enum paradee_status { + PARADEE_OK = 0, + PARADEE_E_INVALID_ARG = 1, // null model, bad ids, bad speed, malformed input + PARADEE_E_UNKNOWN_SYMBOL = 2, // a phoneme outside the model vocabulary + PARADEE_E_TOO_LONG = 3, // more than 510 phonemes (512 ids with the pads) + PARADEE_E_RUNTIME = 4, // forward pass failed + PARADEE_CANCELLED = 5, // the cancel callback returned true + PARADEE_E_OOM = 6, // allocation failed + PARADEE_E_CANCELLED = PARADEE_CANCELLED, // alias +}; + +const char * paradee_status_str(paradee_status st) noexcept; + +// Load a Paradee GGUF written by convert_paradee_pt_to_gguf.py. The file must +// declare general.architecture == "paradee"; every hparam the graph reads is +// required. Returns nullptr and sets `err` on failure. +paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string & err) noexcept; + +// 24000 for Paradee-8M v1.0 (0 for a null model). +int paradee_sample_rate(const paradee_model * model) noexcept; + +// Map a misaki phoneme string (UTF-8, one vocabulary symbol per code point) +// to model input ids: [0] + ids + [0]. Rejects empty input, invalid UTF-8, +// any symbol outside the vocabulary (PARADEE_E_UNKNOWN_SYMBOL, the code point +// is named in `err`) and more than 510 symbols (PARADEE_E_TOO_LONG). No +// symbol is dropped silently. +paradee_status paradee_phonemes_to_ids( + const paradee_model * model, + const std::string & misaki_utf8, + std::vector & ids, + std::string & err) noexcept; + +using paradee_cancel_fn = bool (*)(void * user); +using paradee_cancel_cb = paradee_cancel_fn; // alias + +// The same mapping without the pads and without the length limit, for +// callers that split long phoneme strings into model-sized chunks: ids are +// the vocabulary ids of every symbol, in order. Rejects empty input, invalid +// UTF-8 and symbols outside the vocabulary exactly like +// paradee_phonemes_to_ids. +paradee_status paradee_phonemes_to_symbol_ids( + const paradee_model * model, + const std::string & misaki_utf8, + std::vector & ids, + std::string & err) noexcept; + +// The same unpadded, unlimited mapping, but symbols outside the vocabulary +// are skipped (and counted in `n_dropped`) instead of rejected, as Kokoro and +// misaki do; empty input (or input that is all dropped) gives no ids and +// PARADEE_OK. Invalid UTF-8 is still rejected. This is the libelizainference +// contract ("characters outside the model vocab are dropped"). +paradee_status paradee_phonemes_to_symbol_ids_lenient( + const paradee_model * model, + const std::string & misaki_utf8, + std::vector & ids, + size_t & n_dropped, + std::string & err) noexcept; + +// Synthesize from input ids (with the pad at each end, 3..512 ids, every id +// in [0, n_token)). `speed` finite and in [PARADEE_MIN_SPEED, +// PARADEE_MAX_SPEED] (1.0 = native rate; larger is faster). `cancel` (optional) is polled between stages and inside the +// generator loops; when it returns true the call returns PARADEE_CANCELLED +// with `pcm` empty. Calls on one model are serialised. +paradee_status paradee_synthesize_ids( + const paradee_model * model, + const int32_t * ids, size_t n, + float speed, + paradee_cancel_fn cancel, void * user, + std::vector & pcm, + std::string & err) noexcept; + +// Convenience form of paradee_synthesize_ids for callers that keep options +// and audio in structs. +struct paradee_synth_options { + float speed = 1.0f; + paradee_cancel_fn cancel = nullptr; + void * cancel_user_data = nullptr; +}; + +struct paradee_audio { + std::vector samples; + int sample_rate = 24000; +}; + +paradee_status paradee_synthesize( + const paradee_model * model, + const std::vector & input_ids, + const paradee_synth_options & opts, + paradee_audio & out, + std::string & err) noexcept; + +// --------------------------------------------------------------------------- +// Parity / diagnostics. Stage outputs of one synthesis, in the reference +// dump's layouts (see tools/dump_paradee_reference.py). +// --------------------------------------------------------------------------- +struct paradee_stages { + int T_phon = 0; // number of input ids + int T_frame = 0; // sum of the integer durations + int d_model = 0, hid = 0, sty = 0, asr_dim = 0; + std::vector bert; // [T_phon, d_model] ALBERT last hidden state + std::vector d; // [T_phon, hid+sty] duration-encoder output + std::vector dur; // [T_phon] sigmoid-sum durations / speed + std::vector dur_int; // [T_phon] clamp(round(dur), min 1) + std::vector F0; // [2*T_frame] + std::vector N; // [2*T_frame] + std::vector asr; // [asr_dim, T_frame] aligned asr features (channel-major) + std::vector dec; // [2*T_frame*300] generator output before the phase lock + std::vector src; // [2*T_frame*300] harmonic source (m_source output) + std::vector wav; // final waveform +}; + +// paradee_synthesize_ids that also fills `stages` (when non-null). +paradee_status paradee_synthesize_ids_staged( + const paradee_model * model, + const int32_t * ids, size_t n, + float speed, + paradee_cancel_fn cancel, void * user, + std::vector & pcm, + std::string & err, + paradee_stages * stages) noexcept; + +// Decoder + generator + phase lock from given text-side outputs (the parity +// gate feeds the reference's asr / F0 / N here to test the acoustic half on +// its own). asr is [asr_dim, T_frame] channel-major, F0 and N are +// [2*T_frame]. `src` (optional, [2*T_frame*300]) replaces the harmonic +// source the generator would compute. Fills stages->dec / src / wav (stages +// may not be null). +paradee_status paradee_decode_staged( + const paradee_model * model, + const float * asr, int asr_dim, int T_frame, + const float * F0, const float * N, + const std::vector * src, + paradee_cancel_fn cancel, void * user, + paradee_stages * stages, + std::string & err) noexcept; + +// Phase-locking filter parameters (read from the GGUF; defaults are v1.0). +struct paradee_lock_params { + int n_fft = 1024; + int hop = 256; + float band_lo_hz = 2000.0f; + float band_hi_hz = 8000.0f; + int smooth_frames = 33; // L + float voiced_f0_hz = 60.0f; + int f0_hop = 300; // samples per F0 value + int sample_rate = 24000; +}; + +// The filter alone (training/scripts/phase_lock.py lock()): s = generator +// output, src = its harmonic source, f0 = the F0 curve at f0_hop samples per +// value. out has min(ns, nsrc) samples. Returns false with out = s when the +// parameters are invalid, and false with out empty when `cancel` stopped it. +// Signals no longer than n_fft/2 pass through unchanged (as torch.stft +// cannot reflect-pad them). +bool paradee_phase_lock( + const float * s, size_t ns, + const float * src, size_t nsrc, + const float * f0, size_t nf0, + const paradee_lock_params & p, + std::vector & out, + paradee_cancel_fn cancel = nullptr, void * user = nullptr); + +const paradee_lock_params * paradee_get_lock_params(const paradee_model * model) noexcept; + +} // namespace eliza_paradee diff --git a/tools/paradee/pins.json b/tools/paradee/pins.json new file mode 100644 index 000000000..321e4bbdd --- /dev/null +++ b/tools/paradee/pins.json @@ -0,0 +1,93 @@ +{ + "model": { + "repo": "sahilmahendrakar/Paradee-8M-v1.0", + "revision": "c7f83f86c04e84d0ae94cbe1aad7618e0544f80a", + "tag": "v1.0", + "tag_commit": "8f34b01ef8adcb0bec470b89bf2fd62a7b2369e6", + "tag_note": "The v1.0 tag (8f34b01e) and the pinned main revision (c7f83f86) differ only in README.md; every model, config, tokenizer and sample blob is identical.", + "license": "Apache-2.0", + "params": 8070000 + }, + "files": { + "pytorch/text_side.pt": { + "size": 16998347, + "sha256": "0995bee746043e4f06f72be68fb8634530b95d6467b4a141759e721122648062" + }, + "pytorch/decoder.pt": { + "size": 15564135, + "sha256": "f231b9765ffdddd30df798530af0c557c06cb9cf029a475b8bfdca7b051b9d6c" + }, + "config.json": { + "size": 1774, + "sha256": "f24046974a3a8c747affefb45c7c504263a99d5081787908b16abe8f5ac94fcd" + }, + "tokenizer.json": { + "size": 3497, + "sha256": "77a02c8e164413299b4b4c403b14f8e0e1c1b727db4d46a09d6327b861060a34" + }, + "samples/sentences.txt": { + "size": 621, + "sha256": "2d411facb6b0429ad05f6ea3abf9c6a79ba6613990cadf54e2213e11a8c9fd1b" + }, + "samples/1_paradee.wav": { + "size": 468044, + "sha256": "5b4bf9265c0cb108d1d9c631c26a0c785aa1dd9672565fa680393fbb37133d7e" + }, + "samples/2_paradee.wav": { + "size": 283244, + "sha256": "2e40a0504a06f417d32761009eac1882fb122beb8e4a05e0b3e7374fb87d4fc1" + }, + "samples/3_paradee.wav": { + "size": 361244, + "sha256": "4aaf75652796ba650b9eb0b324a9f4c7e311ee5c74a888deda9ac0b90edf81d8" + }, + "samples/4_paradee.wav": { + "size": 457244, + "sha256": "374596ff98b2a61713c5521861edf6a2a65e460aebccfd9934029ceb9c1888df" + }, + "samples/5_paradee.wav": { + "size": 512444, + "sha256": "9164e9e329bf227cccc0bc75ec5f309c6295748ed799ea4775444ee66b940b1b" + } + }, + "source": { + "repo": "https://github.com/sahilmahendrakar/paradee", + "commit": "9c8b4d7504cbee7e64de2d0341bb690f0b0ab708", + "license": "Apache-2.0", + "files": { + "training/scripts/student.py": "90456f28ac3d6dd06e76ad8f253f9bcadd9060cdecb6880268651e0ff8ba0f8b", + "training/scripts/student_decoder.py": "2443f14fe8ff27be5702d88dc80f8c566e92c03273b6216825fa7787cd66b97c", + "training/scripts/phase_lock.py": "d502744ccb2ede9fc10f56d481020b368a414d4f35b964ffbace14bcd37bbe86", + "training/scripts/export_paradee.py": "5656e4eb725fdd97bb46fc00cbe824a1e9adac4bc4b46440ba27a5b8f326a4db", + "training/scripts/common.py": "76faab4b54f633c1dfee28bfb08c981d4adb3f4ee99b42271cf80a3fc0e9fc9c" + } + }, + "reference_env": { + "note": "training/uv.lock pins kokoro 0.9.4, misaki 0.9.4 and torch 2.14.0. spaCy, en_core_web_sm and espeak-ng are the versions used to generate tests/fixtures/sentences.misaki.json.", + "python": "3.12", + "torch": "2.14.0", + "kokoro": "0.9.4", + "misaki": "0.9.4", + "spacy": "3.8.16", + "en_core_web_sm": "3.8.0", + "espeak-ng": "1.52.0" + }, + "gguf": { + "f32": { + "file": "paradee-8m-v1.0-f32.gguf", + "size": 32072768, + "sha256": "9a18ae710211fab04129e00bc0cde04484c0cf8d80bb19e51df7e331ae83a254", + "note": "convert_paradee_pt_to_gguf.py --outtype f32 output; byte-reproducible (check with --check-output). Changes whenever the converter changes, because its git blob sha is in the metadata." + } + }, + "fixtures": { + "tests/fixtures/sentences.misaki.json": { + "sha256": "30ab9f9cd034b6bf8650e13af4728e658a7e57ccd75ff412c62a41bc4e42bc9d", + "made_by": "tools/dump_paradee_reference.py phonemes" + }, + "tests/fixtures/phase_lock_synthetic.f32": { + "sha256": "691b032dd084e0c9c28f363acef8fb3029880041c9e5a282452626733ef3ee34", + "made_by": "tools/dump_paradee_reference.py lock-fixture" + } + } +} diff --git a/tools/paradee/src/paradee-phase-lock.cpp b/tools/paradee/src/paradee-phase-lock.cpp new file mode 100644 index 000000000..a6f43fde6 --- /dev/null +++ b/tools/paradee/src/paradee-phase-lock.cpp @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT +// +// paradee-phase-lock.cpp — Paradee's parameter-free phase-locking filter. +// +// Port of training/scripts/phase_lock.py lock() (sahilmahendrakar/paradee, +// Apache-2.0, commit 9c8b4d75), the filter the Paradee-8M v1.0 graph applies +// after the decoder (export_paradee.py Lock, L = 33): +// +// S = STFT(s), E = STFT(src) n_fft 1024, hop 256, periodic Hann, +// centre frames with reflect padding +// voiced frame j <=> f0[min(j*hop / f0_hop, len-1)] > 60 Hz +// R = S * conj(E) / max(|E|, 1e-6) +// R~ = R smoothed over frames with hann(L+2)[1:-1] / sum (zero-padded) +// in voiced frames, for bins with 2 kHz <= f < 8 kHz: +// phase = angle(E) + angle(R~), magnitude = |S| +// out = iSTFT (window-envelope normalised, length = len(s)) +// +// The inverse is applied to the change only (out = s + iSTFT(delta)), as the +// exported graph does, so bins outside the band and unvoiced frames pass +// through untouched. Everything runs in double with a radix-2 FFT; the two +// real forward transforms share one complex FFT. + +#include "paradee.h" + +#include +#include +#include +#include +#include + +namespace eliza_paradee { + +namespace { + +using cd = std::complex; +constexpr double K_PI = 3.14159265358979323846; + +// In-place iterative radix-2 FFT (n a power of two). inverse = conjugate +// twiddles, no 1/n scaling. +class Fft { +public: + explicit Fft(int n) : n_(n), rev_((size_t) n), tw_((size_t) n / 2) { + int bits = 0; + while ((1 << bits) < n) ++bits; + for (int i = 0; i < n; ++i) { + int r = 0; + for (int b = 0; b < bits; ++b) if (i & (1 << b)) r |= 1 << (bits - 1 - b); + rev_[(size_t) i] = r; + } + for (int k = 0; k < n / 2; ++k) { + const double a = -2.0 * K_PI * (double) k / (double) n; + tw_[(size_t) k] = cd(std::cos(a), std::sin(a)); + } + } + void run(std::vector & a, bool inverse) const { + const int n = n_; + for (int i = 0; i < n; ++i) { + const int r = rev_[(size_t) i]; + if (i < r) std::swap(a[(size_t) i], a[(size_t) r]); + } + for (int len = 2; len <= n; len <<= 1) { + const int half = len >> 1, step = n / len; + for (int i = 0; i < n; i += len) { + for (int j = 0; j < half; ++j) { + cd w = tw_[(size_t) (j * step)]; + if (inverse) w = std::conj(w); + const cd u = a[(size_t) (i + j)]; + const cd v = a[(size_t) (i + j + half)] * w; + a[(size_t) (i + j)] = u + v; + a[(size_t) (i + j + half)] = u - v; + } + } + } + } +private: + int n_; + std::vector rev_; + std::vector tw_; +}; + +// unit(z) = z / |z|, with angle(0) = 0 as torch.angle defines it. +inline cd unit(cd z) { + const double m = std::abs(z); + return m > 0.0 ? z / m : cd(1.0, 0.0); +} + +} // namespace + +bool paradee_phase_lock( + const float * s, size_t ns, + const float * src, size_t nsrc, + const float * f0, size_t nf0, + const paradee_lock_params & p, + std::vector & out, + paradee_cancel_fn cancel, void * user) { + const size_t k = std::min(ns, nsrc); + out.clear(); + if (!s) return false; + out.assign(s, s + k); + const int N = p.n_fft, hop = p.hop, L = p.smooth_frames; + if (!src || !f0 || nf0 == 0 || N < 4 || N > (1 << 16) || (N & (N - 1)) != 0 || hop <= 0 || + L < 1 || p.f0_hop <= 0 || p.sample_rate <= 0) { + return false; + } + const size_t pad = (size_t) N / 2; + // torch.stft(center=True) needs reflect padding narrower than the signal. + if (k <= pad) return true; + + // Band bins: FR = bin * sr / n_fft, lo <= FR < hi. + const int n_bins = N / 2 + 1; + int b_lo = n_bins, b_hi = 0; // [b_lo, b_hi) + for (int b = 0; b < n_bins; ++b) { + const double fr = (double) b * (double) p.sample_rate / (double) N; + if (fr >= p.band_lo_hz && fr < p.band_hi_hz) { + b_lo = std::min(b_lo, b); + b_hi = std::max(b_hi, b + 1); + } + } + if (b_lo >= b_hi) return true; + const int nb = b_hi - b_lo; + + // Reflect-padded signals. + const size_t np_len = k + 2 * pad; + std::vector ps(np_len), pe(np_len); + for (size_t i = 0; i < k; ++i) { ps[pad + i] = s[i]; pe[pad + i] = src[i]; } + for (size_t i = 0; i < pad; ++i) { + ps[pad - 1 - i] = s[i + 1]; pe[pad - 1 - i] = src[i + 1]; + ps[pad + k + i] = s[k - 2 - i]; pe[pad + k + i] = src[k - 2 - i]; + } + + // Periodic Hann window (torch.hann_window(N)). + std::vector w((size_t) N); + for (int i = 0; i < N; ++i) w[(size_t) i] = 0.5 - 0.5 * std::cos(2.0 * K_PI * (double) i / (double) N); + + const size_t nf = 1 + k / (size_t) hop; + Fft fft(N); + + // Forward STFTs of the band bins: S and E share one complex FFT of + // z = s + i*e: S[b] = (Z[b] + conj(Z[N-b])) / 2, E[b] = (Z[b] - conj(Z[N-b])) / 2i. + std::vector S(nf * (size_t) nb), E(nf * (size_t) nb); + std::vector z((size_t) N); + for (size_t j = 0; j < nf; ++j) { + if (cancel && (j & 31) == 0 && cancel(user)) { out.clear(); return false; } + const size_t off = j * (size_t) hop; + for (int n = 0; n < N; ++n) { + z[(size_t) n] = cd(ps[off + (size_t) n] * w[(size_t) n], pe[off + (size_t) n] * w[(size_t) n]); + } + fft.run(z, false); + for (int b = b_lo; b < b_hi; ++b) { + const cd zb = z[(size_t) b]; + const cd zm = std::conj(z[(size_t) ((N - b) % N)]); + S[j * (size_t) nb + (size_t) (b - b_lo)] = (zb + zm) * 0.5; + E[j * (size_t) nb + (size_t) (b - b_lo)] = (zb - zm) * cd(0.0, -0.5); + } + } + + // Voiced frames. + std::vector voiced(nf, 0); + bool any_voiced = false; + for (size_t j = 0; j < nf; ++j) { + const size_t fi = std::min((j * (size_t) hop) / (size_t) p.f0_hop, nf0 - 1); + voiced[j] = f0[fi] > p.voiced_f0_hz; + any_voiced = any_voiced || voiced[j]; + } + if (!any_voiced) return true; + + // R = S conj(E) / max(|E|, 1e-6), all frames (the smoothing reads + // unvoiced neighbours too). + std::vector R(nf * (size_t) nb); + for (size_t i = 0; i < R.size(); ++i) { + R[i] = S[i] * std::conj(E[i]) / std::max(std::abs(E[i]), 1e-6); + } + + // Smoothing kernel: hann_window(L + 2) (periodic)[1:-1], normalised. + std::vector ker((size_t) L); + double ksum = 0.0; + for (int m = 0; m < L; ++m) { + ker[(size_t) m] = 0.5 - 0.5 * std::cos(2.0 * K_PI * (double) (m + 1) / (double) (L + 2)); + ksum += ker[(size_t) m]; + } + for (double & v : ker) v /= ksum; + const int half = L / 2; + + // Window envelope sum_j w^2(n - j*hop) over the padded signal. + std::vector env(np_len, 0.0); + for (size_t j = 0; j < nf; ++j) { + const size_t off = j * (size_t) hop; + for (int n = 0; n < N; ++n) env[off + (size_t) n] += w[(size_t) n] * w[(size_t) n]; + } + + // Inverse STFT of the change in voiced frames, overlap-added. + std::vector ola(np_len, 0.0); + std::vector dz((size_t) N); + for (size_t j = 0; j < nf; ++j) { + if (!voiced[j]) continue; + if (cancel && (j & 31) == 0 && cancel(user)) { out.clear(); return false; } + std::fill(dz.begin(), dz.end(), cd(0.0, 0.0)); + for (int bi = 0; bi < nb; ++bi) { + cd sm(0.0, 0.0); + for (int m = 0; m < L; ++m) { + const long jj = (long) j + m - half; + if (jj < 0 || jj >= (long) nf) continue; + sm += ker[(size_t) m] * R[(size_t) jj * (size_t) nb + (size_t) bi]; + } + const cd Sv = S[j * (size_t) nb + (size_t) bi]; + const cd Ev = E[j * (size_t) nb + (size_t) bi]; + const cd locked = std::abs(Sv) * unit(Ev) * unit(sm); + const cd d = locked - Sv; + const int b = b_lo + bi; + dz[(size_t) b] += d; + if (b != 0 && b != N / 2) dz[(size_t) (N - b)] += std::conj(d); + } + fft.run(dz, true); + const size_t off = j * (size_t) hop; + for (int n = 0; n < N; ++n) { + ola[off + (size_t) n] += dz[(size_t) n].real() / (double) N * w[(size_t) n]; + } + } + for (size_t i = 0; i < k; ++i) { + const double e = env[pad + i]; + out[i] = (float) ((double) s[i] + (e > 1e-11 ? ola[pad + i] / e : 0.0)); + } + return true; +} + +} // namespace eliza_paradee diff --git a/tools/paradee/src/paradee.cpp b/tools/paradee/src/paradee.cpp new file mode 100644 index 000000000..a00d72323 --- /dev/null +++ b/tools/paradee/src/paradee.cpp @@ -0,0 +1,670 @@ +// SPDX-License-Identifier: MIT +// +// paradee.cpp — Paradee-8M v1.0 loader and synthesis (see include/paradee.h). +// +// The text side and decoder are Kokoro's modules at smaller widths, so this +// file only binds the GGUF to the generalised tools/kokoro layers +// (predictor_forward / decoder_forward) with the student-specific options: +// +// - two learned styles of different sizes (text 32, decoder 16) instead of +// one 256-d voice row, +// - the asr_proj GELU MLP (192 -> 512 -> 512) between the text encoder and +// the decoder, +// - the trained InstanceNorm affine inside every AdaIN1d, +// - torch.round (half to even) for the integer durations, +// - the canonical source STFT (exact zeros for near-empty bins), +// - the phase-locking filter after the generator. + +#include "paradee.h" + +#include "kokoro-decoder.h" +#include "kokoro-layers.h" +#include "kokoro-model-internal.h" +#include "kokoro-predictor.h" + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eliza_paradee { + +using eliza_kokoro::DecoderConfig; +using eliza_kokoro::PredictorConfig; +using eliza_kokoro::PredictorOut; + +const char * paradee_status_str(paradee_status st) noexcept { + switch (st) { + case PARADEE_OK: return "ok"; + case PARADEE_E_INVALID_ARG: return "invalid argument"; + case PARADEE_E_UNKNOWN_SYMBOL: return "phoneme outside the vocabulary"; + case PARADEE_E_TOO_LONG: return "too many phonemes"; + case PARADEE_E_RUNTIME: return "runtime error"; + case PARADEE_CANCELLED: return "cancelled"; + case PARADEE_E_OOM: return "out of memory"; + } + return "unknown"; +} + +struct paradee_hparams { + int text_n_layer = 0, text_n_head = 0, text_d_model = 0, text_d_ff = 0; + int text_vocab_size = 0, text_max_position = 0; + int hidden_dim = 0, predictor_n_layer = 0, predictor_max_dur = 0, text_encoder_kernel = 0; + int style_text_dim = 0, style_decoder_dim = 0, asr_dim = 0; + int up_rates[2] = { 0, 0 }; + int istft_hop = 0; + int sample_rate = 0; + int pad_id = 0, max_ids = 0; +}; + +struct paradee_model { + paradee_hparams hp; + paradee_lock_params lock; + std::unordered_map vocab; // code point -> id + gguf_context * gguf = nullptr; + ggml_context * file_ctx = nullptr; // tensors as stored + ggml_context * ctx = nullptr; // all-F32 view the layers read + std::vector style_text, style_dec; + std::mutex mu; +}; + +void paradee_model_deleter::operator()(paradee_model * m) const noexcept { + if (!m) return; + if (m->ctx && m->ctx != m->file_ctx) ggml_free(m->ctx); + if (m->file_ctx) ggml_free(m->file_ctx); + if (m->gguf) gguf_free(m->gguf); + delete m; +} + +namespace { + +bool read_i32(gguf_context * g, const char * key, int & out, std::string & err) { + const int64_t id = gguf_find_key(g, key); + if (id < 0) { err = std::string("missing GGUF key ") + key; return false; } + switch (gguf_get_kv_type(g, id)) { + case GGUF_TYPE_INT32: out = gguf_get_val_i32(g, id); return true; + case GGUF_TYPE_UINT32: out = (int) gguf_get_val_u32(g, id); return true; + default: err = std::string("GGUF key ") + key + " is not an integer"; return false; + } +} + +bool read_f32(gguf_context * g, const char * key, float & out, std::string & err) { + const int64_t id = gguf_find_key(g, key); + if (id < 0) { err = std::string("missing GGUF key ") + key; return false; } + if (gguf_get_kv_type(g, id) != GGUF_TYPE_FLOAT32) { err = std::string("GGUF key ") + key + " is not f32"; return false; } + out = gguf_get_val_f32(g, id); + return true; +} + +bool read_i32_array(gguf_context * g, const char * key, std::vector & out, std::string & err) { + const int64_t id = gguf_find_key(g, key); + if (id < 0) { err = std::string("missing GGUF key ") + key; return false; } + if (gguf_get_kv_type(g, id) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, id) != GGUF_TYPE_INT32) { + err = std::string("GGUF key ") + key + " is not an int32 array"; + return false; + } + const size_t n = gguf_get_arr_n(g, id); + const int32_t * data = (const int32_t *) gguf_get_arr_data(g, id); + out.assign(data, data + n); + return true; +} + +// Decode one UTF-8 code point at s[i]; advances i. Returns false on invalid input. +bool utf8_next(const std::string & s, size_t & i, uint32_t & cp) { + const unsigned char c = (unsigned char) s[i]; + int len = 0; + if (c < 0x80) { cp = c; len = 1; } + else if ((c & 0xE0) == 0xC0) { cp = c & 0x1F; len = 2; } + else if ((c & 0xF0) == 0xE0) { cp = c & 0x0F; len = 3; } + else if ((c & 0xF8) == 0xF0) { cp = c & 0x07; len = 4; } + else return false; + if (i + (size_t) len > s.size()) return false; + for (int k = 1; k < len; ++k) { + const unsigned char cc = (unsigned char) s[i + (size_t) k]; + if ((cc & 0xC0) != 0x80) return false; + cp = (cp << 6) | (cc & 0x3F); + } + // Reject overlong forms, surrogates and out-of-range values. + if ((len == 2 && cp < 0x80) || (len == 3 && cp < 0x800) || (len == 4 && cp < 0x10000) || + cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) { + return false; + } + i += (size_t) len; + return true; +} + +bool load_vocab(gguf_context * g, int n_token, std::unordered_map & vocab, std::string & err) { + const int64_t sid = gguf_find_key(g, "paradee.vocab.symbols"); + std::vector ids; + if (sid < 0 || gguf_get_kv_type(g, sid) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, sid) != GGUF_TYPE_STRING) { + err = "missing GGUF string array paradee.vocab.symbols"; + return false; + } + if (!read_i32_array(g, "paradee.vocab.ids", ids, err)) return false; + const size_t n = gguf_get_arr_n(g, sid); + if (n != ids.size() || n == 0) { err = "paradee.vocab.symbols / ids length mismatch"; return false; } + for (size_t i = 0; i < n; ++i) { + const std::string sym = gguf_get_arr_str(g, sid, i); + size_t pos = 0; + uint32_t cp = 0; + if (sym.empty() || !utf8_next(sym, pos, cp) || pos != sym.size()) { + err = "vocabulary symbol " + std::to_string(i) + " is not one code point"; + return false; + } + const int id = ids[i]; + if (id <= 0 || id >= n_token) { err = "vocabulary id out of range"; return false; } + if (!vocab.emplace(cp, id).second) { err = "duplicate vocabulary symbol"; return false; } + } + return true; +} + +std::string codepoint_label(uint32_t cp) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "U+%04X", (unsigned) cp); + return buf; +} + +} // namespace + +paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string & err) noexcept { + err.clear(); + try { + paradee_model_ptr m(new paradee_model()); + gguf_init_params params = { /* no_alloc = */ false, /* ctx = */ &m->file_ctx }; + m->gguf = gguf_init_from_file(gguf_path.c_str(), params); + if (!m->gguf || !m->file_ctx) { + err = "gguf_init_from_file failed for '" + gguf_path + "'"; + return nullptr; + } + gguf_context * g = m->gguf; + { + const int64_t id = gguf_find_key(g, "general.architecture"); + const char * arch = (id >= 0 && gguf_get_kv_type(g, id) == GGUF_TYPE_STRING) ? gguf_get_val_str(g, id) : ""; + if (std::string(arch) != "paradee") { + err = std::string("expected general.architecture='paradee', got '") + arch + "'"; + return nullptr; + } + } + paradee_hparams & h = m->hp; + std::vector rates; + bool ok = + read_i32(g, "paradee.text.n_layer", h.text_n_layer, err) && + read_i32(g, "paradee.text.n_head", h.text_n_head, err) && + read_i32(g, "paradee.text.d_model", h.text_d_model, err) && + read_i32(g, "paradee.text.d_ff", h.text_d_ff, err) && + read_i32(g, "paradee.text.vocab_size", h.text_vocab_size, err) && + read_i32(g, "paradee.text.max_position", h.text_max_position, err) && + read_i32(g, "paradee.hidden_dim", h.hidden_dim, err) && + read_i32(g, "paradee.predictor.n_layer", h.predictor_n_layer, err) && + read_i32(g, "paradee.predictor.max_dur", h.predictor_max_dur, err) && + read_i32(g, "paradee.text_encoder.kernel", h.text_encoder_kernel, err) && + read_i32(g, "paradee.style.text_dim", h.style_text_dim, err) && + read_i32(g, "paradee.style.decoder_dim", h.style_decoder_dim, err) && + read_i32(g, "paradee.asr.dim", h.asr_dim, err) && + read_i32_array(g, "paradee.decoder.upsample_rates", rates, err) && + read_i32(g, "paradee.decoder.istft_hop", h.istft_hop, err) && + read_i32(g, "paradee.audio.sample_rate", h.sample_rate, err) && + read_i32(g, "paradee.input.pad_id", h.pad_id, err) && + read_i32(g, "paradee.input.max_ids", h.max_ids, err) && + read_i32(g, "paradee.lock.n_fft", m->lock.n_fft, err) && + read_i32(g, "paradee.lock.hop", m->lock.hop, err) && + read_i32(g, "paradee.lock.smooth_frames", m->lock.smooth_frames, err) && + read_i32(g, "paradee.lock.f0_hop", m->lock.f0_hop, err) && + read_f32(g, "paradee.lock.voiced_f0_hz", m->lock.voiced_f0_hz, err); + if (!ok) return nullptr; + int band_lo = 0, band_hi = 0; + if (!read_i32(g, "paradee.lock.band_lo_hz", band_lo, err) || + !read_i32(g, "paradee.lock.band_hi_hz", band_hi, err)) { + return nullptr; + } + m->lock.band_lo_hz = (float) band_lo; + m->lock.band_hi_hz = (float) band_hi; + m->lock.sample_rate = h.sample_rate; + const int widths[] = { + h.text_n_layer, h.text_n_head, h.text_d_model, h.text_d_ff, h.text_vocab_size, + h.text_max_position, h.hidden_dim, h.predictor_n_layer, h.predictor_max_dur, + h.text_encoder_kernel, h.style_text_dim, h.style_decoder_dim, h.asr_dim, h.istft_hop, + }; + const bool widths_ok = std::all_of(std::begin(widths), std::end(widths), + [](int v) { return v > 0 && v <= (1 << 16); }); + // Phase-lock parameters come from the file too; bound them so a + // corrupt GGUF cannot ask for giant FFT buffers. + const paradee_lock_params & lp = m->lock; + const bool lock_ok = lp.n_fft >= 16 && lp.n_fft <= 8192 && (lp.n_fft & (lp.n_fft - 1)) == 0 && + lp.hop > 0 && lp.hop <= lp.n_fft && lp.smooth_frames >= 1 && + lp.smooth_frames <= 1025 && lp.f0_hop > 0 && std::isfinite(lp.voiced_f0_hz); + if (!widths_ok || !lock_ok || rates.size() != 2 || rates[0] <= 0 || rates[0] > 64 || + rates[1] <= 0 || rates[1] > 64 || + h.pad_id != 0 || h.max_ids < 3 || h.max_ids > h.text_max_position || + h.text_d_model % h.text_n_head != 0 || h.sample_rate <= 0 || h.sample_rate > 192000) { + err = "unsupported Paradee hparams"; + return nullptr; + } + h.up_rates[0] = rates[0]; + h.up_rates[1] = rates[1]; + if (!load_vocab(g, h.text_vocab_size, m->vocab, err)) return nullptr; + + // All-F32 working context (the converter's default output is already + // F32; an F16 file is widened once here). + bool all_f32 = true; + const int64_t n_tensors = gguf_get_n_tensors(g); + size_t f32_bytes = 0; + for (int64_t i = 0; i < n_tensors; ++i) { + ggml_tensor * t = ggml_get_tensor(m->file_ctx, gguf_get_tensor_name(g, i)); + if (!t) continue; + if (t->type != GGML_TYPE_F32) all_f32 = false; + f32_bytes += GGML_PAD((size_t) ggml_nelements(t) * sizeof(float), GGML_MEM_ALIGN); + } + if (all_f32) { + m->ctx = m->file_ctx; + } else { + ggml_init_params p = { f32_bytes + (size_t) (n_tensors + 1) * ggml_tensor_overhead(), nullptr, false }; + m->ctx = ggml_init(p); + if (!m->ctx) { err = "ggml_init failed"; return nullptr; } + for (int64_t i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(g, i); + ggml_tensor * src = ggml_get_tensor(m->file_ctx, name); + if (!src) continue; + ggml_tensor * dst = ggml_new_tensor(m->ctx, GGML_TYPE_F32, ggml_n_dims(src), src->ne); + ggml_set_name(dst, name); + const int64_t n = ggml_nelements(src); + if (src->type == GGML_TYPE_F32) { + std::memcpy(dst->data, src->data, (size_t) n * sizeof(float)); + } else if (src->type == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((const ggml_fp16_t *) src->data, (float *) dst->data, n); + } else { + err = std::string("unsupported tensor type for '") + name + "'"; + return nullptr; + } + } + } + + ggml_tensor * st = ggml_get_tensor(m->ctx, "paradee.style.text"); + ggml_tensor * sd = ggml_get_tensor(m->ctx, "paradee.style.decoder"); + if (!st || !sd || ggml_nelements(st) != h.style_text_dim || ggml_nelements(sd) != h.style_decoder_dim) { + err = "missing or mis-sized paradee.style.text / paradee.style.decoder"; + return nullptr; + } + m->style_text.assign((const float *) st->data, (const float *) st->data + h.style_text_dim); + m->style_dec.assign((const float *) sd->data, (const float *) sd->data + h.style_decoder_dim); + + const char * required[] = { + "paradee.bert.token_embd.weight", "paradee.bert.layer.attn_q.weight", + "paradee.bert_encoder.weight", "paradee.predictor.duration_proj.weight", + "paradee.predictor.F0_proj.weight", "paradee.text_encoder.embd.weight", + "paradee.asr_proj.0.weight", "paradee.asr_proj.1.weight", + "paradee.decoder.encode.conv1.weight", "paradee.gen.conv_post.weight", + }; + for (const char * name : required) { + if (!ggml_get_tensor(m->ctx, name)) { + err = std::string("required tensor missing: ") + name; + return nullptr; + } + } + ggml_tensor * be = ggml_get_tensor(m->ctx, "paradee.bert_encoder.weight"); + ggml_tensor * ap = ggml_get_tensor(m->ctx, "paradee.asr_proj.0.weight"); + ggml_tensor * tok = ggml_get_tensor(m->ctx, "paradee.bert.token_embd.weight"); + if ((int) be->ne[1] != h.hidden_dim || (int) be->ne[0] != h.text_d_model || + (int) ap->ne[1] != h.asr_dim || (int) tok->ne[1] != h.text_vocab_size) { + err = "tensor shapes do not match the Paradee hparams"; + return nullptr; + } + return m; + } catch (const std::exception & e) { + err = std::string("paradee_load_model: ") + e.what(); + return nullptr; + } +} + +int paradee_sample_rate(const paradee_model * model) noexcept { + return model ? model->hp.sample_rate : 0; +} + +const paradee_lock_params * paradee_get_lock_params(const paradee_model * model) noexcept { + return model ? &model->lock : nullptr; +} + +namespace { + +// Vocabulary ids of every symbol; `max_symbols` = 0 means no limit. With +// `n_dropped` non-null, symbols outside the vocabulary are skipped and +// counted instead of rejected, and empty input maps to no ids. +paradee_status map_symbols( + const paradee_model * model, + const std::string & misaki_utf8, + size_t max_symbols, + bool pads, + std::vector & ids, + std::string & err, + size_t * n_dropped = nullptr) noexcept { + err.clear(); + ids.clear(); + if (n_dropped) *n_dropped = 0; + if (!model) { err = "null model"; return PARADEE_E_INVALID_ARG; } + if (misaki_utf8.empty()) { + if (n_dropped) return PARADEE_OK; + err = "empty phoneme string"; + return PARADEE_E_INVALID_ARG; + } + try { + if (pads) ids.push_back(model->hp.pad_id); + const size_t lead = ids.size(); + size_t i = 0; + while (i < misaki_utf8.size()) { + uint32_t cp = 0; + const size_t at = i; + if (!utf8_next(misaki_utf8, i, cp)) { + err = "invalid UTF-8 at byte " + std::to_string(at); + ids.clear(); + return PARADEE_E_INVALID_ARG; + } + const auto it = model->vocab.find(cp); + if (it == model->vocab.end()) { + if (n_dropped) { ++*n_dropped; continue; } + err = "phoneme " + codepoint_label(cp) + " at byte " + std::to_string(at) + + " is not in the Paradee vocabulary"; + ids.clear(); + return PARADEE_E_UNKNOWN_SYMBOL; + } + if (max_symbols && ids.size() - lead >= max_symbols) { + err = "more than " + std::to_string(max_symbols) + " phonemes"; + ids.clear(); + return PARADEE_E_TOO_LONG; + } + ids.push_back(it->second); + } + if (pads) ids.push_back(model->hp.pad_id); + } catch (const std::bad_alloc &) { + err = "out of memory"; + ids.clear(); + return PARADEE_E_OOM; + } catch (const std::exception & e) { + err = e.what(); + ids.clear(); + return PARADEE_E_RUNTIME; + } + return PARADEE_OK; +} + +} // namespace + +paradee_status paradee_phonemes_to_ids( + const paradee_model * model, + const std::string & misaki_utf8, + std::vector & ids, + std::string & err) noexcept { + if (!model) { ids.clear(); err = "null model"; return PARADEE_E_INVALID_ARG; } + return map_symbols(model, misaki_utf8, (size_t) model->hp.max_ids - 2, /*pads*/ true, ids, err); +} + +paradee_status paradee_phonemes_to_symbol_ids( + const paradee_model * model, + const std::string & misaki_utf8, + std::vector & ids, + std::string & err) noexcept { + return map_symbols(model, misaki_utf8, 0, /*pads*/ false, ids, err); +} + +paradee_status paradee_phonemes_to_symbol_ids_lenient( + const paradee_model * model, + const std::string & misaki_utf8, + std::vector & ids, + size_t & n_dropped, + std::string & err) noexcept { + return map_symbols(model, misaki_utf8, 0, /*pads*/ false, ids, err, &n_dropped); +} + +namespace { + +// Layer strategies Paradee opts into (kokoro-layers.h): per-tap SGEMM +// convolutions and the polynomial Snake activation. PARADEE_EXACT_LAYERS=1 +// turns them off (the arithmetic Kokoro-82M uses), for A/B checks. +eliza_kokoro::detail::KokoroLayerOptions fast_layers() { + eliza_kokoro::detail::KokoroLayerOptions o; + const char * exact = std::getenv("PARADEE_EXACT_LAYERS"); + const bool fast = !(exact && exact[0] == '1'); + o.tap_gemm_conv = fast; + o.fast_snake = fast; + return o; +} + +struct cancel_state { + paradee_cancel_fn fn; + void * user; +}; + +bool cancel_trampoline(void * p) { + const cancel_state * c = (const cancel_state *) p; + return c && c->fn && c->fn(c->user); +} + +} // namespace + +namespace { + +// Decoder + generator + phase lock. Caller holds model->mu. +paradee_status decode_locked( + const paradee_model * model, + const float * asr_ct, int A, int T, + const float * F0, const float * N, + paradee_cancel_fn cancel, void * user, + std::vector & dec, std::vector & src, std::vector & pcm, + std::string & err, const std::vector * src_in = nullptr) { + const paradee_hparams & h = model->hp; + cancel_state cs{ cancel, user }; + DecoderConfig dc; + dc.ns = "paradee"; + dc.instnorm_affine = true; + dc.canonical_source_stft = true; + dc.precise_source = true; + dc.up_rates[0] = h.up_rates[0]; + dc.up_rates[1] = h.up_rates[1]; + dc.istft_hop = h.istft_hop; + dc.har_source_out = &src; + dc.har_source_in = src_in; + dc.cancel = cancel ? cancel_trampoline : nullptr; + dc.cancel_user = &cs; + if (!eliza_kokoro::decoder_forward(model->ctx, dc, asr_ct, A, T, F0, N, + model->style_dec.data(), dec, err)) { + if (dc.cancelled) { err = "cancelled"; return PARADEE_CANCELLED; } + if (err.empty()) err = "decoder failed"; + return PARADEE_E_RUNTIME; + } + if (!paradee_phase_lock(dec.data(), dec.size(), src.data(), src.size(), + F0, (size_t) 2 * (size_t) T, model->lock, pcm, cancel, user)) { + if (cancel && cancel(user)) { err = "cancelled"; return PARADEE_CANCELLED; } + err = "phase lock rejected its parameters"; + return PARADEE_E_RUNTIME; + } + if (cancel && cancel(user)) { err = "cancelled"; return PARADEE_CANCELLED; } + return PARADEE_OK; +} + +} // namespace + +paradee_status paradee_synthesize_ids_staged( + const paradee_model * model, + const int32_t * ids, size_t n, + float speed, + paradee_cancel_fn cancel, void * user, + std::vector & pcm, + std::string & err, + paradee_stages * stages) noexcept { + err.clear(); + pcm.clear(); + if (!model) { err = "null model"; return PARADEE_E_INVALID_ARG; } + const paradee_hparams & h = model->hp; + if (!ids || n < 3 || n > (size_t) h.max_ids) { + err = "expected 3.." + std::to_string(h.max_ids) + " input ids, got " + std::to_string(n); + return n > (size_t) h.max_ids ? PARADEE_E_TOO_LONG : PARADEE_E_INVALID_ARG; + } + if (ids[0] != h.pad_id || ids[n - 1] != h.pad_id) { + err = "input ids must start and end with the pad id 0"; + return PARADEE_E_INVALID_ARG; + } + for (size_t i = 0; i < n; ++i) { + if (ids[i] < 0 || ids[i] >= h.text_vocab_size) { + err = "input id " + std::to_string(ids[i]) + " out of range"; + return PARADEE_E_INVALID_ARG; + } + } + if (!std::isfinite(speed) || speed < PARADEE_MIN_SPEED || speed > PARADEE_MAX_SPEED) { + err = "speed must be finite and in [" + std::to_string(PARADEE_MIN_SPEED) + ", " + + std::to_string(PARADEE_MAX_SPEED) + "]"; + return PARADEE_E_INVALID_ARG; + } + + try { + std::lock_guard lk(const_cast(model)->mu); + eliza_kokoro::detail::KokoroLayerOptionsScope layer_scope(fast_layers()); + cancel_state cs{ cancel, user }; + if (cancel && cancel(user)) { err = "cancelled"; return PARADEE_CANCELLED; } + + // 1. Text side. + PredictorConfig pc; + pc.ns = "paradee"; + pc.text_n_layer = h.text_n_layer; + pc.text_n_head = h.text_n_head; + pc.text_d_model = h.text_d_model; + pc.text_d_ff = h.text_d_ff; + pc.n_layer = h.predictor_n_layer; + pc.max_dur = h.predictor_max_dur; + pc.text_kernel = h.text_encoder_kernel; + pc.style = model->style_text.data(); + pc.torch_round = true; + pc.instnorm_affine = true; + pc.asr_proj_mlp = true; + pc.keep_bert = stages != nullptr; + pc.cancel = cancel ? cancel_trampoline : nullptr; + pc.cancel_user = &cs; + PredictorOut pred; + const std::vector id_vec(ids, ids + n); + if (!eliza_kokoro::predictor_forward(model->ctx, pc, id_vec, speed, pred, err)) { + if (pred.cancelled) { pcm.clear(); return PARADEE_CANCELLED; } + if (err.empty()) err = "text side failed"; + return PARADEE_E_RUNTIME; + } + if (pred.sty != h.style_text_dim || pred.asr_dim != h.asr_dim || pred.T_frame <= 0) { + err = "text side produced unexpected widths"; + return PARADEE_E_RUNTIME; + } + const int T = pred.T_frame; + const int A = pred.asr_dim; + std::vector asr_ct((size_t) A * (size_t) T); + for (int t = 0; t < T; ++t) { + const float * row = pred.asr.data() + (size_t) t * A; + for (int c = 0; c < A; ++c) asr_ct[(size_t) c * (size_t) T + t] = row[c]; + } + if (cancel && cancel(user)) { err = "cancelled"; return PARADEE_CANCELLED; } + + // 2. Decoder + generator, 3. phase lock. + std::vector src, dec; + const paradee_status dst = decode_locked(model, asr_ct.data(), A, T, pred.F0_pred.data(), + pred.N_pred.data(), cancel, user, dec, src, pcm, err); + if (dst != PARADEE_OK) { pcm.clear(); return dst; } + + if (stages) { + stages->T_phon = pred.T_phon; + stages->T_frame = T; + stages->d_model = h.text_d_model; + stages->hid = pred.hid; + stages->sty = pred.sty; + stages->asr_dim = A; + stages->bert = std::move(pred.bert); + stages->d = std::move(pred.d); + stages->dur = std::move(pred.duration); + stages->dur_int = std::move(pred.pred_dur); + stages->F0 = std::move(pred.F0_pred); + stages->N = std::move(pred.N_pred); + stages->asr = std::move(asr_ct); + stages->dec = std::move(dec); + stages->src = std::move(src); + stages->wav = pcm; + } + return PARADEE_OK; + } catch (const std::bad_alloc &) { + pcm.clear(); + err = "paradee synthesis: out of memory"; + return PARADEE_E_OOM; + } catch (const std::exception & e) { + pcm.clear(); + err = std::string("paradee synthesis: ") + e.what(); + return PARADEE_E_RUNTIME; + } +} + +paradee_status paradee_synthesize_ids( + const paradee_model * model, + const int32_t * ids, size_t n, + float speed, + paradee_cancel_fn cancel, void * user, + std::vector & pcm, + std::string & err) noexcept { + return paradee_synthesize_ids_staged(model, ids, n, speed, cancel, user, pcm, err, nullptr); +} + +} // namespace eliza_paradee + +namespace eliza_paradee { + +paradee_status paradee_synthesize( + const paradee_model * model, + const std::vector & input_ids, + const paradee_synth_options & opts, + paradee_audio & out, + std::string & err) noexcept { + out.sample_rate = model ? model->hp.sample_rate : 24000; + return paradee_synthesize_ids_staged(model, input_ids.data(), input_ids.size(), opts.speed, + opts.cancel, opts.cancel_user_data, out.samples, err, nullptr); +} + +} // namespace eliza_paradee + +namespace eliza_paradee { + +paradee_status paradee_decode_staged( + const paradee_model * model, + const float * asr, int asr_dim, int T_frame, + const float * F0, const float * N, + const std::vector * src, + paradee_cancel_fn cancel, void * user, + paradee_stages * stages, + std::string & err) noexcept { + err.clear(); + if (!model || !asr || !F0 || !N || !stages || T_frame <= 0 || asr_dim != model->hp.asr_dim) { + err = "invalid decode arguments"; + return PARADEE_E_INVALID_ARG; + } + try { + std::lock_guard lk(const_cast(model)->mu); + eliza_kokoro::detail::KokoroLayerOptionsScope layer_scope(fast_layers()); + stages->T_frame = T_frame; + stages->asr_dim = asr_dim; + const paradee_hparams & h = model->hp; + const size_t n_src = (size_t) T_frame * 2 * (size_t) (h.up_rates[0] * h.up_rates[1] * h.istft_hop); + if (src && src->size() != n_src) { + err = "src must have " + std::to_string(n_src) + " samples"; + return PARADEE_E_INVALID_ARG; + } + return decode_locked(model, asr, asr_dim, T_frame, F0, N, cancel, user, + stages->dec, stages->src, stages->wav, err, src); + } catch (const std::bad_alloc &) { + err = "out of memory"; + return PARADEE_E_OOM; + } catch (const std::exception & e) { + err = e.what(); + return PARADEE_E_RUNTIME; + } +} + +} // namespace eliza_paradee diff --git a/tools/paradee/tests/CMakeLists.txt b/tools/paradee/tests/CMakeLists.txt new file mode 100644 index 000000000..3017d41df --- /dev/null +++ b/tools/paradee/tests/CMakeLists.txt @@ -0,0 +1,46 @@ +# tools/paradee/tests/CMakeLists.txt +# +# test-paradee-phase-lock self-contained (committed fixture), always runs. +# test-paradee-api needs the Paradee GGUF. +# test-paradee-parity needs the GGUF and a reference dump from +# tools/dump_paradee_reference.py. +# +# The model-backed tests read PARADEE_GGUF / PARADEE_REF_DIR from the +# environment, or from the cache variables of the same names, and report +# SKIP (exit 77) when they are not set. See tools/paradee/README.md. + +set(PARADEE_GGUF "" CACHE FILEPATH "Paradee GGUF for test-paradee-api / test-paradee-parity") +set(PARADEE_REF_DIR "" CACHE PATH "dump_paradee_reference.py output for test-paradee-parity") + +set(PARADEE_FIXTURE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fixtures) + +add_executable(test-paradee-phase-lock test_paradee_phase_lock.cpp) +target_link_libraries(test-paradee-phase-lock PRIVATE paradee_lib) +target_compile_definitions(test-paradee-phase-lock PRIVATE PARADEE_FIXTURE_DIR="${PARADEE_FIXTURE_DIR}") +add_test(NAME test-paradee-phase-lock COMMAND test-paradee-phase-lock) + +add_executable(test-paradee-api test_paradee_api.cpp) +find_package(Threads REQUIRED) +target_link_libraries(test-paradee-api PRIVATE paradee_lib Threads::Threads) +add_test(NAME test-paradee-api COMMAND test-paradee-api) + +add_executable(test-paradee-parity test_paradee_parity.cpp) +target_link_libraries(test-paradee-parity PRIVATE paradee_lib) +target_include_directories(test-paradee-parity PRIVATE ${PROJECT_SOURCE_DIR}/vendor) +target_compile_definitions(test-paradee-parity PRIVATE PARADEE_FIXTURE_DIR="${PARADEE_FIXTURE_DIR}") +add_test(NAME test-paradee-parity COMMAND test-paradee-parity) + +set(_paradee_env "") +if (PARADEE_GGUF) + list(APPEND _paradee_env "PARADEE_GGUF=${PARADEE_GGUF}") +endif() +if (PARADEE_REF_DIR) + list(APPEND _paradee_env "PARADEE_REF_DIR=${PARADEE_REF_DIR}") +endif() +foreach(t test-paradee-api test-paradee-parity) + set_tests_properties(${t} PROPERTIES SKIP_RETURN_CODE 77) + if (_paradee_env) + set_tests_properties(${t} PROPERTIES ENVIRONMENT "${_paradee_env}") + endif() +endforeach() +set_tests_properties(test-paradee-parity PROPERTIES TIMEOUT 900) diff --git a/tools/paradee/tests/fixtures/phase_lock_synthetic.f32 b/tools/paradee/tests/fixtures/phase_lock_synthetic.f32 new file mode 100644 index 000000000..04897e800 Binary files /dev/null and b/tools/paradee/tests/fixtures/phase_lock_synthetic.f32 differ diff --git a/tools/paradee/tests/fixtures/sentences.misaki.json b/tools/paradee/tests/fixtures/sentences.misaki.json new file mode 100644 index 000000000..7af532586 --- /dev/null +++ b/tools/paradee/tests/fixtures/sentences.misaki.json @@ -0,0 +1,758 @@ +{ + "source": "sahilmahendrakar/Paradee-8M-v1.0@c7f83f86c04e84d0ae94cbe1aad7618e0544f80a samples/sentences.txt", + "g2p": "misaki en.G2P(trf=False, british=False, fallback=EspeakFallback(british=False), unk='')", + "versions": { + "misaki": "0.9.4", + "spacy": "3.8.16", + "en_core_web_sm": "3.8.0", + "espeak-ng": "1.52.0" + }, + "sentences": [ + { + "index": 1, + "text": "As of August 2015, there were 169 proposed targets for these goals and 304 proposed indicators to show compliance.", + "phonemes": "ˌæz ʌv ˈɔɡəst twˈɛnti fˌɪftˈin, ðɛɹ wɜɹ wˈʌn hˈʌndɹəd sˈɪksti nˈIn pɹəpˈOzd tˈɑɹɡəts fɔɹ ðiz ɡˈOlz ænd θɹˈi hˈʌndɹəd fˈɔɹ pɹəpˈOzd ˈɪndəkˌATəɹz tə ʃˈO kəmplˈIəns.", + "ids": [ + 0, + 157, + 72, + 68, + 16, + 138, + 64, + 16, + 156, + 76, + 92, + 83, + 61, + 62, + 16, + 62, + 65, + 156, + 86, + 56, + 62, + 51, + 16, + 48, + 157, + 102, + 48, + 62, + 156, + 51, + 56, + 3, + 16, + 81, + 86, + 123, + 16, + 65, + 87, + 123, + 16, + 65, + 156, + 138, + 56, + 16, + 50, + 156, + 138, + 56, + 46, + 123, + 83, + 46, + 16, + 61, + 156, + 102, + 53, + 61, + 62, + 51, + 16, + 56, + 156, + 25, + 56, + 16, + 58, + 123, + 83, + 58, + 156, + 31, + 68, + 46, + 16, + 62, + 156, + 69, + 123, + 92, + 83, + 62, + 61, + 16, + 48, + 76, + 123, + 16, + 81, + 51, + 68, + 16, + 92, + 156, + 31, + 54, + 68, + 16, + 72, + 56, + 46, + 16, + 119, + 123, + 156, + 51, + 16, + 50, + 156, + 138, + 56, + 46, + 123, + 83, + 46, + 16, + 48, + 156, + 76, + 123, + 16, + 58, + 123, + 83, + 58, + 156, + 31, + 68, + 46, + 16, + 156, + 102, + 56, + 46, + 83, + 53, + 157, + 24, + 36, + 83, + 123, + 68, + 16, + 62, + 83, + 16, + 131, + 156, + 31, + 16, + 53, + 83, + 55, + 58, + 54, + 156, + 25, + 83, + 56, + 61, + 4, + 0 + ] + }, + { + "index": 2, + "text": "White dwarf stars, if they have a near companion, may then become Type Ia supernovae.", + "phonemes": "wˈIt dwˈɔɹf stˈɑɹz, ɪf ðA hæv ɐ nˌɪɹ kəmpˈænjən, mˈA ðˈɛn bəkˈʌm tˈIp ˈIə sˌupəɹnˈOvˌi.", + "ids": [ + 0, + 65, + 156, + 25, + 62, + 16, + 46, + 65, + 156, + 76, + 123, + 48, + 16, + 61, + 62, + 156, + 69, + 123, + 68, + 3, + 16, + 102, + 48, + 16, + 81, + 24, + 16, + 50, + 72, + 64, + 16, + 70, + 16, + 56, + 157, + 102, + 123, + 16, + 53, + 83, + 55, + 58, + 156, + 72, + 56, + 52, + 83, + 56, + 3, + 16, + 55, + 156, + 24, + 16, + 81, + 156, + 86, + 56, + 16, + 44, + 83, + 53, + 156, + 138, + 55, + 16, + 62, + 156, + 25, + 58, + 16, + 156, + 25, + 83, + 16, + 61, + 157, + 63, + 58, + 83, + 123, + 56, + 156, + 31, + 64, + 157, + 51, + 4, + 0 + ] + }, + { + "index": 3, + "text": "According to Herodotus, this was because they had decided in council that they could not beat the Allies in a naval battle.", + "phonemes": "əkˈɔɹdɪŋ tə hɛɹˈɑdəTəs, ðɪs wʌz bəkˈʌz ðA hæd dəsˈIdᵻd ɪn kˈWnsᵊl ðæt ðA kʊd nˌɑt bˈit ði ˈælˌIz ɪn ɐ nˈAvᵊl bˈæTᵊl.", + "ids": [ + 0, + 83, + 53, + 156, + 76, + 123, + 46, + 102, + 112, + 16, + 62, + 83, + 16, + 50, + 86, + 123, + 156, + 69, + 46, + 83, + 36, + 83, + 61, + 3, + 16, + 81, + 102, + 61, + 16, + 65, + 138, + 68, + 16, + 44, + 83, + 53, + 156, + 138, + 68, + 16, + 81, + 24, + 16, + 50, + 72, + 46, + 16, + 46, + 83, + 61, + 156, + 25, + 46, + 177, + 46, + 16, + 102, + 56, + 16, + 53, + 156, + 39, + 56, + 61, + 42, + 54, + 16, + 81, + 72, + 62, + 16, + 81, + 24, + 16, + 53, + 135, + 46, + 16, + 56, + 157, + 69, + 62, + 16, + 44, + 156, + 51, + 62, + 16, + 81, + 51, + 16, + 156, + 72, + 54, + 157, + 25, + 68, + 16, + 102, + 56, + 16, + 70, + 16, + 56, + 156, + 24, + 64, + 42, + 54, + 16, + 44, + 156, + 72, + 36, + 42, + 54, + 4, + 0 + ] + }, + { + "index": 4, + "text": "In 1997 Hurlbut's career focused on light as applied to photography and film, and he owned a lighting business in Pasadena, California.", + "phonemes": "ɪn nˌIntˈin nˈIndi sˈɛvən hˈɜɹlbʌts kəɹˈɪɹ fˈOkəst ˌɔn lˈIt æz əplˈId tə fətˈɑɡɹəfi ænd fˈɪlm, ænd hi ˈOnd ɐ lˈITɪŋ bˈɪznəs ɪn pˌæsədˈinə, kˌæləfˈɔɹnjə.", + "ids": [ + 0, + 102, + 56, + 16, + 56, + 157, + 25, + 56, + 62, + 156, + 51, + 56, + 16, + 56, + 156, + 25, + 56, + 46, + 51, + 16, + 61, + 156, + 86, + 64, + 83, + 56, + 16, + 50, + 156, + 87, + 123, + 54, + 44, + 138, + 62, + 61, + 16, + 53, + 83, + 123, + 156, + 102, + 123, + 16, + 48, + 156, + 31, + 53, + 83, + 61, + 62, + 16, + 157, + 76, + 56, + 16, + 54, + 156, + 25, + 62, + 16, + 72, + 68, + 16, + 83, + 58, + 54, + 156, + 25, + 46, + 16, + 62, + 83, + 16, + 48, + 83, + 62, + 156, + 69, + 92, + 123, + 83, + 48, + 51, + 16, + 72, + 56, + 46, + 16, + 48, + 156, + 102, + 54, + 55, + 3, + 16, + 72, + 56, + 46, + 16, + 50, + 51, + 16, + 156, + 31, + 56, + 46, + 16, + 70, + 16, + 54, + 156, + 25, + 36, + 102, + 112, + 16, + 44, + 156, + 102, + 68, + 56, + 83, + 61, + 16, + 102, + 56, + 16, + 58, + 157, + 72, + 61, + 83, + 46, + 156, + 51, + 56, + 83, + 3, + 16, + 53, + 157, + 72, + 54, + 83, + 48, + 156, + 76, + 123, + 56, + 52, + 83, + 4, + 0 + ] + }, + { + "index": 5, + "text": "Harvey ended with a century on his Ashes debut, scoring 112 from 183 balls in an innings noted for powerful driving on both sides of the wicket.", + "phonemes": "hˈɑɹvi ˈɛndᵻd wɪð ɐ sˈɛnʧəɹi ˌɔn hɪz ˈæʃᵻz dAbjˈu, skˈɔɹɪŋ wˈʌn hˈʌndɹəd twˈɛlv fɹʌm wˈʌn hˈʌndɹəd ˈATi θɹˈi bˈɔlz ɪn ɐn ˈɪnɪŋz nˈOTᵻd fɔɹ pˈWəɹfᵊl dɹˈIvɪŋ ˌɔn bˈOθ sˈIdz ʌv ðə wˈɪkət.", + "ids": [ + 0, + 50, + 156, + 69, + 123, + 64, + 51, + 16, + 156, + 86, + 56, + 46, + 177, + 46, + 16, + 65, + 102, + 81, + 16, + 70, + 16, + 61, + 156, + 86, + 56, + 133, + 83, + 123, + 51, + 16, + 157, + 76, + 56, + 16, + 50, + 102, + 68, + 16, + 156, + 72, + 131, + 177, + 68, + 16, + 46, + 24, + 44, + 52, + 156, + 63, + 3, + 16, + 61, + 53, + 156, + 76, + 123, + 102, + 112, + 16, + 65, + 156, + 138, + 56, + 16, + 50, + 156, + 138, + 56, + 46, + 123, + 83, + 46, + 16, + 62, + 65, + 156, + 86, + 54, + 64, + 16, + 48, + 123, + 138, + 55, + 16, + 65, + 156, + 138, + 56, + 16, + 50, + 156, + 138, + 56, + 46, + 123, + 83, + 46, + 16, + 156, + 24, + 36, + 51, + 16, + 119, + 123, + 156, + 51, + 16, + 44, + 156, + 76, + 54, + 68, + 16, + 102, + 56, + 16, + 70, + 56, + 16, + 156, + 102, + 56, + 102, + 112, + 68, + 16, + 56, + 156, + 31, + 36, + 177, + 46, + 16, + 48, + 76, + 123, + 16, + 58, + 156, + 39, + 83, + 123, + 48, + 42, + 54, + 16, + 46, + 123, + 156, + 25, + 64, + 102, + 112, + 16, + 157, + 76, + 56, + 16, + 44, + 156, + 31, + 119, + 16, + 61, + 156, + 25, + 46, + 68, + 16, + 138, + 64, + 16, + 81, + 83, + 16, + 65, + 156, + 102, + 53, + 83, + 62, + 4, + 0 + ] + } + ] +} diff --git a/tools/paradee/tests/test_paradee_api.cpp b/tools/paradee/tests/test_paradee_api.cpp new file mode 100644 index 000000000..d058371f5 --- /dev/null +++ b/tools/paradee/tests/test_paradee_api.cpp @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT +// +// test-paradee-api — the eliza_paradee C++ API contract on the real model: +// loading, phoneme mapping (pads, unknown symbols, length limit, UTF-8), +// input validation, determinism, speed, and cancellation (before the call, +// between stages and inside the generator loops). +// +// Needs PARADEE_GGUF (environment); exits 77 (CTest SKIP) without it. + +#include "paradee.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace eliza_paradee; + +static int failures = 0; +#define CHECK(cond, ...) do { if (!(cond)) { std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + std::fprintf(stderr, __VA_ARGS__); std::fprintf(stderr, "\n"); ++failures; } } while (0) + +struct counter { std::atomic polls{0}; int stop_after = 0; }; +static bool cancel_after(void * u) { + counter * c = (counter *) u; + return ++c->polls > c->stop_after; +} + +int main() { + const char * gguf = std::getenv("PARADEE_GGUF"); + if (!gguf || !*gguf) { + std::printf("test-paradee-api: PARADEE_GGUF not set, skipping\n"); + return 77; + } + std::string err; + + // Loader rejections. + CHECK(!paradee_load_model("/nonexistent/paradee.gguf", err) && !err.empty(), "missing file accepted"); + auto model = paradee_load_model(gguf, err); + if (!model) { std::fprintf(stderr, "load failed: %s\n", err.c_str()); return 1; } + CHECK(paradee_sample_rate(model.get()) == 24000, "sample rate %d", paradee_sample_rate(model.get())); + CHECK(paradee_sample_rate(nullptr) == 0, "null model sample rate"); + const paradee_lock_params * lp = paradee_get_lock_params(model.get()); + CHECK(lp && lp->n_fft == 1024 && lp->hop == 256 && lp->smooth_frames == 33 && lp->f0_hop == 300 && + lp->band_lo_hz == 2000.0f && lp->band_hi_hz == 8000.0f && lp->voiced_f0_hz == 60.0f, + "lock parameters not read from the GGUF"); + + // Phonemes -> ids. + std::vector ids; + CHECK(paradee_phonemes_to_ids(model.get(), "həlˈO wˈɜɹld.", ids, err) == PARADEE_OK, "%s", err.c_str()); + const std::vector want = { 0, 50, 83, 54, 156, 31, 16, 65, 156, 87, 123, 54, 46, 4, 0 }; + CHECK(ids == want, "hello world ids differ (got %zu ids)", ids.size()); + CHECK(paradee_phonemes_to_ids(model.get(), "hɛlo#", ids, err) == PARADEE_E_UNKNOWN_SYMBOL && + err.find("U+0023") != std::string::npos && ids.empty(), "unknown symbol: %s", err.c_str()); + CHECK(paradee_phonemes_to_ids(model.get(), "ɡ", ids, err) == PARADEE_OK && ids.size() == 3 && ids[1] == 92, + "U+0261 script g"); + CHECK(paradee_phonemes_to_ids(model.get(), "g", ids, err) == PARADEE_E_UNKNOWN_SYMBOL, "ASCII g is not in the vocab"); + CHECK(paradee_phonemes_to_ids(model.get(), "", ids, err) == PARADEE_E_INVALID_ARG, "empty accepted"); + CHECK(paradee_phonemes_to_ids(model.get(), std::string("a\xC3", 2), ids, err) == PARADEE_E_INVALID_ARG, + "truncated UTF-8 accepted"); + CHECK(paradee_phonemes_to_ids(model.get(), std::string("\xC0\xAF"), ids, err) == PARADEE_E_INVALID_ARG, + "overlong UTF-8 accepted"); + CHECK(paradee_phonemes_to_ids(nullptr, "a", ids, err) == PARADEE_E_INVALID_ARG, "null model"); + { + std::string ok510(510, 'a'), long511(511, 'a'); + CHECK(paradee_phonemes_to_ids(model.get(), ok510, ids, err) == PARADEE_OK && ids.size() == 512, + "510 phonemes must give 512 ids"); + CHECK(paradee_phonemes_to_ids(model.get(), long511, ids, err) == PARADEE_E_TOO_LONG && ids.empty(), + "511 phonemes accepted"); + } + + // Unpadded, unlimited form for chunkers. + { + std::vector sym; + CHECK(paradee_phonemes_to_symbol_ids(model.get(), "həlˈO wˈɜɹld.", sym, err) == PARADEE_OK && + sym == std::vector(want.begin() + 1, want.end() - 1), "symbol ids without pads"); + CHECK(paradee_phonemes_to_symbol_ids(model.get(), std::string(2000, 'a'), sym, err) == PARADEE_OK && + sym.size() == 2000, "symbol ids must not be length-limited"); + CHECK(paradee_phonemes_to_symbol_ids(model.get(), "a#", sym, err) == PARADEE_E_UNKNOWN_SYMBOL && sym.empty(), + "symbol ids must reject unknown symbols"); + + // Lenient form (the libelizainference contract): unknown symbols are + // dropped and counted, empty input is OK, bad UTF-8 is still rejected. + size_t dropped = 99; + CHECK(paradee_phonemes_to_symbol_ids_lenient(model.get(), "h#əlˈO g", sym, dropped, err) == PARADEE_OK && + dropped == 2 && sym.size() == 6, "lenient drop: %zu ids, %zu dropped", sym.size(), dropped); + CHECK(paradee_phonemes_to_symbol_ids_lenient(model.get(), "", sym, dropped, err) == PARADEE_OK && + sym.empty() && dropped == 0, "lenient empty"); + CHECK(paradee_phonemes_to_symbol_ids_lenient(model.get(), "##", sym, dropped, err) == PARADEE_OK && + sym.empty() && dropped == 2, "lenient all dropped"); + CHECK(paradee_phonemes_to_symbol_ids_lenient(model.get(), std::string("a\xC3", 2), sym, dropped, err) == + PARADEE_E_INVALID_ARG && sym.empty(), "lenient truncated UTF-8 accepted"); + CHECK(paradee_phonemes_to_symbol_ids_lenient(nullptr, "a", sym, dropped, err) == PARADEE_E_INVALID_ARG, + "lenient null model"); + } + + // Synthesis input validation. + std::vector pcm; + paradee_phonemes_to_ids(model.get(), "həlˈO wˈɜɹld.", ids, err); + { + std::vector nopad(ids.begin() + 1, ids.end()); + CHECK(paradee_synthesize_ids(model.get(), nopad.data(), nopad.size(), 1.0f, nullptr, nullptr, pcm, err) == + PARADEE_E_INVALID_ARG, "missing leading pad accepted"); + std::vector bad = ids; bad[3] = 178; + CHECK(paradee_synthesize_ids(model.get(), bad.data(), bad.size(), 1.0f, nullptr, nullptr, pcm, err) == + PARADEE_E_INVALID_ARG, "id 178 accepted"); + const int32_t two[2] = { 0, 0 }; + CHECK(paradee_synthesize_ids(model.get(), two, 2, 1.0f, nullptr, nullptr, pcm, err) == PARADEE_E_INVALID_ARG, + "2 ids accepted"); + std::vector big(513, 43); big.front() = 0; big.back() = 0; + CHECK(paradee_synthesize_ids(model.get(), big.data(), big.size(), 1.0f, nullptr, nullptr, pcm, err) == + PARADEE_E_TOO_LONG, "513 ids accepted"); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 0.0f, nullptr, nullptr, pcm, err) == + PARADEE_E_INVALID_ARG, "speed 0 accepted"); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), NAN, nullptr, nullptr, pcm, err) == + PARADEE_E_INVALID_ARG, "speed NaN accepted"); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), PARADEE_MIN_SPEED * 0.5f, nullptr, nullptr, + pcm, err) == PARADEE_E_INVALID_ARG, "speed below the minimum accepted"); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), PARADEE_MAX_SPEED * 2.0f, nullptr, nullptr, + pcm, err) == PARADEE_E_INVALID_ARG, "speed above the maximum accepted"); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), INFINITY, nullptr, nullptr, pcm, err) == + PARADEE_E_INVALID_ARG, "speed inf accepted"); + CHECK(paradee_synthesize_ids(nullptr, ids.data(), ids.size(), 1.0f, nullptr, nullptr, pcm, err) == + PARADEE_E_INVALID_ARG, "null model"); + } + + // Synthesis: deterministic, sane level, speed shortens. + std::vector a, b, fast; + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, a, err) == PARADEE_OK, + "%s", err.c_str()); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, b, err) == PARADEE_OK, + "%s", err.c_str()); + CHECK(!a.empty() && a == b, "synthesis is not deterministic"); + CHECK(a.size() % 600 == 0, "length %zu is not a whole number of frames", a.size()); + double rms = 0, peak = 0; + for (float v : a) { rms += (double) v * v; peak = std::max(peak, (double) std::fabs(v)); } + rms = std::sqrt(rms / (double) a.size()); + CHECK(rms > 0.01 && rms < 0.3 && peak < 1.5, "level rms %.4f peak %.3f", rms, peak); + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.5f, nullptr, nullptr, fast, err) == PARADEE_OK && + fast.size() < a.size(), "speed 1.5 not shorter (%zu vs %zu)", fast.size(), a.size()); + + // Convenience form gives the same audio. + { + paradee_synth_options opts; + paradee_audio audio; + CHECK(paradee_synthesize(model.get(), ids, opts, audio, err) == PARADEE_OK && audio.samples == a && + audio.sample_rate == 24000, "paradee_synthesize differs"); + } + + // Concurrent calls on one model are serialised and give the same audio. + { + std::vector t1, t2; + paradee_status s1 = PARADEE_E_RUNTIME, s2 = PARADEE_E_RUNTIME; + std::string e1, e2; + std::thread th1([&] { s1 = paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, t1, e1); }); + std::thread th2([&] { s2 = paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.0f, nullptr, nullptr, t2, e2); }); + th1.join(); + th2.join(); + CHECK(s1 == PARADEE_OK && s2 == PARADEE_OK && t1 == a && t2 == a, "concurrent synthesis differs"); + } + + // Cancellation: before the call, after the text side, inside the generator. + { + int total_polls = 0; + counter count; count.stop_after = 1 << 30; + CHECK(paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.0f, cancel_after, &count, pcm, err) == PARADEE_OK, + "%s", err.c_str()); + total_polls = count.polls.load(); + CHECK(total_polls >= 8, "only %d cancel polls in one synthesis", total_polls); + for (int stop : { 0, 2, total_polls / 2, total_polls - 2 }) { + counter c; c.stop_after = stop; + pcm.assign(10, 1.0f); + const paradee_status st = paradee_synthesize_ids(model.get(), ids.data(), ids.size(), 1.0f, cancel_after, &c, pcm, err); + CHECK(st == PARADEE_CANCELLED && pcm.empty(), "cancel after %d polls: status %s, %zu samples", + stop, paradee_status_str(st), pcm.size()); + } + } + + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("test-paradee-api: OK (%zu samples, rms %.4f)\n", a.size(), rms); + return 0; +} diff --git a/tools/paradee/tests/test_paradee_parity.cpp b/tools/paradee/tests/test_paradee_parity.cpp new file mode 100644 index 000000000..73b916312 --- /dev/null +++ b/tools/paradee/tests/test_paradee_parity.cpp @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: MIT +// +// test-paradee-parity — native Paradee against the pinned PyTorch reference +// (tools/dump_paradee_reference.py dump) on the misaki phonemes of the model +// card's samples/sentences.txt (fixtures/sentences.misaki.json). +// +// Gates, per sentence: +// ids paradee_phonemes_to_ids == the fixture's ids +// durations integer durations identical +// text side bert, d, dur, F0, N, asr: max|err| <= 1e-3 * RMS(reference) +// source harmonic source from the reference F0: max|err| <= 1e-3 * RMS +// decoder generator output from the reference asr / F0 / N / source: +// max|err| <= 1e-3 * RMS +// lock phase lock alone on the reference decoder output and source: +// max|err| <= 1e-4 against phase_lock.lock() in float64, and no +// further from the shipped float32 phase_lock.lock() than that +// float32 evaluation is from float64 (+1e-4) +// end to end final waveform SNR >= 35 dB against the reference +// +// Each acoustic stage is fed the reference's inputs, so an error is charged +// to the stage that made it: the harmonic source integrates F0 over seconds, +// and float32 rounding differences in F0 (~1e-6 relative) would otherwise +// show up as source phase differences. The end-to-end check covers the chain. +// +// Needs PARADEE_GGUF and PARADEE_REF_DIR (environment); exits 77 (CTest +// SKIP) without them. PARADEE_PARITY_OUT= also writes the native +// waveforms (.wav.f32) for `dump_paradee_reference.py report`. + +#include "paradee.h" + +#include "nlohmann/json.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PARADEE_FIXTURE_DIR +#error "PARADEE_FIXTURE_DIR must be defined" +#endif + +using json = nlohmann::json; +using namespace eliza_paradee; + +static int failures = 0; +#define CHECK(cond, ...) do { if (!(cond)) { std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + std::fprintf(stderr, __VA_ARGS__); std::fprintf(stderr, "\n"); ++failures; } } while (0) + +template +static std::vector read_raw(const std::string & path) { + std::ifstream f(path, std::ios::binary | std::ios::ate); + if (!f) { std::fprintf(stderr, "cannot read %s\n", path.c_str()); std::exit(1); } + std::vector v((size_t) f.tellg() / sizeof(T)); + f.seekg(0); + f.read((char *) v.data(), (std::streamsize) (v.size() * sizeof(T))); + return v; +} + +static json read_json(const std::string & path) { + std::ifstream f(path); + if (!f) { std::fprintf(stderr, "cannot read %s\n", path.c_str()); std::exit(1); } + std::stringstream ss; ss << f.rdbuf(); + return json::parse(ss.str()); +} + +struct err_stats { double max_abs = 0, rms_ref = 0, snr_db = 0; bool same_size = true; }; + +static err_stats compare(const std::vector & ref, const std::vector & got) { + err_stats s; + if (ref.size() != got.size() || ref.empty()) { s.same_size = false; return s; } + double e2 = 0, r2 = 0; + for (size_t i = 0; i < ref.size(); ++i) { + const double d = (double) got[i] - (double) ref[i]; + s.max_abs = std::max(s.max_abs, std::fabs(d)); + e2 += d * d; + r2 += (double) ref[i] * ref[i]; + } + s.rms_ref = std::sqrt(r2 / (double) ref.size()); + s.snr_db = e2 > 0 ? 10.0 * std::log10(r2 / e2) : 300.0; + return s; +} + +static void gate_stage(int sent, const char * name, const std::vector & ref, const std::vector & got) { + const err_stats s = compare(ref, got); + if (!s.same_size) { + CHECK(false, "sentence %d %s: size %zu != reference %zu", sent, name, got.size(), ref.size()); + return; + } + const double ratio = s.max_abs / std::max(s.rms_ref, 1e-30); + std::printf(" %-6s max|err| %9.3g rms %9.4g ratio %9.3g snr %6.1f dB\n", name, s.max_abs, s.rms_ref, ratio, s.snr_db); + CHECK(ratio <= 1e-3, "sentence %d %s: max|err| %.3g > 1e-3 * rms %.4g", sent, name, s.max_abs, s.rms_ref); +} + +int main() { + const char * gguf = std::getenv("PARADEE_GGUF"); + const char * ref_dir_c = std::getenv("PARADEE_REF_DIR"); + if (!gguf || !*gguf || !ref_dir_c || !*ref_dir_c) { + std::printf("test-paradee-parity: PARADEE_GGUF / PARADEE_REF_DIR not set, skipping\n"); + return 77; + } + const std::string ref_dir = ref_dir_c; + const char * out_dir = std::getenv("PARADEE_PARITY_OUT"); + + const json manifest = read_json(ref_dir + "/manifest.json"); + const json fixture = read_json(std::string(PARADEE_FIXTURE_DIR) + "/sentences.misaki.json"); + const float speed = manifest.value("speed", 1.0f); + + std::string err; + auto model = paradee_load_model(gguf, err); + if (!model) { std::fprintf(stderr, "load failed: %s\n", err.c_str()); return 1; } + const paradee_lock_params lock = *paradee_get_lock_params(model.get()); + + CHECK(manifest["sentences"].size() == fixture["sentences"].size(), "reference and fixture sentence counts differ"); + double total_audio = 0, total_secs = 0; + for (size_t k = 0; k < manifest["sentences"].size(); ++k) { + const json & m = manifest["sentences"][k]; + const json & fx = fixture["sentences"][k]; + const int sent = m["index"].get(); + const std::string phonemes = fx["phonemes"].get(); + auto file = [&](const char * name) { return ref_dir + "/" + m["files"][name]["file"].get(); }; + std::printf("sentence %d (%zu bytes of phonemes)\n", sent, phonemes.size()); + CHECK(m["phonemes"].get() == phonemes, "sentence %d: reference was dumped from other phonemes", sent); + + // ids + std::vector ids; + CHECK(paradee_phonemes_to_ids(model.get(), phonemes, ids, err) == PARADEE_OK, "%s", err.c_str()); + CHECK(ids == fx["ids"].get>(), "sentence %d: ids differ from the fixture", sent); + CHECK(ids == read_raw(file("ids")), "sentence %d: ids differ from the reference", sent); + + // full synthesis + paradee_stages st; + std::vector pcm; + const auto t0 = std::chrono::steady_clock::now(); + const paradee_status rc = paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), speed, + nullptr, nullptr, pcm, err, &st); + const double secs = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + if (rc != PARADEE_OK) { CHECK(false, "sentence %d: synthesis failed: %s", sent, err.c_str()); continue; } + total_secs += secs; + total_audio += (double) pcm.size() / 24000.0; + + const std::vector ref_dur = read_raw(file("dur_int")); + CHECK(st.dur_int == ref_dur, "sentence %d: integer durations differ", sent); + std::printf(" durations identical: %s (%d frames)\n", st.dur_int == ref_dur ? "yes" : "NO", st.T_frame); + gate_stage(sent, "bert", read_raw(file("bert")), st.bert); + gate_stage(sent, "d", read_raw(file("d")), st.d); + gate_stage(sent, "dur", read_raw(file("dur")), st.dur); + gate_stage(sent, "F0", read_raw(file("F0")), st.F0); + gate_stage(sent, "N", read_raw(file("N")), st.N); + gate_stage(sent, "asr", read_raw(file("asr")), st.asr); + + // acoustic half, teacher-forced stage by stage + const std::vector ref_asr = read_raw(file("asr")); + const std::vector ref_F0 = read_raw(file("F0")); + const std::vector ref_N = read_raw(file("N")); + const std::vector ref_src = read_raw(file("src")); + const std::vector ref_dec = read_raw(file("dec")); + const std::vector ref_wav = read_raw(file("wav")); + const std::vector ref_l32 = read_raw(file("lock_f32")); + const int T = (int) ref_F0.size() / 2; + { + paradee_stages a; + CHECK(paradee_decode_staged(model.get(), ref_asr.data(), 512, T, ref_F0.data(), ref_N.data(), nullptr, + nullptr, nullptr, &a, err) == PARADEE_OK, "%s", err.c_str()); + gate_stage(sent, "src", ref_src, a.src); + paradee_stages b; + CHECK(paradee_decode_staged(model.get(), ref_asr.data(), 512, T, ref_F0.data(), ref_N.data(), &ref_src, + nullptr, nullptr, &b, err) == PARADEE_OK, "%s", err.c_str()); + gate_stage(sent, "dec", ref_dec, b.dec); + } + { + std::vector locked; + CHECK(paradee_phase_lock(ref_dec.data(), ref_dec.size(), ref_src.data(), ref_src.size(), + ref_F0.data(), ref_F0.size(), lock, locked), "lock rejected its input"); + const err_stats e64 = compare(ref_wav, locked); + const err_stats e32 = compare(ref_l32, locked); + const err_stats f32 = compare(ref_wav, ref_l32); + std::printf(" lock max|err| %9.3g vs float64 phase_lock (gate 1e-4); %9.3g vs float32 phase_lock " + "(float32 itself: %.3g)\n", e64.max_abs, e32.max_abs, f32.max_abs); + CHECK(e64.same_size && e64.max_abs <= 1e-4, "sentence %d lock: %.3g > 1e-4", sent, e64.max_abs); + CHECK(e32.same_size && e32.max_abs <= f32.max_abs + 1e-4, "sentence %d lock vs float32: %.3g > %.3g", + sent, e32.max_abs, f32.max_abs + 1e-4); + } + + // end to end + const err_stats e = compare(ref_wav, pcm); + std::printf(" end to end: snr %.1f dB, max|err| %.3g, %zu samples, %.3f s synth (%.1fx realtime)\n", + e.snr_db, e.max_abs, pcm.size(), secs, (double) pcm.size() / 24000.0 / secs); + CHECK(e.same_size && e.snr_db >= 35.0, "sentence %d: end-to-end SNR %.1f dB < 35 dB", sent, e.snr_db); + + if (out_dir && *out_dir) { + std::ofstream f(std::string(out_dir) + "/" + std::to_string(sent) + ".wav.f32", std::ios::binary); + f.write((const char *) pcm.data(), (std::streamsize) (pcm.size() * sizeof(float))); + } + } + std::printf("total: %.2f s of audio in %.2f s (%.1fx realtime)\n", total_audio, total_secs, total_audio / total_secs); + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("test-paradee-parity: OK\n"); + return 0; +} diff --git a/tools/paradee/tests/test_paradee_phase_lock.cpp b/tools/paradee/tests/test_paradee_phase_lock.cpp new file mode 100644 index 000000000..3017a672a --- /dev/null +++ b/tools/paradee/tests/test_paradee_phase_lock.cpp @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: MIT +// +// test-paradee-phase-lock — the native phase-locking filter against +// training/scripts/phase_lock.py lock() on deterministic synthetic input. +// +// fixtures/phase_lock_synthetic.f32 holds two outputs of the reference on the +// inputs built below (dump_paradee_reference.py lock-fixture): +// [0, n) phase_lock.lock() evaluated in float64 -> gate: max |err| <= 1e-4 +// [n, 2n) phase_lock.lock() as shipped (float32) -> gate: within float32's +// own distance from the float64 result (+1e-4) +// Also checks pass-through (unvoiced input, short input) and cancellation. + +#include "paradee.h" + +#include +#include +#include +#include +#include +#include + +#ifndef PARADEE_FIXTURE_DIR +#error "PARADEE_FIXTURE_DIR must be defined" +#endif + +static constexpr int N = 9600; +static constexpr double SR = 24000.0; +static constexpr double PI = 3.14159265358979323846; + +static void make_inputs(std::vector & s, std::vector & src, std::vector & f0) { + const int nf0 = N / 300; + std::vector f0d((size_t) nf0); + for (int j = 0; j < nf0; ++j) f0d[(size_t) j] = (j >= 12 && j < 16) ? 0.0 : 110.0 + 110.0 * j / (double) (nf0 - 1); + s.assign(N, 0.0f); src.assign(N, 0.0f); f0.assign((size_t) nf0, 0.0f); + for (int j = 0; j < nf0; ++j) f0[(size_t) j] = (float) f0d[(size_t) j]; + double ph = 0.0; + for (int t = 0; t < N; ++t) { + const double f = f0d[(size_t) (t / 300)]; + ph += 2 * PI * f / SR; + const bool voiced = f > 0; + src[(size_t) t] = (float) (std::tanh(0.8 * std::sin(ph)) * (voiced ? 1.0 : 0.0) + 0.01 * std::sin(0.37 * t)); + double acc = 0.0; + for (int h = 1; h < 40; ++h) { + if (f * h >= 11000) break; + acc += std::sin(h * ph + 0.002 * h * t / 7.0 + 0.5 * h) / h; + } + s[(size_t) t] = (float) (0.2 * acc * (voiced ? 1.0 : 0.3) + 0.003 * std::sin(1.3 * t)); + } +} + +static int failures = 0; +#define CHECK(cond, ...) do { if (!(cond)) { std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + std::fprintf(stderr, __VA_ARGS__); std::fprintf(stderr, "\n"); ++failures; } } while (0) + +static double max_abs(const std::vector & a, const float * b, size_t n) { + double m = 0; + for (size_t i = 0; i < n; ++i) m = std::max(m, std::fabs((double) a[i] - (double) b[i])); + return m; +} + +static bool cancel_now(void *) { return true; } + +int main() { + std::vector s, src, f0; + make_inputs(s, src, f0); + + std::vector ref((size_t) 2 * N); + { + std::ifstream in(std::string(PARADEE_FIXTURE_DIR) + "/phase_lock_synthetic.f32", std::ios::binary); + in.read((char *) ref.data(), (std::streamsize) (ref.size() * sizeof(float))); + if (!in) { std::fprintf(stderr, "cannot read phase_lock_synthetic.f32\n"); return 1; } + } + const float * ref64 = ref.data(); + const float * ref32 = ref.data() + N; + + eliza_paradee::paradee_lock_params p; // v1.0 defaults + std::vector out; + CHECK(eliza_paradee::paradee_phase_lock(s.data(), s.size(), src.data(), src.size(), f0.data(), f0.size(), p, out), + "lock returned false"); + CHECK(out.size() == (size_t) N, "output length %zu", out.size()); + const double e64 = max_abs(out, ref64, N); + const double e32 = max_abs(out, ref32, N); + double f32_floor = 0; + for (int i = 0; i < N; ++i) f32_floor = std::max(f32_floor, std::fabs((double) ref32[i] - ref64[i])); + double change = 0; + for (int i = 0; i < N; ++i) change += ((double) ref64[i] - s[(size_t) i]) * ((double) ref64[i] - s[(size_t) i]); + change = std::sqrt(change / N); + std::printf("lock vs phase_lock.lock float64: max|err| = %.3g (gate 1e-4)\n", e64); + std::printf("lock vs phase_lock.lock float32: max|err| = %.3g (float32 vs float64 itself: %.3g)\n", e32, f32_floor); + std::printf("the filter changes this input by rms %.4g\n", change); + CHECK(e64 <= 1e-4, "float64 reference error %.3g > 1e-4", e64); + CHECK(e32 <= f32_floor + 1e-4, "float32 reference error %.3g > %.3g", e32, f32_floor + 1e-4); + CHECK(change > 1e-3, "fixture does not exercise the filter"); + + // Unvoiced everywhere: output == input. + { + std::vector zf0(f0.size(), 0.0f), o2; + eliza_paradee::paradee_phase_lock(s.data(), s.size(), src.data(), src.size(), zf0.data(), zf0.size(), p, o2); + CHECK(max_abs(o2, s.data(), N) == 0.0, "unvoiced input changed"); + } + // Shorter than n_fft/2: passes through (torch cannot reflect-pad it). + { + std::vector o3; + CHECK(eliza_paradee::paradee_phase_lock(s.data(), 500, src.data(), 500, f0.data(), f0.size(), p, o3), + "short input rejected"); + CHECK(o3.size() == 500 && max_abs(o3, s.data(), 500) == 0.0, "short input changed"); + } + // Lengths differ: output has min(ns, nsrc) samples. + { + std::vector o4; + eliza_paradee::paradee_phase_lock(s.data(), N, src.data(), N - 1000, f0.data(), f0.size(), p, o4); + CHECK(o4.size() == (size_t) N - 1000, "min length %zu", o4.size()); + } + // Cancellation empties the output. + { + std::vector o5; + const bool ok = eliza_paradee::paradee_phase_lock(s.data(), s.size(), src.data(), src.size(), + f0.data(), f0.size(), p, o5, cancel_now, nullptr); + CHECK(!ok && o5.empty(), "cancel did not stop the filter"); + } + // Invalid parameters: false, output = input. + { + eliza_paradee::paradee_lock_params bad = p; + bad.n_fft = 1000; + std::vector o6; + CHECK(!eliza_paradee::paradee_phase_lock(s.data(), s.size(), src.data(), src.size(), f0.data(), f0.size(), bad, o6), + "non power-of-two n_fft accepted"); + CHECK(o6.size() == (size_t) N && max_abs(o6, s.data(), N) == 0.0, "rejected call changed the signal"); + } + + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("test-paradee-phase-lock: OK\n"); + return 0; +} diff --git a/tools/paradee/tools/dump_paradee_reference.py b/tools/paradee/tools/dump_paradee_reference.py new file mode 100644 index 000000000..3561d7836 --- /dev/null +++ b/tools/paradee/tools/dump_paradee_reference.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +""" +dump_paradee_reference.py - PyTorch reference for the tools/paradee parity gate. + +Three subcommands: + + phonemes Phonemize samples/sentences.txt with misaki 0.9.4 + (en.G2P(trf=False, british=False, fallback=EspeakFallback(british=False), unk=""), + the setup paradee/tts.py and kokoro.KPipeline(lang_code="a") use) and write + tests/fixtures/sentences.misaki.json. The fixture is committed; `dump` reads it. + + dump Run the pinned PyTorch Paradee (TextStudent 's+mlp' + StudentDecoder 'A' from + the pinned github.com/sahilmahendrakar/paradee checkout, kokoro 0.9.4 modules) + on each fixture sentence and write every stage plus manifest.json for + test-paradee-parity. + + lock-fixture Write tests/fixtures/phase_lock_synthetic.f32: phase_lock.lock() (float64 + and the unmodified float32) on deterministic synthetic input, for the + self-contained test-paradee-phase-lock. + + report Compare native waveforms (written by test-paradee-parity with + PARADEE_PARITY_OUT) with the deterministic reference and with the published, + seeded samples/*_paradee.wav: log-mel L1 (raw and DTW). With --whisper-cli, + also the word error rate of whisper.cpp on the native and the published audio. + Report only. + +The reference is deterministic. Three documented changes make it so and make it a +well-conditioned target; the native graph implements the same three: + + 1. SineGen: rand_ini = 0 and no additive noise (tools/kokoro does the same). + 2. SineGen runs in float64. In float32 the accumulated phase of the 9th harmonic + reaches ~1e5 rad in a few seconds, where one float32 step is ~1e-2 rad; float64 + makes the source a defined value rather than an accident of float32 rounding. + 3. The harmonic source's STFT (the generator's `har` input) is computed in float64 + and values below the DFT's rounding floor are set to exactly zero before the + phase is taken. Without noise, an unvoiced source is a constant, its STFT bins + 2..n_fft/2 are pure rounding noise, and their phase is otherwise arbitrary. + The inverse STFT is export_paradee.py's ConvSTFT.inverse (= torch.istft). + +The final waveform applies phase_lock.lock() in float64. lock_f32 is the unmodified +float32 phase_lock.lock() on the same decoder output, for the Lock-only check. + +usage: + dump_paradee_reference.py phonemes --model-dir HF_SNAPSHOT --out tools/paradee/tests/fixtures/sentences.misaki.json + dump_paradee_reference.py dump --model-dir HF_SNAPSHOT --paradee-src PARADEE_CHECKOUT --out REF_DIR + dump_paradee_reference.py report --model-dir HF_SNAPSHOT --ref REF_DIR --native NATIVE_DIR +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import importlib.metadata +import json +import math +import sys +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +PARADEE_DIR = HERE.parent +PINS = json.loads((PARADEE_DIR / "pins.json").read_text(encoding="utf-8")) +DEFAULT_FIXTURE = PARADEE_DIR / "tests" / "fixtures" / "sentences.misaki.json" +SR = 24000 + + +def sha256_file(p: Path) -> str: + return hashlib.sha256(p.read_bytes()).hexdigest() + + +def check_file(root: Path, rel: str, pin: dict | str) -> Path: + path = root / rel + want = pin if isinstance(pin, str) else pin["sha256"] + got = sha256_file(path) + if got != want: + raise SystemExit(f"{path}: sha256 {got} != pinned {want}") + return path + + +def check_versions(*names: str) -> None: + env = PINS["reference_env"] + for name in names: + got = importlib.metadata.version(name) + if got != env[name]: + raise SystemExit(f"{name} {got} != pinned {env[name]} (see pins.json reference_env)") + + +def read_sentences(model_dir: Path) -> list[str]: + path = check_file(model_dir, "samples/sentences.txt", PINS["files"]["samples/sentences.txt"]) + out = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + num, _, text = line.partition(". ") # "1. As of August 2015, ..." + if not num.isdigit(): + raise SystemExit(f"unexpected sentences.txt line {line!r}") + out.append(text) + return out + + +# --------------------------------------------------------------------------- +# phonemes +# --------------------------------------------------------------------------- + +def cmd_phonemes(args) -> int: + check_versions("misaki", "spacy", "en_core_web_sm") + import espeakng_loader + if args.espeak_lib: + espeakng_loader.get_library_path = lambda: args.espeak_lib + if args.espeak_data: + espeakng_loader.get_data_path = lambda: args.espeak_data + from misaki import en, espeak + + fb = espeak.EspeakFallback(british=False) + g2p = en.G2P(trf=False, british=False, fallback=fb, unk="") + espeak_version = fb.backend.version() + vocab = json.loads(check_file(args.model_dir, "config.json", PINS["files"]["config.json"]).read_text(encoding="utf-8"))["vocab"] + items = [] + for i, text in enumerate(read_sentences(args.model_dir), 1): + ps, _ = g2p(text) + missing = sorted({c for c in ps if c not in vocab}) + if missing: + raise SystemExit(f"sentence {i}: phonemes outside the vocabulary: {missing}") + items.append({"index": i, "text": text, "phonemes": ps, + "ids": [0] + [vocab[c] for c in ps] + [0]}) + doc = { + "source": f"{PINS['model']['repo']}@{PINS['model']['revision']} samples/sentences.txt", + "g2p": "misaki en.G2P(trf=False, british=False, fallback=EspeakFallback(british=False), unk='')", + "versions": {"misaki": importlib.metadata.version("misaki"), "spacy": importlib.metadata.version("spacy"), + "en_core_web_sm": importlib.metadata.version("en_core_web_sm"), + "espeak-ng": ".".join(str(x) for x in espeak_version)}, + "sentences": items, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"wrote {args.out} sha256 {sha256_file(args.out)}") + return 0 + + +# --------------------------------------------------------------------------- +# reference model +# --------------------------------------------------------------------------- + +def load_reference(model_dir: Path, paradee_src: Path): + check_versions("torch", "kokoro", "misaki") + for rel, digest in PINS["source"]["files"].items(): + check_file(paradee_src, rel, digest) + for rel in ("pytorch/text_side.pt", "pytorch/decoder.pt"): + check_file(model_dir, rel, PINS["files"][rel]) + + import torch + import torch.nn.functional as F + sys.path.insert(0, str(paradee_src / "training" / "scripts")) + student = importlib.import_module("student") + student_decoder = importlib.import_module("student_decoder") + phase_lock = importlib.import_module("phase_lock") + from kokoro.istftnet import SineGen + from torch.nn.utils import remove_weight_norm + from torch.nn.utils.parametrize import is_parametrized, remove_parametrizations + + torch.set_grad_enabled(False) + + def strip(m): # export_paradee.py strip() + for x in m.modules(): + try: + remove_weight_norm(x) + except Exception: + pass + if is_parametrized(x, "weight"): + try: + remove_parametrizations(x, "weight", leave_parametrized=True) + except Exception: + pass + + # (1) + (2): deterministic float64 SineGen (kokoro 0.9.4 SineGen with + # flag_for_pulse=False, minus rand_ini and noise). + def f02sine_det(self, f0_values): + rad_values = (f0_values / self.sampling_rate) % 1 + rad_values = F.interpolate(rad_values.transpose(1, 2), scale_factor=1 / self.upsample_scale, + mode="linear").transpose(1, 2) + phase = torch.cumsum(rad_values, dim=1) * 2 * torch.pi + phase = F.interpolate(phase.transpose(1, 2) * self.upsample_scale, scale_factor=self.upsample_scale, + mode="linear").transpose(1, 2) + return torch.sin(phase) + + def sinegen_forward_det(self, f0): + assert not self.flag_for_pulse + f0d = f0.double() + fn = f0d * torch.arange(1, self.harmonic_num + 2, dtype=torch.float64)[None, None, :] + sine_waves = f02sine_det(self, fn) * self.sine_amp + uv = (f0d > self.voiced_threshold).double() + sine_waves = sine_waves * uv + return sine_waves.float(), uv.float(), torch.zeros_like(sine_waves, dtype=torch.float32) + + SineGen.forward = sinegen_forward_det + + # (3): canonical float64 source STFT + ConvSTFT inverse (export_paradee.py). + class RefSTFT(torch.nn.Module): + def __init__(self, n, hop): + super().__init__() + self.n, self.hop = n, hop + w = torch.hann_window(n, dtype=torch.float64) + k = torch.arange(n // 2 + 1, dtype=torch.float64)[:, None] + ang = 2 * math.pi * k * torch.arange(n, dtype=torch.float64) / n + self.re_b, self.im_b = torch.cos(ang) * w, -torch.sin(ang) * w + self.w64 = w + # inverse: ConvSTFT.inverse, float32 like the exported graph + wf = torch.hann_window(n) + kf = torch.arange(n // 2 + 1).float()[:, None] + angf = 2 * torch.pi * kf * torch.arange(n).float() / n + c = torch.full((n // 2 + 1, 1), 2.0); c[0] = c[-1] = 1 + self.inv = (torch.cat([torch.cos(angf) * c, -torch.sin(angf) * c]) / n * wf)[:, None] + self.w2 = (wf ** 2)[None, None] + + def transform(self, x): + xd = F.pad(x.double()[:, None], (self.n // 2, self.n // 2), mode="reflect")[:, 0] + frames = xd.unfold(-1, self.n, self.hop) # [B, T, n] + re = frames @ self.re_b.T # [B, T, K] + im = frames @ self.im_b.T + floor = 1e-12 * (frames * self.w64).abs().sum(-1, keepdim=True) + re = torch.where(re.abs() <= floor, torch.zeros_like(re), re) + im = torch.where(im.abs() <= floor, torch.zeros_like(im), im) + mag = torch.sqrt(re ** 2 + im ** 2) + ph = torch.where((im == 0) & (re < 0), torch.full_like(re, math.pi), torch.atan2(im, re)) + return mag.transpose(1, 2).float(), ph.transpose(1, 2).float() + + def inverse(self, mag, phase): + z = torch.cat([mag * torch.cos(phase), mag * torch.sin(phase)], 1) + y = F.conv_transpose1d(z, self.inv, stride=self.hop)[:, 0] + env = F.conv_transpose1d(torch.ones(1, 1, mag.shape[-1]), self.w2, stride=self.hop)[:, 0] + return (y / env.clamp_min(1e-8))[:, self.n // 2: -(self.n // 2)][:, None] + + ts = student.TextStudent("s+mlp").eval() + ts.load_state_dict(torch.load(model_dir / "pytorch/text_side.pt", map_location="cpu", weights_only=True)["model"]) + ds = student_decoder.load_student_decoder("A", str(model_dir / "pytorch/decoder.pt"), disable_complex=True) + strip(ts) + strip(ds) + g = ds.generator + g.stft = RefSTFT(g.stft.filter_length, g.stft.hop_length) + cap = {} + g.m_source.register_forward_hook(lambda m, i, o: cap.__setitem__("src", o[0])) + return torch, ts, ds, cap, phase_lock, student + + +def lock64(torch, phase_lock, s, src, f0, L): + """phase_lock.lock() evaluated in float64 (same operations; the module's + window and smoothing kernel are float32 constants, so it cannot take + float64 input as is).""" + import torch.nn.functional as F + n, hop = phase_lock.n, phase_lock.hop + W = torch.hann_window(n, dtype=torch.float64) + k = min(len(s), len(src)) + s, src = s[:k].double(), src[:k].double() + S = torch.stft(s, n, hop, window=W, return_complex=True) + E = torch.stft(src, n, hop, window=W, return_complex=True) + v = torch.tensor([float(f0[min(j * hop // 300, len(f0) - 1)]) > 60 for j in range(S.shape[1])])[None] + rel = S * E.conj() / E.abs().clamp_min(1e-6) + ker = torch.hann_window(L + 2, dtype=torch.float64)[1:-1] + ker = (ker / ker.sum())[None, None] + sm = torch.complex(F.conv1d(rel.real[:, None], ker, padding=L // 2)[:, 0], + F.conv1d(rel.imag[:, None], ker, padding=L // 2)[:, 0]) + ph = torch.where(phase_lock.BAND & v, E.angle() + sm.angle(), S.angle()) + return torch.istft(torch.polar(S.abs(), ph), n, hop, window=W, length=k) + + +def run_reference(torch, ts, ds, cap, phase_lock, student, ids: list[int], speed: float) -> dict: + """export_paradee.py Paradee.text() + forward(), with the stages kept.""" + import torch.nn as nn + x_ids = torch.tensor([ids], dtype=torch.long) + p, s = ts.predictor, ts.style + bert = ts.bert(x_ids, attention_mask=torch.ones_like(x_ids)) # [1,T,256] + d_en = ts.bert_encoder(bert).transpose(-1, -2) + T = x_ids.shape[1] + sx = s[:, :, None].expand(-1, -1, T) + x = torch.cat([d_en, sx], 1) + for b in p.text_encoder.lstms: + if isinstance(b, nn.LSTM): + x = b(x.transpose(-1, -2))[0].transpose(-1, -2) + else: + x = torch.cat([b(x.transpose(-1, -2), s).transpose(-1, -2), sx], 1) + d = x.transpose(-1, -2) # [1,T,224] + dur = torch.sigmoid(p.duration_proj(p.lstm(d)[0])).sum(-1)[0] # [T] + e = ts.text_encoder + h = e.embedding(x_ids).transpose(1, 2) + for c in e.cnn: + h = c(h) + t_en = e.lstm(h.transpose(1, 2))[0] + asr_tok = ts.asr_proj(t_en).transpose(-1, -2) # [1,512,T] + n = torch.clamp(torch.round(dur / speed), min=1).long() + aln = student.alignment(n)[None] # [1,T,F] + F0, N = p.F0Ntrain(d.transpose(-1, -2) @ aln, s) + asr = asr_tok @ aln # [1,512,F] + dec = ds(asr, F0, N).squeeze() # [L] + src = cap["src"].squeeze() + k = min(len(dec), len(src)) + lock_f32 = phase_lock.lock(dec, src, F0[0], 33) + wav = lock64(torch, phase_lock, dec, src, F0[0], 33).float() + f = lambda t: t.detach().contiguous().float().numpy() + return { + "ids": np.asarray(ids, dtype=np.int32), + "bert": f(bert[0]), "d": f(d[0]), "dur": f(dur / speed), "dur_int": n.numpy().astype(np.int32), + "F0": f(F0[0]), "N": f(N[0]), "asr": f(asr[0]), "dec": f(dec[:k]), "src": f(src[:k]), + "wav": f(wav), "lock_f32": f(lock_f32), + } + + +def cmd_dump(args) -> int: + fixture = json.loads(args.phonemes.read_text(encoding="utf-8")) + torch, ts, ds, cap, phase_lock, student = load_reference(args.model_dir, args.paradee_src) + args.out.mkdir(parents=True, exist_ok=True) + manifest = { + "model": PINS["model"], "source_commit": PINS["source"]["commit"], + "phonemes_fixture_sha256": sha256_file(args.phonemes), + "speed": args.speed, + "versions": {k: importlib.metadata.version(k) for k in ("torch", "kokoro", "misaki")}, + "reference": "deterministic: SineGen rand_ini=0, no noise, float64; canonical float64 source STFT; float64 phase lock", + "sentences": [], + } + for item in fixture["sentences"]: + out = run_reference(torch, ts, ds, cap, phase_lock, student, item["ids"], args.speed) + files = {} + for name, arr in out.items(): + ext = "i32" if arr.dtype == np.int32 else "f32" + fn = f"{item['index']}.{name}.{ext}" + arr.astype(" 220 Hz with an unvoiced gap; src is a tanh- + shaped harmonic excitation of that F0; s is a sum of its harmonics with a + slowly drifting phase offset, so the lock has something to correct.""" + n = LOCK_FIXTURE_SAMPLES + nf0 = n // 300 + f0 = np.array([0.0 if 12 <= j < 16 else 110.0 + 110.0 * j / (nf0 - 1) for j in range(nf0)]) + ph = 0.0 + s = np.zeros(n); src = np.zeros(n) + for t in range(n): + f = f0[t // 300] + ph += 2 * np.pi * f / SR + voiced = f > 0 + src[t] = np.tanh(0.8 * np.sin(ph)) * (1.0 if voiced else 0.0) + 0.01 * np.sin(0.37 * t) + acc = 0.0 + for h in range(1, 40): + if f * h >= 11000: + break + acc += np.sin(h * ph + 0.002 * h * t / 7.0 + 0.5 * h) / h + s[t] = 0.2 * acc * (1.0 if voiced else 0.3) + 0.003 * np.sin(1.3 * t) + return s.astype(np.float32), src.astype(np.float32), f0.astype(np.float32) + + +def cmd_lock_fixture(args) -> int: + for rel, digest in PINS["source"]["files"].items(): + check_file(args.paradee_src, rel, digest) + import torch + sys.path.insert(0, str(args.paradee_src / "training" / "scripts")) + phase_lock = importlib.import_module("phase_lock") + s, src, f0 = lock_fixture_inputs() + ts_, tsrc, tf0 = torch.from_numpy(s), torch.from_numpy(src), torch.from_numpy(f0) + with torch.no_grad(): + out64 = lock64(torch, phase_lock, ts_, tsrc, tf0, 33).float().numpy() + out32 = phase_lock.lock(ts_, tsrc, tf0, 33).numpy() + args.out.parent.mkdir(parents=True, exist_ok=True) + np.concatenate([out64, out32]).astype(" np.ndarray: + """HTK-scale triangular mel filters (torchaudio MelSpectrogram defaults: htk, no norm).""" + hz2mel = lambda f: 2595.0 * np.log10(1.0 + f / 700.0) + mel2hz = lambda m: 700.0 * (10.0 ** (m / 2595.0) - 1.0) + freqs = np.linspace(0, sr / 2, n_fft // 2 + 1) + pts = mel2hz(np.linspace(hz2mel(0.0), hz2mel(sr / 2), n_mels + 2)) + fb = np.zeros((n_fft // 2 + 1, n_mels)) + for m in range(n_mels): + lo, c, hi = pts[m], pts[m + 1], pts[m + 2] + up = (freqs - lo) / (c - lo) + down = (hi - freqs) / (hi - c) + fb[:, m] = np.maximum(0.0, np.minimum(up, down)) + return fb + + +def logmel(x: np.ndarray) -> np.ndarray: + """log(MelSpectrogram(24000, n_fft=1024, hop=256, n_mels=80)) as training/scripts/common.py.""" + n_fft, hop = 1024, 256 + x = x / max(np.sqrt(np.mean(x.astype(np.float64) ** 2)), 1e-8) + xp = np.pad(x.astype(np.float64), (n_fft // 2, n_fft // 2), mode="reflect") + w = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(n_fft) / n_fft) + frames = np.lib.stride_tricks.sliding_window_view(xp, n_fft)[::hop] * w + power = np.abs(np.fft.rfft(frames, axis=-1)) ** 2 + return np.log(np.maximum(power @ mel_filterbank(SR, n_fft, 80), 1e-5)).T # [80, T] + + +def dtw_l1(a: np.ndarray, b: np.ndarray) -> float: + a, b = a.T, b.T + C = np.abs(a[:, None, :] - b[None, :, :]).mean(-1) + n, m = C.shape + D = np.full((n + 1, m + 1), np.inf); D[0, 0] = 0 + for i in range(1, n + 1): + Di, Dp, Ci = D[i], D[i - 1], C[i - 1] + for j in range(1, m + 1): + Di[j] = Ci[j - 1] + min(Dp[j], Di[j - 1], Dp[j - 1]) + i, j, path = n, m, 0 + while i > 0 and j > 0: + path += 1 + k = int(np.argmin([D[i - 1, j - 1], D[i - 1, j], D[i, j - 1]])) + i, j = (i - 1, j - 1) if k == 0 else (i - 1, j) if k == 1 else (i, j - 1) + return float(D[n, m] / path) + + +def read_wav_f32(path: Path) -> np.ndarray: + import wave + with wave.open(str(path), "rb") as w: + assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path + return np.frombuffer(w.readframes(w.getnframes()), dtype=" tuple[float, float]: + la, lb = logmel(a), logmel(b) + k = min(la.shape[1], lb.shape[1]) + return float(np.abs(la[:, :k] - lb[:, :k]).mean()), dtw_l1(la, lb) + + +def write_wav16k(path: Path, x: np.ndarray) -> None: + """24 kHz float -> 16 kHz 16-bit WAV (band-limited FFT resampling) for whisper.cpp.""" + import wave + n = len(x); m = int(round(n * 16000 / SR)) + X = np.fft.rfft(x.astype(np.float64)) + Y = np.zeros(m // 2 + 1, complex); k = min(len(X), len(Y)); Y[:k] = X[:k] + y = np.clip(np.fft.irfft(Y, m) * (m / n), -1.0, 1.0) + with wave.open(str(path), "wb") as w: + w.setnchannels(1); w.setsampwidth(2); w.setframerate(16000) + w.writeframes((y * 32767).astype(" list[str]: + """Lower-case words with digits spelled out (years as years), punctuation and 'and' dropped, + so '2015' and 'twenty fifteen' compare equal.""" + import re + from num2words import num2words + t = text.lower().replace("-", " ") + t = re.sub(r"(\d+)'s", r"\1s", t) + t = re.sub(r"\d+", lambda m: num2words(int(m.group(0)), to="year") if 1100 <= int(m.group(0)) <= 2099 + else num2words(int(m.group(0))), t) + t = re.sub(r"[^a-z' ]", " ", t.replace("-", " ")) + return [w for w in t.split() if w != "and"] + + +def edit_distance(ref: list[str], hyp: list[str]) -> int: + d = list(range(len(hyp) + 1)) + for i, r in enumerate(ref, 1): + prev, d[0] = d[0], i + for j, h in enumerate(hyp, 1): + prev, d[j] = d[j], min(d[j] + 1, d[j - 1] + 1, prev + (r != h)) + return d[len(hyp)] + + +def report_wer(args, manifest, texts) -> None: + import subprocess + import tempfile + totals = {"native": [0, 0], "published": [0, 0]} + with tempfile.TemporaryDirectory() as tmp: + for s in manifest["sentences"]: + i = s["index"] + audio = {"native": np.fromfile(args.native / f"{i}.wav.f32", dtype=" int: + manifest = json.loads((args.ref / "manifest.json").read_text(encoding="utf-8")) + rows = [] + for s in manifest["sentences"]: + i = s["index"] + ref = np.fromfile(args.ref / s["files"]["wav"]["file"], dtype="2} {'native s':>8} {'pub s':>6} | {'nat-ref raw':>11} {'dtw':>6} | {'nat-pub raw':>11} {'dtw':>6} | {'ref-pub raw':>11} {'dtw':>6}") + for r in rows: + print(f"{r[0]:>2} {r[1]:8.2f} {r[2]:6.2f} | {r[3]:11.4f} {r[4]:6.4f} | {r[5]:11.4f} {r[6]:6.4f} | {r[7]:11.4f} {r[8]:6.4f}") + m = np.mean(np.array([r[3:] for r in rows]), axis=0) + print(f"mean | {m[0]:11.4f} {m[1]:6.4f} | {m[2]:11.4f} {m[3]:6.4f} | {m[4]:11.4f} {m[5]:6.4f}") + if args.whisper_cli: + fixture = json.loads(args.phonemes.read_text(encoding="utf-8")) + texts = {s["index"]: s["text"] for s in fixture["sentences"]} + print(f"\nWER with whisper.cpp ({args.whisper_model.name}), native vs the published samples:") + report_wer(args, manifest, texts) + return 0 + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("phonemes") + p.add_argument("--model-dir", type=Path, required=True) + p.add_argument("--out", type=Path, default=DEFAULT_FIXTURE) + p.add_argument("--espeak-lib", help="libespeak-ng to use instead of espeakng_loader's") + p.add_argument("--espeak-data", help="espeak-ng-data directory to use instead of espeakng_loader's") + p = sub.add_parser("dump") + p.add_argument("--model-dir", type=Path, required=True) + p.add_argument("--paradee-src", type=Path, required=True) + p.add_argument("--phonemes", type=Path, default=DEFAULT_FIXTURE) + p.add_argument("--speed", type=float, default=1.0) + p.add_argument("--out", type=Path, required=True) + p = sub.add_parser("lock-fixture") + p.add_argument("--paradee-src", type=Path, required=True) + p.add_argument("--out", type=Path, default=PARADEE_DIR / "tests" / "fixtures" / "phase_lock_synthetic.f32") + p = sub.add_parser("report") + p.add_argument("--model-dir", type=Path, required=True) + p.add_argument("--ref", type=Path, required=True) + p.add_argument("--native", type=Path, required=True) + p.add_argument("--phonemes", type=Path, default=DEFAULT_FIXTURE) + p.add_argument("--whisper-cli", type=Path, help="whisper.cpp whisper-cli binary: also report WER") + p.add_argument("--whisper-model", type=Path, help="whisper.cpp ggml model (e.g. ggml-base.en.bin)") + args = ap.parse_args(argv) + return {"phonemes": cmd_phonemes, "dump": cmd_dump, "lock-fixture": cmd_lock_fixture, + "report": cmd_report}[args.cmd](args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/paradee/tools/paradee-stage-dump.cpp b/tools/paradee/tools/paradee-stage-dump.cpp new file mode 100644 index 000000000..fd453c908 --- /dev/null +++ b/tools/paradee/tools/paradee-stage-dump.cpp @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: MIT +// +// paradee-stage-dump — run one synthesis and write every stage as raw +// little-endian arrays, in the layouts tools/dump_paradee_reference.py writes +// for the PyTorch reference: +// +// paradee-stage-dump [speed] +// paradee-stage-dump --decode +// +// --decode runs only the decoder, generator and phase lock on given text-side +// outputs (asr [512, T] channel-major, F0 / N [2T]). +// +// writes .{bert,d,dur,F0,N,asr,dec,src,wav}.f32, .dur_int.i32 +// and .ids.i32. + +#include "paradee.h" + +#include +#include +#include +#include +#include +#include + +template +static bool write_raw(const std::string & path, const std::vector & v) { + std::ofstream f(path, std::ios::binary); + f.write((const char *) v.data(), (std::streamsize) (v.size() * sizeof(T))); + return (bool) f; +} + +static std::vector read_f32(const char * path) { + std::ifstream f(path, std::ios::binary | std::ios::ate); + std::vector v((size_t) f.tellg() / sizeof(float)); + f.seekg(0); + f.read((char *) v.data(), (std::streamsize) (v.size() * sizeof(float))); + return v; +} + +int main(int argc, char ** argv) { + if (argc == 7 && std::string(argv[1]) == "--decode") { + std::string err; + auto model = eliza_paradee::paradee_load_model(argv[2], err); + if (!model) { std::fprintf(stderr, "load: %s\n", err.c_str()); return 1; } + const std::vector asr = read_f32(argv[3]), F0 = read_f32(argv[4]), N = read_f32(argv[5]); + const int T = (int) F0.size() / 2; + eliza_paradee::paradee_stages st; + if (eliza_paradee::paradee_decode_staged(model.get(), asr.data(), (int) (asr.size() / (size_t) T), T, + F0.data(), N.data(), nullptr, nullptr, nullptr, &st, err) != eliza_paradee::PARADEE_OK) { + std::fprintf(stderr, "decode: %s\n", err.c_str()); + return 1; + } + const std::string pre = argv[6]; + return write_raw(pre + ".dec.f32", st.dec) && write_raw(pre + ".src.f32", st.src) && + write_raw(pre + ".wav.f32", st.wav) ? 0 : 1; + } + if (argc < 4) { + std::fprintf(stderr, "usage: %s [speed]\n", argv[0]); + return 2; + } + std::ifstream in(argv[2], std::ios::binary); + std::stringstream ss; ss << in.rdbuf(); + std::string ph = ss.str(); + while (!ph.empty() && (ph.back() == '\n' || ph.back() == '\r')) ph.pop_back(); + const float speed = argc > 4 ? std::strtof(argv[4], nullptr) : 1.0f; + const std::string pre = argv[3]; + + std::string err; + auto model = eliza_paradee::paradee_load_model(argv[1], err); + if (!model) { std::fprintf(stderr, "load: %s\n", err.c_str()); return 1; } + std::vector ids; + if (eliza_paradee::paradee_phonemes_to_ids(model.get(), ph, ids, err) != eliza_paradee::PARADEE_OK) { + std::fprintf(stderr, "phonemes: %s\n", err.c_str()); + return 1; + } + eliza_paradee::paradee_stages st; + std::vector pcm; + if (eliza_paradee::paradee_synthesize_ids_staged(model.get(), ids.data(), ids.size(), speed, + nullptr, nullptr, pcm, err, &st) != eliza_paradee::PARADEE_OK) { + std::fprintf(stderr, "synthesis: %s\n", err.c_str()); + return 1; + } + bool ok = write_raw(pre + ".ids.i32", ids) && write_raw(pre + ".bert.f32", st.bert) && + write_raw(pre + ".d.f32", st.d) && write_raw(pre + ".dur.f32", st.dur) && + write_raw(pre + ".dur_int.i32", st.dur_int) && write_raw(pre + ".F0.f32", st.F0) && + write_raw(pre + ".N.f32", st.N) && write_raw(pre + ".asr.f32", st.asr) && + write_raw(pre + ".dec.f32", st.dec) && write_raw(pre + ".src.f32", st.src) && + write_raw(pre + ".wav.f32", st.wav); + std::printf("T_phon=%d T_frame=%d samples=%zu\n", st.T_phon, st.T_frame, st.wav.size()); + return ok ? 0 : 1; +} diff --git a/tools/paradee/tools/paradee-tts.cpp b/tools/paradee/tools/paradee-tts.cpp new file mode 100644 index 000000000..8c69617de --- /dev/null +++ b/tools/paradee/tools/paradee-tts.cpp @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: MIT +// +// paradee-tts — synthesize misaki phonemes with Paradee-8M to a 24 kHz WAV. +// +// paradee-tts --model paradee-8m-v1.0-f32.gguf --phonemes "həlˈO wˈɜɹld." --output hello.wav +// [--speed 1.0] [--f32] [--repeat N] +// +// The input is misaki's phoneme spelling (Kokoro's G2P), not text and not +// eSpeak IPA. --repeat N synthesizes N times and reports the real-time factor +// of the fastest run (thread count: KOKORO_NUM_THREADS; on Apple also +// VECLIB_MAXIMUM_THREADS). + +#include "paradee.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void usage(const char * argv0) { + std::fprintf(stderr, + "usage: %s --model (--phonemes \"...\" | --phonemes-file ) --output \n" + " [--speed 1.0] [--f32] [--repeat N]\n", argv0); +} + +bool write_wav(const std::string & path, const std::vector & samples, int sample_rate, bool f32) { + std::ofstream f(path, std::ios::binary); + if (!f) return false; + const uint32_t n = (uint32_t) samples.size(); + const uint16_t bits = f32 ? 32 : 16, fmt = f32 ? 3 : 1; + const uint32_t bytes = n * (bits / 8u); + auto p32 = [&f](uint32_t v) { char b[4] = { (char) v, (char) (v >> 8), (char) (v >> 16), (char) (v >> 24) }; f.write(b, 4); }; + auto p16 = [&f](uint16_t v) { char b[2] = { (char) v, (char) (v >> 8) }; f.write(b, 2); }; + f.write("RIFF", 4); p32(36 + bytes); f.write("WAVE", 4); + f.write("fmt ", 4); p32(16); p16(fmt); p16(1); p32((uint32_t) sample_rate); + p32((uint32_t) sample_rate * (bits / 8u)); p16(bits / 8); p16(bits); + f.write("data", 4); p32(bytes); + for (uint32_t i = 0; i < n; ++i) { + if (f32) { + f.write((const char *) &samples[i], 4); + } else { + const float v = std::max(-1.0f, std::min(1.0f, samples[i])); + const int16_t s = (int16_t) std::lrintf(v * 32767.0f); + p16((uint16_t) s); + } + } + return (bool) f; +} + +} // namespace + +int main(int argc, char ** argv) { + std::string model_path, phonemes, out_path; + float speed = 1.0f; + bool f32 = false; + int repeat = 1; + for (int i = 1; i < argc; ++i) { + const std::string a = argv[i]; + auto next = [&]() -> const char * { return i + 1 < argc ? argv[++i] : nullptr; }; + const char * v = nullptr; + if (a == "--model" && (v = next())) model_path = v; + else if (a == "--phonemes" && (v = next())) phonemes = v; + else if (a == "--phonemes-file" && (v = next())) { + std::ifstream in(v, std::ios::binary); + std::stringstream ss; ss << in.rdbuf(); phonemes = ss.str(); + while (!phonemes.empty() && (phonemes.back() == '\n' || phonemes.back() == '\r')) phonemes.pop_back(); + } + else if (a == "--output" && (v = next())) out_path = v; + else if (a == "--speed" && (v = next())) speed = std::strtof(v, nullptr); + else if (a == "--repeat" && (v = next())) repeat = std::max(1, std::atoi(v)); + else if (a == "--f32") f32 = true; + else { usage(argv[0]); return 2; } + } + if (model_path.empty() || phonemes.empty() || out_path.empty()) { usage(argv[0]); return 2; } + + std::string err; + auto t0 = std::chrono::steady_clock::now(); + auto model = eliza_paradee::paradee_load_model(model_path, err); + if (!model) { std::fprintf(stderr, "load failed: %s\n", err.c_str()); return 1; } + const double load_ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + + std::vector ids; + eliza_paradee::paradee_status st = eliza_paradee::paradee_phonemes_to_ids(model.get(), phonemes, ids, err); + if (st != eliza_paradee::PARADEE_OK) { + std::fprintf(stderr, "phonemes rejected (%s): %s\n", eliza_paradee::paradee_status_str(st), err.c_str()); + return 1; + } + std::vector pcm; + double best = 1e30; + for (int r = 0; r < repeat; ++r) { + const auto t = std::chrono::steady_clock::now(); + st = eliza_paradee::paradee_synthesize_ids(model.get(), ids.data(), ids.size(), speed, nullptr, nullptr, pcm, err); + best = std::min(best, std::chrono::duration(std::chrono::steady_clock::now() - t).count()); + if (st != eliza_paradee::PARADEE_OK) { + std::fprintf(stderr, "synthesis failed (%s): %s\n", eliza_paradee::paradee_status_str(st), err.c_str()); + return 1; + } + } + const int sr = eliza_paradee::paradee_sample_rate(model.get()); + if (!write_wav(out_path, pcm, sr, f32)) { std::fprintf(stderr, "cannot write %s\n", out_path.c_str()); return 1; } + const double secs = (double) pcm.size() / sr; + std::printf("ids=%zu samples=%zu audio=%.2fs load=%.0fms synth=%.3fs rtf=%.4f (%.1fx realtime)\n", + ids.size(), pcm.size(), secs, load_ms, best, best / secs, secs / best); + return 0; +}