Merge eliza/develop-pin-ea8b3f2 (Parakeet ASR, misaki hardening, CI fixes) into main

CMakeLists.txt

@@ -141,6 +141,12 @@ option(LLAMA_BUILD_PARADEE   "Build the Paradee-8M TTS subtree (tools/paradee/)"
 # subtree lives in tools/misaki-g2p/ and is added when present.
 option(LLAMA_BUILD_MISAKI_G2P "Build the misaki G2P subtree (tools/misaki-g2p/)" ON)
 
+# Parakeet-TDT ASR (FastConformer + TDT) for FermionResearch/Phonon-2, the
+# five-value re-quantisation of nvidia/parakeet-tdt-0.6b-v3 (weights
+# CC-BY-4.0). Produces parakeet_lib + parakeet-cli/parakeet-bench; the fused
+# `elizainference` target folds parakeet_lib in with `if(TARGET ...)`.
+option(LLAMA_BUILD_PARAKEET  "Build the Parakeet-TDT ASR subtree (tools/parakeet/)" ON)
+
 # W3-3 deprecation: ELIZA_FUSE_OMNIVOICE=ON is the pre-W3-3 flag for the
 # fork-root `omnivoice/` graft. The graft is gone (see the deprecation
 # block below); turn the flag into a redirect now so the option ON state
@@ -312,6 +318,12 @@ if (LLAMA_BUILD_PARADEE AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
     endif()
 endif()
 
+# Parakeet-TDT ASR — embed-as-library hook (mirrors kokoro above); precedes
+# tools/omnivoice so the fused target can fold parakeet_lib in.
+if (LLAMA_BUILD_PARAKEET AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
+    add_subdirectory(tools/parakeet)
+endif()
+
 if (LLAMA_BUILD_OMNIVOICE)
     if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/omnivoice/CMakeLists.txt")
         message(FATAL_ERROR

tools/CMakeLists.txt

@@ -61,6 +61,12 @@ else()
             message(STATUS "LLAMA_BUILD_PARADEE: skipped because LLAMA_BUILD_KOKORO=OFF")
         endif()
     endif()
+    if (LLAMA_BUILD_PARAKEET)
+        # Parakeet-TDT ASR (Phonon-2 five-value weights). Like kokoro it
+        # precedes omnivoice so the fused `elizainference` target can fold
+        # parakeet_lib in.
+        add_subdirectory(parakeet)
+    endif()
     if (LLAMA_BUILD_OMNIVOICE)
         # The merged omnivoice subtree lives under tools/omnivoice/. The
         # top-level CMakeLists.txt has a parallel add_subdirectory hook for

tools/espeak-shared/eliza-espeak-shared.h

@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: MIT
+//
+// eliza-espeak-shared.h: the single owner of libespeak-ng's process-global
+// state for every component in this tree that calls it (tools/kokoro and
+// tools/misaki-g2p).
+//
+// libespeak-ng keeps its data path, loaded voice and translator in globals
+// and is not thread-safe. Two components that each guard it with their own
+// mutex and their own "initialised" flag race each other and can leave the
+// other one running on a voice it did not select. Here there is one lock and
+// one record of what libespeak-ng was initialised with:
+//
+//   auto lock = eliza_espeak_shared::lock();
+//   if (!eliza_espeak_shared::prepare(lock, data_dir, "en-us", err)) ...;
+//   espeak_TextToPhonemes(...);           // still under `lock`
+//
+// prepare() initialises libespeak-ng on first use and re-selects the
+// caller's voice whenever another component changed it, so every caller gets
+// its own voice regardless of interleaving. Per-call options (phoneme mode,
+// tie character) are arguments of espeak_TextToPhonemes and need no reset.
+//
+// Data path: an empty `data_dir` means "whatever the process already
+// initialised with, else libespeak-ng's default" (compiled-in path or
+// ESPEAK_DATA_PATH); Kokoro always asks for this. A non-empty, explicitly
+// configured path (misaki_g2p_open's espeak_data_dir) wins: when it differs
+// from the current one libespeak-ng is re-initialised with it, and later
+// default-path callers keep using it.
+//
+// Header-only on purpose: the state lives in a C++17 inline function's
+// static, which the linker folds into one instance per program or shared
+// library, so kokoro_lib and misaki_g2p share it without a CMake target
+// between them (each stays buildable on its own, with or without the other)
+// and without one more static archive for the mobile slices to collect.
+// Include it only when the component links libespeak-ng.
+#pragma once
+
+#include <espeak-ng/speak_lib.h>
+
+#include <mutex>
+#include <string>
+
+namespace eliza_espeak_shared {
+
+struct State {
+    std::mutex  mutex;
+    bool        initialized = false;
+    std::string data_dir;  // "" = libespeak-ng's default path
+    std::string voice;     // voice currently selected, "" when unknown
+};
+
+inline State & state() {
+    static State s;
+    return s;
+}
+
+// Every libespeak-ng call in the process must be made while holding this.
+inline std::unique_lock<std::mutex> lock() {
+    return std::unique_lock<std::mutex>(state().mutex);
+}
+
+// With `held` from lock(): make libespeak-ng ready for `data_dir` (see the
+// data-path rule above) and select `voice`. On failure sets `err`.
+inline bool prepare(const std::unique_lock<std::mutex> & held, const std::string & data_dir, const char * voice,
+                    std::string & err) {
+    State & s = state();
+    if (!held.owns_lock() || held.mutex() != &s.mutex) {
+        err = "eliza_espeak_shared::prepare called without the shared eSpeak lock";
+        return false;
+    }
+    if (!s.initialized || (!data_dir.empty() && data_dir != s.data_dir)) {
+        s.initialized = false;
+        s.voice.clear();
+        // AUDIO_OUTPUT_SYNCHRONOUS: no audio device is opened.
+        const int rate = espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, data_dir.empty() ? nullptr : data_dir.c_str(), 0);
+        if (rate <= 0) {
+            err = "espeak_Initialize failed for data dir '" + data_dir + "'";
+            return false;
+        }
+        s.initialized = true;
+        s.data_dir    = data_dir;
+    }
+    if (s.voice != voice) {
+        if (espeak_SetVoiceByName(voice) != EE_OK) {
+            s.voice.clear();
+            err = std::string("eSpeak NG voice ") + voice + " not found under '" + s.data_dir + "'";
+            return false;
+        }
+        s.voice = voice;
+    }
+    return true;
+}
+
+}  // namespace eliza_espeak_shared

tools/kokoro/CMakeLists.txt

@@ -85,6 +85,9 @@ if(KOKORO_ENABLE_ESPEAK)
         target_include_directories(kokoro_lib PRIVATE ${ESPEAK_NG_INCLUDE_DIR})
         target_link_libraries(kokoro_lib PRIVATE ${ESPEAK_NG_LIBRARY})
         target_compile_definitions(kokoro_lib PRIVATE KOKORO_USE_ESPEAK)
+        # One lock + init owner for the process-global libespeak-ng, shared
+        # with tools/misaki-g2p (header-only; see tools/espeak-shared).
+        target_include_directories(kokoro_lib PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../espeak-shared)
         message(STATUS "Kokoro G2P: libespeak-ng found (${ESPEAK_NG_LIBRARY}) — real IPA path enabled")
     else()
         message(STATUS "Kokoro G2P: libespeak-ng not found — ASCII grapheme fallback; g2p_kind() reports ASCII so the TS layer feeds IPA via kokoro_synthesize_ipa (#11776)")

tools/kokoro/src/kokoro-phonemes.cpp

@@ -24,7 +24,8 @@
 
 #if defined(KOKORO_USE_ESPEAK)
 #include <espeak-ng/speak_lib.h>
-#include <mutex>
+// one lock + init owner shared with tools/misaki-g2p (libespeak-ng is process-global)
+#include "eliza-espeak-shared.h"
 #endif
 
 namespace eliza_kokoro {
@@ -113,31 +114,21 @@ char32_t next_codepoint(const std::string & s, size_t & i) noexcept {
 
 #if defined(KOKORO_USE_ESPEAK)
 
-// espeak-ng is process-global and not thread-safe; serialize init + calls.
-std::mutex & espeak_mutex() {
-    static std::mutex m;
-    return m;
-}
-
-bool ensure_espeak_init() {
-    static bool ok = [] {
-        // AUDIO_OUTPUT_SYNCHRONOUS so no audio device is opened.
-        const int rate = espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, nullptr, 0);
-        if (rate < 0) {
-            return false;
-        }
-        return espeak_SetVoiceByName("en-us") == EE_OK;
-    }();
-    return ok;
-}
+// espeak-ng is process-global and not thread-safe: every call goes through
+// the shared lock, and the voice is re-selected under it on each use because
+// misaki-g2p (same process, fused library) selects its own. Kokoro uses
+// libespeak-ng's default data path, or the explicit one misaki was opened
+// with when that happened first (see eliza-espeak-shared.h).
+constexpr const char * kVoice = "en-us";
 
 // Run espeak over text, accumulating IPA across all clauses. espeak returns
 // one clause per call (stopping at sentence/comma punctuation) and advances
 // the text pointer; we join clauses with a single space to match the
 // `espeak-ng -q --ipa` binary output the reference was derived from.
 std::string espeak_text_to_ipa(const std::string & text) {
-    std::lock_guard<std::mutex> lock(espeak_mutex());
-    if (!ensure_espeak_init()) {
+    auto lock = eliza_espeak_shared::lock();
+    std::string err;
+    if (!eliza_espeak_shared::prepare(lock, std::string(), kVoice, err)) {
         return std::string();
     }
     const void * inptr = static_cast<const void *>(text.c_str());
@@ -231,8 +222,9 @@ std::vector<int32_t> phonemize_to_input_ids(const std::string & text) {
 
 bool espeak_available() noexcept {
 #if defined(KOKORO_USE_ESPEAK)
-    std::lock_guard<std::mutex> lock(espeak_mutex());
-    return ensure_espeak_init();
+    auto lock = eliza_espeak_shared::lock();
+    std::string err;
+    return eliza_espeak_shared::prepare(lock, std::string(), kVoice, err);
 #else
     return false;
 #endif

tools/kokoro/tests/CMakeLists.txt

@@ -50,3 +50,7 @@ add_test(NAME test-kokoro-layer-options COMMAND test-kokoro-layer-options)
 add_executable(test-kokoro-cancel test_kokoro_cancel.cpp)
 target_link_libraries(test-kokoro-cancel PRIVATE kokoro_lib)
 add_test(NAME test-kokoro-cancel COMMAND test-kokoro-cancel)
+
+# "main" is the label CI runs (ctest -L main); unlabelled tests are built but
+# never executed there. Only the self-contained, weight-free, offline tests.
+set_tests_properties(test-kokoro-cancel test-kokoro-layer-options test-kokoro-istft test-kokoro-layers-portable PROPERTIES LABELS main)

tools/kokoro/tests/test_kokoro_layer_options.cpp

@@ -102,10 +102,19 @@ int main() {
             detail::KokoroLayerOptionsScope s(opts(true));
             snake1d_forward(x1.data(), 4, 1000, a.data());
         }
+        // Both paths round the final x + sin^2(a x) / a to float, so each
+        // can be up to half an ulp of the output away from the exact value.
+        // Subtract that one-ulp allowance, then scale by |a| so the result
+        // is in sin^2 units and comparable to the fast_sin2 bound.
         double e = 0;
         for (int c = 0; c < 4; ++c)
-            for (int t = 0; t < 1000; ++t)
-                e = std::max(e, std::fabs((double) x[c * 1000 + t] - x1[c * 1000 + t]) * std::fabs(a[c]));
+            for (int t = 0; t < 1000; ++t) {
+                const float r = x[c * 1000 + t];
+                const double ulp = (double) std::nextafter(std::fabs(r), INFINITY) - std::fabs(r);
+                const double d = std::fabs((double) r - x1[c * 1000 + t]);
+                e = std::max(e, std::max(0.0, d - ulp) * std::fabs(a[c]));
+            }
+        std::printf("snake1d fast vs libm: max excess over output rounding %.3g\n", e);
         CHECK(e < 1e-6, "snake mismatch %.3g", e);
     }
 

tools/misaki-g2p/CMakeLists.txt

@@ -69,6 +69,9 @@ if (MISAKI_G2P_ESPEAK)
     endif()
     if (MISAKI_G2P_HAS_ESPEAK)
         target_compile_definitions(misaki_g2p PRIVATE MISAKI_G2P_USE_ESPEAK)
+        # One lock + init owner for the process-global libespeak-ng, shared
+        # with tools/kokoro (header-only; see tools/espeak-shared).
+        target_include_directories(misaki_g2p PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../espeak-shared)
         message(STATUS "misaki-g2p: libespeak-ng linked (GPL-3.0-or-later): eSpeak + E2M fallback, as in Paradee training")
     else()
         message(STATUS "misaki-g2p: libespeak-ng not found: out-of-lexicon words are spelled out (fallback kind 0)")

tools/misaki-g2p/include/misaki-g2p.h

@@ -8,7 +8,8 @@
  *
  * Thread safety: a misaki_g2p handle may be used from one thread at a time.
  * Separate handles may run concurrently; calls into eSpeak NG are serialised
- * internally because libespeak-ng is process-global.
+ * internally because libespeak-ng is process-global, under the same lock
+ * Kokoro uses (tools/espeak-shared/eliza-espeak-shared.h).
  */
 #ifndef MISAKI_G2P_H
 #define MISAKI_G2P_H

tools/misaki-g2p/src/espeak.cpp

@@ -6,11 +6,12 @@
 // then misaki espeak.EspeakFallback's eSpeak-to-misaki (E2M) table.
 #include "misaki-internal.h"
 
-#include <mutex>
 #include <stdexcept>
 
 #if defined(MISAKI_G2P_USE_ESPEAK)
 #    include <espeak-ng/speak_lib.h>
+// one lock + init owner shared with tools/kokoro (libespeak-ng is process-global)
+#    include "eliza-espeak-shared.h"
 #endif
 
 namespace misaki {
@@ -152,31 +153,8 @@ std::vector<u32> restore(std::vector<u32> text, std::vector<MarkIndex> marks) {
 }
 
 #if defined(MISAKI_G2P_USE_ESPEAK)
-std::mutex & espeak_mutex() {
-    static std::mutex m;
-    return m;
-}
-bool g_espeak_ready = false;
-std::string g_espeak_path;
-
-// Caller holds espeak_mutex(). (Re)initialises libespeak-ng for data_dir.
-bool ensure_espeak_locked(const std::string & data_dir, std::string & err) {
-    if (g_espeak_ready && g_espeak_path == data_dir) return true;
-    g_espeak_ready = false;
-    int rate = espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, data_dir.empty() ? nullptr : data_dir.c_str(), 0);
-    if (rate <= 0) {
-        err = "espeak_Initialize failed for data dir '" + data_dir + "'";
-        return false;
-    }
-    // phonemizer selects the first voice whose language is en-us
-    if (espeak_SetVoiceByName("gmw/en-US") != EE_OK) {
-        err = "eSpeak NG voice gmw/en-US not found under '" + data_dir + "'";
-        return false;
-    }
-    g_espeak_ready = true;
-    g_espeak_path = data_dir;
-    return true;
-}
+// phonemizer selects the first voice whose language is en-us
+constexpr const char * kVoice = "gmw/en-US";
 #endif
 
 }  // namespace
@@ -229,14 +207,14 @@ u32 EspeakFallback::e2m(const u32 & in) {
 
 std::unique_ptr<EspeakFallback> EspeakFallback::open(const std::string & data_dir, std::string & err) {
 #if defined(MISAKI_G2P_USE_ESPEAK)
-    std::lock_guard<std::mutex> lock(espeak_mutex());
+    auto lock = eliza_espeak_shared::lock();
     // espeak-ng keeps its data path in a 160-byte buffer (N_PATH_HOME) and
     // fails on longer ones with a misleading "No such file" error.
     if (data_dir.size() + sizeof("/espeak-ng-data/phontab") > 160) {
         err = "eSpeak NG data path is too long for libespeak-ng (max ~135 bytes): " + data_dir;
         return nullptr;
     }
-    if (!ensure_espeak_locked(data_dir, err)) return nullptr;
+    if (!eliza_espeak_shared::prepare(lock, data_dir, kVoice, err)) return nullptr;
     return std::unique_ptr<EspeakFallback>(new EspeakFallback(data_dir));
 #else
     (void) data_dir;
@@ -249,9 +227,11 @@ EspeakFallback::~EspeakFallback() = default;
 
 u32 EspeakFallback::phonemize_line(const u32 & text) {
 #if defined(MISAKI_G2P_USE_ESPEAK)
-    std::lock_guard<std::mutex> lock(espeak_mutex());
+    // Re-applies this component's voice under the shared lock: Kokoro may
+    // have selected its own since the last call.
+    auto lock = eliza_espeak_shared::lock();
     std::string init_err;
-    if (!ensure_espeak_locked(data_dir_, init_err)) throw std::runtime_error(init_err);
+    if (!eliza_espeak_shared::prepare(lock, data_dir_, kVoice, init_err)) throw std::runtime_error(init_err);
     std::string utf8 = to_utf8(text);
     const void * ptr = utf8.c_str();
     // EspeakWrapper.text_to_phonemes(text, tie=True)
@@ -290,6 +270,7 @@ u32 EspeakFallback::phonemize_line(const u32 & text) {
     return out;
 #else
     (void) text;
+    (void) data_dir_;
     return u32();
 #endif
 }

tools/misaki-g2p/src/g2p.cpp

@@ -425,12 +425,20 @@ class Lexicon {
                 }
             }
         };
+        // Numbers past the named scales are not read (as the guarded cardinal
+        // branch below): misaki's num2words raises there.
+        bool too_long = false;
         auto extend_num = [&](const u32 & num, bool first) {
             std::string d = to_utf8(num);
+            if (!number_fits(d)) {
+                too_long = true;
+                return;
+            }
             extend_words(number_cardinal(d), first);
         };
         const bool digits = is_ascii_digits(word);
         if (digits && is_ordinal_suffix()) {
+            if (!number_fits(to_utf8(word))) return {};
             extend_words(number_ordinal(to_utf8(word)), true);
         } else if (result.empty() && word.size() == 4 && !(currency && (*currency == U"$" || *currency == U"£" || *currency == U"€")) && digits) {
             extend_words(number_year(to_utf8(word)), true);
@@ -514,12 +522,13 @@ class Lexicon {
                     words = U"point";
                     for (size_t i = 1; i < w2.size(); i++) words += U" " + number_cardinal(to_utf8(u32(1, w2[i])));
                 } else {
+                    if (!number_fits(to_utf8(w2.substr(0, w2.find('.'))))) return {};
                     words = number_float(to_utf8(w2));
                 }
             }
             extend_words(words, true);
         }
-        if (result.empty()) return {};
+        if (too_long || result.empty()) return {};
         u32 joined;
         int rating = 99;
         for (size_t i = 0; i < result.size(); i++) {

tools/misaki-g2p/src/lexicon.cpp

@@ -2,6 +2,8 @@
 // Reader for misaki-en-us.lex (written by convert_misaki_lexicon.py).
 #include "misaki-internal.h"
 
+#include <algorithm>
+#include <cerrno>
 #include <cstdio>
 #include <cstring>
 
@@ -123,7 +125,7 @@ std::unique_ptr<LexFile> LexFile::open(const std::string & path, std::string & e
 
 LexFile::~LexFile() {
 #if !defined(MISAKI_NO_MMAP)
-    if (mapped_ && base_) munmap((void *) base_, size_);
+    if (mapped_ && base_) munmap(const_cast<uint8_t *>(base_), size_);
 #endif
 }