tools/parakeet: native Parakeet-TDT ASR (Phonon-2 five-value) behind eliza_inference_asr_*, with streaming

CMakeLists.txt

@@ -141,6 +141,12 @@ option(LLAMA_BUILD_PARADEE   "Build the Paradee-8M TTS subtree (tools/paradee/)"
 # subtree lives in tools/misaki-g2p/ and is added when present.
 option(LLAMA_BUILD_MISAKI_G2P "Build the misaki G2P subtree (tools/misaki-g2p/)" ON)
 
+# Parakeet-TDT ASR (FastConformer + TDT) for FermionResearch/Phonon-2, the
+# five-value re-quantisation of nvidia/parakeet-tdt-0.6b-v3 (weights
+# CC-BY-4.0). Produces parakeet_lib + parakeet-cli/parakeet-bench; the fused
+# `elizainference` target folds parakeet_lib in with `if(TARGET ...)`.
+option(LLAMA_BUILD_PARAKEET  "Build the Parakeet-TDT ASR subtree (tools/parakeet/)" ON)
+
 # W3-3 deprecation: ELIZA_FUSE_OMNIVOICE=ON is the pre-W3-3 flag for the
 # fork-root `omnivoice/` graft. The graft is gone (see the deprecation
 # block below); turn the flag into a redirect now so the option ON state
@@ -312,6 +318,12 @@ if (LLAMA_BUILD_PARADEE AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
     endif()
 endif()
 
+# Parakeet-TDT ASR — embed-as-library hook (mirrors kokoro above); precedes
+# tools/omnivoice so the fused target can fold parakeet_lib in.
+if (LLAMA_BUILD_PARAKEET AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
+    add_subdirectory(tools/parakeet)
+endif()
+
 if (LLAMA_BUILD_OMNIVOICE)
     if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/omnivoice/CMakeLists.txt")
         message(FATAL_ERROR

tools/CMakeLists.txt

@@ -61,6 +61,12 @@ else()
             message(STATUS "LLAMA_BUILD_PARADEE: skipped because LLAMA_BUILD_KOKORO=OFF")
         endif()
     endif()
+    if (LLAMA_BUILD_PARAKEET)
+        # Parakeet-TDT ASR (Phonon-2 five-value weights). Like kokoro it
+        # precedes omnivoice so the fused `elizainference` target can fold
+        # parakeet_lib in.
+        add_subdirectory(parakeet)
+    endif()
     if (LLAMA_BUILD_OMNIVOICE)
         # The merged omnivoice subtree lives under tools/omnivoice/. The
         # top-level CMakeLists.txt has a parallel add_subdirectory hook for

tools/omnivoice/CMakeLists.txt

@@ -247,7 +247,22 @@ function(eliza_inference_fold_paradee target)
     endif()
 endfunction()
 
+# Parakeet-TDT ASR (tools/parakeet, FermionResearch/Phonon-2 five-value
+# weights). Folded in when parakeet_lib exists (added before this directory);
+# it then serves eliza_inference_asr_* for bundles with a parakeet-tdt GGUF
+# under asr/ (see src/eliza-asr-parakeet.inc). PRIVATE like kokoro/paradee.
+function(eliza_inference_fold_parakeet target)
+    if(TARGET parakeet_lib)
+        target_link_libraries(${target} PRIVATE parakeet_lib)
+        target_include_directories(${target} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../parakeet/include)
+        target_compile_definitions(${target} PRIVATE ELIZA_ENABLE_PARAKEET)
+    endif()
+endfunction()
+
 if(TARGET mtmd)
+    if(TARGET parakeet_lib)
+        message(STATUS "elizainference: Parakeet-TDT ASR enabled (streaming via asr_stream_*)")
+    endif()
     if(TARGET paradee_lib AND TARGET misaki_g2p)
         message(STATUS "elizainference: Paradee TTS + misaki G2P enabled (ABI v17)")
     elseif(TARGET paradee_lib OR TARGET misaki_g2p)
@@ -305,6 +320,7 @@ if(TARGET mtmd)
         target_compile_definitions(elizainference PRIVATE ELIZA_ENABLE_KOKORO)
     endif()
     eliza_inference_fold_paradee(elizainference)
+    eliza_inference_fold_parakeet(elizainference)
     set_target_properties(elizainference PROPERTIES
         OUTPUT_NAME              elizainference
         POSITION_INDEPENDENT_CODE ON
@@ -360,6 +376,7 @@ if(TARGET mtmd)
         target_compile_definitions(elizainference_static PRIVATE ELIZA_ENABLE_KOKORO)
     endif()
     eliza_inference_fold_paradee(elizainference_static)
+    eliza_inference_fold_parakeet(elizainference_static)
     set_target_properties(elizainference_static PROPERTIES
         OUTPUT_NAME              elizainference
         POSITION_INDEPENDENT_CODE ON)
@@ -402,6 +419,20 @@ if(TARGET mtmd)
             endif()
         endif()
 
+        if(TARGET parakeet_lib)
+            # Parakeet-TDT ASR through the real library. Skips (passes) unless
+            # ELIZA_TEST_PARAKEET_BUNDLE / ELIZA_TEST_PARAKEET_PCM are set.
+            add_executable(eliza-inference-test-parakeet-asr tests/ffi-parakeet-asr.cpp)
+            target_compile_features(eliza-inference-test-parakeet-asr PRIVATE cxx_std_17)
+            target_include_directories(eliza-inference-test-parakeet-asr PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+            find_package(Threads REQUIRED)
+            target_link_libraries(eliza-inference-test-parakeet-asr PRIVATE elizainference Threads::Threads)
+            if(BUILD_TESTING)
+                add_test(NAME eliza-inference-test-parakeet-asr COMMAND eliza-inference-test-parakeet-asr)
+                set_property(TEST eliza-inference-test-parakeet-asr PROPERTY LABELS main)
+            endif()
+        endif()
+
         add_executable(eliza-inference-test-paradee-asr
             tests/ffi-paradee-asr.cpp
             tests/paradee-contract/fake-paradee.cpp)

tools/omnivoice/include/eliza-inference-ffi.h

@@ -144,6 +144,28 @@ extern "C" {
  *           non-positive capacity instead of creating a zero-sized context.
  *       Additive symbol; no existing signature changes. A v17 library lacks
  *       the symbol, so loaders must treat it as absent.
+ *       Parakeet-TDT ASR backend (additive behaviour, no new symbols, no
+ *       version bump):
+ *       builds that fold tools/parakeet (`ELIZA_ENABLE_PARAKEET`) serve the
+ *       ASR surface from a parakeet-tdt GGUF under `<bundle>/asr/` (e.g.
+ *       FermionResearch/Phonon-2, CC-BY-4.0) when `mmap_acquire("asr")`
+ *       finds one; `ELIZA_ASR_BACKEND=qwen3|parakeet` overrides. With it:
+ *         - `asr_transcribe_timed` returns TDT token-frame word times (80 ms
+ *           resolution) instead of duration-proportional ones;
+ *         - `asr_stream_supported()` returns 1 while at least one context has
+ *           the parakeet backend loaded (0 before `mmap_acquire("asr")`, after
+ *           `mmap_evict("asr")`, and for the Qwen3-ASR backend), and the
+ *           `asr_stream_*` session decodes pause-delimited windows with local
+ *           agreement. `stream_open` fails cleanly without the backend;
+ *           partial/finish on a stream whose region was evicted return
+ *           ELIZA_ERR_INVALID_ARG;
+ *         - stream `out_tokens` are never written (`*io_n_tokens = 0`):
+ *           Parakeet's SentencePiece vocabulary is not the text model's;
+ *         - `asr_cancel` is polled between encoder layers and decode steps;
+ *         - `asr_stream_feed` may run on another thread than partial/finish
+ *           (it only buffers, and never waits for a decode); `close` must not
+ *           race other calls on the same stream. Partial/finish truncate at a
+ *           UTF-8 boundary when `max_text_bytes` is too small.
  *   v17: Paradee-8M TTS with native misaki G2P, plus cancellation and
  *       eviction for every TTS head and for ASR. All additions are new
  *       symbols; no existing signature changes.
@@ -293,7 +315,7 @@ const char * eliza_inference_abi_version(void);
  *   ----------------------  --------------------------------------  -----
  *   batch ASR + cancel      eliza_inference_asr_supported            v17
  *   ASR word timestamps     eliza_inference_asr_timestamps_supported v12
- *   streaming ASR           eliza_inference_asr_stream_supported     v2 (0 today)
+ *   streaming ASR           eliza_inference_asr_stream_supported     v2 (1 with parakeet loaded, v18)
  *   OmniVoice stream+cancel eliza_inference_tts_stream_supported     v2
  *   Kokoro TTS              eliza_inference_kokoro_supported         v10
  *   Kokoro G2P kind         eliza_inference_kokoro_g2p_kind(ctx)     v14

tools/omnivoice/src/eliza-asr-parakeet.inc

@@ -0,0 +1,201 @@
+/* tools/omnivoice/src/eliza-asr-parakeet.inc - Parakeet-TDT (Phonon-2)
+ * backend for the eliza_inference_asr_* surface.
+ *
+ * #included by eliza-inference-ffi.cpp inside `#ifdef ELIZA_ENABLE_PARAKEET`
+ * after EliInferenceContext is defined. Everything here runs on
+ * tools/parakeet (parakeet_lib): native FastConformer encoder + TDT decoder
+ * on the CPU, five-value int8 kernels, no ONNX / MLX / Core ML.
+ *
+ * Selection (mmap_acquire("asr")):
+ *   ELIZA_ASR_BACKEND=parakeet  require a parakeet-tdt GGUF under <bundle>/asr/
+ *   ELIZA_ASR_BACKEND=qwen3     never use parakeet (Qwen3-ASR via mtmd)
+ *   unset / auto                parakeet when <bundle>/asr/ holds a GGUF with
+ *                               general.architecture = "parakeet-tdt",
+ *                               otherwise the Qwen3-ASR path, unchanged.
+ * Threads: ELIZA_ASR_THREADS, else parakeet's default (fast cores, <= 4).
+ *
+ * Streaming: windowed re-decode with local agreement (tools/parakeet/src/
+ * pk-stream.cpp). eliza_inference_asr_stream_supported() returns 1 while at
+ * least one context has the parakeet backend loaded. Token ids are not
+ * returned (*io_n_tokens = 0): Parakeet's 8192-piece SentencePiece vocabulary
+ * is not the text model's.
+ */
+
+#include "parakeet.h"
+
+#include <climits>
+
+static std::atomic<int> g_eliza_parakeet_loaded{ 0 };
+
+struct EliParakeetAsr {
+    pk_model * model = nullptr;
+    std::string path;
+    uint64_t generation = 0;  /* bumped on every load, so open streams notice an evict */
+};
+
+/* Shared by every context (each bumps it under its own asr_mutex). */
+static std::atomic<uint64_t> g_eliza_parakeet_generation{ 0 };
+
+static int eliza_parakeet_cancel_cb(void * user) {
+    auto * ctx = (EliInferenceContext *) user;
+    return ctx && ctx->asr_cancel.load(std::memory_order_acquire) ? 1 : 0;
+}
+
+static std::string eliza_parakeet_backend_pref() {
+    const char * env = std::getenv("ELIZA_ASR_BACKEND");
+    std::string v = env ? env : "";
+    std::transform(v.begin(), v.end(), v.begin(), [](unsigned char c) { return (char) std::tolower(c); });
+    return v;
+}
+
+static std::string eliza_parakeet_find(const std::filesystem::path & bundle_dir) {
+    const auto dir = bundle_dir / "asr";
+    std::error_code ec;
+    if (!std::filesystem::is_directory(dir, ec)) return "";
+    std::vector<std::string> found;
+    for (const auto & e : std::filesystem::directory_iterator(dir, ec)) {
+        if (!e.is_regular_file(ec) || e.path().extension() != ".gguf") continue;
+        if (pk_gguf_is_parakeet(e.path().string().c_str())) found.push_back(e.path().string());
+    }
+    std::sort(found.begin(), found.end());
+    return found.empty() ? "" : found.front();
+}
+
+/* Caller holds asr_mutex. Returns 1 when the parakeet backend is (now)
+ * loaded, 0 when this bundle / preference does not use it, negative on a
+ * load error. */
+static int eliza_parakeet_load_locked(EliInferenceContext * ctx, char ** out_error) {
+    if (ctx->parakeet && ctx->parakeet->model) return 1;
+    const std::string pref = eliza_parakeet_backend_pref();
+    if (pref == "qwen3" || pref == "mtmd") return 0;
+    const std::string path = eliza_parakeet_find(std::filesystem::path(ctx->bundle_dir));
+    if (path.empty()) {
+        if (pref == "parakeet") {
+            eliza_set_error(out_error, std::string("[libelizainference] ELIZA_ASR_BACKEND=parakeet but no parakeet-tdt GGUF under ") +
+                                           (std::filesystem::path(ctx->bundle_dir) / "asr").string());
+            return ELIZA_ERR_INVALID_ARG;
+        }
+        return 0;
+    }
+    pk_params pp = pk_default_params();
+    /* same policy as the Qwen3-ASR path: Android's CPU profile keeps ASR
+     * weights in private memory (ELIZA_ASR_USE_MMAP overrides) */
+    pp.use_mmap = eliza_asr_use_mmap() ? 1 : 0;
+    if (const char * env = std::getenv("ELIZA_ASR_THREADS")) {
+        const int n = std::atoi(env);
+        if (n > 0) pp.n_threads = n;
+    }
+    char err[512] = { 0 };
+    eliza_asr_debug_log("loading parakeet-tdt ASR");
+    pk_model * m = pk_model_load(path.c_str(), pp, err, sizeof(err));
+    if (!m) {
+        eliza_set_error(out_error, std::string("[libelizainference] parakeet ASR load failed: ") + err);
+        return ELIZA_ERR_FFI_FAULT;
+    }
+    if (!ctx->parakeet) ctx->parakeet = new EliParakeetAsr();
+    ctx->parakeet->model = m;
+    ctx->parakeet->path = path;
+    ctx->parakeet->generation = g_eliza_parakeet_generation.fetch_add(1) + 1;
+    g_eliza_parakeet_loaded.fetch_add(1);
+    eliza_asr_debug_log("parakeet-tdt ASR ready");
+    return 1;
+}
+
+/* Caller holds asr_mutex. */
+static void eliza_parakeet_free_locked(EliInferenceContext * ctx) {
+    if (!ctx->parakeet) return;
+    if (ctx->parakeet->model) {
+        pk_model_free(ctx->parakeet->model);
+        g_eliza_parakeet_loaded.fetch_sub(1);
+    }
+    delete ctx->parakeet;
+    ctx->parakeet = nullptr;
+}
+
+static bool eliza_parakeet_active_locked(const EliInferenceContext * ctx) {
+    return ctx->parakeet && ctx->parakeet->model;
+}
+
+/* Caller holds asr_mutex and has cleared asr_cancel. */
+static int eliza_parakeet_transcribe_locked(
+    EliInferenceContext * ctx,
+    const float * pcm,
+    size_t n_samples,
+    int sample_rate_hz,
+    std::string & out_text,
+    std::vector<int> * word_start,
+    std::vector<int> * word_end,
+    char ** out_error) {
+    pk_model * m = ctx->parakeet->model;
+    std::vector<float> audio = eliza_resample_linear(pcm, n_samples, sample_rate_hz, pk_model_sample_rate(m));
+    pk_result r{};
+    const int rc = pk_transcribe(m, audio.data(), audio.size(), eliza_parakeet_cancel_cb, ctx, &r);
+    if (rc == PK_E_CANCELLED) {
+        pk_result_free(&r);
+        eliza_set_error(out_error, "[libelizainference] asr_transcribe: cancelled");
+        return ELIZA_ERR_CANCELLED;
+    }
+    if (rc != PK_OK) {
+        pk_result_free(&r);
+        eliza_set_error(out_error, "[libelizainference] asr_transcribe: parakeet decode failed rc=" + std::to_string(rc));
+        return ELIZA_ERR_FFI_FAULT;
+    }
+    out_text = r.text ? r.text : "";
+    if (word_start && word_end) {
+        size_t n = r.n_tokens + 1;
+        word_start->assign(n, 0);
+        word_end->assign(n, 0);
+        pk_result_words(m, &r, word_start->data(), word_end->data(), &n);
+        word_start->resize(n);
+        word_end->resize(n);
+    }
+    pk_result_free(&r);
+    return 0;
+}
+
+/* ---- streaming ---------------------------------------------------------- */
+
+/* Threading: feed() may run on the audio thread while partial()/finish()
+ * decode on another. feed() only resamples into `pending` under `feed_mu`
+ * (never waits for a decode); partial()/finish() hold ctx->asr_mutex, move
+ * `pending` into the pk_stream under `feed_mu`, then decode without it.
+ * close() must not race any other call on the same stream. */
+struct EliAsrStream {
+    EliInferenceContext * ctx = nullptr;
+    pk_stream * st = nullptr;
+    uint64_t generation = 0;
+    int in_rate = 16000;
+    std::mutex feed_mu;
+    std::vector<float> pending;  /* 16 kHz, not yet handed to st */
+    bool finished = false;       /* under feed_mu */
+    /* continuous linear resampler state (in_rate -> 16 kHz), under feed_mu */
+    double phase = 0.0;
+    float last = 0.0f;
+    bool have_last = false;
+};
+
+/* Appends to `out`. Caller holds s->feed_mu. */
+static void eliza_parakeet_stream_resample(EliAsrStream * s, const float * pcm, size_t n, std::vector<float> & out) {
+    if (s->in_rate == 16000) {
+        out.insert(out.end(), pcm, pcm + n);
+        return;
+    }
+    const double step = (double) s->in_rate / 16000.0;
+    /* input index space: -1 = s->last, 0..n-1 = pcm */
+    double pos = s->phase;
+    for (;;) {
+        const double ip = std::floor(pos);
+        const long i0 = (long) ip;
+        if (i0 + 1 >= (long) n) break;
+        const float a = i0 < 0 ? (s->have_last ? s->last : pcm[0]) : pcm[i0];
+        const float b = pcm[i0 + 1];
+        const float t = (float) (pos - ip);
+        out.push_back(a + (b - a) * t);
+        pos += step;
+    }
+    if (n > 0) {
+        s->last = pcm[n - 1];
+        s->have_last = true;
+        s->phase = pos - (double) n;  /* relative to the next chunk; >= -1 */
+    }
+}