From 8e7ced9ac737060eb6eb067ca2be41331c83c1b9 Mon Sep 17 00:00:00 2001 From: Shaw Date: Tue, 13 Oct 2026 22:04:42 -0400 Subject: [PATCH 1/2] feat(omnivoice): libelizainference ABI v17 (Paradee, ASR cancel, TTS evict) ABI v17 adds these symbols and changes no existing signature: - Paradee-8M TTS with native misaki G2P, guarded by ELIZA_ENABLE_PARADEE. Paradee is linked when both the paradee_lib and misaki_g2p targets exist, into both the shared and the static elizainference. New entries: paradee_supported, load, sample_rate, g2p_kind, phonemize, synthesize_alloc and synthesize_phonemes_alloc. The phoneme string is split into chunks of at most 510 ids (sentence end, then clause, then space, then hard slice). Each chunk is padded with 0 at both ends, and the chunks' PCM is joined into one buffer. The model and lexicon live on the ctx under their own mutex. - cancel_tts now also cancels Kokoro and Paradee. Each engine has its own flag and clears it when a call starts. Paradee checks its flag through the synth cancel callback and between chunks. Kokoro checks before and after the forward pass. - New asr_cancel: an atomic flag set without asr_mutex. It is wired to the llama abort callback on asr_lctx and checked between mtmd chunks (which now use eval_chunk_single) and before each decode token. A cancelled call returns ELIZA_ERR_CANCELLED. New asr_supported returns 1. - mmap_evict("tts") now also frees Kokoro, Paradee and the lexicon. It keeps their paths, and the next synth or mmap_acquire("tts") reloads them. acquire("tts") succeeds with no OmniVoice weights present when a Kokoro or Paradee head is configured. The OmniVoice GGUF picker skips tts/paradee/ and paradee* files. - The header documents the regions and lists one probe per capability. Tests: - eliza-inference-test-paradee-asr builds the bridge against scripted stand-ins for the paradee/misaki contract. It covers chunking, a load/synth/cancel/evict race across threads, evict then reload, and ASR cancel during prefill and during decode. - eliza-inference-test-abi checks the real library. It covers the ABI string "17", the probes, the no-op calls, and an opt-in real-model ASR cancel. Co-Authored-By: Claude Opus 5.5 --- tools/omnivoice/CMakeLists.txt | 80 ++ tools/omnivoice/include/eliza-inference-ffi.h | 195 +++- tools/omnivoice/src/eliza-inference-ffi.cpp | 923 ++++++++++++++++-- tools/omnivoice/tests/ffi-abi.c | 246 +++++ tools/omnivoice/tests/ffi-paradee-asr.cpp | 617 ++++++++++++ .../paradee-contract/fake-paradee-control.h | 28 + .../tests/paradee-contract/fake-paradee.cpp | 178 ++++ .../tests/paradee-contract/misaki-g2p.h | 41 + .../tests/paradee-contract/paradee.h | 72 ++ 9 files changed, 2266 insertions(+), 114 deletions(-) create mode 100644 tools/omnivoice/tests/ffi-abi.c create mode 100644 tools/omnivoice/tests/ffi-paradee-asr.cpp create mode 100644 tools/omnivoice/tests/paradee-contract/fake-paradee-control.h create mode 100644 tools/omnivoice/tests/paradee-contract/fake-paradee.cpp create mode 100644 tools/omnivoice/tests/paradee-contract/misaki-g2p.h create mode 100644 tools/omnivoice/tests/paradee-contract/paradee.h diff --git a/tools/omnivoice/CMakeLists.txt b/tools/omnivoice/CMakeLists.txt index 1e8733ecb..71522cf71 100644 --- a/tools/omnivoice/CMakeLists.txt +++ b/tools/omnivoice/CMakeLists.txt @@ -232,7 +232,28 @@ endif() # (the fused build links mtmd anyway), opt out with -DELIZA_ENABLE_VISION=OFF. option(ELIZA_ENABLE_VISION "Build the fused mmproj vision-describe ABI (v9)" ON) +# Paradee-8M TTS + native misaki G2P (ABI v17). Folded in only when BOTH +# paradee_lib and misaki_g2p exist (tools/paradee, tools/misaki-g2p; they must +# be added before this directory, like kokoro_lib). PRIVATE: the +# eliza_paradee:: / eliza_misaki:: C++ APIs stay behind the +# eliza_inference_paradee_* surface. +function(eliza_inference_fold_paradee target) + if(TARGET paradee_lib AND TARGET misaki_g2p) + target_link_libraries(${target} PRIVATE paradee_lib misaki_g2p) + target_include_directories(${target} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../paradee/include + ${CMAKE_CURRENT_SOURCE_DIR}/../misaki-g2p/include) + target_compile_definitions(${target} PRIVATE ELIZA_ENABLE_PARADEE) + endif() +endfunction() + if(TARGET mtmd) + if(TARGET paradee_lib AND TARGET misaki_g2p) + message(STATUS "elizainference: Paradee TTS + misaki G2P enabled (ABI v17)") + elseif(TARGET paradee_lib OR TARGET misaki_g2p) + message(WARNING "elizainference: Paradee needs both paradee_lib and misaki_g2p; " + "building without Paradee (paradee_supported() == 0)") + endif() add_library(elizainference SHARED ${OMNIVOICE_CORE_SOURCES} ${OMNIVOICE_FFI_SOURCES}) @@ -283,6 +304,7 @@ if(TARGET mtmd) ${CMAKE_CURRENT_SOURCE_DIR}/../kokoro/include) target_compile_definitions(elizainference PRIVATE ELIZA_ENABLE_KOKORO) endif() + eliza_inference_fold_paradee(elizainference) set_target_properties(elizainference PROPERTIES OUTPUT_NAME elizainference POSITION_INDEPENDENT_CODE ON @@ -337,6 +359,7 @@ if(TARGET mtmd) ${CMAKE_CURRENT_SOURCE_DIR}/../kokoro/include) target_compile_definitions(elizainference_static PRIVATE ELIZA_ENABLE_KOKORO) endif() + eliza_inference_fold_paradee(elizainference_static) set_target_properties(elizainference_static PROPERTIES OUTPUT_NAME elizainference POSITION_INDEPENDENT_CODE ON) @@ -345,6 +368,63 @@ if(TARGET mtmd) set_target_properties(elizainference_static PROPERTIES OUTPUT_NAME elizainference_static) endif() add_dependencies(elizainference_static omnivoice-version) + + # FFI tests (ABI v17). + # eliza-inference-test-abi links the real libelizainference: ABI + # string, probes, cancel/evict no-ops, and + # (opt-in, ELIZA_TEST_ASR_DIR) a real-model + # ASR cancel during prefill. + # eliza-inference-test-paradee-asr compiles the FFI bridge into the test + # (ELIZA_INFERENCE_TEST_HOOKS) against the + # scripted Paradee/misaki doubles in + # tests/paradee-contract/: load/synth/cancel + # race, chunking, evict -> reload, ASR cancel + # during prefill and decode. + option(ELIZA_INFERENCE_BUILD_FFI_TESTS "Build the libelizainference FFI tests" ${LLAMA_BUILD_TESTS}) + if(ELIZA_INFERENCE_BUILD_FFI_TESTS) + if(NOT WIN32) # POSIX threads / mkdtemp + add_executable(eliza-inference-test-abi tests/ffi-abi.c) + set_target_properties(eliza-inference-test-abi PROPERTIES + C_STANDARD 11 + C_STANDARD_REQUIRED ON) + target_include_directories(eliza-inference-test-abi PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include) + find_package(Threads REQUIRED) + target_link_libraries(eliza-inference-test-abi PRIVATE elizainference Threads::Threads) + if(TARGET paradee_lib AND TARGET misaki_g2p) + target_compile_definitions(eliza-inference-test-abi PRIVATE ELIZA_TEST_EXPECT_PARADEE=1) + endif() + if(BUILD_TESTING) + add_test(NAME eliza-inference-test-abi COMMAND eliza-inference-test-abi) + endif() + endif() + + add_executable(eliza-inference-test-paradee-asr + tests/ffi-paradee-asr.cpp + tests/paradee-contract/fake-paradee.cpp) + target_compile_features(eliza-inference-test-paradee-asr PRIVATE cxx_std_17) + target_compile_definitions(eliza-inference-test-paradee-asr PRIVATE + ${OMNIVOICE_COMPILE_DEFINITIONS} + ELIZA_ENABLE_PARADEE ELIZA_INFERENCE_TEST_HOOKS) + # The contract doubles come first so "paradee.h" / "misaki-g2p.h" + # resolve to them even when the real libraries are also configured. + target_include_directories(eliza-inference-test-paradee-asr BEFORE PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/tests/paradee-contract) + target_include_directories(eliza-inference-test-paradee-asr PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_BINARY_DIR} + ${OMNIVOICE_LLAMA_PRIVATE_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR}/../mtmd + ${CMAKE_CURRENT_SOURCE_DIR}/../../include + ${CMAKE_CURRENT_SOURCE_DIR}/../../common) + target_link_libraries(eliza-inference-test-paradee-asr PRIVATE + omnivoice_lib llama mtmd eliza_voice_classifiers llama-common) + + if(BUILD_TESTING) + add_test(NAME eliza-inference-test-paradee-asr COMMAND eliza-inference-test-paradee-asr) + endif() + endif() endif() # llama-server wiring — mount the OmniVoice POST /v1/audio/speech route diff --git a/tools/omnivoice/include/eliza-inference-ffi.h b/tools/omnivoice/include/eliza-inference-ffi.h index cbd7ce393..d00f0d2d7 100644 --- a/tools/omnivoice/include/eliza-inference-ffi.h +++ b/tools/omnivoice/include/eliza-inference-ffi.h @@ -1,5 +1,5 @@ /* - * libelizainference FFI ABI v16. + * libelizainference FFI ABI v17. * * (Banner tracks ELIZA_INFERENCE_ABI_VERSION below; the per-version history is * at the end of this header preamble, newest first.) @@ -134,6 +134,34 @@ extern "C" { * load and refuses to bind if they disagree. * * Changelog: + * v17: Paradee-8M TTS with native misaki G2P, plus cancellation and + * eviction for every TTS head and for ASR. All additions are new + * symbols; no existing signature changes. + * - `eliza_inference_paradee_supported/load/sample_rate/g2p_kind/ + * phonemize/synthesize_alloc/synthesize_phonemes_alloc`. Built + * with `ELIZA_ENABLE_PARADEE` (TARGETs paradee_lib + misaki_g2p). + * Text goes through the native misaki lexicon, is split into + * <= 510-id chunks, and the PCM is concatenated. PCM is released + * with `eliza_inference_free_pcm`. + * - `eliza_inference_cancel_tts` now also stops in-flight Kokoro and + * Paradee synthesis (ELIZA_ERR_CANCELLED). Paradee polls the flag + * inside its graph and between chunks; Kokoro checks it before and + * after the forward pass (kokoro_lib has no in-graph hook). + * - `eliza_inference_asr_cancel` + `eliza_inference_asr_supported`. + * Cancel is an atomic flag (no asr_mutex), checked by the llama + * abort callback during prefill/decode, between mtmd chunks, and + * before every decode token. + * - `mmap_evict("tts")` frees Kokoro and Paradee (+ lexicon) as well + * as OmniVoice, keeping their paths; the next synth or + * `mmap_acquire("tts")` reloads them. `mmap_acquire("tts")` on a + * bundle without OmniVoice weights succeeds when a Kokoro or + * Paradee head is configured on the ctx. + * - The OmniVoice GGUF picker skips `tts/paradee/` and `paradee*` + * files, as it already skipped `tts/kokoro/`. + * Streaming ASR (`asr_stream_*`) is unchanged and still reports + * `asr_stream_supported() == 0`. + * v16: context-local text-model GPU selection + * (`eliza_inference_create_with_options`). * v15: Kokoro synthesis can return an exact-size, library-owned PCM buffer * through `eliza_inference_kokoro_synthesize_alloc` and * `eliza_inference_kokoro_synthesize_ipa_alloc`; callers release it with @@ -226,9 +254,9 @@ extern "C" { * v7: real Silero VAD (same symbol surface as v6). * v6: fused wake-word, speaker, diarizer. */ -#define ELIZA_INFERENCE_ABI_VERSION 16 +#define ELIZA_INFERENCE_ABI_VERSION 17 -/* Returns a static, NUL-terminated string of the form "16" matching +/* Returns a static, NUL-terminated string of the form "17" matching * ELIZA_INFERENCE_ABI_VERSION at the time the library was built. The * pointer is owned by the library — do NOT free. */ const char * eliza_inference_abi_version(void); @@ -244,7 +272,34 @@ const char * eliza_inference_abi_version(void); #define ELIZA_ERR_FFI_FAULT -4 /* mmap/madvise/syscall failure */ #define ELIZA_ERR_OOM -5 /* allocation failure */ #define ELIZA_ERR_ABI_MISMATCH -6 /* loader vs library disagree */ -#define ELIZA_ERR_CANCELLED -7 /* caller requested cancellation (chunk cb / cancel_tts) */ +#define ELIZA_ERR_CANCELLED -7 /* caller requested cancellation (chunk cb / cancel_tts / asr_cancel) */ + +/* ---- Capability probes (per capability) --------------------------- * + * + * Probe before binding a surface; an absent symbol (older library) means + * unsupported. All probes are build properties and take no ctx unless noted. + * + * capability probe since + * ---------------------- -------------------------------------- ----- + * batch ASR + cancel eliza_inference_asr_supported v17 + * ASR word timestamps eliza_inference_asr_timestamps_supported v12 + * streaming ASR eliza_inference_asr_stream_supported v2 (0 today) + * OmniVoice stream+cancel eliza_inference_tts_stream_supported v2 + * Kokoro TTS eliza_inference_kokoro_supported v10 + * Kokoro G2P kind eliza_inference_kokoro_g2p_kind(ctx) v14 + * Paradee TTS + misaki eliza_inference_paradee_supported v17 + * misaki fallback kind eliza_inference_paradee_g2p_kind(ctx) v17 + * Silero VAD eliza_inference_vad_supported v3/v7 + * wake word eliza_inference_wakeword_supported v5 + * speaker embedding eliza_inference_speaker_supported v6 + * diarizer eliza_inference_diariz_supported v6 + * streaming LLM eliza_inference_llm_stream_supported v4 + * MTP / KV-cache quant eliza_inference_llm_{mtp,kv_quant}_supported v8 + * embeddings eliza_inference_embed_supported v9 + * vision describe eliza_inference_vision_supported v9 + * tokenize / detokenize eliza_inference_tokenize_supported v9 + * end-of-turn scoring eliza_inference_llm_eot_supported v11 + */ /* ---- Lifecycle ------------------------------------------------------ */ @@ -279,7 +334,10 @@ void eliza_inference_destroy(EliInferenceContext * ctx); * by calling `mmap_evict(...)` on the same region names. * * `region_name` is a stable string in the set: - * - "tts" : OmniVoice weights (mmap of tts/omnivoice-*.gguf) + * - "tts" : every TTS head on the ctx. OmniVoice weights (mmap of + * tts/omnivoice-*.gguf), plus the Kokoro model + voice and the + * Paradee model + misaki lexicon once they have been loaded with + * `kokoro_load` / `paradee_load`. * - "asr" : ASR weights (mmap of asr/...) * - "text" : text+vision weights (kept hot — always acquired) * - "dflash" : drafter weights (kept hot — always acquired) @@ -291,7 +349,20 @@ void eliza_inference_destroy(EliInferenceContext * ctx); * voice-only region to minimize voice-off RSS. Callers must treat an * evicted region as unavailable until a later `mmap_acquire(region)`. * The "text" and "dflash" regions are allowed to be no-ops because the - * text runtime keeps them hot for voice-off text turns. */ + * text runtime keeps them hot for voice-off text turns. + * + * Region semantics (ABI v17): + * evict("tts") frees OmniVoice, Kokoro and Paradee + lexicon. Each head + * is freed under its own lock, so a synth in flight on that + * head finishes first. Kokoro / Paradee paths are kept. + * acquire("tts") reloads any Kokoro / Paradee head evicted above, then + * loads OmniVoice. It succeeds without OmniVoice weights in + * the bundle when a Kokoro or Paradee head is configured. + * Kokoro / Paradee synth calls also reload an evicted head themselves, so + * acquire("tts") is optional for them. OmniVoice synth still requires it. + * evict("asr") frees the ASR model, context, mmproj and sampler + * (unchanged); acquire("asr") reloads them. + * "text", "dflash", "vad": no-ops. */ int eliza_inference_mmap_acquire( EliInferenceContext * ctx, const char * region_name, @@ -373,7 +444,15 @@ int eliza_inference_tts_synthesize_stream( * `eliza_inference_tts_synthesize_stream` on another thread). The * in-flight call returns ELIZA_ERR_CANCELLED at the next kernel * boundary. Returns ELIZA_OK whether or not a forward pass was running - * (cancelling nothing is not an error). */ + * (cancelling nothing is not an error). + * + * ABI v17: also cancels in-flight Kokoro (`kokoro_synthesize*`) and Paradee + * (`paradee_synthesize*`) calls. Each engine has its own flag and clears it + * when a call on that engine starts, so a cancel never carries over into a + * call that starts after it. Paradee stops inside the graph or between + * chunks; Kokoro is checked before and after its forward pass, so a Kokoro + * call finishes its current pass and then returns ELIZA_ERR_CANCELLED with + * no audio. */ int eliza_inference_cancel_tts( EliInferenceContext * ctx, char ** out_error); @@ -483,6 +562,87 @@ int eliza_inference_kokoro_synthesize_ipa_alloc( void eliza_inference_free_pcm(float * pcm); +/* ---- Paradee-8M TTS + native misaki G2P (ABI v17) ----------------- * + * + * Paradee-8M v1.0 (Apache-2.0, distilled from Kokoro-82M): one voice + * (af_heart), 24 kHz mono fp32 out. The model input is misaki phoneme ids + * (the Kokoro vocab) with a 0 pad at each end, at most 512 ids, plus a speed + * scalar. G2P runs natively through the misaki English lexicon + fallback + * linked into this library; no ONNX, no host phonemizer. + * + * Build flag `ELIZA_ENABLE_PARADEE` (TARGETs paradee_lib + misaki_g2p). In a + * build without it `paradee_supported() == 0` and every other entry returns + * ELIZA_ERR_NOT_IMPLEMENTED. Model + lexicon are owned by the ctx, guarded by + * their own lock (independent of OmniVoice / Kokoro), freed by + * `mmap_evict("tts")` (reloaded lazily) and by `eliza_inference_destroy`. + * `eliza_inference_cancel_tts` cancels an in-flight Paradee synth. */ + +/* 1 when this build linked paradee_lib + misaki_g2p; 0 otherwise. */ +int eliza_inference_paradee_supported(void); + +/* Load the Paradee GGUF and the misaki lexicon into `ctx`. A repeat call with + * the same paths is a no-op. `espeak_data_dir` may be NULL: the lexicon then + * uses its rules fallback for out-of-vocabulary words (see `g2p_kind`). + * On failure the previously loaded model, if any, stays loaded. Returns + * ELIZA_OK or a negative ELIZA_* code with `*out_error` set. */ +int eliza_inference_paradee_load( + EliInferenceContext * ctx, + const char * gguf_path, + const char * lexicon_path, + const char * espeak_data_dir, + char ** out_error); + +/* Sample rate of the loaded model (24000 for v1.0). Still answers after + * `mmap_evict("tts")`. Negative ELIZA_* if `paradee_load` never succeeded. */ +int eliza_inference_paradee_sample_rate(EliInferenceContext * ctx); + +/* How the loaded misaki lexicon phonemizes words it does not contain. */ +#define ELIZA_PARADEE_G2P_LEXICON_ONLY 0 /* no fallback */ +#define ELIZA_PARADEE_G2P_RULES 1 /* native letter-to-sound rules */ +#define ELIZA_PARADEE_G2P_ESPEAK 2 /* eSpeak NG, converted to misaki spelling */ +/* Returns one of ELIZA_PARADEE_G2P_*, or a negative ELIZA_* code when no + * lexicon was loaded (or on a non-Paradee build). */ +int eliza_inference_paradee_g2p_kind(EliInferenceContext * ctx); + +/* Run the native G2P only: UTF-8 English text in, misaki phoneme string out. + * On success `*out_utf8` is a NUL-terminated heap string (free it with + * `eliza_inference_free_string`) and the return value is its byte length. */ +int eliza_inference_paradee_phonemize( + EliInferenceContext * ctx, + const char * text, + size_t text_len, + char ** out_utf8, + char ** out_error); + +/* Text -> PCM. Native misaki G2P, then the phoneme string is split into + * <= 510-id chunks (sentence ends, then clause punctuation, then spaces), + * each chunk is padded and synthesized, and the PCM is concatenated. `speed` + * scales the speaking rate (1.0 = native; <= 0 or non-finite means 1.0). + * On success `*out_pcm` is a malloc-owned buffer of `*out_samples` floats at + * `paradee_sample_rate` (NULL only when `*out_samples == 0`); release it with + * `eliza_inference_free_pcm`. Returns ELIZA_OK, ELIZA_ERR_CANCELLED after + * `eliza_inference_cancel_tts`, or another negative ELIZA_* code. */ +int eliza_inference_paradee_synthesize_alloc( + EliInferenceContext * ctx, + const char * text, + size_t text_len, + float speed, + float ** out_pcm, + size_t * out_samples, + char ** out_error); + +/* Same as `paradee_synthesize_alloc`, but the input is a misaki phoneme + * string (as returned by `paradee_phonemize`); G2P is skipped. Characters + * outside the model vocab are dropped. */ +int eliza_inference_paradee_synthesize_phonemes_alloc( + EliInferenceContext * ctx, + const char * misaki_phonemes, + size_t phonemes_len, + float speed, + float ** out_pcm, + size_t * out_samples, + char ** out_error); + /* ---- OmniVoice reference encode (ABI v4) -------------------------- * * * Encode a 24 kHz mono fp32 PCM buffer through the OmniVoice tokenizer @@ -778,6 +938,27 @@ int eliza_inference_asr_transcribe( size_t max_text_bytes, char ** out_error); +/* ---- ASR probe + cancel (ABI v17) --------------------------------- */ + +/* 1 when this build has the batch ASR path (mtmd audio encoder compiled in). + * Always 1 for libelizainference, which is only built with mtmd. Whether the + * bundle has ASR weights is answered by `mmap_acquire("asr")`. */ +int eliza_inference_asr_supported(void); + +/* Cancel the `asr_transcribe` / `asr_transcribe_timed` call in flight on + * `ctx` (from another thread). Sets an atomic flag without taking the ASR + * lock. The flag is checked by the llama abort callback during the prefill + * and decode graphs, between prompt chunks, and before every decode token; + * the audio encoder pass itself has no abort hook, so a cancel during it + * lands when that pass ends. The cancelled call returns ELIZA_ERR_CANCELLED. + * Each transcription clears the flag when it starts (after it holds the ASR + * lock), so a cancel with nothing in flight has no effect on later calls. + * Returns ELIZA_OK (cancelling nothing is not an error), or + * ELIZA_ERR_INVALID_ARG for a NULL ctx. */ +int eliza_inference_asr_cancel( + EliInferenceContext * ctx, + char ** out_error); + /* ---- ASR word timestamps (ABI v12) -------------------------------- */ /* Capability probe: returns 1 when this build can emit per-word timestamps diff --git a/tools/omnivoice/src/eliza-inference-ffi.cpp b/tools/omnivoice/src/eliza-inference-ffi.cpp index 0a0ea77c3..8ec811696 100644 --- a/tools/omnivoice/src/eliza-inference-ffi.cpp +++ b/tools/omnivoice/src/eliza-inference-ffi.cpp @@ -63,6 +63,15 @@ extern "C" { #include "kokoro.h" #endif +#ifdef ELIZA_ENABLE_PARADEE +/* Paradee-8M TTS + native misaki G2P (ABI v17). Both are C++ APIs + * (namespaces eliza_paradee / eliza_misaki) linked from paradee_lib and + * misaki_g2p; the omnivoice CMakeLists adds their include dirs and this + * define only when both targets exist. */ +#include "paradee.h" +#include "misaki-g2p.h" +#endif + #include #include #include @@ -133,6 +142,17 @@ struct EliInferenceContext { int asr_sample_rate = 0; int asr_n_batch = 512; std::atomic tts_cancel{false}; + /* Per-engine cancel flags (ABI v17). eliza_inference_cancel_tts sets all + * three; each engine clears only its own flag when a call starts, so an + * OmniVoice call starting on another thread cannot clear a pending + * Paradee cancel (and the reverse). */ + std::atomic kokoro_cancel{false}; + std::atomic paradee_cancel{false}; + /* ASR cancel (ABI v17). Written by eliza_inference_asr_cancel without + * taking asr_mutex; read by the decode loop, by the llama abort callback + * on asr_lctx (prefill + decode graphs), and between mtmd chunks. Cleared + * at the start of each transcription, after asr_mutex is held. */ + std::atomic asr_cancel{false}; std::mutex tts_mutex; std::mutex asr_mutex; /* Streaming-LLM text model, loaded lazily on the first @@ -186,8 +206,26 @@ struct EliInferenceContext { * clauses per reply and must not reload per clause. */ std::string kokoro_gguf_path; std::string kokoro_voice_path; + int kokoro_style_dim = 256; + int kokoro_rate = 0; /* sample rate of the last successful load */ std::mutex kokoro_mutex; #endif +#ifdef ELIZA_ENABLE_PARADEE + /* Paradee TTS + misaki lexicon (ABI v17), owned by the ctx and protected + * by paradee_mutex. The paths are kept after mmap_evict("tts") frees the + * model and lexicon, so the next synth / mmap_acquire("tts") reloads + * them. paradee_configured is true once a load has succeeded. */ + eliza_paradee::paradee_model_ptr paradee_model; + int paradee_rate = 0; /* sample rate of the last successful load */ + int paradee_g2p_kind = -1; /* ELIZA_PARADEE_G2P_* of the last load */ + eliza_misaki::misaki_lexicon_ptr paradee_lexicon; + std::string paradee_gguf_path; + std::string paradee_lexicon_path; + std::string paradee_espeak_dir; + bool paradee_has_espeak_dir = false; + bool paradee_configured = false; + std::mutex paradee_mutex; +#endif }; /* ELZ2 magic 'ELZ1' (the ascii bytes 'E','L','Z','1' little-endian). @@ -454,10 +492,19 @@ static std::string eliza_lower_ascii(std::string value) { // before "omnivoice-*") mis-select it as the OmniVoice LM — ov_init then aborts // with "tensor 'llm.embed_tokens.weight' not found" on every tier that bundles // both backends (0_8b/2b/4b/9b). Drop any candidate under a kokoro/ subdir. +/* Kokoro and Paradee (ABI v17) GGUFs live under tts/kokoro/ and + * tts/paradee/; they are separate TTS heads and must never be picked as the + * OmniVoice model or codec. */ static void eliza_drop_kokoro_ggufs(std::vector & ggufs) { ggufs.erase( std::remove_if(ggufs.begin(), ggufs.end(), [](const std::string & p) { - return p.find("/kokoro/") != std::string::npos; + if (p.find("/kokoro/") != std::string::npos || + p.find("/paradee/") != std::string::npos) { + return true; + } + const std::string name = + eliza_lower_ascii(std::filesystem::path(p).filename().string()); + return name.rfind("paradee", 0) == 0; }), ggufs.end()); } @@ -830,6 +877,15 @@ static int eliza_map_ov_status(ov_status rc) { return ELIZA_ERR_FFI_FAULT; } +/* ASR cancel (ABI v17). Installed as the llama abort callback on asr_lctx, so + * a cancel stops the prefill and decode graphs at the next node boundary + * (llama_decode then returns 2). Also polled between mtmd chunks and before + * every decode token. */ +static bool eliza_asr_cancel_requested(void * user_data) { + EliInferenceContext * ctx = (EliInferenceContext *) user_data; + return ctx && ctx->asr_cancel.load(std::memory_order_acquire); +} + static bool eliza_tts_cancel_requested(void * user_data) { EliInferenceContext * ctx = (EliInferenceContext *) user_data; return ctx && ctx->tts_cancel.load(std::memory_order_acquire); @@ -987,6 +1043,8 @@ static int eliza_load_asr(EliInferenceContext * ctx, char ** out_error) { eliza_set_error(out_error, "[libelizainference] failed to initialize ASR llama context"); return ELIZA_ERR_FFI_FAULT; } + /* ABI v17: eliza_inference_asr_cancel aborts prefill / decode graphs. */ + llama_set_abort_callback(ctx->asr_lctx, eliza_asr_cancel_requested, ctx); eliza_asr_debug_log("initialized ASR llama context; loading audio mmproj"); mtmd_context_params aparams = mtmd_context_params_default(); @@ -1714,6 +1772,14 @@ void eliza_inference_destroy(EliInferenceContext * ctx) { ctx->kokoro_model.reset(); /* kokoro_model_deleter frees the GGUF */ ctx->kokoro_loaded = false; } +#endif +#ifdef ELIZA_ENABLE_PARADEE + { + std::lock_guard lock(ctx->paradee_mutex); + ctx->paradee_model.reset(); + ctx->paradee_lexicon.reset(); + ctx->paradee_configured = false; + } #endif delete ctx; } @@ -1724,33 +1790,15 @@ void eliza_inference_destroy(EliInferenceContext * ctx) { * with its own GGUF reader; loaded once per ctx, owned by the ctx. * ===================================================================== */ -int eliza_inference_kokoro_supported(void) { #ifdef ELIZA_ENABLE_KOKORO - return 1; -#else - return 0; -#endif -} - -int eliza_inference_kokoro_load( +/* Load the Kokoro model + voice into ctx. Caller holds kokoro_mutex. On + * failure the previous model (if any) stays loaded. */ +static int eliza_kokoro_load_locked( EliInferenceContext * ctx, - const char * gguf_path, - const char * voice_bin_path, + const std::string & gguf_path, + const std::string & voice_bin_path, int style_dim, char ** out_error) { -#ifdef ELIZA_ENABLE_KOKORO - if (!ctx || !gguf_path || !voice_bin_path) { - eliza_set_error(out_error, "[libelizainference] kokoro_load: null argument"); - return ELIZA_ERR_INVALID_ARG; - } - if (style_dim <= 0) style_dim = 256; - std::lock_guard lock(ctx->kokoro_mutex); - /* Idempotent: already loaded with these exact paths → reuse, no reload. */ - if (ctx->kokoro_loaded && ctx->kokoro_model && - ctx->kokoro_gguf_path == gguf_path && - ctx->kokoro_voice_path == voice_bin_path) { - return ELIZA_OK; - } std::string err; eliza_kokoro::kokoro_model_ptr model = eliza_kokoro::kokoro_load_model(gguf_path, err); @@ -1767,12 +1815,72 @@ int eliza_inference_kokoro_load( std::string("[libelizainference] kokoro_load_voice_preset failed: ") + err); return ELIZA_ERR_BUNDLE_INVALID; } + ctx->kokoro_rate = eliza_kokoro::kokoro_sample_rate(model.get()); ctx->kokoro_model = std::move(model); ctx->kokoro_voice = std::move(voice); ctx->kokoro_gguf_path = gguf_path; ctx->kokoro_voice_path = voice_bin_path; + ctx->kokoro_style_dim = style_dim; ctx->kokoro_loaded = true; return ELIZA_OK; +} + +/* Reload a Kokoro model that mmap_evict("tts") freed. Caller holds + * kokoro_mutex. Returns ELIZA_ERR_INVALID_ARG when kokoro_load never + * succeeded on this ctx. */ +static int eliza_kokoro_ensure_loaded_locked( + EliInferenceContext * ctx, const char * operation, char ** out_error) { + if (ctx->kokoro_loaded && ctx->kokoro_model) return ELIZA_OK; + if (ctx->kokoro_gguf_path.empty()) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + + ": no model loaded (call kokoro_load first)"); + return ELIZA_ERR_INVALID_ARG; + } + /* Copies: eliza_kokoro_load_locked reassigns the members. */ + const std::string gguf = ctx->kokoro_gguf_path; + const std::string voice = ctx->kokoro_voice_path; + return eliza_kokoro_load_locked(ctx, gguf, voice, ctx->kokoro_style_dim, out_error); +} + +/* Free the Kokoro weights + voice but keep their paths (mmap_evict("tts")). */ +static void eliza_kokoro_evict(EliInferenceContext * ctx) { + std::lock_guard lock(ctx->kokoro_mutex); + ctx->kokoro_model.reset(); + ctx->kokoro_voice = eliza_kokoro::kokoro_voice_preset{}; + ctx->kokoro_loaded = false; +} +#endif + +int eliza_inference_kokoro_supported(void) { +#ifdef ELIZA_ENABLE_KOKORO + return 1; +#else + return 0; +#endif +} + +int eliza_inference_kokoro_load( + EliInferenceContext * ctx, + const char * gguf_path, + const char * voice_bin_path, + int style_dim, + char ** out_error) { +#ifdef ELIZA_ENABLE_KOKORO + if (!ctx || !gguf_path || !voice_bin_path) { + eliza_set_error(out_error, "[libelizainference] kokoro_load: null argument"); + return ELIZA_ERR_INVALID_ARG; + } + if (style_dim <= 0) style_dim = 256; + std::lock_guard lock(ctx->kokoro_mutex); + /* Idempotent: already loaded with these exact paths → reuse, no reload. */ + if (ctx->kokoro_loaded && ctx->kokoro_model && + ctx->kokoro_gguf_path == gguf_path && + ctx->kokoro_voice_path == voice_bin_path && + ctx->kokoro_style_dim == style_dim) { + return ELIZA_OK; + } + return eliza_kokoro_load_locked(ctx, gguf_path, voice_bin_path, style_dim, out_error); #else (void) ctx; (void) gguf_path; (void) voice_bin_path; (void) style_dim; eliza_set_error(out_error, @@ -1799,11 +1907,18 @@ static int eliza_kokoro_synthesize_audio( return ELIZA_ERR_INVALID_ARG; } std::lock_guard lock(ctx->kokoro_mutex); - if (!ctx->kokoro_loaded || !ctx->kokoro_model) { + /* A cancel_tts issued before this call held the lock targets an earlier + * call; start this one clean. kokoro_lib has no in-graph cancel hook, so + * the flag is checked after the (possibly lazy) reload and after the + * forward pass: a cancelled call discards its audio and reports + * ELIZA_ERR_CANCELLED. */ + ctx->kokoro_cancel.store(false, std::memory_order_release); + const int load_rc = eliza_kokoro_ensure_loaded_locked(ctx, operation, out_error); + if (load_rc != ELIZA_OK) return load_rc; + if (ctx->kokoro_cancel.load(std::memory_order_acquire)) { eliza_set_error(out_error, - std::string("[libelizainference] ") + operation + - ": no model loaded (call kokoro_load first)"); - return ELIZA_ERR_INVALID_ARG; + std::string("[libelizainference] ") + operation + ": cancelled"); + return ELIZA_ERR_CANCELLED; } if (speed <= 0.0f) speed = 1.0f; const std::string in(input, input_len ? input_len : std::strlen(input)); @@ -1813,6 +1928,12 @@ static int eliza_kokoro_synthesize_audio( ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err) : eliza_kokoro::kokoro_synthesize( ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err); + if (ctx->kokoro_cancel.exchange(false, std::memory_order_acq_rel)) { + audio.samples.clear(); + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": cancelled"); + return ELIZA_ERR_CANCELLED; + } if (st != eliza_kokoro::KOKORO_OK) { eliza_set_error(out_error, std::string("[libelizainference] ") + operation + " failed: " + err); @@ -1898,8 +2019,12 @@ int eliza_inference_kokoro_sample_rate(EliInferenceContext * ctx) { #ifdef ELIZA_ENABLE_KOKORO if (!ctx) return ELIZA_ERR_INVALID_ARG; std::lock_guard lock(ctx->kokoro_mutex); - if (!ctx->kokoro_loaded || !ctx->kokoro_model) return ELIZA_ERR_INVALID_ARG; - return eliza_kokoro::kokoro_sample_rate(ctx->kokoro_model.get()); + if (ctx->kokoro_loaded && ctx->kokoro_model) { + return eliza_kokoro::kokoro_sample_rate(ctx->kokoro_model.get()); + } + /* Evicted by mmap_evict("tts"): report the rate of the last load. */ + if (!ctx->kokoro_gguf_path.empty() && ctx->kokoro_rate > 0) return ctx->kokoro_rate; + return ELIZA_ERR_INVALID_ARG; #else (void) ctx; return ELIZA_ERR_NOT_IMPLEMENTED; @@ -2009,10 +2134,460 @@ int eliza_inference_kokoro_synthesize_ipa_alloc( #endif } +/* ===================================================================== * + * Paradee-8M TTS + native misaki G2P (ABI v17). Paradee is a single-voice + * (af_heart) 24 kHz model distilled from Kokoro-82M. Its input is misaki + * phoneme ids with a 0 pad token at each end, at most 512 ids, plus a speed + * scalar. Text goes through the native misaki lexicon (misaki_g2p), the + * phoneme string is split into model-sized chunks at sentence, clause and + * word boundaries, each chunk runs one forward pass, and the PCM is + * concatenated. Model + lexicon are owned by the ctx under paradee_mutex. + * ===================================================================== */ + +#ifdef ELIZA_ENABLE_PARADEE +static bool eliza_paradee_cancel_requested(void * user_data) { + EliInferenceContext * ctx = (EliInferenceContext *) user_data; + return ctx && ctx->paradee_cancel.load(std::memory_order_acquire); +} + +static int eliza_paradee_map_g2p_kind(eliza_misaki::misaki_fallback_kind kind) { + switch (kind) { + case eliza_misaki::MISAKI_FALLBACK_ESPEAK: return ELIZA_PARADEE_G2P_ESPEAK; + case eliza_misaki::MISAKI_FALLBACK_RULES: return ELIZA_PARADEE_G2P_RULES; + default: return ELIZA_PARADEE_G2P_LEXICON_ONLY; + } +} + +/* Load model + lexicon into ctx. Caller holds paradee_mutex. On failure the + * previously loaded model and lexicon (if any) stay in place. */ +static int eliza_paradee_load_locked( + EliInferenceContext * ctx, + const std::string & gguf_path, + const std::string & lexicon_path, + const char * espeak_dir, + char ** out_error) { + std::string err; + eliza_paradee::paradee_model_ptr model = eliza_paradee::paradee_load_model(gguf_path, err); + if (!model) { + eliza_set_error(out_error, + std::string("[libelizainference] paradee_load_model failed: ") + err); + return ELIZA_ERR_BUNDLE_INVALID; + } + err.clear(); + eliza_misaki::misaki_lexicon_ptr lexicon = + eliza_misaki::misaki_load_lexicon(lexicon_path, espeak_dir, err); + if (!lexicon) { + eliza_set_error(out_error, + std::string("[libelizainference] misaki_load_lexicon failed: ") + err); + return ELIZA_ERR_BUNDLE_INVALID; + } + ctx->paradee_rate = eliza_paradee::paradee_sample_rate(model.get()); + ctx->paradee_g2p_kind = + eliza_paradee_map_g2p_kind(eliza_misaki::misaki_get_fallback_kind(lexicon.get())); + ctx->paradee_model = std::move(model); + ctx->paradee_lexicon = std::move(lexicon); + ctx->paradee_gguf_path = gguf_path; + ctx->paradee_lexicon_path = lexicon_path; + ctx->paradee_has_espeak_dir = espeak_dir != nullptr; + ctx->paradee_espeak_dir = espeak_dir ? std::string(espeak_dir) : std::string(); + ctx->paradee_configured = true; + return ELIZA_OK; +} + +/* Reload what mmap_evict("tts") freed. Caller holds paradee_mutex. */ +static int eliza_paradee_ensure_loaded_locked( + EliInferenceContext * ctx, const char * operation, char ** out_error) { + if (ctx->paradee_model && ctx->paradee_lexicon) return ELIZA_OK; + if (!ctx->paradee_configured) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + + ": no model loaded (call paradee_load first)"); + return ELIZA_ERR_INVALID_ARG; + } + const std::string gguf = ctx->paradee_gguf_path; + const std::string lexicon = ctx->paradee_lexicon_path; + const std::string espeak = ctx->paradee_espeak_dir; + return eliza_paradee_load_locked( + ctx, gguf, lexicon, ctx->paradee_has_espeak_dir ? espeak.c_str() : nullptr, out_error); +} + +static void eliza_paradee_evict(EliInferenceContext * ctx) { + std::lock_guard lock(ctx->paradee_mutex); + ctx->paradee_model.reset(); + ctx->paradee_lexicon.reset(); +} + +/* Split `text` after any of `boundaries`, keeping the boundary and the spaces + * after it in the left piece. Concatenating the pieces gives `text` back. */ +static std::vector eliza_split_after( + const std::string & text, const std::vector & boundaries) { + std::vector pieces; + size_t start = 0; + size_t i = 0; + while (i < text.size()) { + size_t matched = 0; + for (const std::string & b : boundaries) { + if (!b.empty() && text.compare(i, b.size(), b) == 0) { + matched = b.size(); + break; + } + } + if (matched == 0) { + ++i; + continue; + } + i += matched; + while (i < text.size() && text[i] == ' ') ++i; + pieces.push_back(text.substr(start, i - start)); + start = i; + } + if (start < text.size()) pieces.push_back(text.substr(start)); + return pieces; +} + +/* Pack the misaki phoneme string into chunks of at most `max_ids` vocab ids. + * Splits at sentence ends first, then clause punctuation, then spaces, and + * slices a single over-long word as a last resort. Vocab mapping is per + * character, so the ids of a concatenation are the concatenation of ids. */ +static int eliza_paradee_chunk( + const eliza_paradee::paradee_model * model, + const std::string & phonemes, + size_t max_ids, + int level, + std::vector> & chunks, + std::string & err) { + static const std::vector> kLevels = { + { ".", "!", "?", "\xE2\x80\xA6" /* … */ }, + { ",", ";", ":", "\xE2\x80\x94" /* — */ }, + { " " }, + }; + std::vector ids; + if (eliza_paradee::paradee_phonemes_to_ids(model, phonemes, ids, err) != eliza_paradee::PARADEE_OK) { + return ELIZA_ERR_INVALID_ARG; + } + if (ids.empty()) return ELIZA_OK; + auto append_fit = [&](std::vector piece) { + if (!chunks.empty() && chunks.back().size() + piece.size() <= max_ids) { + chunks.back().insert(chunks.back().end(), piece.begin(), piece.end()); + } else { + chunks.push_back(std::move(piece)); + } + }; + if (ids.size() <= max_ids) { + append_fit(std::move(ids)); + return ELIZA_OK; + } + if (level >= (int) kLevels.size()) { + for (size_t off = 0; off < ids.size(); off += max_ids) { + const size_t end = std::min(ids.size(), off + max_ids); + chunks.emplace_back(ids.begin() + (std::ptrdiff_t) off, ids.begin() + (std::ptrdiff_t) end); + } + return ELIZA_OK; + } + const std::vector pieces = eliza_split_after(phonemes, kLevels[(size_t) level]); + if (pieces.size() <= 1) { + return eliza_paradee_chunk(model, phonemes, max_ids, level + 1, chunks, err); + } + for (const std::string & piece : pieces) { + const int rc = eliza_paradee_chunk(model, piece, max_ids, level, chunks, err); + if (rc != ELIZA_OK) return rc; + } + return ELIZA_OK; +} + +static int eliza_paradee_synthesize_owned( + EliInferenceContext * ctx, + const char * input, + size_t input_len, + bool phoneme_input, + float speed, + float ** out_pcm, + size_t * out_samples, + char ** out_error) { + const char * operation = phoneme_input + ? "paradee_synthesize_phonemes_alloc" + : "paradee_synthesize_alloc"; + if (!out_pcm || !out_samples) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": null output argument"); + return ELIZA_ERR_INVALID_ARG; + } + *out_pcm = nullptr; + *out_samples = 0; + if (!ctx || !input) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": null argument"); + return ELIZA_ERR_INVALID_ARG; + } + if (!(speed > 0.0f) || !std::isfinite(speed)) speed = 1.0f; + const std::string in(input, input_len ? input_len : std::strlen(input)); + + std::lock_guard lock(ctx->paradee_mutex); + /* Start clean: a cancel issued before this call held the lock was aimed + * at an earlier call. */ + ctx->paradee_cancel.store(false, std::memory_order_release); + const int load_rc = eliza_paradee_ensure_loaded_locked(ctx, operation, out_error); + if (load_rc != ELIZA_OK) return load_rc; + auto cancelled = [&]() { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": cancelled"); + return ELIZA_ERR_CANCELLED; + }; + if (ctx->paradee_cancel.load(std::memory_order_acquire)) return cancelled(); + + std::string phonemes; + std::string err; + if (phoneme_input) { + phonemes = in; + } else if (!eliza_misaki::misaki_phonemize(ctx->paradee_lexicon.get(), in, phonemes, err)) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": misaki_phonemize failed: " + err); + return ELIZA_ERR_FFI_FAULT; + } + + std::vector> chunks; + const size_t max_ids = (size_t) eliza_paradee::PARADEE_MAX_INPUT_IDS - 2; + if (eliza_paradee_chunk(ctx->paradee_model.get(), phonemes, max_ids, 0, chunks, err) != ELIZA_OK) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": phoneme mapping failed: " + err); + return ELIZA_ERR_INVALID_ARG; + } + + eliza_paradee::paradee_synth_options opts; + opts.speed = speed; + opts.cancel = eliza_paradee_cancel_requested; + opts.cancel_user_data = ctx; + std::vector pcm; + std::vector input_ids; + for (const std::vector & chunk : chunks) { + if (ctx->paradee_cancel.load(std::memory_order_acquire)) return cancelled(); + input_ids.clear(); + input_ids.reserve(chunk.size() + 2); + input_ids.push_back(0); + input_ids.insert(input_ids.end(), chunk.begin(), chunk.end()); + input_ids.push_back(0); + eliza_paradee::paradee_audio audio; + err.clear(); + const eliza_paradee::paradee_status st = + eliza_paradee::paradee_synthesize(ctx->paradee_model.get(), input_ids, opts, audio, err); + if (st == eliza_paradee::PARADEE_E_CANCELLED) return cancelled(); + if (st != eliza_paradee::PARADEE_OK) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": paradee_synthesize failed: " + err); + return st == eliza_paradee::PARADEE_E_OOM ? ELIZA_ERR_OOM : ELIZA_ERR_FFI_FAULT; + } + pcm.insert(pcm.end(), audio.samples.begin(), audio.samples.end()); + } + if (ctx->paradee_cancel.exchange(false, std::memory_order_acq_rel)) return cancelled(); + if (pcm.empty()) return ELIZA_OK; + + float * owned = static_cast(std::malloc(pcm.size() * sizeof(float))); + if (!owned) { + eliza_set_error(out_error, + std::string("[libelizainference] ") + operation + ": PCM allocation failed"); + return ELIZA_ERR_OOM; + } + std::memcpy(owned, pcm.data(), pcm.size() * sizeof(float)); + *out_pcm = owned; + *out_samples = pcm.size(); + return ELIZA_OK; +} +#endif + +#define ELIZA_PARADEE_NOT_BUILT(out_error) \ + do { \ + eliza_set_error((out_error), \ + "[libelizainference] paradee not built (ELIZA_ENABLE_PARADEE off)"); \ + return ELIZA_ERR_NOT_IMPLEMENTED; \ + } while (0) + +int eliza_inference_paradee_supported(void) { +#ifdef ELIZA_ENABLE_PARADEE + return 1; +#else + return 0; +#endif +} + +int eliza_inference_paradee_load( + EliInferenceContext * ctx, + const char * gguf_path, + const char * lexicon_path, + const char * espeak_data_dir, + char ** out_error) { +#ifdef ELIZA_ENABLE_PARADEE + if (!ctx || !gguf_path || !lexicon_path || !gguf_path[0] || !lexicon_path[0]) { + eliza_set_error(out_error, "[libelizainference] paradee_load: null argument"); + return ELIZA_ERR_INVALID_ARG; + } + if (espeak_data_dir && !espeak_data_dir[0]) espeak_data_dir = nullptr; + std::lock_guard lock(ctx->paradee_mutex); + if (ctx->paradee_model && ctx->paradee_lexicon && + ctx->paradee_gguf_path == gguf_path && + ctx->paradee_lexicon_path == lexicon_path && + ctx->paradee_has_espeak_dir == (espeak_data_dir != nullptr) && + (!espeak_data_dir || ctx->paradee_espeak_dir == espeak_data_dir)) { + return ELIZA_OK; + } + return eliza_paradee_load_locked(ctx, gguf_path, lexicon_path, espeak_data_dir, out_error); +#else + (void) ctx; (void) gguf_path; (void) lexicon_path; (void) espeak_data_dir; + ELIZA_PARADEE_NOT_BUILT(out_error); +#endif +} + +int eliza_inference_paradee_sample_rate(EliInferenceContext * ctx) { +#ifdef ELIZA_ENABLE_PARADEE + if (!ctx) return ELIZA_ERR_INVALID_ARG; + std::lock_guard lock(ctx->paradee_mutex); + if (!ctx->paradee_configured || ctx->paradee_rate <= 0) return ELIZA_ERR_INVALID_ARG; + return ctx->paradee_rate; +#else + (void) ctx; + return ELIZA_ERR_NOT_IMPLEMENTED; +#endif +} + +int eliza_inference_paradee_g2p_kind(EliInferenceContext * ctx) { +#ifdef ELIZA_ENABLE_PARADEE + if (!ctx) return ELIZA_ERR_INVALID_ARG; + std::lock_guard lock(ctx->paradee_mutex); + if (!ctx->paradee_configured || ctx->paradee_g2p_kind < 0) return ELIZA_ERR_INVALID_ARG; + return ctx->paradee_g2p_kind; +#else + (void) ctx; + return ELIZA_ERR_NOT_IMPLEMENTED; +#endif +} + +int eliza_inference_paradee_phonemize( + EliInferenceContext * ctx, + const char * text, + size_t text_len, + char ** out_utf8, + char ** out_error) { +#ifdef ELIZA_ENABLE_PARADEE + if (!out_utf8) { + eliza_set_error(out_error, "[libelizainference] paradee_phonemize: null output argument"); + return ELIZA_ERR_INVALID_ARG; + } + *out_utf8 = nullptr; + if (!ctx || !text) { + eliza_set_error(out_error, "[libelizainference] paradee_phonemize: null argument"); + return ELIZA_ERR_INVALID_ARG; + } + const std::string in(text, text_len ? text_len : std::strlen(text)); + std::lock_guard lock(ctx->paradee_mutex); + const int load_rc = eliza_paradee_ensure_loaded_locked(ctx, "paradee_phonemize", out_error); + if (load_rc != ELIZA_OK) return load_rc; + std::string phonemes; + std::string err; + if (!eliza_misaki::misaki_phonemize(ctx->paradee_lexicon.get(), in, phonemes, err)) { + eliza_set_error(out_error, + std::string("[libelizainference] paradee_phonemize: misaki_phonemize failed: ") + err); + return ELIZA_ERR_FFI_FAULT; + } + if (phonemes.size() > (size_t) INT32_MAX) { + eliza_set_error(out_error, "[libelizainference] paradee_phonemize: output too large"); + return ELIZA_ERR_INVALID_ARG; + } + char * owned = eliza_strdup(phonemes); + if (!owned) { + eliza_set_error(out_error, "[libelizainference] paradee_phonemize: allocation failed"); + return ELIZA_ERR_OOM; + } + *out_utf8 = owned; + return (int) phonemes.size(); +#else + (void) ctx; (void) text; (void) text_len; + if (out_utf8) *out_utf8 = nullptr; + ELIZA_PARADEE_NOT_BUILT(out_error); +#endif +} + +int eliza_inference_paradee_synthesize_alloc( + EliInferenceContext * ctx, + const char * text, + size_t text_len, + float speed, + float ** out_pcm, + size_t * out_samples, + char ** out_error) { +#ifdef ELIZA_ENABLE_PARADEE + return eliza_paradee_synthesize_owned( + ctx, text, text_len, false, speed, out_pcm, out_samples, out_error); +#else + (void) ctx; (void) text; (void) text_len; (void) speed; + if (out_pcm) *out_pcm = nullptr; + if (out_samples) *out_samples = 0; + ELIZA_PARADEE_NOT_BUILT(out_error); +#endif +} + +int eliza_inference_paradee_synthesize_phonemes_alloc( + EliInferenceContext * ctx, + const char * misaki_phonemes, + size_t phonemes_len, + float speed, + float ** out_pcm, + size_t * out_samples, + char ** out_error) { +#ifdef ELIZA_ENABLE_PARADEE + return eliza_paradee_synthesize_owned( + ctx, misaki_phonemes, phonemes_len, true, speed, out_pcm, out_samples, out_error); +#else + (void) ctx; (void) misaki_phonemes; (void) phonemes_len; (void) speed; + if (out_pcm) *out_pcm = nullptr; + if (out_samples) *out_samples = 0; + ELIZA_PARADEE_NOT_BUILT(out_error); +#endif +} + void eliza_inference_free_pcm(float * pcm) { std::free(pcm); } +/* mmap_acquire("tts"): reload any Kokoro / Paradee head that a previous + * mmap_evict("tts") freed, then load OmniVoice. A bundle with no OmniVoice + * GGUF is fine when a Kokoro or Paradee head is configured on this ctx: the + * region is then those heads only, and the OmniVoice entries keep reporting + * "TTS region is not acquired". */ +static int eliza_acquire_tts_region(EliInferenceContext * ctx, char ** out_error) { + bool other_head = false; +#ifdef ELIZA_ENABLE_KOKORO + { + std::lock_guard lock(ctx->kokoro_mutex); + if (!ctx->kokoro_gguf_path.empty()) { + other_head = true; + const int rc = eliza_kokoro_ensure_loaded_locked(ctx, "mmap_acquire(tts)", out_error); + if (rc != ELIZA_OK) return rc; + } + } +#endif +#ifdef ELIZA_ENABLE_PARADEE + { + std::lock_guard lock(ctx->paradee_mutex); + if (ctx->paradee_configured) { + other_head = true; + const int rc = eliza_paradee_ensure_loaded_locked(ctx, "mmap_acquire(tts)", out_error); + if (rc != ELIZA_OK) return rc; + } + } +#endif + if (other_head) { + std::string tts_path; + std::string codec_path; + bool have_omnivoice = false; + { + std::lock_guard lock(ctx->tts_mutex); + have_omnivoice = ctx->ov != nullptr || !ctx->tts_model_path.empty() || + eliza_pick_voice_files(std::filesystem::path(ctx->bundle_dir), tts_path, codec_path); + } + if (!have_omnivoice) return ELIZA_OK; + } + return eliza_load_tts(ctx, out_error); +} + int eliza_inference_mmap_acquire( EliInferenceContext * ctx, const char * region_name, @@ -2026,7 +2601,7 @@ int eliza_inference_mmap_acquire( return ELIZA_ERR_INVALID_ARG; } if (std::strcmp(region_name, "tts") == 0) { - return eliza_load_tts(ctx, out_error); + return eliza_acquire_tts_region(ctx, out_error); } if (std::strcmp(region_name, "asr") == 0) { return eliza_load_asr(ctx, out_error); @@ -2047,11 +2622,24 @@ int eliza_inference_mmap_evict( return ELIZA_ERR_INVALID_ARG; } if (std::strcmp(region_name, "tts") == 0) { - std::lock_guard lock(ctx->tts_mutex); - if (ctx->ov) { - ov_free(ctx->ov); - ctx->ov = nullptr; + /* The "tts" region is every TTS head on the ctx: OmniVoice, Kokoro, + * and Paradee + its misaki lexicon. Each is freed under its own mutex + * (an in-flight synth on that engine finishes first). Kokoro and + * Paradee keep their paths so the next synth or + * mmap_acquire("tts") reloads them. */ + { + std::lock_guard lock(ctx->tts_mutex); + if (ctx->ov) { + ov_free(ctx->ov); + ctx->ov = nullptr; + } } +#ifdef ELIZA_ENABLE_KOKORO + eliza_kokoro_evict(ctx); +#endif +#ifdef ELIZA_ENABLE_PARADEE + eliza_paradee_evict(ctx); +#endif } if (std::strcmp(region_name, "asr") == 0) { std::lock_guard lock(ctx->asr_mutex); @@ -2132,89 +2720,88 @@ int eliza_inference_tts_synthesize( return written; } -/* Shared ASR decode core. Takes the asr_mutex, runs the audio-in/text-out - * decode, and yields the cleaned transcript. Both eliza_inference_asr_transcribe - * and eliza_inference_asr_transcribe_timed call this so the decode lives in - * exactly one place. Returns 0 on success or a negative ELIZA_* code. */ -static int eliza_asr_decode_core( - EliInferenceContext * ctx, - const float * pcm, - size_t n_samples, - int sample_rate_hz, - size_t max_text_bytes, - std::string & out_transcript, - char ** out_error) { - out_transcript.clear(); - if (n_samples == 0) { - return 0; - } - - std::lock_guard lock(ctx->asr_mutex); - if (!ctx->asr_model || !ctx->asr_lctx || !ctx->asr_mtmd || !ctx->asr_sampler) { - eliza_set_error(out_error, "[libelizainference] asr_transcribe: ASR region is not acquired; call mmap_acquire(\"asr\") after arming voice input"); - return ELIZA_ERR_INVALID_ARG; - } +/* The model-facing half of one transcription. The real implementation wraps + * mtmd + llama on the ctx's ASR region; the FFI unit test (built with + * ELIZA_INFERENCE_TEST_HOOKS) substitutes a scripted one so the cancel paths + * in eliza_asr_run_engine are tested without model weights. */ +struct EliAsrEngine { + virtual ~EliAsrEngine() = default; + virtual size_t n_chunks() = 0; + /* Evaluate prompt chunk `i` (text or audio). 0 on success. */ + virtual int32_t eval_chunk(size_t i, bool logits_last) = 0; + virtual llama_token sample() = 0; + virtual bool is_eog(llama_token token) = 0; + virtual std::string piece(llama_token token) = 0; + virtual void accept(llama_token token) = 0; + /* Feed one sampled token back. 0 on success. */ + virtual int32_t decode(llama_token token) = 0; +}; - std::vector audio = eliza_resample_linear(pcm, n_samples, sample_rate_hz, ctx->asr_sample_rate); - std::unique_ptr bitmap( - mtmd_bitmap_init_from_audio(audio.size(), audio.data()), - mtmd_bitmap_free); - if (!bitmap) { - eliza_set_error(out_error, "[libelizainference] asr_transcribe: failed to create audio bitmap"); - return ELIZA_ERR_FFI_FAULT; - } +struct EliAsrMtmdEngine final : EliAsrEngine { + EliInferenceContext * ctx; + mtmd_input_chunks * chunks; + const llama_vocab * vocab; + llama_pos n_past = 0; - std::string prompt = eliza_format_asr_prompt(ctx->asr_model); - mtmd_input_text text = { prompt.c_str(), true, true }; - const mtmd_bitmap * bitmaps[] = { bitmap.get() }; - std::unique_ptr chunks( - mtmd_input_chunks_init(), - mtmd_input_chunks_free); - if (!chunks) { - eliza_set_error(out_error, "[libelizainference] asr_transcribe: failed to allocate input chunks"); - return ELIZA_ERR_FFI_FAULT; - } - int32_t tok_rc = mtmd_tokenize(ctx->asr_mtmd, chunks.get(), &text, bitmaps, 1); - if (tok_rc != 0) { - eliza_set_error(out_error, "[libelizainference] asr_transcribe: mtmd_tokenize failed rc=" + std::to_string(tok_rc)); - return ELIZA_ERR_FFI_FAULT; + EliAsrMtmdEngine(EliInferenceContext * c, mtmd_input_chunks * ch) + : ctx(c), chunks(ch), vocab(llama_model_get_vocab(c->asr_model)) {} + + size_t n_chunks() override { return mtmd_input_chunks_size(chunks); } + int32_t eval_chunk(size_t i, bool logits_last) override { + /* mtmd_helper_eval_chunk_single = one iteration of + * mtmd_helper_eval_chunks. Its llama_decode calls honour the abort + * callback; the audio encoder (mtmd_encode_chunk) has no abort hook, + * so cancel lands at the end of the encoder pass at the latest. */ + return mtmd_helper_eval_chunk_single( + ctx->asr_mtmd, ctx->asr_lctx, mtmd_input_chunks_get(chunks, i), + n_past, 0, ctx->asr_n_batch, logits_last, &n_past); + } + llama_token sample() override { return llama_sampler_sample(ctx->asr_sampler, ctx->asr_lctx, -1); } + bool is_eog(llama_token token) override { return llama_vocab_is_eog(vocab, token); } + std::string piece(llama_token token) override { return eliza_llama_token_piece(vocab, token); } + void accept(llama_token token) override { llama_sampler_accept(ctx->asr_sampler, token); } + int32_t decode(llama_token token) override { + llama_batch batch = llama_batch_get_one(&token, 1); + return llama_decode(ctx->asr_lctx, batch); } +}; - llama_memory_clear(llama_get_memory(ctx->asr_lctx), true); - llama_sampler_reset(ctx->asr_sampler); +static int eliza_asr_cancelled(char ** out_error) { + eliza_set_error(out_error, "[libelizainference] asr_transcribe: cancelled"); + return ELIZA_ERR_CANCELLED; +} - llama_pos n_past = 0; - int32_t eval_rc = mtmd_helper_eval_chunks( - ctx->asr_mtmd, - ctx->asr_lctx, - chunks.get(), - n_past, - 0, - ctx->asr_n_batch, - true, - &n_past); - if (eval_rc != 0) { - eliza_set_error(out_error, "[libelizainference] asr_transcribe: mtmd_helper_eval_chunks failed rc=" + std::to_string(eval_rc)); - return ELIZA_ERR_FFI_FAULT; +/* Prefill + greedy decode over `engine`. Caller holds asr_mutex and has + * cleared ctx->asr_cancel. Returns 0 or a negative ELIZA_* code. */ +static int eliza_asr_run_engine( + EliInferenceContext * ctx, + EliAsrEngine & engine, + int max_decode_tokens, + size_t max_text_bytes, + std::string & out_transcript, + char ** out_error) { + const size_t n_chunks = engine.n_chunks(); + for (size_t i = 0; i < n_chunks; ++i) { + if (eliza_asr_cancel_requested(ctx)) return eliza_asr_cancelled(out_error); + const int32_t eval_rc = engine.eval_chunk(i, i + 1 == n_chunks); + if (eval_rc != 0) { + if (eliza_asr_cancel_requested(ctx)) return eliza_asr_cancelled(out_error); + eliza_set_error(out_error, "[libelizainference] asr_transcribe: mtmd_helper_eval_chunks failed rc=" + std::to_string(eval_rc)); + return ELIZA_ERR_FFI_FAULT; + } } - const llama_vocab * vocab = llama_model_get_vocab(ctx->asr_model); std::string transcript; transcript.reserve(std::min(max_text_bytes, 256)); - const int max_decode_tokens = std::min( - 4096, - std::max( - 192, - 64 + (int) (((audio.size() + (size_t) ctx->asr_sample_rate - 1) / - (size_t) ctx->asr_sample_rate) * 32))); bool completed = false; for (int i = 0; i < max_decode_tokens; ++i) { - llama_token token = llama_sampler_sample(ctx->asr_sampler, ctx->asr_lctx, -1); - if (llama_vocab_is_eog(vocab, token)) { + if (eliza_asr_cancel_requested(ctx)) return eliza_asr_cancelled(out_error); + llama_token token = engine.sample(); + if (engine.is_eog(token)) { completed = true; break; } - std::string piece = eliza_llama_token_piece(vocab, token); + std::string piece = engine.piece(token); if (!piece.empty()) { if (transcript.size() + piece.size() + 1 > max_text_bytes) { eliza_set_error(out_error, "[libelizainference] asr_transcribe: output buffer too small"); @@ -2238,14 +2825,15 @@ static int eliza_asr_decode_core( } } } - llama_sampler_accept(ctx->asr_sampler, token); - llama_batch batch = llama_batch_get_one(&token, 1); - int32_t decode_rc = llama_decode(ctx->asr_lctx, batch); + engine.accept(token); + int32_t decode_rc = engine.decode(token); if (decode_rc != 0) { + if (eliza_asr_cancel_requested(ctx)) return eliza_asr_cancelled(out_error); eliza_set_error(out_error, "[libelizainference] asr_transcribe: llama_decode failed rc=" + std::to_string(decode_rc)); return ELIZA_ERR_FFI_FAULT; } } + if (eliza_asr_cancel_requested(ctx)) return eliza_asr_cancelled(out_error); if (!completed) { // Decode hit the token cap without an explicit end marker. The old // behaviour hard-failed here, which surfaced as a spurious transcription @@ -2266,6 +2854,71 @@ static int eliza_asr_decode_core( return 0; } +/* Shared ASR decode core. Takes the asr_mutex, runs the audio-in/text-out + * decode, and yields the cleaned transcript. Both eliza_inference_asr_transcribe + * and eliza_inference_asr_transcribe_timed call this so the decode lives in + * exactly one place. Returns 0 on success or a negative ELIZA_* code + * (ELIZA_ERR_CANCELLED after eliza_inference_asr_cancel). */ +static int eliza_asr_decode_core( + EliInferenceContext * ctx, + const float * pcm, + size_t n_samples, + int sample_rate_hz, + size_t max_text_bytes, + std::string & out_transcript, + char ** out_error) { + out_transcript.clear(); + if (n_samples == 0) { + return 0; + } + + std::lock_guard lock(ctx->asr_mutex); + /* Cleared only once this call owns the ASR region: a cancel issued while + * this call waited for the mutex was aimed at the call that held it. */ + ctx->asr_cancel.store(false, std::memory_order_release); + if (!ctx->asr_model || !ctx->asr_lctx || !ctx->asr_mtmd || !ctx->asr_sampler) { + eliza_set_error(out_error, "[libelizainference] asr_transcribe: ASR region is not acquired; call mmap_acquire(\"asr\") after arming voice input"); + return ELIZA_ERR_INVALID_ARG; + } + + std::vector audio = eliza_resample_linear(pcm, n_samples, sample_rate_hz, ctx->asr_sample_rate); + std::unique_ptr bitmap( + mtmd_bitmap_init_from_audio(audio.size(), audio.data()), + mtmd_bitmap_free); + if (!bitmap) { + eliza_set_error(out_error, "[libelizainference] asr_transcribe: failed to create audio bitmap"); + return ELIZA_ERR_FFI_FAULT; + } + + std::string prompt = eliza_format_asr_prompt(ctx->asr_model); + mtmd_input_text text = { prompt.c_str(), true, true }; + const mtmd_bitmap * bitmaps[] = { bitmap.get() }; + std::unique_ptr chunks( + mtmd_input_chunks_init(), + mtmd_input_chunks_free); + if (!chunks) { + eliza_set_error(out_error, "[libelizainference] asr_transcribe: failed to allocate input chunks"); + return ELIZA_ERR_FFI_FAULT; + } + int32_t tok_rc = mtmd_tokenize(ctx->asr_mtmd, chunks.get(), &text, bitmaps, 1); + if (tok_rc != 0) { + eliza_set_error(out_error, "[libelizainference] asr_transcribe: mtmd_tokenize failed rc=" + std::to_string(tok_rc)); + return ELIZA_ERR_FFI_FAULT; + } + + llama_memory_clear(llama_get_memory(ctx->asr_lctx), true); + llama_sampler_reset(ctx->asr_sampler); + + const int max_decode_tokens = std::min( + 4096, + std::max( + 192, + 64 + (int) (((audio.size() + (size_t) ctx->asr_sample_rate - 1) / + (size_t) ctx->asr_sample_rate) * 32))); + EliAsrMtmdEngine engine(ctx, chunks.get()); + return eliza_asr_run_engine(ctx, engine, max_decode_tokens, max_text_bytes, out_transcript, out_error); +} + int eliza_inference_asr_transcribe( EliInferenceContext * ctx, const float * pcm, @@ -2292,6 +2945,57 @@ int eliza_inference_asr_transcribe( return (int) transcript.size(); } +/* ABI v17. elizainference is only built when the mtmd target exists (see + * tools/omnivoice/CMakeLists.txt), and mtmd carries the audio encoder, so any + * build of this file has the batch ASR path. Whether a bundle has ASR weights + * is a separate, runtime question answered by mmap_acquire("asr"). */ +int eliza_inference_asr_supported(void) { + return 1; +} + +int eliza_inference_asr_cancel( + EliInferenceContext * ctx, + char ** out_error) { + if (!ctx) { + eliza_set_error(out_error, "[libelizainference] asr_cancel: ctx is NULL"); + return ELIZA_ERR_INVALID_ARG; + } + /* No asr_mutex: the in-flight transcription holds it. */ + ctx->asr_cancel.store(true, std::memory_order_release); + return ELIZA_OK; +} + +#ifdef ELIZA_INFERENCE_TEST_HOOKS +/* Test-only (tests/ffi-paradee-asr.cpp #includes this file with + * ELIZA_INFERENCE_TEST_HOOKS defined; no shipped target defines it): run the + * shared prefill/decode control flow over a scripted engine, with the same + * mutex + cancel reset as eliza_asr_decode_core. */ +static int eliza_test_asr_run_engine( + EliInferenceContext * ctx, + EliAsrEngine * engine, + int max_decode_tokens, + char * out_text, + size_t max_text_bytes, + char ** out_error) { + if (!ctx || !engine || !out_text || max_text_bytes == 0) return ELIZA_ERR_INVALID_ARG; + std::lock_guard lock(ctx->asr_mutex); + ctx->asr_cancel.store(false, std::memory_order_release); + std::string transcript; + const int rc = eliza_asr_run_engine(ctx, *engine, max_decode_tokens, max_text_bytes, transcript, out_error); + if (rc < 0) return rc; + if (transcript.size() + 1 > max_text_bytes) return ELIZA_ERR_INVALID_ARG; + std::memcpy(out_text, transcript.data(), transcript.size()); + out_text[transcript.size()] = '\0'; + return (int) transcript.size(); +} + +/* The abort callback installed on asr_lctx, so a scripted engine can poll it + * exactly the way llama_decode does. */ +static bool eliza_test_asr_abort_requested(EliInferenceContext * ctx) { + return eliza_asr_cancel_requested(ctx); +} +#endif + int eliza_inference_asr_timestamps_supported(void) { return 1; } @@ -2536,7 +3240,12 @@ int eliza_inference_cancel_tts( char ** out_error) { (void) out_error; if (ctx) { + /* One flag per TTS engine (OmniVoice, Kokoro, Paradee). Each engine + * clears its own flag when a call starts, so a set flag only ever + * stops the call in flight on that engine. */ ctx->tts_cancel.store(true, std::memory_order_release); + ctx->kokoro_cancel.store(true, std::memory_order_release); + ctx->paradee_cancel.store(true, std::memory_order_release); } // Cancelling nothing is not an error. return ELIZA_OK; diff --git a/tools/omnivoice/tests/ffi-abi.c b/tools/omnivoice/tests/ffi-abi.c new file mode 100644 index 000000000..03c997b45 --- /dev/null +++ b/tools/omnivoice/tests/ffi-abi.c @@ -0,0 +1,246 @@ +/* tools/omnivoice/tests/ffi-abi.c + * + * C11 test against the real libelizainference (ABI v17). Always runs: + * - ABI string "17" and the v17 capability probes + * - cancel_tts / asr_cancel are no-ops with nothing in flight + * - mmap_evict / mmap_acquire("tts") on a bundle with no TTS weights + * - Paradee entries before any load (or NOT_IMPLEMENTED when not built) + * + * Opt-in real-model ASR cancel (skipped unless both are set): + * ELIZA_TEST_ASR_DIR bundle dir containing asr/eliza-1-asr.gguf + + * asr/eliza-1-asr-mmproj.gguf + * ELIZA_TEST_ASR_WAV 16-bit PCM mono WAV + * It transcribes uncancelled (after one warm-up run), cancels a run 20 ms in + * (prefill) and a run late in its decode, expecting ELIZA_ERR_CANCELLED, + * then checks an uncancelled run still returns the same text. + */ +#include "eliza-inference-ffi.h" + +#include +#include +#include +#include +#include +#include +#include + +static int g_fail = 0; +static int g_checks = 0; + +#define CHECK(cond) do { \ + ++g_checks; \ + if (!(cond)) { ++g_fail; fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); } \ + } while (0) + +#define CHECK_RC(expr, want) do { \ + int rc_ = (expr); ++g_checks; \ + if (rc_ != (want)) { ++g_fail; fprintf(stderr, "FAIL %s:%d: %s = %d, want %d\n", __FILE__, __LINE__, #expr, rc_, (int) (want)); } \ + } while (0) + +static void clear_err(char ** err) { + if (*err) { eliza_inference_free_string(*err); *err = NULL; } +} + +static double now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double) ts.tv_sec * 1000.0 + (double) ts.tv_nsec / 1e6; +} + +static void test_probes_and_noops(void) { + char * err = NULL; + CHECK(strcmp(eliza_inference_abi_version(), "17") == 0); + CHECK_RC(eliza_inference_asr_supported(), 1); +#if defined(ELIZA_TEST_EXPECT_PARADEE) + CHECK_RC(eliza_inference_paradee_supported(), 1); +#else + CHECK_RC(eliza_inference_paradee_supported(), 0); +#endif + + char dir_template[] = "/tmp/eliza-ffi-abi-XXXXXX"; + char * dir = mkdtemp(dir_template); + CHECK(dir != NULL); + if (!dir) return; + EliInferenceContext * ctx = eliza_inference_create(dir, &err); + CHECK(ctx != NULL); + clear_err(&err); + if (!ctx) return; + + CHECK_RC(eliza_inference_cancel_tts(ctx, &err), ELIZA_OK); + CHECK_RC(eliza_inference_asr_cancel(ctx, &err), ELIZA_OK); + CHECK_RC(eliza_inference_asr_cancel(NULL, &err), ELIZA_ERR_INVALID_ARG); + clear_err(&err); + CHECK_RC(eliza_inference_mmap_evict(ctx, "tts", &err), ELIZA_OK); + CHECK_RC(eliza_inference_mmap_evict(ctx, "asr", &err), ELIZA_OK); + /* No TTS weights and no configured Kokoro/Paradee head. */ + CHECK_RC(eliza_inference_mmap_acquire(ctx, "tts", &err), ELIZA_ERR_BUNDLE_INVALID); + clear_err(&err); + + float * pcm = NULL; + size_t n = 0; + char * ph = NULL; +#if defined(ELIZA_TEST_EXPECT_PARADEE) + const int want = ELIZA_ERR_INVALID_ARG; +#else + const int want = ELIZA_ERR_NOT_IMPLEMENTED; +#endif + CHECK_RC(eliza_inference_paradee_synthesize_alloc(ctx, "hi", 2, 1.0f, &pcm, &n, &err), want); + CHECK(pcm == NULL && n == 0); + clear_err(&err); + CHECK_RC(eliza_inference_paradee_synthesize_phonemes_alloc(ctx, "hi", 2, 1.0f, &pcm, &n, &err), want); + clear_err(&err); + CHECK_RC(eliza_inference_paradee_phonemize(ctx, "hi", 2, &ph, &err), want); + CHECK(ph == NULL); + clear_err(&err); + CHECK(eliza_inference_paradee_sample_rate(ctx) < 0); + CHECK(eliza_inference_paradee_g2p_kind(ctx) < 0); +#if !defined(ELIZA_TEST_EXPECT_PARADEE) + CHECK_RC(eliza_inference_paradee_load(ctx, "a.gguf", "lex.bin", NULL, &err), ELIZA_ERR_NOT_IMPLEMENTED); + clear_err(&err); +#endif + + eliza_inference_destroy(ctx); + rmdir(dir); +} + +/* ---- opt-in real-model ASR cancel ---------------------------------- */ + +static float * read_wav_mono16(const char * path, size_t * n_out, int * rate_out) { + FILE * f = fopen(path, "rb"); + if (!f) return NULL; + unsigned char hdr[12]; + if (fread(hdr, 1, 12, f) != 12 || memcmp(hdr, "RIFF", 4) != 0 || memcmp(hdr + 8, "WAVE", 4) != 0) { + fclose(f); + return NULL; + } + int rate = 0, channels = 0, bits = 0; + float * out = NULL; + for (;;) { + unsigned char ch[8]; + if (fread(ch, 1, 8, f) != 8) break; + uint32_t size = (uint32_t) ch[4] | ((uint32_t) ch[5] << 8) | ((uint32_t) ch[6] << 16) | ((uint32_t) ch[7] << 24); + if (memcmp(ch, "fmt ", 4) == 0) { + unsigned char fmt[16]; + if (size < 16 || fread(fmt, 1, 16, f) != 16) break; + channels = fmt[2] | (fmt[3] << 8); + rate = (int) ((uint32_t) fmt[4] | ((uint32_t) fmt[5] << 8) | ((uint32_t) fmt[6] << 16) | ((uint32_t) fmt[7] << 24)); + bits = fmt[14] | (fmt[15] << 8); + if (size > 16) fseek(f, (long) (size - 16), SEEK_CUR); + } else if (memcmp(ch, "data", 4) == 0) { + if (channels != 1 || bits != 16) break; + size_t n = size / 2; + int16_t * raw = (int16_t *) malloc(size); + out = (float *) malloc(n * sizeof(float)); + if (!raw || !out || fread(raw, 1, size, f) != size) { free(raw); free(out); out = NULL; break; } + for (size_t i = 0; i < n; ++i) out[i] = (float) raw[i] / 32768.0f; + free(raw); + *n_out = n; + *rate_out = rate; + break; + } else { + fseek(f, (long) (size + (size & 1)), SEEK_CUR); + } + } + fclose(f); + return out; +} + +struct asr_job { + EliInferenceContext * ctx; + const float * pcm; + size_t n; + int rate; + char text[4096]; + int rc; + double ms; +}; + +static void * asr_thread(void * arg) { + struct asr_job * job = (struct asr_job *) arg; + char * err = NULL; + const double t0 = now_ms(); + job->rc = eliza_inference_asr_transcribe(job->ctx, job->pcm, job->n, job->rate, job->text, sizeof(job->text), &err); + job->ms = now_ms() - t0; + if (job->rc < 0 && job->rc != ELIZA_ERR_CANCELLED) fprintf(stderr, "asr error: %s\n", err ? err : "(none)"); + clear_err(&err); + return NULL; +} + +static void run_cancel_after(struct asr_job * job, double after_ms) { + pthread_t th; + char * err = NULL; + pthread_create(&th, NULL, asr_thread, job); + usleep((useconds_t) (after_ms * 1000.0)); + CHECK_RC(eliza_inference_asr_cancel(job->ctx, &err), ELIZA_OK); + pthread_join(th, NULL); +} + +static void test_real_asr_cancel(void) { + const char * dir = getenv("ELIZA_TEST_ASR_DIR"); + const char * wav = getenv("ELIZA_TEST_ASR_WAV"); + if (!dir || !wav) { + fprintf(stderr, "skip: real-model ASR cancel (set ELIZA_TEST_ASR_DIR and ELIZA_TEST_ASR_WAV)\n"); + return; + } + size_t n = 0; + int rate = 0; + float * pcm = read_wav_mono16(wav, &n, &rate); + CHECK(pcm != NULL); + if (!pcm) return; + char * err = NULL; + EliInferenceContext * ctx = eliza_inference_create(dir, &err); + CHECK(ctx != NULL); + clear_err(&err); + if (!ctx) { free(pcm); return; } + CHECK_RC(eliza_inference_mmap_acquire(ctx, "asr", &err), ELIZA_OK); + if (err) fprintf(stderr, "acquire asr: %s\n", err); + clear_err(&err); + + static struct asr_job warm, base, early, late, again; + warm.ctx = ctx; warm.pcm = pcm; warm.n = n; warm.rate = rate; + asr_thread(&warm); /* first run pays graph / kernel warm-up */ + base = warm; + asr_thread(&base); + CHECK(base.rc > 0); + fprintf(stderr, "real asr: uncancelled %.0f ms (warm-up run %.0f ms): \"%s\"\n", base.ms, warm.ms, base.text); + + /* Early cancel: lands in prefill (audio encode / audio-embedding decode). */ + early = base; early.text[0] = '\0'; + run_cancel_after(&early, 20.0); + CHECK_RC(early.rc, ELIZA_ERR_CANCELLED); + fprintf(stderr, "real asr: cancel at 20 ms -> rc=%d after %.0f ms\n", early.rc, early.ms); + + /* Late cancel: aims at the token decode. Run time varies with machine + * load, so if the run finished before the cancel landed, retry earlier. + * A run that was not cancelled must still return the full transcript. */ + const double fractions[] = { 0.7, 0.5, 0.35, 0.2 }; + int late_cancelled = 0; + for (size_t i = 0; i < sizeof(fractions) / sizeof(fractions[0]) && !late_cancelled; ++i) { + late = base; late.text[0] = '\0'; + const double at = base.ms * fractions[i]; + run_cancel_after(&late, at); + fprintf(stderr, "real asr: cancel at %.0f ms -> rc=%d after %.0f ms\n", at, late.rc, late.ms); + if (late.rc == ELIZA_ERR_CANCELLED) { + late_cancelled = 1; + } else { + CHECK(late.rc == base.rc && strcmp(late.text, base.text) == 0); + } + } + CHECK(late_cancelled); + + /* The cancel flag does not leak into the next transcription. */ + again = base; again.text[0] = '\0'; + asr_thread(&again); + CHECK(again.rc == base.rc); + CHECK(strcmp(again.text, base.text) == 0); + + eliza_inference_destroy(ctx); + free(pcm); +} + +int main(void) { + test_probes_and_noops(); + test_real_asr_cancel(); + fprintf(stderr, "%s: %d checks, %d failures\n", g_fail ? "FAIL" : "PASS", g_checks, g_fail); + return g_fail ? 1 : 0; +} diff --git a/tools/omnivoice/tests/ffi-paradee-asr.cpp b/tools/omnivoice/tests/ffi-paradee-asr.cpp new file mode 100644 index 000000000..789b89364 --- /dev/null +++ b/tools/omnivoice/tests/ffi-paradee-asr.cpp @@ -0,0 +1,617 @@ +// tools/omnivoice/tests/ffi-paradee-asr.cpp +// +// Unit test for the ABI v17 additions in src/eliza-inference-ffi.cpp: +// - Paradee load / g2p kind / phonemize / synth, <=510-id chunking +// - cancel_tts during an in-flight Paradee synth, plus a load/synth/ +// cancel/evict race across threads +// - mmap_evict("tts") frees Paradee + lexicon, next synth or +// mmap_acquire("tts") reloads +// - eliza_inference_asr_cancel during prefill and during decode, over a +// scripted ASR engine that polls the same abort callback llama_decode +// uses +// - ABI string "17" +// +// The bridge is compiled into this TU (ELIZA_INFERENCE_TEST_HOOKS) against +// the Paradee / misaki doubles in tests/paradee-contract/, so no weights are +// needed. The real-library and real-model checks are in tests/ffi-abi.c. + +#include "../src/eliza-inference-ffi.cpp" + +#include "fake-paradee-control.h" + +#include +#include +#include +#include +#include + +static int g_failures = 0; +static int g_checks = 0; + +#define CHECK(cond) do { \ + ++g_checks; \ + if (!(cond)) { \ + ++g_failures; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +#define CHECK_RC(rc, want) do { \ + const int _rc = (rc); \ + ++g_checks; \ + if (_rc != (want)) { \ + ++g_failures; \ + std::fprintf(stderr, "FAIL %s:%d: %s = %d, want %d\n", __FILE__, __LINE__, #rc, _rc, (int) (want)); \ + } \ + } while (0) + +namespace fs = std::filesystem; +using clock_type = std::chrono::steady_clock; + +static void clear_err(char ** err) { + if (*err) { + eliza_inference_free_string(*err); + *err = nullptr; + } +} + +static fs::path make_bundle(const char * name) { + fs::path dir = fs::temp_directory_path() / (std::string("eliza-ffi-test-") + name + "-" + + std::to_string((long long) clock_type::now().time_since_epoch().count())); + fs::create_directories(dir); + return dir; +} + +static bool wait_until(const std::atomic & flag, int timeout_ms) { + const auto deadline = clock_type::now() + std::chrono::milliseconds(timeout_ms); + while (!flag.load()) { + if (clock_type::now() > deadline) return false; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; +} + +static int synth_text(EliInferenceContext * ctx, const std::string & text, size_t * n_out, char ** err) { + float * pcm = nullptr; + size_t n = 0; + const int rc = eliza_inference_paradee_synthesize_alloc(ctx, text.c_str(), text.size(), 1.0f, &pcm, &n, err); + if (rc == ELIZA_OK && n > 0) CHECK(pcm != nullptr); + if (rc != ELIZA_OK) CHECK(pcm == nullptr && n == 0); + eliza_inference_free_pcm(pcm); + if (n_out) *n_out = n; + return rc; +} + +static void test_abi_and_probes() { + CHECK(std::string(eliza_inference_abi_version()) == "17"); + CHECK_RC(eliza_inference_paradee_supported(), 1); + CHECK_RC(eliza_inference_asr_supported(), 1); + CHECK_RC(eliza_inference_asr_stream_supported(), 0); +} + +static void test_paradee_load_and_synth() { + fake_paradee_reset(); + const fs::path bundle = make_bundle("load"); + char * err = nullptr; + EliInferenceContext * ctx = eliza_inference_create(bundle.string().c_str(), &err); + CHECK(ctx != nullptr); + clear_err(&err); + + // Nothing loaded yet. + CHECK_RC(synth_text(ctx, "hello", nullptr, &err), ELIZA_ERR_INVALID_ARG); + CHECK(err != nullptr); + clear_err(&err); + CHECK(eliza_inference_paradee_sample_rate(ctx) < 0); + CHECK(eliza_inference_paradee_g2p_kind(ctx) < 0); + CHECK_RC(eliza_inference_paradee_load(ctx, nullptr, "lex.bin", nullptr, &err), ELIZA_ERR_INVALID_ARG); + clear_err(&err); + + // A failed load reports BUNDLE_INVALID and leaves the ctx unconfigured. + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/missing.gguf", "/x/lex.bin", nullptr, &err), ELIZA_ERR_BUNDLE_INVALID); + CHECK(err != nullptr && std::string(err).find("paradee_load_model") != std::string::npos); + clear_err(&err); + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/missing-lex.bin", nullptr, &err), ELIZA_ERR_BUNDLE_INVALID); + clear_err(&err); + CHECK_RC(fake_paradee().live_models.load(), 0); + + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/lex.bin", nullptr, &err), ELIZA_OK); + CHECK_RC(fake_paradee().model_loads.load(), 2); // the failed lexicon attempt loaded a model too + CHECK_RC(fake_paradee().live_models.load(), 1); + // Same paths: no reload. + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/lex.bin", nullptr, &err), ELIZA_OK); + CHECK_RC(fake_paradee().model_loads.load(), 2); + CHECK_RC(eliza_inference_paradee_sample_rate(ctx), 24000); + CHECK_RC(eliza_inference_paradee_g2p_kind(ctx), ELIZA_PARADEE_G2P_RULES); + // A failed reload keeps the loaded model. + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/missing.gguf", "/x/lex.bin", nullptr, &err), ELIZA_ERR_BUNDLE_INVALID); + clear_err(&err); + CHECK_RC(eliza_inference_paradee_sample_rate(ctx), 24000); + // eSpeak data dir -> ESPEAK fallback kind (different config -> reload). + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/lex.bin", "/x/espeak-ng-data", &err), ELIZA_OK); + CHECK_RC(eliza_inference_paradee_g2p_kind(ctx), ELIZA_PARADEE_G2P_ESPEAK); + CHECK_RC(fake_paradee().live_models.load(), 1); + CHECK_RC(fake_paradee().live_lexicons.load(), 1); + + // Phonemize. + char * ph = nullptr; + CHECK_RC(eliza_inference_paradee_phonemize(ctx, "Hello, World", 12, &ph, &err), 12); + CHECK(ph != nullptr && std::string(ph) == "hello, world"); + eliza_inference_free_string(ph); + ph = nullptr; + CHECK_RC(eliza_inference_paradee_phonemize(ctx, "\x01" "fail", 5, &ph, &err), ELIZA_ERR_FFI_FAULT); + CHECK(ph == nullptr); + clear_err(&err); + + // Short text: one forward pass, 0-padded input, exact sample count. + { + std::lock_guard lock(fake_paradee().sizes_mutex); + fake_paradee().input_sizes.clear(); + } + size_t n = 0; + CHECK_RC(synth_text(ctx, "Hi there.", &n, &err), ELIZA_OK); + CHECK_RC((int) n, 9 * FAKE_PARADEE_SAMPLES_PER_ID); + { + std::lock_guard lock(fake_paradee().sizes_mutex); + CHECK(fake_paradee().input_sizes.size() == 1 && fake_paradee().input_sizes[0] == 11); + } + // Text whose phonemes are all out of vocab / dropped -> zero samples, OK. + CHECK_RC(synth_text(ctx, "12345", &n, &err), ELIZA_OK); + CHECK_RC((int) n, 0); + CHECK_RC(synth_text(ctx, "\x01" "fail", &n, &err), ELIZA_ERR_FFI_FAULT); + clear_err(&err); + + // Phoneme input path: '#' is out of vocab and dropped. + { + float * pcm = nullptr; + const std::string phon = "h#\xC9\x99lo\xCA\x8A"; // h # ə l o ʊ + CHECK_RC(eliza_inference_paradee_synthesize_phonemes_alloc(ctx, phon.c_str(), phon.size(), 1.2f, &pcm, &n, &err), ELIZA_OK); + CHECK_RC((int) n, 5 * FAKE_PARADEE_SAMPLES_PER_ID); + CHECK(pcm != nullptr && pcm[0] == (float) (43 + ('h' - 'a')) / 1000.0f); + eliza_inference_free_pcm(pcm); + } + CHECK_RC(eliza_inference_paradee_synthesize_alloc(ctx, "x", 1, 1.0f, nullptr, &n, &err), ELIZA_ERR_INVALID_ARG); + clear_err(&err); + + eliza_inference_destroy(ctx); + CHECK_RC(fake_paradee().live_models.load(), 0); + CHECK_RC(fake_paradee().live_lexicons.load(), 0); + fs::remove_all(bundle); +} + +static void test_paradee_chunking() { + fake_paradee_reset(); + const fs::path bundle = make_bundle("chunk"); + char * err = nullptr; + EliInferenceContext * ctx = eliza_inference_create(bundle.string().c_str(), &err); + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/lex.bin", nullptr, &err), ELIZA_OK); + + // 40 sentences of 30 chars each (1200 ids): must split at sentence ends, + // every chunk <= 510 ids + 2 pads, and lose nothing. + std::string text; + for (int i = 0; i < 40; ++i) text += "the quick brown fox jumps now. "; + const size_t expected_ids = text.size(); + { + std::lock_guard lock(fake_paradee().sizes_mutex); + fake_paradee().input_sizes.clear(); + } + size_t n = 0; + CHECK_RC(synth_text(ctx, text, &n, &err), ELIZA_OK); + CHECK_RC((int) n, (int) (expected_ids * FAKE_PARADEE_SAMPLES_PER_ID)); + { + std::lock_guard lock(fake_paradee().sizes_mutex); + const auto & sizes = fake_paradee().input_sizes; + CHECK(sizes.size() == 3); + size_t total = 0; + for (size_t s : sizes) { + CHECK(s <= 512); + total += s - 2; + } + CHECK(total == expected_ids); + // Greedy sentence packing: 31-char sentences, 16 per 510-id chunk. + CHECK(!sizes.empty() && sizes[0] == 16 * 31 + 2); + } + + // One 1300-letter "word": no boundaries at all -> hard slices. + const std::string word(1300, 'a'); + { + std::lock_guard lock(fake_paradee().sizes_mutex); + fake_paradee().input_sizes.clear(); + } + CHECK_RC(synth_text(ctx, word, &n, &err), ELIZA_OK); + CHECK_RC((int) n, 1300 * FAKE_PARADEE_SAMPLES_PER_ID); + { + std::lock_guard lock(fake_paradee().sizes_mutex); + const auto & sizes = fake_paradee().input_sizes; + CHECK(sizes.size() == 3 && sizes[0] == 512 && sizes[1] == 512 && sizes[2] == 1300 - 1020 + 2); + } + + // Clause fallback: one 600-char sentence with commas. + std::string clause_text; + for (int i = 0; i < 20; ++i) clause_text += "and then some more words, "; + clause_text += "done."; + { + std::lock_guard lock(fake_paradee().sizes_mutex); + fake_paradee().input_sizes.clear(); + } + CHECK_RC(synth_text(ctx, clause_text, &n, &err), ELIZA_OK); + CHECK_RC((int) n, (int) (clause_text.size() * FAKE_PARADEE_SAMPLES_PER_ID)); + { + std::lock_guard lock(fake_paradee().sizes_mutex); + const auto & sizes = fake_paradee().input_sizes; + CHECK(sizes.size() == 2); + // 26-char clauses: 19 fit in 510. + CHECK(!sizes.empty() && sizes[0] == 19 * 26 + 2); + } + + eliza_inference_destroy(ctx); + fs::remove_all(bundle); +} + +static void test_paradee_cancel() { + fake_paradee_reset(); + const fs::path bundle = make_bundle("cancel"); + char * err = nullptr; + EliInferenceContext * ctx = eliza_inference_create(bundle.string().c_str(), &err); + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/lex.bin", nullptr, &err), ELIZA_OK); + + // In-flight cancel stops the forward pass well before it would finish. + fake_paradee().block_ms = 5000; + int rc = 1; + char * thread_err = nullptr; + const auto t0 = clock_type::now(); + std::thread synth([&]() { rc = synth_text(ctx, "a long utterance.", nullptr, &thread_err); }); + CHECK(wait_until(fake_paradee().in_flight, 2000)); + CHECK_RC(eliza_inference_cancel_tts(ctx, &err), ELIZA_OK); + synth.join(); + const auto ms = std::chrono::duration_cast(clock_type::now() - t0).count(); + CHECK_RC(rc, ELIZA_ERR_CANCELLED); + CHECK(thread_err != nullptr && std::string(thread_err).find("cancelled") != std::string::npos); + clear_err(&thread_err); + CHECK(ms < 2000); + + // The flag does not carry over into the next call. + fake_paradee().block_ms = 0; + CHECK_RC(synth_text(ctx, "next.", nullptr, &err), ELIZA_OK); + // Cancel with nothing in flight is not an error and does not poison the + // next call either. + CHECK_RC(eliza_inference_cancel_tts(ctx, &err), ELIZA_OK); + CHECK_RC(synth_text(ctx, "after idle cancel.", nullptr, &err), ELIZA_OK); + + // Cancel during a multi-chunk synth: the current chunk stops and no later + // chunk runs. + fake_paradee().block_ms = 300; + std::string long_text; + for (int i = 0; i < 40; ++i) long_text += "the quick brown fox jumps now. "; + const int calls_before = fake_paradee().synth_calls.load(); + fake_paradee().in_flight = false; + std::thread synth2([&]() { rc = synth_text(ctx, long_text, nullptr, &thread_err); }); + CHECK(wait_until(fake_paradee().in_flight, 2000)); + CHECK_RC(eliza_inference_cancel_tts(ctx, &err), ELIZA_OK); + synth2.join(); + clear_err(&thread_err); + CHECK_RC(rc, ELIZA_ERR_CANCELLED); + CHECK_RC(fake_paradee().synth_calls.load() - calls_before, 1); + + // Flag isolation: an OmniVoice call starting (ElizaScopedTtsForward + // clears tts_cancel) must not clear a cancel aimed at the Paradee call in + // flight. + fake_paradee().block_ms = 3000; + fake_paradee().in_flight = false; + std::thread synth3([&]() { rc = synth_text(ctx, "isolated.", nullptr, &thread_err); }); + CHECK(wait_until(fake_paradee().in_flight, 2000)); + CHECK_RC(eliza_inference_cancel_tts(ctx, &err), ELIZA_OK); + { ElizaScopedTtsForward omnivoice_call_starts(ctx); } + synth3.join(); + clear_err(&thread_err); + CHECK_RC(rc, ELIZA_ERR_CANCELLED); + + eliza_inference_destroy(ctx); + fs::remove_all(bundle); +} + +static void test_paradee_race() { + fake_paradee_reset(); + const fs::path bundle = make_bundle("race"); + char * err = nullptr; + EliInferenceContext * ctx = eliza_inference_create(bundle.string().c_str(), &err); + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/a.gguf", "/x/lex.bin", nullptr, &err), ELIZA_OK); + fake_paradee().block_ms = 2; + + std::atomic bad{0}; + std::atomic ok{0}; + std::atomic cancelled{0}; + auto worker = [&](int seed) { + std::mt19937 rng((unsigned) seed); + char * e = nullptr; + for (int i = 0; i < 150; ++i) { + int rc = ELIZA_OK; + switch (rng() % 5) { + case 0: + rc = eliza_inference_paradee_load(ctx, (rng() & 1) ? "/x/a.gguf" : "/x/b.gguf", "/x/lex.bin", nullptr, &e); + break; + case 1: + case 2: + rc = synth_text(ctx, "race text, with a clause. and more.", nullptr, &e); + break; + case 3: + rc = eliza_inference_cancel_tts(ctx, &e); + break; + default: + rc = (rng() & 1) ? eliza_inference_mmap_evict(ctx, "tts", &e) + : eliza_inference_mmap_acquire(ctx, "tts", &e); + break; + } + if (rc == ELIZA_OK) ok.fetch_add(1); + else if (rc == ELIZA_ERR_CANCELLED) cancelled.fetch_add(1); + else { + bad.fetch_add(1); + std::fprintf(stderr, "race: unexpected rc=%d err=%s\n", rc, e ? e : "(none)"); + } + if (e) { eliza_inference_free_string(e); e = nullptr; } + } + }; + std::vector threads; + for (int t = 0; t < 6; ++t) threads.emplace_back(worker, 1234 + t); + for (auto & t : threads) t.join(); + CHECK_RC(bad.load(), 0); + CHECK(ok.load() > 0); + CHECK(fake_paradee().live_models.load() <= 1); + CHECK(fake_paradee().live_lexicons.load() <= 1); + std::fprintf(stderr, "race: ok=%d cancelled=%d\n", ok.load(), cancelled.load()); + + // Still usable afterwards. + fake_paradee().block_ms = 0; + CHECK_RC(synth_text(ctx, "after the race.", nullptr, &err), ELIZA_OK); + eliza_inference_destroy(ctx); + CHECK_RC(fake_paradee().live_models.load(), 0); + fs::remove_all(bundle); +} + +static void test_evict_reload() { + fake_paradee_reset(); + const fs::path bundle = make_bundle("evict"); + char * err = nullptr; + EliInferenceContext * ctx = eliza_inference_create(bundle.string().c_str(), &err); + + // No TTS head configured and no OmniVoice weights: acquire fails as before. + CHECK_RC(eliza_inference_mmap_acquire(ctx, "tts", &err), ELIZA_ERR_BUNDLE_INVALID); + clear_err(&err); + CHECK_RC(eliza_inference_mmap_evict(ctx, "tts", &err), ELIZA_OK); + + CHECK_RC(eliza_inference_paradee_load(ctx, "/x/paradee.gguf", "/x/lex.bin", nullptr, &err), ELIZA_OK); + CHECK_RC(fake_paradee().model_loads.load(), 1); + + // Evict frees model + lexicon; rate / g2p kind still answer. + CHECK_RC(eliza_inference_mmap_evict(ctx, "tts", &err), ELIZA_OK); + CHECK_RC(fake_paradee().live_models.load(), 0); + CHECK_RC(fake_paradee().live_lexicons.load(), 0); + CHECK_RC(eliza_inference_paradee_sample_rate(ctx), 24000); + CHECK_RC(eliza_inference_paradee_g2p_kind(ctx), ELIZA_PARADEE_G2P_RULES); + // Evicting "asr" leaves TTS alone. + CHECK_RC(eliza_inference_mmap_evict(ctx, "asr", &err), ELIZA_OK); + + // Next synth reloads lazily. + size_t n = 0; + CHECK_RC(synth_text(ctx, "reloaded.", &n, &err), ELIZA_OK); + CHECK_RC((int) n, 9 * FAKE_PARADEE_SAMPLES_PER_ID); + CHECK_RC(fake_paradee().model_loads.load(), 2); + CHECK_RC(fake_paradee().lexicon_loads.load(), 2); + CHECK_RC(fake_paradee().live_models.load(), 1); + + // Phonemize reloads too. + CHECK_RC(eliza_inference_mmap_evict(ctx, "tts", &err), ELIZA_OK); + char * ph = nullptr; + CHECK_RC(eliza_inference_paradee_phonemize(ctx, "ab", 2, &ph, &err), 2); + eliza_inference_free_string(ph); + CHECK_RC(fake_paradee().model_loads.load(), 3); + + // acquire("tts") reloads; with no OmniVoice weights in the bundle it + // still succeeds because a Paradee head is configured. Repeat acquire is + // a no-op. + CHECK_RC(eliza_inference_mmap_evict(ctx, "tts", &err), ELIZA_OK); + CHECK_RC(eliza_inference_mmap_acquire(ctx, "tts", &err), ELIZA_OK); + clear_err(&err); + CHECK_RC(fake_paradee().model_loads.load(), 4); + CHECK_RC(eliza_inference_mmap_acquire(ctx, "tts", &err), ELIZA_OK); + CHECK_RC(fake_paradee().model_loads.load(), 4); + // OmniVoice entries still report the region as not acquired. + float buf[16]; + CHECK_RC(eliza_inference_tts_synthesize(ctx, "x", 1, nullptr, buf, 16, &err), ELIZA_ERR_INVALID_ARG); + clear_err(&err); + + // A Paradee GGUF under tts/ is never picked as the OmniVoice model. + fs::create_directories(bundle / "tts" / "paradee"); + { std::ofstream(bundle / "tts" / "paradee" / "paradee-8m.gguf") << "x"; } + { std::ofstream(bundle / "tts" / "paradee-8m-v1.0.gguf") << "x"; } + std::string tts_model, codec_model; + CHECK(!eliza_pick_voice_files(bundle, tts_model, codec_model)); + + // Evict while a synth is in flight: the synth finishes, then the model is freed. + fake_paradee().block_ms = 200; + fake_paradee().in_flight = false; + int rc = 1; + char * thread_err = nullptr; + std::thread synth([&]() { rc = synth_text(ctx, "in flight.", &n, &thread_err); }); + CHECK(wait_until(fake_paradee().in_flight, 2000)); + CHECK_RC(eliza_inference_mmap_evict(ctx, "tts", &err), ELIZA_OK); + synth.join(); + clear_err(&thread_err); + CHECK_RC(rc, ELIZA_OK); + CHECK_RC((int) n, 10 * FAKE_PARADEE_SAMPLES_PER_ID); + CHECK_RC(fake_paradee().live_models.load(), 0); + + eliza_inference_destroy(ctx); + fs::remove_all(bundle); +} + +// Scripted ASR engine. Prefill: `n_chunks` chunks; chunk `slow_chunk` spins +// until the abort callback fires (like a long audio-embedding llama_decode) +// and returns 2. Decode: emits `tokens` then EOG; when `slow_decode` is set, +// each decode() polls the abort callback for up to 5 s (like a long +// llama_decode) before returning. +struct ScriptedAsrEngine final : EliAsrEngine { + EliInferenceContext * ctx; + size_t chunks = 3; + long slow_chunk = -1; + std::vector tokens; + bool infinite = false; + bool slow_decode = false; + size_t pos = 0; + std::atomic in_prefill{false}; + std::atomic in_decode{false}; + std::atomic decoded{0}; + + explicit ScriptedAsrEngine(EliInferenceContext * c) : ctx(c) {} + size_t n_chunks() override { return chunks; } + int32_t eval_chunk(size_t i, bool) override { + if ((long) i != slow_chunk) return 0; + in_prefill = true; + for (int t = 0; t < 5000; ++t) { + if (eliza_test_asr_abort_requested(ctx)) return 2; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return 0; + } + llama_token sample() override { return (llama_token) pos; } + bool is_eog(llama_token t) override { return !infinite && (size_t) t >= tokens.size(); } + std::string piece(llama_token t) override { return infinite ? std::string("a ") : tokens[(size_t) t]; } + void accept(llama_token) override {} + int32_t decode(llama_token) override { + ++pos; + decoded.fetch_add(1); + in_decode = true; + if (slow_decode) { + for (int t = 0; t < 5000; ++t) { + if (eliza_test_asr_abort_requested(ctx)) return 2; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } else { + std::this_thread::sleep_for(std::chrono::microseconds(200)); + } + return 0; + } +}; + +static void test_asr_cancel() { + const fs::path bundle = make_bundle("asr"); + char * err = nullptr; + EliInferenceContext * ctx = eliza_inference_create(bundle.string().c_str(), &err); + char text[256]; + + CHECK_RC(eliza_inference_asr_cancel(nullptr, &err), ELIZA_ERR_INVALID_ARG); + clear_err(&err); + + // Uncancelled run produces the transcript. + { + ScriptedAsrEngine e(ctx); + e.tokens = { "hello", " world" }; + CHECK_RC(eliza_test_asr_run_engine(ctx, &e, 64, text, sizeof(text), &err), 11); + CHECK(std::string(text) == "hello world"); + } + // A cancel with nothing in flight does not affect the next call. + CHECK_RC(eliza_inference_asr_cancel(ctx, &err), ELIZA_OK); + { + ScriptedAsrEngine e(ctx); + e.tokens = { "still", " fine" }; + CHECK_RC(eliza_test_asr_run_engine(ctx, &e, 64, text, sizeof(text), &err), 10); + } + + // Cancel during prefill (abort callback inside a chunk eval). + { + ScriptedAsrEngine e(ctx); + e.slow_chunk = 1; + e.tokens = { "never" }; + int rc = 1; + char * terr = nullptr; + const auto t0 = clock_type::now(); + std::thread th([&]() { rc = eliza_test_asr_run_engine(ctx, &e, 64, text, sizeof(text), &terr); }); + CHECK(wait_until(e.in_prefill, 2000)); + CHECK_RC(eliza_inference_asr_cancel(ctx, &err), ELIZA_OK); + th.join(); + CHECK_RC(rc, ELIZA_ERR_CANCELLED); + CHECK(terr != nullptr && std::string(terr).find("cancelled") != std::string::npos); + clear_err(&terr); + CHECK_RC(e.decoded.load(), 0); + CHECK(std::chrono::duration_cast(clock_type::now() - t0).count() < 2000); + } + + // Cancel during decode (per-token check): an endless token stream. + { + ScriptedAsrEngine e(ctx); + e.infinite = true; + int rc = 1; + char * terr = nullptr; + char big[1 << 16]; + std::thread th([&]() { rc = eliza_test_asr_run_engine(ctx, &e, 1 << 20, big, sizeof(big), &terr); }); + CHECK(wait_until(e.in_decode, 2000)); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + CHECK_RC(eliza_inference_asr_cancel(ctx, &err), ELIZA_OK); + th.join(); + clear_err(&terr); + CHECK_RC(rc, ELIZA_ERR_CANCELLED); + CHECK(e.decoded.load() > 0); + } + + // Cancel during one long decode step (abort callback inside llama_decode). + { + ScriptedAsrEngine e(ctx); + e.tokens = { "one", " two", " three" }; + e.slow_decode = true; + int rc = 1; + char * terr = nullptr; + const auto t0 = clock_type::now(); + std::thread th([&]() { rc = eliza_test_asr_run_engine(ctx, &e, 64, text, sizeof(text), &terr); }); + CHECK(wait_until(e.in_decode, 2000)); + CHECK_RC(eliza_inference_asr_cancel(ctx, &err), ELIZA_OK); + th.join(); + clear_err(&terr); + CHECK_RC(rc, ELIZA_ERR_CANCELLED); + CHECK_RC(e.decoded.load(), 1); + CHECK(std::chrono::duration_cast(clock_type::now() - t0).count() < 2000); + } + + // A cancel issued while a second call waits for the ASR lock stops the + // call holding the lock, not the waiter. + { + ScriptedAsrEngine first(ctx); + first.slow_chunk = 0; + ScriptedAsrEngine second(ctx); + second.tokens = { "second" }; + int rc1 = 1, rc2 = 1; + char * e1 = nullptr; + char * e2 = nullptr; + char text2[64]; + std::thread t1([&]() { rc1 = eliza_test_asr_run_engine(ctx, &first, 64, text, sizeof(text), &e1); }); + CHECK(wait_until(first.in_prefill, 2000)); + std::thread t2([&]() { rc2 = eliza_test_asr_run_engine(ctx, &second, 64, text2, sizeof(text2), &e2); }); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + CHECK_RC(eliza_inference_asr_cancel(ctx, &err), ELIZA_OK); + t1.join(); + t2.join(); + clear_err(&e1); + clear_err(&e2); + CHECK_RC(rc1, ELIZA_ERR_CANCELLED); + CHECK_RC(rc2, 6); + } + + // The real entry still requires an acquired ASR region. + float pcm[160] = {0}; + CHECK_RC(eliza_inference_asr_transcribe(ctx, pcm, 160, 16000, text, sizeof(text), &err), ELIZA_ERR_INVALID_ARG); + clear_err(&err); + + eliza_inference_destroy(ctx); + fs::remove_all(bundle); +} + +int main() { + test_abi_and_probes(); + test_paradee_load_and_synth(); + test_paradee_chunking(); + test_paradee_cancel(); + test_paradee_race(); + test_evict_reload(); + test_asr_cancel(); + std::fprintf(stderr, "%s: %d checks, %d failures\n", + g_failures ? "FAIL" : "PASS", g_checks, g_failures); + return g_failures ? 1 : 0; +} diff --git a/tools/omnivoice/tests/paradee-contract/fake-paradee-control.h b/tools/omnivoice/tests/paradee-contract/fake-paradee-control.h new file mode 100644 index 000000000..e42eff37f --- /dev/null +++ b/tools/omnivoice/tests/paradee-contract/fake-paradee-control.h @@ -0,0 +1,28 @@ +// Scripting + counters for the Paradee / misaki test doubles in +// fake-paradee.cpp. Only the FFI unit test uses this. +#pragma once + +#include +#include +#include +#include + +struct fake_paradee_state { + std::atomic model_loads{0}; + std::atomic live_models{0}; + std::atomic lexicon_loads{0}; + std::atomic live_lexicons{0}; + std::atomic synth_calls{0}; + std::atomic in_flight{false}; + // > 0: each paradee_synthesize call runs this long, polling the cancel + // callback every millisecond (stand-in for a slow forward pass). + std::atomic block_ms{0}; + std::mutex sizes_mutex; + std::vector input_sizes; // input_ids.size() per synth call +}; + +fake_paradee_state & fake_paradee(); +void fake_paradee_reset(); + +// Samples the fake emits per non-pad input id. +constexpr int FAKE_PARADEE_SAMPLES_PER_ID = 100; diff --git a/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp b/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp new file mode 100644 index 000000000..e7b675466 --- /dev/null +++ b/tools/omnivoice/tests/paradee-contract/fake-paradee.cpp @@ -0,0 +1,178 @@ +// Scripted implementation of the Paradee / misaki contract headers in this +// directory. Deterministic, weight-free: it lets the FFI unit test drive +// load / synth / cancel / evict paths of src/eliza-inference-ffi.cpp. +// +// Vocab: the Kokoro ids the real model uses for punctuation and space +// (; 1, : 2, , 3, . 4, ! 5, ? 6, — 9, … 10, space 16); 'a'..'z' -> 43..68; +// '#' is out of vocab and dropped; any other code point -> 100. + +#include "paradee.h" +#include "misaki-g2p.h" +#include "fake-paradee-control.h" + +#include +#include +#include + +static fake_paradee_state g_state; + +fake_paradee_state & fake_paradee() { return g_state; } + +void fake_paradee_reset() { + g_state.model_loads = 0; + g_state.lexicon_loads = 0; + g_state.synth_calls = 0; + g_state.in_flight = false; + g_state.block_ms = 0; + std::lock_guard lock(g_state.sizes_mutex); + g_state.input_sizes.clear(); +} + +namespace eliza_paradee { + +struct paradee_model { + std::string path; +}; + +void paradee_model_deleter::operator()(paradee_model * m) const noexcept { + if (!m) return; + g_state.live_models.fetch_sub(1); + delete m; +} + +paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string & err_out) noexcept { + if (gguf_path.find("missing") != std::string::npos) { + err_out = "fake: no such file: " + gguf_path; + return nullptr; + } + g_state.model_loads.fetch_add(1); + g_state.live_models.fetch_add(1); + return paradee_model_ptr(new paradee_model{ gguf_path }); +} + +int paradee_sample_rate(const paradee_model * model) noexcept { + return model ? 24000 : -1; +} + +paradee_status paradee_phonemes_to_ids( + const paradee_model * model, + const std::string & s, + std::vector & ids, + std::string & err_out) noexcept { + if (!model) { err_out = "fake: null model"; return PARADEE_E_INVALID_ARG; } + ids.clear(); + for (size_t i = 0; i < s.size();) { + const unsigned char c = (unsigned char) s[i]; + size_t len = 1; + if (c >= 0xF0) len = 4; else if (c >= 0xE0) len = 3; else if (c >= 0xC0) len = 2; + const std::string cp = s.substr(i, len); + i += len; + if (cp == ";") ids.push_back(1); + else if (cp == ":") ids.push_back(2); + else if (cp == ",") ids.push_back(3); + else if (cp == ".") ids.push_back(4); + else if (cp == "!") ids.push_back(5); + else if (cp == "?") ids.push_back(6); + else if (cp == "\xE2\x80\x94") ids.push_back(9); + else if (cp == "\xE2\x80\xA6") ids.push_back(10); + else if (cp == " ") ids.push_back(16); + else if (cp == "#") continue; + else if (len == 1 && c >= 'a' && c <= 'z') ids.push_back(43 + (c - 'a')); + else ids.push_back(100); + } + return PARADEE_OK; +} + +paradee_status paradee_synthesize( + const paradee_model * model, + const std::vector & input_ids, + const paradee_synth_options & opts, + paradee_audio & out, + std::string & err_out) noexcept { + if (!model) { err_out = "fake: null model"; return PARADEE_E_INVALID_ARG; } + if (input_ids.size() < 3 || input_ids.size() > (size_t) PARADEE_MAX_INPUT_IDS || + input_ids.front() != 0 || input_ids.back() != 0) { + err_out = "fake: bad input_ids (size " + std::to_string(input_ids.size()) + ")"; + return PARADEE_E_INVALID_ARG; + } + if (!(opts.speed > 0.0f)) { err_out = "fake: bad speed"; return PARADEE_E_INVALID_ARG; } + g_state.synth_calls.fetch_add(1); + { + std::lock_guard lock(g_state.sizes_mutex); + g_state.input_sizes.push_back(input_ids.size()); + } + const int block_ms = g_state.block_ms.load(); + g_state.in_flight = true; + for (int t = 0; t < block_ms; ++t) { + if (opts.cancel && opts.cancel(opts.cancel_user_data)) { + g_state.in_flight = false; + return PARADEE_E_CANCELLED; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + if (opts.cancel && opts.cancel(opts.cancel_user_data)) { + g_state.in_flight = false; + return PARADEE_E_CANCELLED; + } + out.sample_rate = 24000; + out.samples.clear(); + for (size_t i = 1; i + 1 < input_ids.size(); ++i) { + for (int k = 0; k < FAKE_PARADEE_SAMPLES_PER_ID; ++k) { + out.samples.push_back((float) input_ids[i] / 1000.0f); + } + } + g_state.in_flight = false; + return PARADEE_OK; +} + +} // namespace eliza_paradee + +namespace eliza_misaki { + +struct misaki_lexicon { + misaki_fallback_kind kind; +}; + +void misaki_lexicon_deleter::operator()(misaki_lexicon * l) const noexcept { + if (!l) return; + g_state.live_lexicons.fetch_sub(1); + delete l; +} + +misaki_lexicon_ptr misaki_load_lexicon( + const std::string & lexicon_path, + const char * espeak_data_dir, + std::string & err_out) noexcept { + if (lexicon_path.find("missing") != std::string::npos) { + err_out = "fake: no such lexicon: " + lexicon_path; + return nullptr; + } + g_state.lexicon_loads.fetch_add(1); + g_state.live_lexicons.fetch_add(1); + return misaki_lexicon_ptr(new misaki_lexicon{ + espeak_data_dir ? MISAKI_FALLBACK_ESPEAK : MISAKI_FALLBACK_RULES }); +} + +misaki_fallback_kind misaki_get_fallback_kind(const misaki_lexicon * lexicon) noexcept { + return lexicon ? lexicon->kind : MISAKI_FALLBACK_NONE; +} + +// "G2P": lowercase ASCII letters and keep punctuation / spaces; digits are +// dropped. Text containing "\x01fail" reports an error. +bool misaki_phonemize( + const misaki_lexicon * lexicon, + const std::string & text, + std::string & out, + std::string & err_out) noexcept { + if (!lexicon) { err_out = "fake: null lexicon"; return false; } + if (text.find("\x01" "fail") != std::string::npos) { err_out = "fake: g2p failure"; return false; } + out.clear(); + for (unsigned char c : text) { + if (std::isalpha(c)) out.push_back((char) std::tolower(c)); + else if (std::isdigit(c)) continue; + else out.push_back((char) c); + } + return true; +} + +} // namespace eliza_misaki diff --git a/tools/omnivoice/tests/paradee-contract/misaki-g2p.h b/tools/omnivoice/tests/paradee-contract/misaki-g2p.h new file mode 100644 index 000000000..f0bf58492 --- /dev/null +++ b/tools/omnivoice/tests/paradee-contract/misaki-g2p.h @@ -0,0 +1,41 @@ +// tools/omnivoice/tests/paradee-contract/misaki-g2p.h +// +// Test double for the public misaki G2P header +// (tools/misaki-g2p/include/misaki-g2p.h). The FFI bridge is written against +// exactly these declarations; keep this copy in sync with the real header. +#pragma once + +#include +#include + +namespace eliza_misaki { + +// How words missing from the lexicon are phonemized. +enum misaki_fallback_kind { + MISAKI_FALLBACK_NONE = 0, // lexicon only; unknown words are dropped or spelled + MISAKI_FALLBACK_RULES = 1, // native letter-to-sound rules, misaki spelling + MISAKI_FALLBACK_ESPEAK = 2, // eSpeak NG output converted to misaki spelling +}; + +struct misaki_lexicon; +struct misaki_lexicon_deleter { void operator()(misaki_lexicon *) const noexcept; }; +using misaki_lexicon_ptr = std::unique_ptr; + +// `espeak_data_dir` may be nullptr: the lexicon then uses the rules fallback +// (or none, if the build has no rules). +misaki_lexicon_ptr misaki_load_lexicon( + const std::string & lexicon_path, + const char * espeak_data_dir, + std::string & err_out) noexcept; + +misaki_fallback_kind misaki_get_fallback_kind(const misaki_lexicon * lexicon) noexcept; + +// English text -> misaki phoneme string (UTF-8). Returns false and sets +// `err_out` on failure. +bool misaki_phonemize( + const misaki_lexicon * lexicon, + const std::string & text, + std::string & phonemes_out, + std::string & err_out) noexcept; + +} // namespace eliza_misaki diff --git a/tools/omnivoice/tests/paradee-contract/paradee.h b/tools/omnivoice/tests/paradee-contract/paradee.h new file mode 100644 index 000000000..f7e795b7e --- /dev/null +++ b/tools/omnivoice/tests/paradee-contract/paradee.h @@ -0,0 +1,72 @@ +// tools/omnivoice/tests/paradee-contract/paradee.h +// +// Test double for the public Paradee header (tools/paradee/include/paradee.h). +// The FFI bridge in src/eliza-inference-ffi.cpp is written against exactly +// these declarations. The FFI unit test (tests/ffi-paradee-asr.cpp) compiles +// the bridge against this header and the scripted fake in fake-paradee.cpp, +// so cancel / race / evict-reload behaviour is tested without model weights. +// +// Keep this file in sync with tools/paradee/include/paradee.h. If the real +// header changes a signature used by the bridge, change this copy too. +#pragma once + +#include +#include +#include +#include + +namespace eliza_paradee { + +// Model input limit, including the 0 pad token at each end. +constexpr int PARADEE_MAX_INPUT_IDS = 512; + +struct paradee_model; +struct paradee_model_deleter { void operator()(paradee_model *) const noexcept; }; +using paradee_model_ptr = std::unique_ptr; + +struct paradee_audio { + std::vector samples; + int sample_rate = 24000; +}; + +enum paradee_status { + PARADEE_OK = 0, + PARADEE_E_INVALID_ARG = 1, + PARADEE_E_LOAD_FAIL = 2, + PARADEE_E_OOM = 3, + PARADEE_E_CANCELLED = 4, + PARADEE_E_RUNTIME = 5, +}; + +// Polled by the synthesis graph between stages. Returning true stops the +// forward pass and makes paradee_synthesize return PARADEE_E_CANCELLED. +using paradee_cancel_cb = bool (*)(void * user_data); + +struct paradee_synth_options { + float speed = 1.0f; + paradee_cancel_cb cancel = nullptr; + void * cancel_user_data = nullptr; +}; + +paradee_model_ptr paradee_load_model(const std::string & gguf_path, std::string & err_out) noexcept; + +int paradee_sample_rate(const paradee_model * model) noexcept; + +// Map a misaki phoneme string to model vocab ids (no pad tokens added). +// Characters outside the vocab are dropped, as kokoro/misaki do. +paradee_status paradee_phonemes_to_ids( + const paradee_model * model, + const std::string & misaki_phonemes, + std::vector & ids_out, + std::string & err_out) noexcept; + +// Run one forward pass. `input_ids` is the full model input including the +// leading and trailing 0 pad token, at most PARADEE_MAX_INPUT_IDS entries. +paradee_status paradee_synthesize( + const paradee_model * model, + const std::vector & input_ids, + const paradee_synth_options & opts, + paradee_audio & out, + std::string & err_out) noexcept; + +} // namespace eliza_paradee From 76ed981218cd22aac9c778aef5fcfab9acee0e30 Mon Sep 17 00:00:00 2001 From: Shaw Date: Tue, 13 Oct 2026 22:18:55 -0400 Subject: [PATCH 2/2] test(omnivoice): run the v17 FFI tests in CI and fix racy check counters The cancel and race tests call CHECK from worker threads, but the counters were plain ints. TSan reported the races, the check total varied between runs (509/514/518), and a lost ++g_failures could turn a failure into a PASS. The counters are now std::atomic. eliza-inference-test-abi and eliza-inference-test-paradee-asr were registered with ctest but had no label, so CI (ctest -L main) built them and never ran them. Both now carry the main label. Co-Authored-By: Claude Opus 5.5 --- tools/omnivoice/CMakeLists.txt | 4 ++++ tools/omnivoice/tests/ffi-paradee-asr.cpp | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/omnivoice/CMakeLists.txt b/tools/omnivoice/CMakeLists.txt index 71522cf71..2be48b753 100644 --- a/tools/omnivoice/CMakeLists.txt +++ b/tools/omnivoice/CMakeLists.txt @@ -396,6 +396,9 @@ if(TARGET mtmd) endif() if(BUILD_TESTING) add_test(NAME eliza-inference-test-abi COMMAND eliza-inference-test-abi) + # "main" is the label CI runs (ctest -L main); unlabelled + # tests are built but never executed there. + set_property(TEST eliza-inference-test-abi PROPERTY LABELS main) endif() endif() @@ -423,6 +426,7 @@ if(TARGET mtmd) if(BUILD_TESTING) add_test(NAME eliza-inference-test-paradee-asr COMMAND eliza-inference-test-paradee-asr) + set_property(TEST eliza-inference-test-paradee-asr PROPERTY LABELS main) endif() endif() endif() diff --git a/tools/omnivoice/tests/ffi-paradee-asr.cpp b/tools/omnivoice/tests/ffi-paradee-asr.cpp index 789b89364..28c95c3fd 100644 --- a/tools/omnivoice/tests/ffi-paradee-asr.cpp +++ b/tools/omnivoice/tests/ffi-paradee-asr.cpp @@ -19,14 +19,17 @@ #include "fake-paradee-control.h" +#include #include #include #include #include #include -static int g_failures = 0; -static int g_checks = 0; +/* Atomic: CHECK runs on the worker threads of the cancel / race tests too, and + * a lost ++g_failures there would turn a failure into a PASS. */ +static std::atomic g_failures{0}; +static std::atomic g_checks{0}; #define CHECK(cond) do { \ ++g_checks; \ @@ -612,6 +615,6 @@ int main() { test_evict_reload(); test_asr_cancel(); std::fprintf(stderr, "%s: %d checks, %d failures\n", - g_failures ? "FAIL" : "PASS", g_checks, g_failures); - return g_failures ? 1 : 0; + g_failures.load() ? "FAIL" : "PASS", g_checks.load(), g_failures.load()); + return g_failures.load() ? 1 : 0; }