diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c734451..3ba384e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Moonshine streaming: incremental frontend** (`moonshine_stream_jni.c`, `libskainet_moonshine_stream.so` both + ABIs). With an optional `frontend_step.vmfb` next to `frontend.vmfb` (the frontend exported with + `MOONSHINE_FE_SAMPLES=6400`) the frontend runs in 0.32 s chunks as the audio arrives, windows run only the encoder + and adapter on cached features, and frames that can only see zero padding take the frontend's constant silence + feature. On an arm32 Mali device, 40 German commands paced like a microphone, against 0.62.2: features + bit-identical (141/141 windows), transcripts identical (40/40), the end-of-utterance flush p50 319 → 172 ms, end of + audio → final text p50 −275 ms. Without the file nothing changes. `IreeMoonshineStream.OPTIONAL_FILE_NAMES` names + it; `stats()` gains `feStepCalls` / `feStepMs`. + +### Added + +- `native/moonshine_stream_check.c`: drives `libskainet_moonshine_stream.so` on a device without an app (fake + JNIEnv), feeding PCM files paced like a microphone and printing transcript and stats per file. + +### Fixed + +- `native/build-moonshine-stream.sh` always passes `CMAKE_C_FLAGS`, so a plain build after a `GEOMETRY=…` one no + longer keeps the previous flags from the CMake cache. + ### Added - **`IreeModuleRunner` + `libskainet_iree_module.so`** (`runtime-iree-android`, `arm64-v8a` / `armeabi-v7a`, diff --git a/llm-runtime/iree-android/native/build-moonshine-stream.sh b/llm-runtime/iree-android/native/build-moonshine-stream.sh index 861e34d8..4ff1ca54 100755 --- a/llm-runtime/iree-android/native/build-moonshine-stream.sh +++ b/llm-runtime/iree-android/native/build-moonshine-stream.sh @@ -17,7 +17,9 @@ set -euo pipefail ABI="${1:-arm64-v8a}"; shift || true VULKAN=""; EXTRA=() -[ -n "${GEOMETRY:-}" ] && EXTRA=(--cmake "-DCMAKE_C_FLAGS=${GEOMETRY}") +# Always passed, empty included: the CMake cache in the build dir keeps the last value otherwise, so a plain +# build after a GEOMETRY="-DMOONSHINE_FE_VERIFY" one would silently still carry the verify code. +EXTRA=(--cmake "-DCMAKE_C_FLAGS=${GEOMETRY:-}") while [ $# -gt 0 ]; do case "$1" in --vulkan) VULKAN="--vulkan" ;; diff --git a/llm-runtime/iree-android/native/moonshine_stream_check.c b/llm-runtime/iree-android/native/moonshine_stream_check.c new file mode 100644 index 00000000..3d39e5c9 --- /dev/null +++ b/llm-runtime/iree-android/native/moonshine_stream_check.c @@ -0,0 +1,115 @@ +/* moonshine_stream_check.c — drives libskainet_moonshine_stream.so on a device without an app. + * + * A minimal JNIEnv (only the functions the library calls) backed by plain C data: dlopen the library, + * nativeCreate once, then for every PCM file feed it in 40 ms chunks paced like a microphone, nativeFinish, + * and print one line per file: name transcript stats. Stage timings also go to logcat + * (tag moonshine-timing), as they do in an app. + * + * moonshine_stream_check [--fast] + * device vulkan | local-task + * model dir the eight files of a materialized model (frontend.vmfb ... dec_embed.bin), plus + * frontend_step.vmfb when the library should use the incremental frontend + * pcm headerless little-endian s16 mono 16 kHz; a directory = every *.pcm in it, name order + * --fast feed without pacing (timings then no longer resemble a live stream) + * + * Build (same image as the library): + * docker run --rm -v "$PWD/native":/src -w /src --entrypoint sh skainet/iree-android:3.11.0 -c \ + * '/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi29-clang \ + * -O2 -o out/moonshine_stream_check moonshine_stream_check.c -ldl' + */ +#include +#include +#include +#include +#include +#include +#include + +typedef struct { jsize len; void* data; } Arr; + +static const char* GetStringUTFChars(JNIEnv* e, jstring s, jboolean* c) { if (c) *c = 0; return (const char*)s; } +static void ReleaseStringUTFChars(JNIEnv* e, jstring s, const char* p) {} +static jstring NewStringUTF(JNIEnv* e, const char* u) { return (jstring)strdup(u ? u : ""); } +static jsize GetArrayLength(JNIEnv* e, jarray a) { return ((Arr*)a)->len; } +static jfloat* GetFloatArrayElements(JNIEnv* e, jfloatArray a, jboolean* c) { if (c) *c = 0; return ((Arr*)a)->data; } +static void ReleaseFloatArrayElements(JNIEnv* e, jfloatArray a, jfloat* p, jint m) {} +static jclass FindClass(JNIEnv* e, const char* n) { return (jclass)n; } +static jint ThrowNew(JNIEnv* e, jclass c, const char* m) { fprintf(stderr, "%s: %s\n", (const char*)c, m); return 0; } + +static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; } +static void sleep_until(double t) { + double d = t - now_ms(); if (d <= 0) return; + struct timespec ts = { (time_t)(d / 1000), (long)((d - (time_t)(d / 1000) * 1000) * 1e6) }; nanosleep(&ts, NULL); +} + +static float* read_pcm(const char* path, int* n) { + FILE* f = fopen(path, "rb"); if (!f) return NULL; + fseek(f, 0, SEEK_END); long bytes = ftell(f); fseek(f, 0, SEEK_SET); + short* s = malloc(bytes); if (fread(s, 1, bytes, f) != (size_t)bytes) { fclose(f); free(s); return NULL; } + fclose(f); + *n = (int)(bytes / 2); + float* x = malloc(sizeof(float) * (*n)); + for (int i = 0; i < *n; ++i) x[i] = s[i] / 32768.0f; + free(s); return x; +} + +static int is_pcm(const struct dirent* d) { size_t l = strlen(d->d_name); return l > 4 && !strcmp(d->d_name + l - 4, ".pcm"); } + +int main(int argc, char** argv) { + if (argc < 5) { fprintf(stderr, "usage: %s [--fast]\n", argv[0]); return 2; } + int fast = argc > 5 && !strcmp(argv[5], "--fast"); + void* lib = dlopen(argv[1], RTLD_NOW); if (!lib) { fprintf(stderr, "dlopen: %s\n", dlerror()); return 3; } +#define SYM(n) dlsym(lib, "Java_sk_ainet_transformers_iree_android_IreeMoonshineStream_" n) + jlong (*create)(JNIEnv*, jobject, jstring, jstring, jstring, jstring, jstring, jstring, jstring, jstring, jstring) = SYM("nativeCreate"); + jstring (*feed)(JNIEnv*, jobject, jlong, jfloatArray) = SYM("nativeFeedPcm"); + jstring (*finish)(JNIEnv*, jobject, jlong) = SYM("nativeFinish"); + jstring (*stats)(JNIEnv*, jobject, jlong) = SYM("nativeStats"); + void (*destroy)(JNIEnv*, jobject, jlong) = SYM("nativeDestroy"); + if (!create || !feed || !finish || !stats || !destroy) { fprintf(stderr, "dlsym failed\n"); return 3; } + + struct JNINativeInterface ni; memset(&ni, 0, sizeof ni); + ni.GetStringUTFChars = GetStringUTFChars; ni.ReleaseStringUTFChars = ReleaseStringUTFChars; ni.NewStringUTF = NewStringUTF; + ni.GetArrayLength = GetArrayLength; ni.GetFloatArrayElements = GetFloatArrayElements; + ni.ReleaseFloatArrayElements = ReleaseFloatArrayElements; ni.FindClass = FindClass; ni.ThrowNew = ThrowNew; + const struct JNINativeInterface* nip = ∋ + JNIEnv* env = (JNIEnv*)&nip; + + const char* names[] = {"frontend.vmfb", "encoder.vmfb", "adapter.vmfb", "prefill.vmfb", "step.vmfb", "params.irpa", "vocab.bin", "dec_embed.bin"}; + static char p[8][1024]; + for (int i = 0; i < 8; ++i) snprintf(p[i], sizeof p[i], "%s/%s", argv[3], names[i]); + double t0 = now_ms(); + jlong h = create(env, NULL, (jstring)argv[2], (jstring)p[0], (jstring)p[1], (jstring)p[2], (jstring)p[3], (jstring)p[4], (jstring)p[5], (jstring)p[6], (jstring)p[7]); + if (!h) { fprintf(stderr, "nativeCreate failed\n"); return 4; } + fprintf(stderr, "create %.0fms\n", now_ms() - t0); + + struct dirent** list = NULL; int nfiles = 0; + DIR* d = opendir(argv[4]); + if (d) { closedir(d); nfiles = scandir(argv[4], &list, is_pcm, alphasort); } + for (int k = 0; k < (d ? nfiles : 1); ++k) { + char path[2048]; + if (d) snprintf(path, sizeof path, "%s/%s", argv[4], list[k]->d_name); else snprintf(path, sizeof path, "%s", argv[4]); + int n = 0; float* x = read_pcm(path, &n); + if (!x) { fprintf(stderr, "cannot read %s\n", path); continue; } + const int CH = 640; /* 40 ms */ + double start = now_ms(); + for (int off = 0; off < n; off += CH) { + if (!fast) sleep_until(start + off / 16.0); + Arr a = { n - off < CH ? n - off : CH, x + off }; + jstring part = feed(env, NULL, h, (jfloatArray)&a); + if (part) free((void*)part); + } + if (!fast) sleep_until(start + n / 16.0); + double tf = now_ms(); + jstring out = finish(env, NULL, h); + double fin = now_ms() - tf; + jstring st = stats(env, NULL, h); + const char* base = strrchr(path, '/'); + printf("%s\t%s\tfinishWallMs=%.0f,%s\n", base ? base + 1 : path, out ? (const char*)out : "", fin, st ? (const char*)st : ""); + fflush(stdout); + if (out) free((void*)out); + if (st) free((void*)st); + free(x); + } + destroy(env, NULL, h); + return 0; +} diff --git a/llm-runtime/iree-android/native/moonshine_stream_jni.c b/llm-runtime/iree-android/native/moonshine_stream_jni.c index 18c27027..7ba4c862 100644 --- a/llm-runtime/iree-android/native/moonshine_stream_jni.c +++ b/llm-runtime/iree-android/native/moonshine_stream_jni.c @@ -83,6 +83,17 @@ #define FE_IN ((CHUNK + FE_LC) * SPF) /* 21760 samples */ #define FE_OUT (CHUNK + FE_LC) /* 68 frames out */ #define MAXMEM 256 /* compiled cross-memory pad (5.12 s of finalized speech) */ +/* Incremental frontend (optional 9th graph `frontend_step.vmfb` next to `frontend.vmfb`): the same frontend + * exported for FE_STEP + FE_LC frames, run as the audio arrives, so a window only runs the encoder and + * adapter on features that already exist. Without it the frontend reruns over the whole 68-frame window + * at every hop and once more over the tail at finish(), which is most of the flush. + * The frontend is causal: output frame f reads 200 Hz frames 4f-12..4f, i.e. 3 frames of left context, + * so FE_LC = 4 frames of real audio in front of a chunk makes its kept frames exact. */ +#define FE_STEP 16 /* frames computed per incremental frontend call (0.32 s) */ +#define FES_IN ((FE_STEP + FE_LC) * SPF) /* 6400 samples */ +#define FES_OUT (FE_STEP + FE_LC) /* 20 frames out */ +#define MAXF (MAXMEM + CHUNK + FES_OUT) /* feature cache, frames */ +#define FE_RF 4 /* frames past `produced` that can still see audio (3 of context + a partly fed last frame) */ #define MAXTOK 48 /* 1 = skip the exact re-decode at the end of an utterance and keep the incremental result. * Off by default; switched on for the latency experiment via -DMOONSHINE_FAST_FINISH=1. */ @@ -123,6 +134,11 @@ typedef struct { iree_hal_device_t* dev; iree_hal_allocator_t* alloc; iree_runtime_session_t* fe, *enc, *adp, *pre, *step; + iree_runtime_session_t* fes; /* incremental frontend, NULL = whole-window frontend */ + float (*feat)[ENC_DIM]; /* [MAXF] frontend features of this utterance (fes only) */ + int fe_done; /* features [0, fe_done) are exact: backed by audio, never recomputed */ + double fe_feed_ms; int fe_calls; /* incremental frontend cost while the audio streams in */ + float fe_silence[ENC_DIM]; /* the frontend's output for all-zero input: every frame past FE_RF of the padding */ iree_vm_module_t* params; Vocab vocab; float* embed; /* [vocab.n][DIM] */ @@ -339,6 +355,43 @@ static int finish_budget(int npcm) { return b; } +/* One incremental frontend call for the chunk starting at frame `s`, zero-padded past the fed audio. + * Fed from sample 0 when s < FE_LC (the model's own padding is then the utterance start, see + * process_window). Writes frames s .. s+n-1 into e->feat and returns n, or -1 on failure. */ +static int fe_chunk(Engine* e, int s) { + static float in[FES_IN]; + static float out[FES_OUT][ENC_DIM]; + const int lc = s < FE_LC ? s : FE_LC; + const int s0 = (s - lc) * SPF; + for (int i = 0; i < FES_IN; ++i) { + int si = s0 + i; + in[i] = (si >= 0 && si < e->npcm) ? e->pcm[si] : 0.0f; + } + iree_hal_buffer_view_t* v = mkview(e, (iree_hal_dim_t[]){1, FES_IN}, 2, IREE_HAL_ELEMENT_TYPE_FLOAT_32, in, sizeof(in)); + if (!v) return -1; + iree_hal_buffer_view_t* o = run1(e, e->fes, FN_FE, v); + iree_hal_buffer_view_release(v); + if (!o) return -1; + int rc = pull(e, o, out, sizeof(out)); + iree_hal_buffer_view_release(o); + if (rc) return -1; + int n = FES_OUT - lc; + if (s + n > MAXF) n = MAXF - s; + for (int f = 0; f < n; ++f) memcpy(e->feat[s + f], out[lc + f], ENC_DIM * sizeof(float)); + return n; +} + +/* Extend the exact features as far as the fed audio allows, in whole FE_STEP chunks. */ +static void fe_advance(Engine* e, int produced) { + while (e->fes && e->fe_done + FE_STEP <= produced && e->fe_done + FES_OUT <= MAXF) { + double t0 = now_ms(); + int n = fe_chunk(e, e->fe_done); + if (n <= 0) break; + e->fe_done += n < produced - e->fe_done ? n : produced - e->fe_done; + e->fe_feed_ms += now_ms() - t0; e->fe_calls++; + } +} + /* Process one encoder window starting at feature frame `start`; finalize rows into e->mem. */ static int process_window(Engine* e, int start, int produced, int flush, int peek) { double t0 = now_ms(); @@ -350,27 +403,70 @@ static int process_window(Engine* e, int start, int produced, int flush, int pee * utterance — an empty final where the reference reads "Fail again, fail better.". So the window at frame 0 is fed * from sample 0 and its first CHUNK output frames are kept: the graph's own convolution padding is then exactly the * reference start. (The peek is a window at frame 0 too.) */ - static float fein[FE_IN]; const int lc = start == 0 ? 0 : FE_LC; - int s0 = (start - lc) * SPF; - for (int i = 0; i < FE_IN; ++i) { - int si = s0 + i; - fein[i] = (si >= 0 && si < e->npcm) ? e->pcm[si] : 0.0f; - } - iree_hal_buffer_view_t* fev = mkview(e, (iree_hal_dim_t[]){1, FE_IN}, 2, - IREE_HAL_ELEMENT_TYPE_FLOAT_32, fein, sizeof(fein)); - if (!fev) return -1; - iree_hal_buffer_view_t* feo = run1(e, e->fe, FN_FE, fev); - iree_hal_buffer_view_release(fev); - if (!feo) return -1; static float feats[FE_OUT][ENC_DIM]; - if (pull(e, feo, feats, sizeof(feats))) { iree_hal_buffer_view_release(feo); return -1; } - iree_hal_buffer_view_release(feo); + const float* encsrc; + if (e->fes) { + /* features up to the fed audio exist already; only the part of the window past them (a peek's or + * the flush's zero-padded tail) is computed here. Frames before `produced` come out exact either + * way, so fe_done may move up to it. */ + fe_advance(e, produced); + /* run the graph only as far as a frame can still see audio; the rest of the window is padding, + * whose feature is a constant (from warmup). The window at frame 0 keeps the graph throughout: + * there the zero-padded start is the model's own and FE_RF does not describe it. */ + int need = start == 0 ? start + CHUNK : produced + FE_RF; + if (need > start + CHUNK) need = start + CHUNK; + int s = e->fe_done; + while (s < need) { + int n = fe_chunk(e, s); + if (n <= 0) return -1; + s += n; + } + for (int f = s > need ? need : s; f < start + CHUNK; ++f) memcpy(e->feat[f], e->fe_silence, sizeof(e->fe_silence)); + if (produced > e->fe_done) e->fe_done = produced < s ? produced : s; + encsrc = e->feat[start]; + } else { + static float fein[FE_IN]; + int s0 = (start - lc) * SPF; + for (int i = 0; i < FE_IN; ++i) { + int si = s0 + i; + fein[i] = (si >= 0 && si < e->npcm) ? e->pcm[si] : 0.0f; + } + iree_hal_buffer_view_t* fev = mkview(e, (iree_hal_dim_t[]){1, FE_IN}, 2, + IREE_HAL_ELEMENT_TYPE_FLOAT_32, fein, sizeof(fein)); + if (!fev) return -1; + iree_hal_buffer_view_t* feo = run1(e, e->fe, FN_FE, fev); + iree_hal_buffer_view_release(fev); + if (!feo) return -1; + if (pull(e, feo, feats, sizeof(feats))) { iree_hal_buffer_view_release(feo); return -1; } + iree_hal_buffer_view_release(feo); + encsrc = feats[lc]; + } +#ifdef MOONSHINE_FE_VERIFY + /* Check the incremental features against the whole-window frontend over the same samples. */ + if (e->fes) { + static float fein[FE_IN]; + int s0 = (start - lc) * SPF; + for (int i = 0; i < FE_IN; ++i) { int si = s0 + i; fein[i] = (si >= 0 && si < e->npcm) ? e->pcm[si] : 0.0f; } + iree_hal_buffer_view_t* fev = mkview(e, (iree_hal_dim_t[]){1, FE_IN}, 2, IREE_HAL_ELEMENT_TYPE_FLOAT_32, fein, sizeof(fein)); + iree_hal_buffer_view_t* feo = fev ? run1(e, e->fe, FN_FE, fev) : NULL; + if (fev) iree_hal_buffer_view_release(fev); + if (feo && !pull(e, feo, feats, sizeof(feats))) { + double md = 0, ref = 0; + for (int f = 0; f < CHUNK; ++f) for (int k = 0; k < ENC_DIM; ++k) { + double d = fabs((double)feats[lc + f][k] - e->feat[start + f][k]); if (d > md) md = d; + if (fabs(feats[lc + f][k]) > ref) ref = fabs(feats[lc + f][k]); + } + TLOG("fe-verify win %d start %d produced %d maxdiff %.3g (max |x| %.3g)\n", e->win, start, produced, md, ref); + } + if (feo) iree_hal_buffer_view_release(feo); + } +#endif double t1 = now_ms(); /* encoder on the clean 64 frames (drop the lc context rows) */ iree_hal_buffer_view_t* encin = mkview(e, (iree_hal_dim_t[]){1, CHUNK, ENC_DIM}, 3, - IREE_HAL_ELEMENT_TYPE_FLOAT_32, feats[lc], CHUNK * ENC_DIM * sizeof(float)); + IREE_HAL_ELEMENT_TYPE_FLOAT_32, encsrc, CHUNK * ENC_DIM * sizeof(float)); iree_hal_buffer_view_t* enco = encin ? run1(e, e->enc, FN_ENC, encin) : NULL; if (encin) iree_hal_buffer_view_release(encin); if (!enco) return -1; @@ -591,6 +687,11 @@ static void warmup(Engine* e) { iree_hal_buffer_view_t* o = v ? run1(e, e->fe, FN_FE, v) : NULL; if (v) iree_hal_buffer_view_release(v); if (o) iree_hal_buffer_view_release(o); + if (e->fes) { + /* also captures the padding feature: a chunk at s >= FE_LC over zeros, any kept frame */ + e->npcm = 0; + if (fe_chunk(e, FE_LC) > 0) memcpy(e->fe_silence, e->feat[FE_LC], sizeof(e->fe_silence)); + } static float zf[CHUNK][DIM]; memset(zf, 0, sizeof(zf)); v = mkview(e, (iree_hal_dim_t[]){1, CHUNK, DIM}, 3, @@ -630,6 +731,7 @@ static void warmup(Engine* e) { static void reset_utterance(Engine* e) { e->npcm = 0; e->win = 0; e->nmem = 0; + e->fe_done = 0; e->fe_feed_ms = 0.0; e->fe_calls = 0; e->peeked = 0; e->pending = 0; e->halted = 0; e->shown_len = 0; e->t_first_feed = 0.0; e->dropped = 0; e->windows = 0; e->hops = 0; e->nevt = 0; release_self(e); release_cross(e); @@ -666,6 +768,18 @@ JNIEXPORT jlong JNICALL JNIFN(nativeCreate)(JNIEnv* env, jobject thiz, TLOG("create: fe(%s) -> %s\n", fe, e->fe ? "ok" : "FAIL"); if (e->fe && e->params) { e->alloc = iree_runtime_session_device_allocator(e->fe); + /* optional incremental frontend, by convention next to frontend.vmfb */ + char fesp[1024]; + const char* slash = strrchr(fe, '/'); + snprintf(fesp, sizeof(fesp), "%.*sfrontend_step.vmfb", slash ? (int)(slash - fe + 1) : 0, fe); + FILE* probe = fopen(fesp, "rb"); + if (probe) { + fclose(probe); + e->fes = load_session(e, fesp, 0); + e->feat = e->fes ? malloc(sizeof(float[MAXF][ENC_DIM])) : NULL; + if (!e->feat && e->fes) { iree_runtime_session_release(e->fes); e->fes = NULL; } + } + TLOG("create: fes(%s) -> %s\n", fesp, e->fes ? "ok (incremental frontend)" : "absent (whole-window frontend)"); e->enc = load_session(e, enc, 0); TLOG("create: enc(%s) -> %s\n", enc, e->enc ? "ok" : "FAIL"); e->adp = load_session(e, adp, 0); @@ -700,7 +814,7 @@ JNIEXPORT jlong JNICALL JNIFN(nativeCreate)(JNIEnv* env, jobject thiz, (*env)->ReleaseStringUTFChars(env, jpre, pre); (*env)->ReleaseStringUTFChars(env, jstep, stp); (*env)->ReleaseStringUTFChars(env, jparams, par); (*env)->ReleaseStringUTFChars(env, jvocab, voc); (*env)->ReleaseStringUTFChars(env, jembed, emb); - if (!ret && e) { free(e->embed); free(e->pcm); free(e); } + if (!ret && e) { free(e->embed); free(e->pcm); free(e->feat); free(e); } return ret; } @@ -718,6 +832,7 @@ JNIEXPORT jstring JNICALL JNIFN(nativeFeedPcm)(JNIEnv* env, jobject thiz, int produced = e->npcm / SPF; int changed = 0; + fe_advance(e, produced); /* How far behind the microphone this stream is: wall clock since the first sample minus the audio * that has arrived. Positive means the caller is waiting on us, and every millisecond of it is still * owed at the moment the utterance ends. Only the optional decode work is given up for it. @@ -831,21 +946,24 @@ JNIEXPORT jstring JNICALL JNIFN(nativeFinish)(JNIEnv* env, jobject thiz, jlong h const char* p = e->text; while (*p == ' ') ++p; out = (*env)->NewStringUTF(env, p); /* flush and re-decode reported apart, so the cost of each half is visible rather than inferred. */ - TLOG("finish %.0fms (flush %.0fms decode %.0fms) nmem %d toks %d/%d fast=%d dropped %d lag %.0fms \"%s\"\n", + TLOG("finish %.0fms (flush %.0fms decode %.0fms) nmem %d toks %d/%d fast=%d dropped %d lag %.0fms \"%s\" festep %d/%.0fms\n", now_ms() - t0, t_flush - t0, now_ms() - t_flush, e->nmem, e->ntoks, finish_budget(e->npcm), MOONSHINE_FAST_FINISH, e->dropped, - e->t_first_feed > 0.0 ? t0 - e->t_first_feed - (double)e->npcm * 1000.0 / 16000.0 : 0.0, p); + e->t_first_feed > 0.0 ? t0 - e->t_first_feed - (double)e->npcm * 1000.0 / 16000.0 : 0.0, p, + e->fe_calls, e->fe_feed_ms); } /* Same figures the line above logs, kept for nativeStats — a caller should not have to parse logcat * to learn what recognition cost. Written before the reset, which clears the counters. */ int n = snprintf(e->stats, sizeof(e->stats), "flushMs=%.0f,decodeMs=%.0f,finishMs=%.0f,finishAtMs=%.0f,lagMs=%.0f,droppedDecodes=%d," - "windows=%d,hops=%d,memFrames=%d,tokens=%d,tokenBudget=%d,audioMs=%.0f,fastFinish=%d,lagBudgetMs=%d", + "windows=%d,hops=%d,memFrames=%d,tokens=%d,tokenBudget=%d,audioMs=%.0f,fastFinish=%d,lagBudgetMs=%d," + "feStepCalls=%d,feStepMs=%.0f", t_flush - t0, now_ms() - t_flush, now_ms() - t0, e->t_first_feed > 0.0 ? t0 - e->t_first_feed : 0.0, e->t_first_feed > 0.0 ? t0 - e->t_first_feed - (double)e->npcm * 1000.0 / 16000.0 : 0.0, e->dropped, e->windows, e->hops, e->nmem, e->ntoks, finish_budget(e->npcm), - (double)e->npcm * 1000.0 / 16000.0, MOONSHINE_FAST_FINISH, MOONSHINE_LAG_BUDGET_MS); + (double)e->npcm * 1000.0 / 16000.0, MOONSHINE_FAST_FINISH, MOONSHINE_LAG_BUDGET_MS, + e->fe_calls, e->fe_feed_ms); /* The utterance's timeline, so the shape can be drawn and not just totalled: * timeline=::::|... * kind w = encoder window (a/b/c = frontend, encoder, adapter), p = early peek, h = partial @@ -879,6 +997,7 @@ JNIEXPORT void JNICALL JNIFN(nativeDestroy)(JNIEnv* env, jobject thiz, jlong han Engine* e = (Engine*)(intptr_t)handle; if (!e) return; reset_utterance(e); if (e->fe) iree_runtime_session_release(e->fe); + if (e->fes) iree_runtime_session_release(e->fes); if (e->enc) iree_runtime_session_release(e->enc); if (e->adp) iree_runtime_session_release(e->adp); if (e->pre) iree_runtime_session_release(e->pre); @@ -887,5 +1006,5 @@ JNIEXPORT void JNICALL JNIFN(nativeDestroy)(JNIEnv* env, jobject thiz, jlong han if (e->dev) iree_hal_device_release(e->dev); if (e->inst) iree_runtime_instance_release(e->inst); for (int i = 0; i < e->vocab.n; ++i) free(e->vocab.t[i]); - free(e->vocab.t); free(e->embed); free(e->pcm); free(e); + free(e->vocab.t); free(e->embed); free(e->pcm); free(e->feat); free(e); } diff --git a/llm-runtime/iree-android/src/main/jniLibs/arm64-v8a/libskainet_moonshine_stream.so b/llm-runtime/iree-android/src/main/jniLibs/arm64-v8a/libskainet_moonshine_stream.so index 2b705ff0..4690023f 100755 Binary files a/llm-runtime/iree-android/src/main/jniLibs/arm64-v8a/libskainet_moonshine_stream.so and b/llm-runtime/iree-android/src/main/jniLibs/arm64-v8a/libskainet_moonshine_stream.so differ diff --git a/llm-runtime/iree-android/src/main/jniLibs/armeabi-v7a/libskainet_moonshine_stream.so b/llm-runtime/iree-android/src/main/jniLibs/armeabi-v7a/libskainet_moonshine_stream.so index c0791e82..1922ffa3 100755 Binary files a/llm-runtime/iree-android/src/main/jniLibs/armeabi-v7a/libskainet_moonshine_stream.so and b/llm-runtime/iree-android/src/main/jniLibs/armeabi-v7a/libskainet_moonshine_stream.so differ diff --git a/llm-runtime/iree-android/src/main/kotlin/sk/ainet/transformers/iree/android/IreeMoonshineStream.kt b/llm-runtime/iree-android/src/main/kotlin/sk/ainet/transformers/iree/android/IreeMoonshineStream.kt index 650b5606..22ab5ad3 100644 --- a/llm-runtime/iree-android/src/main/kotlin/sk/ainet/transformers/iree/android/IreeMoonshineStream.kt +++ b/llm-runtime/iree-android/src/main/kotlin/sk/ainet/transformers/iree/android/IreeMoonshineStream.kt @@ -11,6 +11,11 @@ package sk.ainet.transformers.iree.android * flushes the tail, runs one exact full re-decode over the final memory, returns the final transcript * and resets for the next utterance. [reset] aborts the current utterance and keeps the engine. * + * Optional ninth file: `frontend_step.vmfb` next to `frontend.vmfb` (the same frontend exported for 6400 samples, + * `MOONSHINE_FE_SAMPLES=6400`). When it is there the frontend runs in 0.32 s chunks as the audio arrives and a window + * only runs the encoder and adapter, which takes most of the frontend out of [finish]. The features are bit-identical + * either way, so the transcript does not change; without the file the runtime behaves as before. See [OPTIONAL_FILE_NAMES]. + * * One instance = one utterance at a time; callers serialize access. Per-hop stage timings are logged * under the `moonshine-timing` tag. * @@ -74,6 +79,8 @@ public class IreeMoonshineStream( * partial decodes actually run, and encoder windows processed. Windows are the mandatory work; * a dropped decode costs a partial and cannot change the final transcript. * - `memFrames`, `tokens`, `tokenBudget`, `audioMs` — the utterance's size in each unit. + * - `feStepCalls`, `feStepMs` — incremental frontend calls made while the audio streamed in, and their cost; + * both 0 without `frontend_step.vmfb`. * - `fastFinish`, `lagBudgetMs` — what this binary was compiled with, so a measurement can say * which build produced it. * @@ -111,6 +118,12 @@ public class IreeMoonshineStream( "frontend.vmfb", "encoder.vmfb", "adapter.vmfb", "prefill.vmfb", "step.vmfb", "params.irpa", "vocab.bin", "dec_embed.bin", ) + /** + * Files the runtime picks up when they sit next to `frontend.vmfb`, and works without: `frontend_step.vmfb`, the + * incremental frontend. A materializer should copy them when the model has them. + */ + public val OPTIONAL_FILE_NAMES: List = listOf("frontend_step.vmfb") + /** [Files] for a directory that holds the eight files under their standard names. */ public fun filesIn(dir: java.io.File): Files { val p = FILE_NAMES.map { java.io.File(dir, it).absolutePath }