diff --git a/KernelRace/gradle/libs.versions.toml b/KernelRace/gradle/libs.versions.toml index b37f620..7fee674 100644 --- a/KernelRace/gradle/libs.versions.toml +++ b/KernelRace/gradle/libs.versions.toml @@ -18,11 +18,10 @@ kotlinxIo = "0.9.1" # IosModelProvider: skainet-data-source is JVM-only (KtorRemoteDataSourceFetcher lives in # its jvmMain), so the iOS download path talks to Ktor's Darwin engine on its own. ktorClientCore = "3.5.2" -# skainet-transformers has not released against the 0.40.x core line yet -# (tops out at 0.39.1 on Maven Central), so this sample — like KllamaDemo — -# stays one core version behind the transformers-free samples. -skainet = "0.40.1" -skainet-transformers = "0.40.2" +# SKaiNET-transformers 0.61.0 is built against engine 0.60.0; keep the two pins +# moving together (the transformers release notes name the engine version). +skainet = "0.60.0" +skainet-transformers = "0.61.0" [libraries] kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } diff --git a/KernelRace/shared/build.gradle.kts b/KernelRace/shared/build.gradle.kts index 4ccf302..8780fdf 100644 --- a/KernelRace/shared/build.gradle.kts +++ b/KernelRace/shared/build.gradle.kts @@ -51,7 +51,7 @@ kotlin { api(libs.skainet.io.gguf) // SKaiNET-transformers: Llama inference + generation. agent and - // runtime-kllama both publish wasmJs targets as of 0.39.1, so the + // runtime-kllama both publish wasmJs targets (since 0.39.1), so the // whole engine (unlike KllamaDemo's 0.34.1-era jvmMain-only split) // lives in commonMain. api(project.dependencies.platform(libs.skainet.transformers.bom)) diff --git a/KernelRace/shared/src/androidMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.android.kt b/KernelRace/shared/src/androidMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.android.kt index 6a20fe1..f037956 100644 --- a/KernelRace/shared/src/androidMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.android.kt +++ b/KernelRace/shared/src/androidMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.android.kt @@ -5,15 +5,14 @@ import sk.ainet.apps.llm.OptimizedLLMRuntime import sk.ainet.apps.llm.tokenizer.TokenizerFactory import sk.ainet.context.ExecutionContext import sk.ainet.io.AndroidRandomAccessSource -import sk.ainet.io.model.QuantPolicy import sk.ainet.lang.types.FP32 -import sk.ainet.models.llama.DecoderGgufWeightLoader +import sk.ainet.lang.nn.dsl.decoder.DecoderGgufWeightLoader import sk.ainet.models.llama.LlamaNetworkLoader import sk.ainet.samples.kernelrace.model.ModelData /** * The "five lines" fast path: random-access reads straight off the file, - * `QuantPolicy.NATIVE_OPTIMIZED` keeps Q8_0 packed for the hand-written ARM NEON kernels + * the loader's default `WeightForm` (packed, mapped) keeps Q8_0 packed for the hand-written ARM NEON kernels * (skainet-backend-jni-cpu), which register themselves through the kernel SPI just by being * on the classpath. */ @@ -21,7 +20,6 @@ actual suspend fun buildLlamaComponents(ctx: ExecutionContext, model: ModelData) val path = (model as ModelData.FilePath).path val weights = DecoderGgufWeightLoader( randomAccessProvider = { AndroidRandomAccessSource.open(path) }, - quantPolicy = QuantPolicy.NATIVE_OPTIMIZED, acceptedArchitectures = setOf("llama", "mistral"), // SmolLM2 is llama-family ).loadToMapStreaming(ctx) val runtime = OptimizedLLMRuntime( diff --git a/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.kt b/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.kt index f979f13..5cd6037 100644 --- a/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.kt +++ b/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.kt @@ -6,7 +6,6 @@ import sk.ainet.apps.llm.Tokenizer import sk.ainet.apps.llm.tokenizer.TokenizerFactory import sk.ainet.context.ExecutionContext import sk.ainet.io.RandomAccessSource -import sk.ainet.io.model.QuantPolicy import sk.ainet.lang.types.FP32 import sk.ainet.models.llama.LlamaNetworkLoader import sk.ainet.samples.kernelrace.model.ModelData @@ -47,7 +46,7 @@ private class InMemoryRandomAccessSource(private val bytes: ByteArray) : RandomA /** * Platform-split loader: * - Android takes the "five lines" fast path: random-access reads straight off the file, - * `QuantPolicy.NATIVE_OPTIMIZED` keeps Q8_0 packed for the hand-written ARM NEON kernels. + * the loader's default `WeightForm` (packed, mapped) keeps Q8_0 packed for the hand-written ARM NEON kernels. * - Desktop JVM takes the same file-based path (no NEON kernels to dispatch to on non-Android * hardware, but still avoids materializing the whole file on-heap). * - wasmJs has no filesystem, so it falls back to [buildLlamaComponentsFallback]: the whole GGUF @@ -74,7 +73,7 @@ suspend fun buildLlamaComponentsFallback(ctx: ExecutionContext, bytes: ByteArray val tokenizer = TokenizerFactory.fromGgufSource(InMemoryRandomAccessSource(bytes)) val randomAccessProvider = { InMemoryRandomAccessSource(bytes) } val model = LlamaNetworkLoader - .fromGguf(randomAccessProvider, QuantPolicy.NATIVE_OPTIMIZED, false) + .fromGguf(randomAccessProvider) .load(ctx) val runtime = OptimizedLLMRuntime( model = model, diff --git a/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlmEngine.kt b/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlmEngine.kt index 89774df..09a47fc 100644 --- a/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlmEngine.kt +++ b/KernelRace/shared/src/commonMain/kotlin/sk/ainet/samples/kernelrace/engine/LlmEngine.kt @@ -47,9 +47,9 @@ class LlmEngine private constructor(private val components: LlamaComponents) : G onToken = { tokenId -> if (tokenCount == 0) firstTokenAt = TimeSource.Monotonic.markNow() tokenCount++ - onToken(tokenizer.decode(tokenId)) + onToken(tokenizer.decodeToken(tokenId)) }, - decode = { tokenizer.decode(it) }, + decode = { tokenizer.decodeToken(it) }, ) val totalMs = genStart.elapsedNow().inWholeMilliseconds val decodeS = firstTokenAt?.elapsedNow()?.inWholeMilliseconds?.let { it / 1000.0 } ?: 0.0 diff --git a/KernelRace/shared/src/iosMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.ios.kt b/KernelRace/shared/src/iosMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.ios.kt index 74135df..0dd9acf 100644 --- a/KernelRace/shared/src/iosMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.ios.kt +++ b/KernelRace/shared/src/iosMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.ios.kt @@ -5,15 +5,14 @@ import sk.ainet.apps.llm.OptimizedLLMRuntime import sk.ainet.apps.llm.tokenizer.TokenizerFactory import sk.ainet.context.ExecutionContext import sk.ainet.io.PosixPreadRandomAccessSource -import sk.ainet.io.model.QuantPolicy import sk.ainet.lang.types.FP32 -import sk.ainet.models.llama.DecoderGgufWeightLoader +import sk.ainet.lang.nn.dsl.decoder.DecoderGgufWeightLoader import sk.ainet.models.llama.LlamaNetworkLoader import sk.ainet.samples.kernelrace.model.ModelData /** Same file-based random-access shape as the JVM/Android actuals, backed by POSIX `pread(2)` * (shared by macOS/iOS/Linux native in skainet-io-core's native64Main) instead of a JDK/ART - * file API. `QuantPolicy.NATIVE_OPTIMIZED` keeps Q8_0 packed for the Apple native-cinterop + * file API. the loader's default `WeightForm` (packed, mapped) keeps Q8_0 packed for the Apple native-cinterop * kernels — see kllama's registerPlatformBackends()/installNativeKernels() wiring, which * installs the same provider this app relies on. */ actual suspend fun buildLlamaComponents(ctx: ExecutionContext, model: ModelData): LlamaComponents { @@ -21,7 +20,6 @@ actual suspend fun buildLlamaComponents(ctx: ExecutionContext, model: ModelData) fun openSource() = checkNotNull(PosixPreadRandomAccessSource.open(path)) { "Cannot open GGUF at $path" } val weights = DecoderGgufWeightLoader( randomAccessProvider = { openSource() }, - quantPolicy = QuantPolicy.NATIVE_OPTIMIZED, acceptedArchitectures = setOf("llama", "mistral"), // SmolLM2 is llama-family ).loadToMapStreaming(ctx) val runtime = OptimizedLLMRuntime( diff --git a/KernelRace/shared/src/jvmMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.jvm.kt b/KernelRace/shared/src/jvmMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.jvm.kt index 917f357..9452d9a 100644 --- a/KernelRace/shared/src/jvmMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.jvm.kt +++ b/KernelRace/shared/src/jvmMain/kotlin/sk/ainet/samples/kernelrace/engine/LlamaRuntimeBuilder.jvm.kt @@ -5,9 +5,8 @@ import sk.ainet.apps.llm.OptimizedLLMRuntime import sk.ainet.apps.llm.tokenizer.TokenizerFactory import sk.ainet.context.ExecutionContext import sk.ainet.io.JvmRandomAccessSource -import sk.ainet.io.model.QuantPolicy import sk.ainet.lang.types.FP32 -import sk.ainet.models.llama.DecoderGgufWeightLoader +import sk.ainet.lang.nn.dsl.decoder.DecoderGgufWeightLoader import sk.ainet.models.llama.LlamaNetworkLoader import sk.ainet.samples.kernelrace.model.ModelData @@ -17,7 +16,6 @@ actual suspend fun buildLlamaComponents(ctx: ExecutionContext, model: ModelData) val path = (model as ModelData.FilePath).path val weights = DecoderGgufWeightLoader( randomAccessProvider = { JvmRandomAccessSource.open(path) }, - quantPolicy = QuantPolicy.NATIVE_OPTIMIZED, acceptedArchitectures = setOf("llama", "mistral"), ).loadToMapStreaming(ctx) val runtime = OptimizedLLMRuntime(