diff --git a/.github/workflows/publish-performance-pack.yml b/.github/workflows/publish-performance-pack.yml index 19c82d126..ff82c0eb5 100644 --- a/.github/workflows/publish-performance-pack.yml +++ b/.github/workflows/publish-performance-pack.yml @@ -70,7 +70,12 @@ jobs: run: ./scripts/fetch-win-binaries.ps1 - name: Build Kev server runtime (model weights excluded) shell: pwsh - run: ./scripts/build-kev-runtime.ps1 + run: | + ./scripts/build-kev-runtime.ps1 + . ./scripts/config/windows-vc-runtime.ps1 + Get-ChildItem resources/bin/kev-runtime -Recurse -File -Filter '*.exe' | + Select-Object -ExpandProperty DirectoryName -Unique | + ForEach-Object { Copy-VCRuntime $_ } - name: Check staged Windows runtimes shell: bash run: | @@ -166,6 +171,13 @@ jobs: rm -rf "$BIN_ROOT/llama-cuda" "$BIN_ROOT/cuda-runtime" cp -a "$RUNNER_TEMP/engine/llama-cuda" "$RUNNER_TEMP/engine/cuda-runtime" "$BIN_ROOT/" cat "$RUNNER_TEMP/engine/engine.json" + if [ "$PLATFORM" = win32 ]; then + # llama-cuda was staged on Linux; reuse the VC redist from the Windows job. + for dll in MSVCP140.dll MSVCP140_1.dll MSVCP140_2.dll MSVCP140_ATOMIC_WAIT.dll \ + MSVCP140_CODECVT_IDS.dll VCRUNTIME140.dll VCRUNTIME140_1.dll CONCRT140.dll VCOMP140.dll; do + cp "$BIN_ROOT/whisper/$dll" "$BIN_ROOT/llama-cuda/$dll" + done + fi - name: Check the Linux llama engines' libraries resolve if: inputs.platform == 'linux' run: | @@ -189,7 +201,8 @@ jobs: run: | python3 -m pip install --no-cache-dir --no-deps --target "$RUNNER_TEMP/onnx-nvidia" \ 'nvidia-cuda-runtime==13.2.86' 'nvidia-cublas==13.2.2.2' \ - 'nvidia-curand==10.4.2.66' 'nvidia-cudnn-cu13==9.23.1.3' + 'nvidia-curand==10.4.2.66' 'nvidia-cudnn-cu13==9.23.1.3' \ + 'nvidia-cufft==12.2.0.57' 'nvidia-cuda-nvrtc==13.2.86' mkdir -p "$BIN_ROOT/onnx-cuda" for folder in cu13 cudnn; do find "$RUNNER_TEMP/onnx-nvidia/nvidia/$folder/lib" -maxdepth 1 -type f -name '*.so.*' \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69c49ddab..2f7d0d2c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,6 +122,8 @@ jobs: build-mac: needs: version runs-on: macos-15 # Standard arm64 runner; matches the pinned native payload. + env: + MACOSX_DEPLOYMENT_TARGET: '13.0' steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -139,13 +141,12 @@ jobs: 'https://github.com/off-grid-ai/OGAD/releases/download/native-deps-2026-09/ogad-mac-native-2026-10-v2.tar.gz' echo '50f13dc0b6318eb0c036ccd9ebe0f21d931912992bf828f1cd56de4113e9e48c '"$RUNNER_TEMP/mac-native.tar.gz" | shasum -a 256 -c - tar -xzf "$RUNNER_TEMP/mac-native.tar.gz" -C resources/bin + - name: Build macOS image engine for macOS 13 + run: bash scripts/build-sd-mac.sh - name: Verify cached macOS engines run: | for variant in llama llama-prism; do test -x "resources/bin/$variant/llama-server" - minos=$(otool -l "resources/bin/$variant/llama-server" | awk '/LC_BUILD_VERSION/{f=1} f&&/minos/{print $2; exit}') - test -n "$minos" - case "$minos" in 1[0-3].*|[1-9].*) ;; *) echo "Unsupported $variant minos $minos" >&2; exit 1 ;; esac DYLD_LIBRARY_PATH="resources/bin/$variant" "resources/bin/$variant/llama-server" --help >/dev/null done test -x resources/bin/whisper/whisper-cli @@ -286,6 +287,7 @@ jobs: ref: 45d7fd41a1cbf268fa3e712aa5558f08fa1a75d9 # ExecuTorch 1.5.1; the same commit on every platform path: _executorch_speech lfs: true # Mac releases bundle the default English voices for offline first use. + submodules: recursive persist-credentials: false - name: Put the speech runtime beside this checkout run: | @@ -295,6 +297,17 @@ jobs: fi rm -rf ../executorch-speech mv _executorch_speech ../executorch-speech + - name: Build macOS speech runtime for macOS 13 + run: | + npm --prefix ../executorch-speech ci + # Use the bundled Python and its existing torchgen, YAML, and Jinja dependencies. + speech_python="$GITHUB_WORKSPACE/resources/bin/kev-runtime/python/bin/python3" + "$speech_python" -c 'import torchgen, yaml, jinja2' + cmake -S ../executorch-speech/native -B ../executorch-speech/native/build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DFLATCC_ALLOW_WERROR=OFF -DPYTHON_EXECUTABLE="$speech_python" + cmake --build ../executorch-speech/native/build --config Release --parallel 4 \ + --target executorch-speech - name: Let npm read the private UI package (core and Pro) # @offgrid/operator-ui is a private Wednesday package that core and Pro both render with # (skin, primitives, motion). It is listed as optional, so npm would skip it silently @@ -385,14 +398,15 @@ jobs: printf '%s' "$APPLE_API_KEY_CONTENT" > "$RUNNER_TEMP/AuthKey.p8" export APPLE_API_KEY="$RUNNER_TEMP/AuthKey.p8" npm run prepare:speech-defaults + node scripts/check-macos-minos.mjs 13.0 # Build every artifact with publication disabled. The artifact hook verifies # both the updater ZIP and DMG before this command can complete. npx electron-builder --mac \ - -c.dmg.artifactName='OffGrid-${version}.${ext}' \ + -c.dmg.artifactName='OffGrid-${version}-${arch}.${ext}' \ --publish never # electron-builder notarizes the app before it creates the DMG. Submit the # completed, signed DMG too, then staple its ticket for offline Gatekeeper. - DMG="dist/OffGrid-$VERSION.dmg" + DMG="dist/OffGrid-$VERSION-arm64.dmg" xcrun notarytool submit "$DMG" \ --key "$APPLE_API_KEY" \ --key-id "$APPLE_API_KEY_ID" \ @@ -488,7 +502,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} run: | - cp "dist/OffGrid-$VERSION.dmg" dist/OffGrid-latest.dmg + cp "dist/OffGrid-$VERSION-arm64.dmg" dist/OffGrid-latest.dmg gh release upload "v$VERSION" dist/OffGrid-latest.dmg --clobber # Constant nightly download link. /releases/latest EXCLUDES pre-releases, so it # can never point at a beta build (that is exactly what keeps the STABLE link @@ -502,7 +516,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} run: | - cp "dist/OffGrid-$VERSION.dmg" dist/OffGrid-nightly.dmg + cp "dist/OffGrid-$VERSION-arm64.dmg" dist/OffGrid-nightly.dmg # Ensure the rolling 'nightly' pre-release exists (kept separate from the # versioned -beta.N releases electron-builder makes for the updater feed), # then clobber its dmg so the link always serves the freshest nightly. @@ -684,6 +698,14 @@ jobs: fi - name: Install dependencies run: npm ci + - name: Install Vulkan SDK for Whisper + uses: humbletim/install-vulkan-sdk@v1.2 + with: + version: '1.4.321.1' + cache: true + - name: Build Windows Vulkan Whisper + shell: pwsh + run: ./scripts/build-whisper-vulkan-win.ps1 - name: Fetch Windows native binaries (llama/whisper/sd/ffmpeg) shell: pwsh env: @@ -700,7 +722,7 @@ jobs: GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} MICROSOFT_CLIENT_ID: ${{ vars.MICROSOFT_CLIENT_ID }} - run: node scripts/stage-native.mjs && npx electron-vite build && node scripts/check-windows-onnx-preload.mjs + run: node scripts/stage-native.mjs && npx electron-vite build - name: Package Windows web installer, payload, and update metadata shell: bash env: @@ -714,10 +736,10 @@ jobs: shell: bash run: | test -s dist/win-unpacked/resources/performance-packs.json - for binary in llama/llama-server.exe llama-cpu/llama-server.exe llama-prism/llama-server.exe llama-prism-cpu/llama-server.exe whisper-cpu/whisper-cli.exe sd/sd-cli.exe sd-cpu/sd-cli.exe ffmpeg.exe; do + for binary in llama/llama-server.exe llama-cpu/llama-server.exe llama-prism/llama-server.exe llama-prism-cpu/llama-server.exe whisper-cpu/whisper-cli.exe whisper-vulkan/whisper-cli.exe whisper-vulkan/whisper-server.exe sd/sd-cli.exe sd-cpu/sd-cli.exe ffmpeg.exe; do test -s "dist/win-unpacked/resources/bin/$binary" done - for folder in llama-cuda llama-prism-cuda cuda-runtime sd-cuda whisper kev-runtime; do + for folder in llama-cuda llama-prism-cuda cuda-runtime sd-cuda whisper kev-runtime onnx-cuda; do test ! -e "dist/win-unpacked/resources/bin/$folder" done - name: Report Windows native payload sizes @@ -931,6 +953,14 @@ jobs: run: npm ci - name: Install Linux GPU runtime dependencies run: sudo apt-get update -qq && sudo apt-get install -y libgomp1 libvulkan1 libvulkan-dev glslc spirv-headers unzip + - name: Stage Vulkan loaders beside cached Linux engines + run: | + source="$(ldconfig -p | awk '$1 == "libvulkan.so.1" && /x86-64/ && !found { found=$NF } END { print found }')" + test -n "$source" + for folder in llama llama-prism whisper; do + install -m 755 "$(readlink -f "$source")" "build/linux-bin/$folder/libvulkan.so.1" + test -s "build/linux-bin/$folder/libvulkan.so.1" + done - name: Typecheck Linux main process env: OFFGRID_FORCE_CORE: '0' @@ -959,8 +989,21 @@ jobs: - name: Verify Linux package inputs run: | test -s dist/linux-unpacked/resources/performance-packs.json + provider="$(find dist/linux-unpacked/resources/app.asar.unpacked/node_modules \ + -path '*/onnxruntime-node/*' -name libonnxruntime_providers_cuda.so -type f -print -quit)" + test -n "$provider" + test -s "$provider" + for folder in llama llama-prism whisper; do + test -s "dist/linux-unpacked/resources/bin/$folder/libvulkan.so.1" + done + while IFS= read -r -d '' binary; do + if file "$binary" | grep -q 'ELF.*executable'; then + glibc="$(objdump -T "$binary" | grep -o 'GLIBC_[0-9.]*' | sort -V | tail -1 || true)" + printf '[ci] %s highest GLIBC: %s\n' "$binary" "${glibc:-none (static binary)}" + fi + done < <(find dist/linux-unpacked/resources/bin -type f -print0) cmp resources/bin/kev-local-server.py dist/linux-unpacked/resources/bin/kev-local-server.py - for folder in llama-cuda llama-prism-cuda cuda-runtime sd-cuda whisper-cuda kev-runtime; do + for folder in llama-cuda llama-prism-cuda cuda-runtime sd-cuda whisper-cuda kev-runtime onnx-cuda; do test ! -e "dist/linux-unpacked/resources/bin/$folder" done for variant in llama llama-cpu llama-prism llama-prism-cpu; do @@ -1013,9 +1056,6 @@ jobs: const root = path.resolve("dist/linux-unpacked/resources/app.asar/node_modules") const Database = require(path.join(root, "better-sqlite3-multiple-ciphers")) new Database(":memory:").close() - if (typeof require(path.join(root, "@lancedb/lancedb")).connect !== "function") { - throw new Error("LanceDB native package did not load") - } if (typeof require(path.join(root, "sharp")) !== "function") { throw new Error("Sharp native package did not load") } @@ -1038,7 +1078,7 @@ jobs: test -n "$bundled" cmp "dist/linux-unpacked/resources/bin/$variant/llama-server" "$bundled" done - for helper in whisper/whisper-cli whisper-cpu/whisper-cli whisper/LICENSE ffmpeg licenses/ffmpeg.txt sd/sd-cli sd/sd-server sd/libggml-vulkan.so sd/libgomp.so.1 sd/libvulkan.so.1 executorch-speech licenses/libgomp1.txt licenses/libvulkan1.txt; do + for helper in llama/libvulkan.so.1 llama-prism/libvulkan.so.1 whisper/libvulkan.so.1 whisper/whisper-cli whisper-cpu/whisper-cli whisper/LICENSE ffmpeg licenses/ffmpeg.txt sd/sd-cli sd/sd-server sd/libggml-vulkan.so sd/libgomp.so.1 sd/libvulkan.so.1 executorch-speech licenses/libgomp1.txt licenses/libvulkan1.txt; do bundled="$(find "$extraction/squashfs-root" -path "*/resources/bin/$helper" -type f -print -quit)" test -n "$bundled" cmp "dist/linux-unpacked/resources/bin/$helper" "$bundled" @@ -1055,7 +1095,7 @@ jobs: test -n "$bundled" cmp "dist/linux-unpacked/resources/bin/$variant/llama-server" "$bundled" done - for helper in whisper/whisper-cli whisper-cpu/whisper-cli whisper/LICENSE ffmpeg licenses/ffmpeg.txt sd/sd-cli sd/sd-server sd/libggml-vulkan.so sd/libgomp.so.1 sd/libvulkan.so.1 executorch-speech licenses/libgomp1.txt licenses/libvulkan1.txt; do + for helper in llama/libvulkan.so.1 llama-prism/libvulkan.so.1 whisper/libvulkan.so.1 whisper/whisper-cli whisper-cpu/whisper-cli whisper/LICENSE ffmpeg licenses/ffmpeg.txt sd/sd-cli sd/sd-server sd/libggml-vulkan.so sd/libgomp.so.1 sd/libvulkan.so.1 executorch-speech licenses/libgomp1.txt licenses/libvulkan1.txt; do bundled="$(find "$extraction/deb" -path "*/resources/bin/$helper" -type f -print -quit)" test -n "$bundled" cmp "dist/linux-unpacked/resources/bin/$helper" "$bundled" diff --git a/docs/RELEASE_DESKTOP.md b/docs/RELEASE_DESKTOP.md index 6af1a1c6a..c59cd0855 100644 --- a/docs/RELEASE_DESKTOP.md +++ b/docs/RELEASE_DESKTOP.md @@ -240,3 +240,9 @@ quickly. No EV needed. - [ ] Windows: bundled llama-server doesn't start — fix in progress separately. - [ ] Azure Trusted Signing validated + `azureSignOptions` wired (§4) — resumes with Windows. + +### Linux AppImage + +The AppImage and deb packages are x64 only. On Ubuntu 24.04, install +`libfuse2t64` before you start the AppImage. Headless servers and some sandboxes +need the `--no-sandbox` option. diff --git a/e2e/fixtures/gpu-fault-llama-server.mjs b/e2e/fixtures/gpu-fault-llama-server.mjs new file mode 100644 index 000000000..6041bcc5f --- /dev/null +++ b/e2e/fixtures/gpu-fault-llama-server.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node + +/* eslint-disable @typescript-eslint/explicit-function-return-type -- executable JavaScript fixture */ + +// The GPU fault journey's only fake: the native llama-server boundary. Everything that decides what +// happens after a fault (the HTTP client's classifier, the chat fallback policy, the same-model +// restart, the backend ladder, the CPU-only launch flags, the runtime status and Settings) is +// production code. Like a real engine, this one reports where it put the model in its load log, +// and like a GPU that has lost its context it answers "500 Compute error" for as many questions as +// faults.json beside it still lists. A fault is spent when it is answered, across restarts. + +import fs from 'node:fs' +import http from 'node:http' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const args = process.argv.slice(2) +const portFlag = Math.max(args.indexOf('--port'), args.indexOf('-p')) +const port = portFlag >= 0 ? Number(args[portFlag + 1]) : 8439 +const deviceFlag = args.indexOf('--device') +const cpuOnly = deviceFlag >= 0 && args[deviceFlag + 1] === 'none' +const here = path.dirname(fileURLToPath(import.meta.url)) +const FAULTS = path.join(here, 'faults.json') +const LOG = path.join(here, 'launches.jsonl') + +fs.appendFileSync(LOG, `${JSON.stringify({ args })}\n`) + +// The load log the app reads to name the backend and the layers on the GPU. +if (cpuOnly) { + process.stderr.write('load_tensors: CPU model buffer size = 120.00 MiB\n') +} else { + process.stderr.write('load_tensors: offloaded 28/28 layers to GPU\n') + process.stderr.write('load_tensors: Metal model buffer size = 120.00 MiB\n') +} + +function takeFault() { + try { + const plan = JSON.parse(fs.readFileSync(FAULTS, 'utf8')) + if (!(plan.remaining > 0)) return false + fs.writeFileSync(FAULTS, JSON.stringify({ remaining: plan.remaining - 1 })) + return true + } catch { + return false + } +} + +function lastUser(messages) { + const user = [...(messages ?? [])].reverse().find((message) => message.role === 'user') + const content = user?.content + return typeof content === 'string' + ? content + : Array.isArray(content) + ? content.map((part) => part?.text ?? '').join('') + : '' +} + +const delta = (payload) => `data: ${JSON.stringify({ choices: [{ delta: payload }] })}\n\n` + +const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) + return + } + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'gpu-fault-model' }] })) + return + } + if (request.method !== 'POST' || !String(request.url).includes('/chat/completions')) { + response.writeHead(404) + response.end() + return + } + let body = '' + request.on('data', (chunk) => { + body += chunk + }) + request.on('end', () => { + let payload = {} + try { + payload = JSON.parse(body) + } catch { + /* answer the default */ + } + const question = lastUser(payload.messages) + // Only the journey's own questions fault; follow-up suggestions and titles answer normally. + const faulted = question.startsWith('GPU check') && takeFault() + fs.appendFileSync( + LOG, + `${JSON.stringify({ question: question.slice(0, 80), stream: payload.stream !== false, faulted, cpuOnly })}\n` + ) + if (faulted) { + response.writeHead(500, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ error: { code: 500, message: 'Compute error', type: 'server_error' } }) + ) + return + } + const text = question.startsWith('GPU check') + ? `Answered on the ${cpuOnly ? 'CPU' : 'GPU'}.` + : 'Okay.' + if (payload.stream === false) { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [ + { index: 0, finish_reason: 'stop', message: { role: 'assistant', content: text } } + ] + }) + ) + return + } + response.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }) + response.write(delta({ content: text })) + response.write( + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: 'stop' }] })}\n\n` + ) + response.write('data: [DONE]\n\n') + response.end() + }) +}) + +server.listen(port, '127.0.0.1') diff --git a/e2e/fixtures/unified-retrieval-llama-server.mjs b/e2e/fixtures/unified-retrieval-llama-server.mjs new file mode 100644 index 000000000..3063ff3f9 --- /dev/null +++ b/e2e/fixtures/unified-retrieval-llama-server.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node + +/* eslint-disable @typescript-eslint/explicit-function-return-type -- executable JavaScript fixture */ + +// The unified retrieval journey's only model fake: the native llama-server HTTP boundary. The +// rendered app, chat storage, the search index (FTS + trigram), the record search service, the +// search_memory tool, citation numbering, the date line in the system prompt, IPC and the chat UI +// remain production code. Like a small local model, this boundary answers only from what reached +// it: a fact it repeats must have come back through the real search_memory result, and the date it +// states must have come from the real system prompt. + +import fs from 'node:fs' +import http from 'node:http' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const args = process.argv.slice(2) +const portFlag = Math.max(args.indexOf('--port'), args.indexOf('-p')) +const port = portFlag >= 0 ? Number(args[portFlag + 1]) : 8439 + +// What reached the model, one line per request, beside this executable: the spec prints it when a +// journey fails, so a wrong answer shows what retrieval actually returned. +const LOG = path.join(path.dirname(fileURLToPath(import.meta.url)), 'requests.jsonl') + +const delta = (payload) => `data: ${JSON.stringify({ choices: [{ delta: payload }] })}\n\n` + +function textOf(content) { + if (typeof content === 'string') return content + if (Array.isArray(content)) return content.map((part) => part?.text ?? '').join('') + return '' +} + +function lastUser(messages) { + const user = [...messages].reverse().find((message) => message.role === 'user') + return textOf(user?.content) +} + +function replyFor(payload) { + const messages = payload.messages ?? [] + const question = lastUser(messages) + const asked = question.toLowerCase() + + // "What is today's date?": read it from the system prompt's date line, never from a clock here. + // (Only the user's own question: the app's follow-up suggestions quote it after "The user asked:".) + if (asked.startsWith("what is today's date")) { + const system = messages + .filter((message) => message.role === 'system') + .map((message) => textOf(message.content)) + .join('\n') + const date = /Current date: ([^(]+?) \(/.exec(system)?.[1] + return date ? `Today is ${date}.` : 'I do not know the date.' + } + + // A memory question: search first, then answer only from the tool result. + if (asked.startsWith('search my memory')) { + const offered = (payload.tools ?? []).some((tool) => tool?.function?.name === 'search_memory') + const userIndex = messages.map((message) => message.role).lastIndexOf('user') + const results = messages + .slice(userIndex + 1) + .filter((message) => message.role === 'tool') + .map((message) => textOf(message.content)) + .join('\n') + if (!offered) return 'search_memory was not offered.' + if (!results) { + return `${JSON.stringify({ + name: 'search_memory', + arguments: { query: 'Kunal RBI filing promise' } + })}` + } + const source = /]*\bat="([^"]+)"[^>]*>([^<]*RBI filing[^<]*)<\/source>/.exec( + results + ) + if (!source) return 'I found nothing about that in your memory.' + return `You promised Kunal the RBI filing by Friday [S${source[1]}]. Noted ${source[2].slice(0, 10)}.` + } + + // Anything else: acknowledge this exact message, so the journey can wait for its own reply. + return `Noted (${[...question].length}).` +} + +const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) + return + } + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'unified-retrieval-model' }] })) + return + } + if (request.method !== 'POST' || !String(request.url).includes('/chat/completions')) { + response.writeHead(404) + response.end() + return + } + let body = '' + request.on('data', (chunk) => { + body += chunk + }) + request.on('end', () => { + let payload = {} + try { + payload = JSON.parse(body) + } catch { + /* answer the default */ + } + const text = replyFor(payload) + const messages = payload.messages ?? [] + fs.appendFileSync( + LOG, + `${JSON.stringify({ + question: lastUser(messages), + tools: (payload.tools ?? []).map((tool) => tool?.function?.name), + toolResults: messages.filter((m) => m.role === 'tool').map((m) => textOf(m.content)), + reply: text + })}\n` + ) + if (payload.stream === false) { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: text } }] + }) + ) + return + } + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }) + for (const piece of text.match(/.{1,24}/gsu) ?? []) response.write(delta({ content: piece })) + response.write( + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: 'stop' }] })}\n\n` + ) + response.write('data: [DONE]\n\n') + response.end() + }) +}) + +server.listen(port, '127.0.0.1', () => { + console.log(`unified-retrieval fake llama-server listening on ${port}`) +}) diff --git a/e2e/gpu-fault-recovery.spec.ts b/e2e/gpu-fault-recovery.spec.ts new file mode 100644 index 000000000..f35443a94 --- /dev/null +++ b/e2e/gpu-fault-recovery.spec.ts @@ -0,0 +1,135 @@ +/** + * GPU fault recovery: a GPU that stops responding never breaks a chat or changes the model. + * + * The rendered app, the HTTP client's fault classifier, the shared chat fallback policy, the + * same-model restart, the backend ladder, the CPU-only launch flags (`--device none`), the runtime + * status and the Settings row are production code. One fake stands at the native boundary: a + * llama-server that reports where it placed the model and answers "500 Compute error" for as many + * questions as its fault plan lists (e2e/fixtures/gpu-fault-llama-server.mjs). + * + * Proves, from what the user sees: + * - one fault: the same model restarts and the answer arrives, with no notice; + * - two faults: the same model answers on the CPU, the chat says so once, Settings shows CPU, + * and "Try the GPU again" puts it back on the GPU. + */ +import { expect, test, type ElectronApplication, type Page } from '@playwright/test' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { completeOnboarding } from './helpers/onboarding' +import { launchOffGrid, targetIsPackaged } from './helpers/launch' + +const CPU_NOTICE = 'Chat moved to the CPU. The GPU stopped responding. Replies will be slower.' + +let app: ElectronApplication | null = null +let page: Page +let rootDir: string +let llamaDir: string + +function installModelBoundary(profileDir: string): string { + const modelsDir = path.join(profileDir, 'models') + const binDir = path.join(profileDir, 'bin') + llamaDir = path.join(binDir, 'llama') + fs.mkdirSync(modelsDir, { recursive: true }) + fs.mkdirSync(llamaDir, { recursive: true }) + const model = Buffer.alloc(2_048) + model.write('GGUF') + fs.writeFileSync(path.join(modelsDir, 'gpu-fault.gguf'), model) + fs.writeFileSync( + path.join(modelsDir, 'active-model.json'), + JSON.stringify({ id: 'gpu-fault-model', primary: 'gpu-fault.gguf', mmproj: null }) + ) + const executable = path.join(llamaDir, 'llama-server') + fs.copyFileSync( + path.join(process.cwd(), 'e2e', 'fixtures', 'gpu-fault-llama-server.mjs'), + executable + ) + fs.chmodSync(executable, 0o755) + return binDir +} + +function planFaults(remaining: number): void { + fs.writeFileSync(path.join(llamaDir, 'faults.json'), JSON.stringify({ remaining })) +} + +async function ask(text: string, reply: string): Promise { + const composer = page.getByRole('textbox', { name: 'Message', exact: true }) + await composer.fill(text) + await composer.press('Enter') + await expect(page.getByText(text, { exact: true }).last()).toBeVisible() + await expect(page.getByText(reply, { exact: true }).last()).toBeVisible({ timeout: 90_000 }) +} + +/** The chat engine's row in Settings → Text: its "Now:" line. */ +async function openChatBackendRow(): Promise> { + await page.getByTitle('Active models').click() + await page.getByRole('button', { name: 'Settings', exact: true }).last().click() + const row = page.getByText(/^Now: /).first() + await expect(row).toBeVisible() + return row +} + +test.describe('GPU fault recovery', () => { + test.skip(targetIsPackaged(), 'This native-boundary fixture is for the source-built E2E target.') + test.skip(process.platform !== 'darwin', 'The fixture stands in for the macOS Metal engine.') + test.describe.configure({ mode: 'serial', timeout: 240_000 }) + + test.beforeEach(async () => { + rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-gpu-fault-')) + const profileDir = path.join(rootDir, 'profile') + const binDir = installModelBoundary(profileDir) + planFaults(0) + app = await launchOffGrid({ + env: { + ...process.env, + OFFGRID_USER_DATA: profileDir, + OFFGRID_BIN_DIR: binDir, + OFFGRID_EMBEDDINGS_OFFLINE: '1', + NODE_ENV: 'production', + NO_PROXY: '127.0.0.1,localhost' + } + }) + page = await app.firstWindow() + await page.waitForLoadState('domcontentloaded') + await completeOnboarding(page) + await page.getByRole('button', { name: 'Chat', exact: true }).click() + await expect(page.getByRole('textbox', { name: 'Message', exact: true })).toBeVisible() + const captureDismiss = page.getByRole('button', { name: 'Dismiss', exact: true }) + if (await captureDismiss.isVisible().catch(() => false)) await captureDismiss.click() + }) + + // eslint-disable-next-line no-empty-pattern + test.afterEach(async ({}, testInfo) => { + const launches = path.join(llamaDir, 'launches.jsonl') + if (testInfo.status !== testInfo.expectedStatus && fs.existsSync(launches)) { + console.log(fs.readFileSync(launches, 'utf8')) + } + const running = app + app = null + if (running) await running.close() + if (path.basename(rootDir).startsWith('offgrid-gpu-fault-')) { + fs.rmSync(rootDir, { recursive: true, force: true }) + } + }) + + test('one GPU fault restarts the same model and the answer still arrives', async () => { + await ask('GPU check: hello', 'Answered on the GPU.') + planFaults(1) + await ask('GPU check: once more', 'Answered on the GPU.') + await expect(page.getByText(CPU_NOTICE)).toHaveCount(0) + }) + + test('a second fault moves the same model to the CPU, says so once, and the GPU comes back on request', async () => { + await ask('GPU check: hello', 'Answered on the GPU.') + planFaults(2) + await ask('GPU check: keep going', 'Answered on the CPU.') + // The notice sits above the answer (the sidebar preview may repeat its text). + await expect(page.getByText(CPU_NOTICE).first()).toBeVisible() + await expect(page.getByText('Answered on the CPU.', { exact: true }).last()).toBeVisible() + + const row = await openChatBackendRow() + await expect(row).toContainText('Now: CPU') + await page.getByRole('button', { name: 'Try the GPU again' }).first().click() + await expect(row).toContainText('Metal · all layers on GPU', { timeout: 60_000 }) + }) +}) diff --git a/e2e/unified-retrieval.spec.ts b/e2e/unified-retrieval.spec.ts new file mode 100644 index 000000000..5f9ecc7d2 --- /dev/null +++ b/e2e/unified-retrieval.spec.ts @@ -0,0 +1,188 @@ +/** + * Unified retrieval: one search engine for memory, chats and the search palette. + * + * The rendered app, chat storage, the search index (unicode61 FTS with the trigram twin for + * scripts without spaces), the record search service, the search_memory tool, citation + * numbering, the date line at the end of the system prompt, IPC and the chat UI are production + * code. One fake stands at the true boundary: a scripted llama-server that repeats only what + * reached it (e2e/fixtures/unified-retrieval-llama-server.mjs). Embeddings stay offline, so this + * proves the keyword and trigram paths a fresh install has before its embedder is ready. + * + * Proves, on a fresh profile, from what the user sees: + * - a fact told in one chat is found from a new chat, cited as [S#], dated the day it was said, + * and the citation opens the chat it came from; + * - the chat knows today's date (it comes from the prompt's date line, not the model); + * - the search palette finds chat text by a typed prefix, a Hindi word, Japanese text (no + * spaces between words) and the time word "today". + */ +import { expect, test, type ElectronApplication, type Page } from '@playwright/test' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { completeOnboarding } from './helpers/onboarding' +import { launchOffGrid, targetIsPackaged } from './helpers/launch' + +const PROMISE = 'Note for later: I promised Kunal the RBI filing by Friday.' +const ENGLISH = 'Quarterly compliance review with the audit team.' +const HINDI = 'कल की बैठक में बजट पर चर्चा हुई।' +const JAPANESE = '来週の会議は東京で行います。' + +let app: ElectronApplication | null = null +let page: Page +let rootDir: string +let modelLog: string + +function installModelBoundary(profileDir: string): string { + const modelsDir = path.join(profileDir, 'models') + const binDir = path.join(profileDir, 'bin') + const llamaDir = path.join(binDir, 'llama') + fs.mkdirSync(modelsDir, { recursive: true }) + fs.mkdirSync(llamaDir, { recursive: true }) + const model = Buffer.alloc(2_048) + model.write('GGUF') + fs.writeFileSync(path.join(modelsDir, 'unified-retrieval.gguf'), model) + fs.writeFileSync( + path.join(modelsDir, 'active-model.json'), + JSON.stringify({ id: 'unified-retrieval-model', primary: 'unified-retrieval.gguf', mmproj: null }) + ) + const executable = path.join(llamaDir, 'llama-server') + fs.copyFileSync( + path.join(process.cwd(), 'e2e', 'fixtures', 'unified-retrieval-llama-server.mjs'), + executable + ) + fs.chmodSync(executable, 0o755) + return binDir +} + +/** The local calendar day, as the app writes it on a source ("2026-10-15"). */ +function localDay(date = new Date()): string { + const pad = (n: number): string => String(n).padStart(2, '0') + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}` +} + +async function openChat(): Promise { + await page.getByRole('button', { name: 'Chat', exact: true }).click() + await expect(page.getByRole('textbox', { name: 'Message', exact: true })).toBeVisible() + const captureDismiss = page.getByRole('button', { name: 'Dismiss', exact: true }) + if (await captureDismiss.isVisible().catch(() => false)) await captureDismiss.click() +} + +/** The scripted model's acknowledgement of exactly this message. */ +const noted = (text: string): string => `Noted (${[...text].length}).` + +async function say(text: string, reply: string | RegExp = noted(text)): Promise { + const composer = page.getByRole('textbox', { name: 'Message', exact: true }) + await composer.fill(text) + await composer.press('Enter') + await expect(page.getByText(text, { exact: true }).last()).toBeVisible() + await expect(page.getByText(reply).last()).toBeVisible({ timeout: 60_000 }) +} + +async function newChat(): Promise { + await page.getByRole('button', { name: 'New chat', exact: true }).first().click() + await expect(page.getByRole('textbox', { name: 'Message', exact: true })).toHaveValue('') +} + +test.describe('unified retrieval', () => { + test.skip(targetIsPackaged(), 'This native-boundary fixture is for the source-built E2E target.') + test.describe.configure({ mode: 'serial', timeout: 180_000 }) + + test.beforeEach(async () => { + rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-unified-retrieval-')) + const profileDir = path.join(rootDir, 'profile') + const binDir = installModelBoundary(profileDir) + modelLog = path.join(binDir, 'llama', 'requests.jsonl') + app = await launchOffGrid({ + env: { + ...process.env, + OFFGRID_USER_DATA: profileDir, + OFFGRID_BIN_DIR: binDir, + OFFGRID_EMBEDDINGS_OFFLINE: '1', + // Memory search across chats is a Pro tool. + OFFGRID_PRO: '1', + NODE_ENV: 'production', + NO_PROXY: '127.0.0.1,localhost' + } + }) + page = await app.firstWindow() + await page.waitForLoadState('domcontentloaded') + await completeOnboarding(page) + await openChat() + }) + + // Playwright requires the first callback argument to use object destructuring. + // eslint-disable-next-line no-empty-pattern + test.afterEach(async ({}, testInfo) => { + // On failure, show what reached the model: the question, the tools offered, the tool results. + if (testInfo.status !== testInfo.expectedStatus && fs.existsSync(modelLog)) { + await testInfo.attach('model-requests', { path: modelLog, contentType: 'application/json' }) + console.log(fs.readFileSync(modelLog, 'utf8')) + } + const running = app + app = null + if (running) await running.close() + if (path.basename(rootDir).startsWith('offgrid-unified-retrieval-')) { + fs.rmSync(rootDir, { recursive: true, force: true }) + } + }) + + test('a fact from another chat comes back cited, dated, and opens its source', async () => { + await say(PROMISE) + + await newChat() + await say( + 'Search my memory: what did I promise Kunal?', + /You promised Kunal the RBI filing by Friday/ + ) + + // The answer repeats only what search_memory returned, with the day it was said. + const answer = page.getByText(/You promised Kunal the RBI filing by Friday/).last() + await expect(answer).toContainText(`Noted ${localDay()}.`) + + // The [S#] tag renders as a citation button; it opens the chat the fact came from. + const citation = answer.getByRole('button', { name: /^S\d+$/ }) + await expect(citation).toBeVisible() + await citation.click() + await expect(page.getByText(PROMISE, { exact: true }).last()).toBeVisible() + }) + + test("the chat states today's date from the prompt's date line", async () => { + // Built from parts: Node and the app's Chromium differ on the comma after the weekday. + const now = new Date() + const part = (options: Intl.DateTimeFormatOptions): string => + now.toLocaleDateString('en-GB', options) + const today = new RegExp( + `^Today is ${part({ weekday: 'long' })},? ${now.getDate()} ${part({ month: 'long' })} ${now.getFullYear()}\\.$` + ) + await say("What is today's date?", today) + }) + + test('the search palette finds chat text by prefix, Hindi, Japanese and "today"', async () => { + await say(ENGLISH) + await say(HINDI) + await say(JAPANESE) + + await page.keyboard.press('Meta+K') + const dialog = page.getByRole('dialog', { name: 'Search Off Grid AI' }) + await expect(dialog).toBeVisible() + const search = dialog.getByPlaceholder(/^Search everything/) + + // A word still being typed matches as a prefix. + await search.fill('quarterl') + await expect(dialog.getByText(/Quarterly compliance review/).first()).toBeVisible() + + // Hindi: whole words, vowel signs kept (the default tokenizer splits them apart). + await search.fill('बजट') + await expect(dialog.getByText(/बजट पर चर्चा/).first()).toBeVisible() + + // Japanese has no spaces between words: the trigram index finds text inside a run. + await search.fill('東京で行') + await expect(dialog.getByText(/会議は東京で行います/).first()).toBeVisible() + + // A time word alone lists what happened in that window. + await search.fill('today') + await expect( + dialog.getByText(/Quarterly compliance review|बजट पर चर्चा|会議は東京で行います/).first() + ).toBeVisible() + }) +}) diff --git a/electron-builder.yml b/electron-builder.yml index 5dea618b3..a889dbc52 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -60,6 +60,7 @@ win: - '!llama-cuda/**' - '!llama-prism-cuda/**' - '!cuda-runtime/**' + - '!onnx-cuda/**' - '!sd-cuda/**' - '!whisper/**' - '!kev-runtime/**' @@ -109,6 +110,9 @@ mac: extraResources: - from: resources/bin to: bin + filter: + - '**/*' + - '!onnx-cuda/**' - from: ../executorch-speech/native/bin/executorch-speech to: bin/executorch-speech - from: ../executorch-speech/generated/default-assets @@ -120,7 +124,7 @@ mac: # Let electron-builder's signing stand. dmg: sign: true - artifactName: ${name}-${version}.${ext} + artifactName: ${name}-${version}-${arch}.${ext} # The app is currently ~3.3 GB before compression. macOS 26 adds an EFI/GPT # partition to large HFS images, so dmgbuild's computed size can leave less usable # space than the bundle needs. Allocate deterministic headroom, then shrink after @@ -142,6 +146,7 @@ linux: - '!llama-cuda/**' - '!llama-prism-cuda/**' - '!cuda-runtime/**' + - '!onnx-cuda/**' - '!sd-cuda/**' - '!whisper-cuda/**' - '!kev-runtime/**' @@ -168,6 +173,7 @@ deb: - libsecret-1-0 - libasound2t64 - libgbm1 + - libvulkan1 - xdotool - x11-utils - wtype @@ -186,6 +192,8 @@ deb: - xdg-desktop-portal - grim - kde-spectacle + recommends: + - mesa-vulkan-drivers # Electron fuses (flipped at package time). ASAR integrity makes a tampered app.asar # refuse to load — the local-first anti-tamper lever (we don't phone home, so this is # the bar against cracking the offline license check). Only the integrity fuses are diff --git a/electron.vite.config.ts b/electron.vite.config.ts index ab9a1a8c5..de349e99b 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -126,12 +126,6 @@ export default defineConfig({ // embeddings.ts can spawn it by path. It must not be folded into the main chunk: the point // is that native ONNX inference runs off the thread that owns the window. rollupOptions: { - // Rollup hoists external require() calls above source imports. The CUDA - // pack must be selected before its first ONNX Runtime import, including - // in the embedding and speech workers. - output: { - banner: readFileSync(resolve('scripts/windows-onnx-cuda-preload.cjs'), 'utf8') - }, input: { index: resolve('src/main/index.ts'), 'embeddings-worker': resolve('src/main/embeddings-worker.ts'), diff --git a/electron/accessibility/main.swift b/electron/accessibility/main.swift deleted file mode 100644 index e7f4dc59a..000000000 --- a/electron/accessibility/main.swift +++ /dev/null @@ -1,111 +0,0 @@ -import Cocoa -import ApplicationServices - -// Model for JSON Output -struct WindowContext: Codable { - let appName: String - let title: String - let selectedText: String - let content: String - let isTrusted: Bool -} - -func getAttribute(element: AXUIElement, attribute: String) -> CFTypeRef? { - var value: CFTypeRef? - let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) - if result == .success { - return value - } - return nil -} - -func getFocusedElement() -> WindowContext? { - let systemWide = AXUIElementCreateSystemWide() - - // Check trust - if !AXIsProcessTrusted() { - return WindowContext(appName: "Unknown", title: "Permission Denied", selectedText: "", content: "", isTrusted: false) - } - // print("DEBUG: Process trusted") - - // Get focused app - var appElement: AXUIElement! - var appName = "Unknown" - - // Try NSWorkspace first (more robust) - if let frontApp = NSWorkspace.shared.frontmostApplication { - appName = frontApp.localizedName ?? "Unknown" - appElement = AXUIElementCreateApplication(frontApp.processIdentifier) - } else { - // Fallback to AXSystemWide - guard let focusedApp = getAttribute(element: systemWide, attribute: kAXFocusedApplicationAttribute) else { - return nil - } - appElement = (focusedApp as! AXUIElement) - } - - // Get focused UI Element directly from the app element (more reliable than system-wide) - guard let focusedElementRef = getAttribute(element: appElement, attribute: kAXFocusedUIElementAttribute) else { - - // Fallback: Try to get the focused window if specific element fails - if let windowRef = getAttribute(element: appElement, attribute: kAXFocusedWindowAttribute) { - // We can at least get the title from the window - let window = windowRef as! AXUIElement - var windowTitle = "" - if let title = getAttribute(element: window, attribute: kAXTitleAttribute) as? String { - windowTitle = title - } - // Return just the window info if we can't get the element - return WindowContext(appName: appName, title: windowTitle, selectedText: "", content: "", isTrusted: true) - } - - return nil - } - let focusedElement = focusedElementRef as! AXUIElement - - // Try to get Value (content) or Selected Text - var selectedText = "" - var content = "" - - if let selected = getAttribute(element: focusedElement, attribute: kAXSelectedTextAttribute) as? String { - selectedText = selected - } - - if let val = getAttribute(element: focusedElement, attribute: kAXValueAttribute) as? String { - content = val - } - - // Get Window Title - var windowTitle = "" - if let window = getAttribute(element: focusedElement, attribute: kAXWindowAttribute) { - let windowElem = window as! AXUIElement - if let title = getAttribute(element: windowElem, attribute: kAXTitleAttribute) as? String { - windowTitle = title - } - } - // print("DEBUG: Found context for \(appName)") - - return WindowContext(appName: appName, title: windowTitle, selectedText: selectedText, content: content, isTrusted: true) -} - -// Main Loop -setbuf(stdout, nil) // Unbuffer stdout - -let timer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: true) { _ in - if let context = getFocusedElement() { - // Relaxed condition: Print if we have at least an app name - if !context.appName.isEmpty { - let encoder = JSONEncoder() - if let data = try? encoder.encode(context), let json = String(data: data, encoding: .utf8) { - print(json) - } - } - } else { - // Check permissions silently? - if !AXIsProcessTrusted() { - print("{\"isTrusted\": false}") - } - } -} - -RunLoop.main.run() diff --git a/package-lock.json b/package-lock.json index 39d5dcb96..97003da4e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,12 +12,10 @@ "dependencies": { "@base-ui/react": "1.8.0", "@dnd-kit/core": "^6.3.1", - "@dnd-kit/sortable": "^10.0.0", "@electron-toolkit/preload": "^3.0.2", "@electron-toolkit/utils": "^4.0.0", "@gutenye/ocr-node": "^1.4.9", "@huggingface/transformers": "4.3.1", - "@lancedb/lancedb": "^0.40.0", "@langchain/core": "^1.2.9", "@langchain/langgraph": "^1.4.12", "@modelcontextprotocol/sdk": "^1.29.0", @@ -37,14 +35,11 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/vite": "^4.1.18", "@tanstack/react-virtual": "3.14.13", - "apache-arrow": "^18.1.0", "async-mutex": "^0.5.0", "better-sqlite3": "^12.6.2", "better-sqlite3-multiple-ciphers": "^12.11.1", "bonjour-service": "^1.4.3", - "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "cmdk": "^1.1.1", "electron-updater": "^6.8.9", "get-windows": "^9.3.0", "hash-wasm": "^4.12.0", @@ -3686,172 +3681,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@lancedb/lancedb": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb/-/lancedb-0.40.0.tgz", - "integrity": "sha512-hCvk6y/gHARQ++NarvguEmX02Hm2zJAUJQgf1Sq8KznBKjI21xsK5YYzw2L0eUArDWxPjtLl4kD329ymLMqLew==", - "cpu": [ - "x64", - "arm64" - ], - "license": "Apache-2.0", - "os": [ - "darwin", - "linux", - "win32" - ], - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "reflect-metadata": "^0.2.2" - }, - "engines": { - "node": ">= 22" - }, - "optionalDependencies": { - "@huggingface/transformers": "3.0.2", - "@lancedb/lancedb-darwin-arm64": "0.40.0", - "@lancedb/lancedb-linux-arm64-gnu": "0.40.0", - "@lancedb/lancedb-linux-arm64-musl": "0.40.0", - "@lancedb/lancedb-linux-x64-gnu": "0.40.0", - "@lancedb/lancedb-linux-x64-musl": "0.40.0", - "@lancedb/lancedb-win32-arm64-msvc": "0.40.0", - "@lancedb/lancedb-win32-x64-msvc": "0.40.0", - "openai": "4.29.2" - }, - "peerDependencies": { - "@types/node": ">=22", - "apache-arrow": ">=15.0.0 <=18.1.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@lancedb/lancedb-darwin-arm64": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.40.0.tgz", - "integrity": "sha512-m/DqH8BUHPXTHQFpvhBfeAYZnrI9aJrz5lVVaQNefhcJO9ULIDk36isKjh6sM6owr29SemXjprbJFfcEskxylA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-arm64-gnu": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-gnu/-/lancedb-linux-arm64-gnu-0.40.0.tgz", - "integrity": "sha512-O9CIe1RNqbHZujn4c8DMTsXJQ+eHTWn5ilTilRbm9ENsnAk02SgMYr2AH+hargTAjsXOwrWKbUogRbRi0VridA==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-arm64-musl": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-musl/-/lancedb-linux-arm64-musl-0.40.0.tgz", - "integrity": "sha512-ttaeDklDfIkpK0fzaz1+SwH01jtaGU15jgPaPn9Dzv9AfpXoYlT7F3FDsa95BH23OSOath2kx6LougodU76lew==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-x64-gnu": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.40.0.tgz", - "integrity": "sha512-Y7lQkz7uuQvKsZla85YNdhz6oPaY3fpzNLeNuNmHb8cEnLH8ArlbpOL/Bbdlp5sBmv6rE4lDYArn6IeBX8OAJQ==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-x64-musl": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-musl/-/lancedb-linux-x64-musl-0.40.0.tgz", - "integrity": "sha512-Ohj8J+iZ9QHXn5DbVacHlGR7JBNyqXdREXJDuKK7rB8zHXFMi3/uRGQQxy6XPBRFeuE0/wnIcgOLkiN1as1AwA==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-win32-arm64-msvc": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.40.0.tgz", - "integrity": "sha512-8gzFTWB4KoTkuEReqiYy4TmT8uQEOMzMoKJlxQhayjB5dsAGk99VucbgL4gw5a2buazmD6QoW1LkMYGngXpoNQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-win32-x64-msvc": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.40.0.tgz", - "integrity": "sha512-DZxho/MhnGJCQc98CN3yY8qWlDCATLTjGUNqD58Fx1sNMctmkbdAf4F1fTRahz0aojfnM9aoUNaSXdY0qaPhbQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 18" - } - }, "node_modules/@langchain/core": { "version": "1.2.17", "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.17.tgz", @@ -8054,15 +7883,6 @@ "react": "^18.0.0 || ^19.0.0" } }, - "node_modules/@swc/helpers": { - "version": "0.5.23", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", - "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/@szmarczak/http-timer": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", @@ -8637,18 +8457,6 @@ "assertion-error": "^2.0.1" } }, - "node_modules/@types/command-line-args": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", - "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", - "license": "MIT" - }, - "node_modules/@types/command-line-usage": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", - "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", - "license": "MIT" - }, "node_modules/@types/d3": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", @@ -9016,17 +8824,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "optional": true, - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, "node_modules/@types/react": { "version": "19.3.0", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", @@ -9769,19 +9566,6 @@ "node": ">= 14" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -9861,6 +9645,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "devOptional": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -9879,35 +9664,6 @@ "license": "MIT", "optional": true }, - "node_modules/apache-arrow": { - "version": "18.1.0", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-18.1.0.tgz", - "integrity": "sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.11", - "@types/command-line-args": "^5.2.3", - "@types/command-line-usage": "^5.0.4", - "@types/node": "^20.13.0", - "command-line-args": "^5.2.1", - "command-line-usage": "^7.0.1", - "flatbuffers": "^24.3.25", - "json-bignum": "^0.0.3", - "tslib": "^2.6.2" - }, - "bin": { - "arrow2csv": "bin/arrow2csv.js" - } - }, - "node_modules/apache-arrow/node_modules/@types/node": { - "version": "20.19.43", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", - "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, "node_modules/app-builder-lib": { "version": "26.16.1", "resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-26.16.1.tgz", @@ -10244,15 +10000,6 @@ "dequal": "^2.0.3" } }, - "node_modules/array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -10565,12 +10312,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==", - "optional": true - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -11224,6 +10965,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -11236,21 +10978,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chalk-template": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", - "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", - "license": "MIT", - "dependencies": { - "chalk": "^4.1.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/chalk-template?sponsor=1" - } - }, "node_modules/character-entities": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", @@ -11291,16 +11018,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": "*" - } - }, "node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -11442,6 +11159,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "devOptional": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -11454,6 +11172,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "devOptional": true, "license": "MIT" }, "node_modules/color-support": { @@ -11489,54 +11208,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/command-line-args": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", - "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", - "license": "MIT", - "dependencies": { - "array-back": "^3.1.0", - "find-replace": "^3.0.0", - "lodash.camelcase": "^4.3.0", - "typical": "^4.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/command-line-usage": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.4.tgz", - "integrity": "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==", - "license": "MIT", - "dependencies": { - "array-back": "^6.2.2", - "chalk-template": "^0.4.0", - "table-layout": "^4.1.1", - "typical": "^7.3.0" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/command-line-usage/node_modules/array-back": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", - "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, - "node_modules/command-line-usage/node_modules/typical": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", - "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, "node_modules/commander": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", @@ -11679,16 +11350,6 @@ "node": ">= 8" } }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": "*" - } - }, "node_modules/css-tree": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", @@ -12591,17 +12252,6 @@ "node": ">=0.3.1" } }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "optional": true, - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, "node_modules/dingbat-to-unicode": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.2.tgz", @@ -14372,18 +14022,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/find-replace": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", - "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", - "license": "MIT", - "dependencies": { - "array-back": "^3.0.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -14415,12 +14053,6 @@ "node": ">=16" } }, - "node_modules/flatbuffers": { - "version": "24.12.23", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.12.23.tgz", - "integrity": "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==", - "license": "Apache-2.0" - }, "node_modules/flatted": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", @@ -14512,13 +14144,6 @@ "node": ">= 6" } }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT", - "optional": true - }, "node_modules/form-data/node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -14559,30 +14184,6 @@ "node": ">=18.14.0" } }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/formdata-node/node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 14" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -15295,6 +14896,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -15845,16 +15447,6 @@ "node": ">= 14" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -16162,13 +15754,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT", - "optional": true - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -16963,14 +16548,6 @@ "node": ">=6" } }, - "node_modules/json-bignum": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", - "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", - "engines": { - "node": ">=0.8" - } - }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -17587,12 +17164,6 @@ "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", "license": "MIT" }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "license": "MIT" - }, "node_modules/lodash.escaperegexp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", @@ -17880,18 +17451,6 @@ "node": ">= 0.4" } }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, "node_modules/mdast-util-find-and-replace": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.3.tgz", @@ -19498,27 +19057,6 @@ "node": ">=10" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "optional": true, - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-exports-info": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.2.tgz", @@ -20053,44 +19591,6 @@ "integrity": "sha512-gBuF6U32YErKIAt+yD7DeGBpjRxhJ1uJwako6ygjokSZULDU6Pd+KWComX8Tumk1hYVHRxjPj7mdMnD0ryAPOw==", "license": "MIT" }, - "node_modules/openai": { - "version": "4.29.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", - "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" - }, - "bin": { - "openai": "bin/cli" - } - }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "optional": true, - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/openai/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT", - "optional": true - }, "node_modules/option": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", @@ -21827,12 +21327,6 @@ "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, - "node_modules/reflect-metadata": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", - "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", - "license": "Apache-2.0" - }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -23419,6 +22913,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -23470,28 +22965,6 @@ "integrity": "sha512-wieBHXygIm7OyQOu5hQlkk62/WyCFYGlWg7L6/ZCUZwx0o398Zkn4pVmMyfYhfMG8kGrj/Krt8eIk6UKC6VzwA==", "license": "MIT" }, - "node_modules/table-layout": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", - "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", - "license": "MIT", - "dependencies": { - "array-back": "^6.2.2", - "wordwrapjs": "^5.1.0" - }, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/table-layout/node_modules/array-back": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", - "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, "node_modules/tailwind-merge": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.7.0.tgz", @@ -24151,15 +23624,6 @@ "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/typical": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", - "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/unbash": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/unbash/-/unbash-5.1.0.tgz", @@ -25332,16 +24796,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 8" - } - }, "node_modules/webcrypto-core": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/webcrypto-core/-/webcrypto-core-1.9.2.tgz", @@ -25545,15 +24999,6 @@ "node": ">=0.10.0" } }, - "node_modules/wordwrapjs": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", - "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 57033730e..8e9a073db 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "dev:linked": "OPERATOR_UI_SRC=../ui-library node scripts/dev.mjs", "dev:inner": "node scripts/stage-native.mjs && electron-vite dev", "gateway": "node scripts/stage-native.mjs && OFFGRID_SERVER_ONLY=1 electron-vite dev", - "build": "node scripts/stage-native.mjs && npm run typecheck && electron-vite build && node scripts/check-windows-onnx-preload.mjs", + "build": "node scripts/stage-native.mjs && npm run typecheck && electron-vite build", "postinstall": "node scripts/patch-windows-onnx-cuda.mjs && electron-builder install-app-deps", "prepare:speech-defaults": "node scripts/prepare-default-speech.mjs", "build:unpack": "npm run build && npm run prepare:speech-defaults && electron-builder --dir", @@ -57,11 +57,8 @@ "knip": "knip --no-gitignore", "coverage:all": "bash scripts/coverage-all.sh" }, - "//overrides": "ONE sharp for the whole tree. sharp ships libvips as `libvips-42.dll`, and Windows resolves a DLL by NAME across the process: whichever copy loads first wins for every later binding. The embedding runtime can load sharp before our own sharp 0.35; two libvips versions in one Windows process then fail with ERR_DLOPEN_FAILED. Every file attachment failed on Windows and only there because macOS binds by path. Proven by load order: sharp alone loads; transformers-then-sharp does not. Deduping to one version is the fix, not load-order juggling. The same holds for ONNX Runtime: LanceDB 0.40 lists Transformers.js 3.0.2 (ONNX Runtime 1.19, about 320 MB) as an optional dependency for an embedding helper the app never calls, so it shares the app's one Transformers.js.", + "//overrides": "ONE sharp for the whole tree. sharp ships libvips as `libvips-42.dll`, and Windows resolves a DLL by NAME across the process: whichever copy loads first wins for every later binding. The embedding runtime can load sharp before our own sharp 0.35; two libvips versions in one Windows process then fail with ERR_DLOPEN_FAILED. Every file attachment failed on Windows and only there because macOS binds by path. Proven by load order: sharp alone loads; transformers-then-sharp does not. Deduping to one version is the fix, not load-order juggling.", "overrides": { - "@lancedb/lancedb": { - "@huggingface/transformers": "$@huggingface/transformers" - }, "kokoro-js": { "@huggingface/transformers": "$@huggingface/transformers" }, @@ -71,12 +68,10 @@ "dependencies": { "@base-ui/react": "1.8.0", "@dnd-kit/core": "^6.3.1", - "@dnd-kit/sortable": "^10.0.0", "@electron-toolkit/preload": "^3.0.2", "@electron-toolkit/utils": "^4.0.0", "@gutenye/ocr-node": "^1.4.9", "@huggingface/transformers": "4.3.1", - "@lancedb/lancedb": "^0.40.0", "@langchain/core": "^1.2.9", "@langchain/langgraph": "^1.4.12", "@modelcontextprotocol/sdk": "^1.29.0", @@ -96,14 +91,11 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/vite": "^4.1.18", "@tanstack/react-virtual": "3.14.13", - "apache-arrow": "^18.1.0", "async-mutex": "^0.5.0", "better-sqlite3": "^12.6.2", "better-sqlite3-multiple-ciphers": "^12.11.1", "bonjour-service": "^1.4.3", - "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "cmdk": "^1.1.1", "electron-updater": "^6.8.9", "get-windows": "^9.3.0", "hash-wasm": "^4.12.0", diff --git a/pro b/pro index fab65c48a..71203914e 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit fab65c48a997c8b04d9dd1a5ba67ac5d275ad539 +Subproject commit 71203914e145c4e91a292760d907a1309b0192bf diff --git a/scripts/build-llama-linux.sh b/scripts/build-llama-linux.sh index 951a6af1f..3e105bf20 100644 --- a/scripts/build-llama-linux.sh +++ b/scripts/build-llama-linux.sh @@ -113,6 +113,11 @@ cp "$SOURCE/LICENSE" "$DEST/" GOMP="$(ldconfig -p | awk '/libgomp\.so\.1 / && !found { found=$NF } END { print found }')" test -n "$GOMP" cp -L "$GOMP" "$DEST/libgomp.so.1" +if [ "$ACCELERATOR" = vulkan ]; then + VULKAN="$(ldconfig -p | awk '$1 == "libvulkan.so.1" && /x86-64/ && !found { found=$NF } END { print found }')" + test -n "$VULKAN" + install -m 755 "$(readlink -f "$VULKAN")" "$DEST/libvulkan.so.1" +fi chmod +x "$DEST/llama-server" file "$DEST/llama-server" | grep -q 'ELF 64-bit.*x86-64' diff --git a/scripts/build-sd-mac.sh b/scripts/build-sd-mac.sh new file mode 100644 index 000000000..da2e427a6 --- /dev/null +++ b/scripts/build-sd-mac.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Preserve master-889-c678dfe and the existing universal three-file payload. +REF=c678dfe704a2230342376b46add9c8ca736a653d +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="${OFFGRID_BUILD_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}" +DEST="$ROOT/resources/bin/sd" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +[ "$(uname -s)" = Darwin ] || { echo '[build-sd-mac] macOS is required' >&2; exit 1; } + +git clone --no-checkout https://github.com/leejet/stable-diffusion.cpp "$WORK/src" +git -C "$WORK/src" checkout --detach "$REF" +git -C "$WORK/src" submodule update --init --recursive +test "$(git -C "$WORK/src" rev-parse HEAD)" = "$REF" +# SD_BUILD_SHARED_GGML_LIB=OFF keeps GGML inside the one existing SD dylib. +cmake -S "$WORK/src" -B "$WORK/build" -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 -DCMAKE_OSX_ARCHITECTURES='arm64;x86_64' \ + -DSD_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_NATIVE=OFF \ + -DGGML_OPENMP=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_SHARED_GGML_LIB=OFF \ + -DSD_BUILD_EXAMPLES=ON -DCMAKE_DISABLE_FIND_PACKAGE_OpenSSL=ON \ + -DCMAKE_DISABLE_FIND_PACKAGE_CURL=ON +cmake --build "$WORK/build" --config Release -j"$(sysctl -n hw.ncpu)" \ + --target sd-cli sd-server stable-diffusion + +mkdir -p "$WORK/stage" +for name in sd-cli sd-server libstable-diffusion.dylib; do + source="$(find "$WORK/build" -name "$name" -type f -print -quit)" + test -n "$source" + cp "$source" "$WORK/stage/$name" + lipo "$WORK/stage/$name" -verify_arch arm64 x86_64 +done +for file in "$WORK/stage/"*; do + while IFS= read -r rpath; do + install_name_tool -delete_rpath "$rpath" "$file" + done < <(otool -l "$file" | awk '/cmd LC_RPATH/{found=1; next} found && $1 == "path" {print $2; found=0}' | sort -u) + install_name_tool -add_rpath @loader_path "$file" + while IFS= read -r dep; do + case "$dep" in + /System/Library/*|/usr/lib/*) ;; + *) + name="$(basename "$dep")" + if [ ! -f "$WORK/stage/$name" ]; then + echo "[build-sd-mac] unbundled dependency: $file -> $dep" >&2 + exit 1 + fi + install_name_tool -change "$dep" "@loader_path/$name" "$file" + ;; + esac + done < <(otool -L "$file" | awk '/compatibility version/ {print $1}' | sort -u) +done +install_name_tool -id @loader_path/libstable-diffusion.dylib "$WORK/stage/libstable-diffusion.dylib" +node "$SCRIPT_DIR/check-macos-minos.mjs" 13.0 "$WORK/stage" +rm -rf "$DEST" +mkdir -p "$DEST" +cp "$WORK/stage/"* "$DEST/" +node "$SCRIPT_DIR/check-macos-minos.mjs" 13.0 "$DEST" +echo '[build-sd-mac] staged universal SD engine for macOS 13+' diff --git a/scripts/build-voice-linux.sh b/scripts/build-voice-linux.sh index d4d483921..c79f142d4 100644 --- a/scripts/build-voice-linux.sh +++ b/scripts/build-voice-linux.sh @@ -39,7 +39,14 @@ if [ "${OFFGRID_BUILD_VOICE_FROM_SOURCE:-0}" != 1 ] && tar -xzf "$VOICE_WORK/$VOICE_ARCHIVE" -C "$ROOT/build/linux-bin" --no-same-owner OFFGRID_REUSE_STAGED_VOICE=1 fi +stage_vulkan_loader() { + local source + source="$(ldconfig -p | awk '$1 == "libvulkan.so.1" && /x86-64/ && !found { found=$NF } END { print found }')" + test -n "$source" + install -m 755 "$(readlink -f "$source")" "$ROOT/build/linux-bin/whisper/libvulkan.so.1" +} if [ "${OFFGRID_REUSE_STAGED_VOICE:-0}" = 1 ]; then + stage_vulkan_loader for binary in "$ROOT/build/linux-bin/whisper-cuda/whisper-cli" \ "$ROOT/build/linux-bin/whisper/whisper-cli" \ "$ROOT/build/linux-bin/whisper-cpu/whisper-cli" \ @@ -151,6 +158,7 @@ mkdir -p "$BIN/whisper-cuda" "$BIN/whisper" "$BIN/whisper-cpu" "$BIN/licenses" install -m 755 "$WHISPER_CUDA_BIN" "$BIN/whisper-cuda/whisper-cli" install -m 755 "$WHISPER_VULKAN_BIN" "$BIN/whisper/whisper-cli" install -m 755 "$WHISPER_CPU_BIN" "$BIN/whisper-cpu/whisper-cli" +stage_vulkan_loader install -m 644 "$WORK/whisper/LICENSE" "$BIN/whisper/LICENSE" install -m 755 "$WORK/$FFMPEG_DIR/bin/ffmpeg" "$BIN/ffmpeg" install -m 644 "$WORK/$FFMPEG_DIR/LICENSE.txt" "$BIN/licenses/ffmpeg.txt" diff --git a/scripts/build-whisper-vulkan-win.ps1 b/scripts/build-whisper-vulkan-win.ps1 new file mode 100644 index 000000000..be48b40d0 --- /dev/null +++ b/scripts/build-whisper-vulkan-win.ps1 @@ -0,0 +1,35 @@ +# Match the pinned Windows whisper download in fetch-win-binaries.ps1. +$ErrorActionPreference = 'Stop' +$WhisperRef = 'b5130' +$WhisperCommit = '927cfce34f31707e17f2bff35c349632fb9e2c3a' +if (-not $env:VULKAN_SDK -or -not (Test-Path "$env:VULKAN_SDK\Bin\glslc.exe")) { + throw 'The Vulkan SDK with glslc is required to build Windows Whisper' +} +. "$PSScriptRoot\config\windows-vc-runtime.ps1" +$dest = Join-Path $PSScriptRoot '..\resources\bin\whisper-vulkan' +$tmpBase = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { $env:TEMP } +$work = Join-Path $tmpBase ("whisper-vulkan-" + [guid]::NewGuid().ToString('N')) +try { + git clone --depth 1 --branch $WhisperRef https://github.com/ggml-org/whisper.cpp "$work\src" + if ($LASTEXITCODE -ne 0) { throw 'Whisper clone failed' } + $commit = git -C "$work\src" rev-parse HEAD + if ($LASTEXITCODE -ne 0 -or $commit -ne $WhisperCommit) { throw 'Whisper source commit mismatch' } + cmake -S "$work\src" -B "$work\build" -A x64 ` + -DGGML_VULKAN=ON -DWHISPER_BUILD_SERVER=ON -DCMAKE_BUILD_TYPE=Release ` + -DBUILD_SHARED_LIBS=ON -DGGML_NATIVE=OFF -DGGML_CUDA=OFF ` + -DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_EXAMPLES=ON + if ($LASTEXITCODE -ne 0) { throw 'Whisper CMake configuration failed' } + cmake --build "$work\build" --config Release --parallel 4 --target whisper-cli whisper-server + if ($LASTEXITCODE -ne 0) { throw 'Whisper Vulkan build failed' } + if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } + New-Item -ItemType Directory -Force -Path $dest | Out-Null + Get-ChildItem "$work\build" -Recurse -File | + Where-Object { $_.Name -in @('whisper-cli.exe', 'whisper-server.exe') -or $_.Extension -eq '.dll' } | + Copy-Item -Destination $dest -Force + Copy-VCRuntime $dest + foreach ($name in @('whisper-cli.exe', 'whisper-server.exe')) { + if (-not (Test-Path "$dest\$name")) { throw "Whisper Vulkan output missing: $name" } + } +} finally { + if (Test-Path $work) { Remove-Item -Recurse -Force $work } +} diff --git a/scripts/check-macos-minos.mjs b/scripts/check-macos-minos.mjs new file mode 100644 index 000000000..89c947c09 --- /dev/null +++ b/scripts/check-macos-minos.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { execFileSync } from 'node:child_process' + +const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const [maximum = '13.0', ...requestedRoots] = process.argv.slice(2) +if (!/^\d+(?:\.\d+){0,2}$/.test(maximum)) throw new Error('Expected a macOS version, e.g. 13.0') +const folderOverrides = new Map([ + // Core ML image generation uses macOS 14 Core ML APIs. + ['coreml-sd', '14.0'], + // The Kev MLX runtime needs macOS 14; the app hides Kev on older macOS. + ['kev-runtime', '14.0'] +]) +const roots = requestedRoots.length + ? requestedRoots.map((root) => path.resolve(root)) + : [ + path.join(repo, 'resources/bin'), + ...[path.resolve(repo, '../executorch-speech/native/bin')].filter((root) => + fs.existsSync(root) + ) + ] +const magicNumbers = new Set([ + 'feedface', + 'cefaedfe', + 'feedfacf', + 'cffaedfe', + 'cafebabe', + 'bebafeca', + 'cafebabf', + 'bfbafeca' +]) +const visited = new Set() +const failures = [] +let checked = 0 +function exceeds(version, target) { + const actual = version.split('.').map(Number) + const allowed = target.split('.').map(Number) + for (let i = 0; i < Math.max(actual.length, allowed.length); i++) { + if ((actual[i] ?? 0) !== (allowed[i] ?? 0)) return (actual[i] ?? 0) > (allowed[i] ?? 0) + } + return false +} +function walk(file, root) { + const real = fs.realpathSync(file) + if (visited.has(real)) return + visited.add(real) + const stat = fs.statSync(file) + if (stat.isDirectory()) { + for (const name of fs.readdirSync(file)) walk(path.join(file, name), root) + return + } + if (!stat.isFile()) return + const header = Buffer.alloc(4) + const fd = fs.openSync(file, 'r') + try { + fs.readSync(fd, header, 0, 4, 0) + } finally { + fs.closeSync(fd) + } + if (!magicNumbers.has(header.toString('hex'))) return + checked++ + const folder = path.relative(root, file).split(path.sep)[0] + const limit = folderOverrides.get(folder) ?? maximum + try { + // Inspect all universal slices, including legacy LC_VERSION_MIN_MACOSX commands. + const output = execFileSync('otool', ['-arch', 'all', '-l', file], { + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024 + }) + const versions = [ + ...output.matchAll( + /cmd LC_BUILD_VERSION\s[\s\S]*?\bminos\s+(\d+(?:\.\d+)*)|cmd LC_VERSION_MIN_MACOSX\s[\s\S]*?\bversion\s+(\d+(?:\.\d+)*)/g + ) + ].map((match) => match[1] ?? match[2]) + if (!versions.length) failures.push(`${file}: no macOS minimum version found`) + for (const version of new Set(versions)) { + if (exceeds(version, limit)) failures.push(`${file}: minos ${version} exceeds ${limit}`) + } + } catch (error) { + failures.push(`${file}: cannot inspect Mach-O (${error.message})`) + } +} +for (const root of roots) walk(root, root) +if (!checked) failures.push('No Mach-O files found') +if (failures.length) { + console.error(failures.join('\n')) + process.exitCode = 1 +} else { + console.log(`[macos-minos] checked ${checked} Mach-O files (default maximum ${maximum})`) +} diff --git a/scripts/check-windows-onnx-preload.mjs b/scripts/check-windows-onnx-preload.mjs deleted file mode 100644 index 7fcec5eff..000000000 --- a/scripts/check-windows-onnx-preload.mjs +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env node -// Verify the Windows pack bootstrap executes before Rollup's hoisted ONNX import. -import fs from 'node:fs' -import path from 'node:path' - -if (process.platform !== 'win32') process.exit(0) - -for (const name of ['index.js', 'embeddings-worker.js', 'tts-onnx-worker.js']) { - const file = path.join(import.meta.dirname, '..', 'out', 'main', name) - const source = fs.readFileSync(file, 'utf8') - const bootstrap = source.indexOf('process.arch === "x64"') - const manifest = source.indexOf('performance-packs.json') - const onnx = source.indexOf('require("@huggingface/transformers")') - if (bootstrap < 0 || manifest < 0 || onnx < 0 || bootstrap >= onnx || manifest >= onnx) { - throw new Error(`Windows ONNX bootstrap does not precede external import in ${name}`) - } -} -console.log('Windows ONNX bootstrap precedes external imports in all three main entries') diff --git a/scripts/config/windows-vc-runtime.ps1 b/scripts/config/windows-vc-runtime.ps1 new file mode 100644 index 000000000..4a69969dc --- /dev/null +++ b/scripts/config/windows-vc-runtime.ps1 @@ -0,0 +1,25 @@ +# App-local VC++ runtime for each native executable, including optional engines. +function Copy-VCRuntime($Destination) { + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + if (-not (Test-Path $vswhere)) { throw "VC++ redist not found: vswhere is missing at $vswhere" } + $installation = & $vswhere -latest -products '*' -property installationPath + if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'VC++ redist not found: install Visual Studio C++ build tools' } + $redistRoot = Join-Path $installation 'VC\Redist\MSVC' + if (-not (Test-Path $redistRoot)) { throw "VC++ redist folder not found: $redistRoot" } + $version = Get-ChildItem $redistRoot -Directory | + Where-Object { $_.Name -match '^\d+\.\d+\.\d+$' } | + Sort-Object { [version]$_.Name } -Descending | Select-Object -First 1 + if (-not $version) { throw "VC++ redist version folder not found: $redistRoot" } + $x64 = Join-Path $version.FullName 'x64' + if (-not (Test-Path $x64)) { throw "VC++ x64 redist folder not found: $x64" } + $folders = @(Get-ChildItem $x64 -Directory | Where-Object { $_.Name -match '^Microsoft\.VC14.*\.(CRT|OpenMP)$' }) + if ($folders.Count -lt 2) { throw "VC++ CRT/OpenMP redist folders not found: $x64" } + foreach ($name in @('MSVCP140.dll', 'MSVCP140_1.dll', 'MSVCP140_2.dll', + 'MSVCP140_ATOMIC_WAIT.dll', 'MSVCP140_CODECVT_IDS.dll', 'VCRUNTIME140.dll', + 'VCRUNTIME140_1.dll', 'CONCRT140.dll', 'VCOMP140.dll')) { + $source = $folders | ForEach-Object { Join-Path $_.FullName $name } | + Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 + if (-not $source) { throw "VC++ redist DLL missing: $name under $x64" } + Copy-Item -LiteralPath $source -Destination $Destination -Force + } +} diff --git a/scripts/fetch-win-binaries.ps1 b/scripts/fetch-win-binaries.ps1 index ce633f2f3..dd13edaf1 100644 --- a/scripts/fetch-win-binaries.ps1 +++ b/scripts/fetch-win-binaries.ps1 @@ -29,6 +29,7 @@ $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' # makes Invoke-WebRequest downloads fast +. "$PSScriptRoot\config\windows-vc-runtime.ps1" $IncludeCuda = $env:OFFGRID_INCLUDE_CUDA -ne '0' $bin = Join-Path $PSScriptRoot '..\resources\bin' @@ -172,9 +173,8 @@ try { } catch { Write-Warning "whisper.cpp CPU fetch failed: $_" } # --- stable-diffusion.cpp (image gen): CUDA, Vulkan, then CPU ------------------- -# The CUDA engine reuses bin/cuda-runtime from llama.cpp. Do not fetch the -# separate 563 MB stable-diffusion CUDA runtime archive: it contains the same -# three CUDA DLLs and previously made the installer too large to build. +# The image engine needs CUDA 12.8, while llama uses 12.4. Keep the image +# runtime beside sd-cuda so the optional pack preserves both versions. # Keep this release pinned: Qwen-Image 2.1 needs the current runtime and a moving # latest release can change the packaged DLL contract without review. $SdRef = 'master-968-8cd731e' @@ -183,6 +183,12 @@ if ($IncludeCuda) { try { $x = Expand-Asset 'leejet/stable-diffusion.cpp' 'bin-win-cuda12-x64\.zip$' $SdRef '8b6f311f22b56db02f545a674f7549a0da283f7134d5fd01c42014b81ab78af0' $dest = Copy-Runtime $x 'sd-cuda' + $runtime = Expand-Asset 'leejet/stable-diffusion.cpp' '^cudart-sd-bin-win-cu12-x64\.zip$' $SdRef 'fe20366827d357c00797eebb58244dddab7fd9a348d70090c3871004c320f38d' + foreach ($name in @('cudart64_12.dll', 'cublas64_12.dll', 'cublasLt64_12.dll')) { + $source = Get-ChildItem $runtime -Recurse -File -Filter $name | Select-Object -First 1 + if (-not $source) { throw "CUDA 12.8 DLL missing from image runtime: $name" } + Copy-Item $source.FullName -Destination $dest -Force + } $cli = Join-Path $dest 'sd-cli.exe' $sd = Join-Path $dest 'sd.exe' if (-not (Test-Path $cli) -and (Test-Path $sd)) { Copy-Item $sd $cli -Force } @@ -225,6 +231,11 @@ try { } catch { Write-Warning "ffmpeg fetch failed: $_" } Write-Host '' +# Include the root FFmpeg executable and every nested engine executable. +Get-ChildItem -Path $bin -Recurse -Filter '*.exe' -File | + Select-Object -ExpandProperty DirectoryName -Unique | + ForEach-Object { Copy-VCRuntime $_ } + Write-Host 'resources/bin now contains (win64):' Get-ChildItem -Path $bin -Recurse -Include *.exe | ForEach-Object { Write-Host " $($_.FullName.Replace($bin, '').TrimStart('\'))" } @@ -267,6 +278,9 @@ if ($IncludeCuda) { (Join-Path $bin 'whisper\cublasLt64_12.dll'), (Join-Path $bin 'whisper\nvrtc64_120_0.dll'), (Join-Path $bin 'sd-cuda\sd-cli.exe'), + (Join-Path $bin 'sd-cuda\cudart64_12.dll'), + (Join-Path $bin 'sd-cuda\cublas64_12.dll'), + (Join-Path $bin 'sd-cuda\cublasLt64_12.dll'), (Join-Path $bin 'sd-cuda\ggml-cuda.dll'))) { if (-not (Test-Path -LiteralPath $p)) { throw "REQUIRED Windows CUDA runtime missing: $p" } } diff --git a/scripts/package-performance-pack.mjs b/scripts/package-performance-pack.mjs index f42bd461e..5b5f931db 100644 --- a/scripts/package-performance-pack.mjs +++ b/scripts/package-performance-pack.mjs @@ -7,27 +7,88 @@ import { createHash } from 'node:crypto' import { spawnSync } from 'node:child_process' const [, , platform, binRoot, archive, version] = process.argv -if (!['win32', 'linux'].includes(platform) || !binRoot || !archive || !/^[a-zA-Z0-9._-]+$/.test(version ?? '')) { - throw new Error('Usage: node scripts/package-performance-pack.mjs ') +if ( + !['win32', 'linux'].includes(platform) || + !binRoot || + !archive || + !/^[a-zA-Z0-9._-]+$/.test(version ?? '') +) { + throw new Error( + 'Usage: node scripts/package-performance-pack.mjs ' + ) } -const folders = platform === 'win32' - ? ['llama-cuda', 'llama-prism-cuda', 'cuda-runtime', 'sd-cuda', 'whisper', 'kev-runtime', 'onnx-cuda'] - : ['llama-cuda', 'llama-prism-cuda', 'cuda-runtime', 'sd-cuda', 'whisper-cuda', 'kev-runtime', 'onnx-cuda'] -const required = platform === 'win32' - ? ['llama-cuda/llama-server.exe', 'llama-prism-cuda/llama-server.exe', 'cuda-runtime/cudart64_12.dll', 'sd-cuda/sd-cli.exe', 'whisper/whisper-cli.exe', 'kev-runtime/python/python.exe', 'onnx-cuda/onnxruntime_binding.node', 'onnx-cuda/onnxruntime.dll', 'onnx-cuda/onnxruntime_providers_cuda.dll', 'onnx-cuda/cublasLt64_12.dll', 'onnx-cuda/cudnn64_9.dll'] - : ['llama-cuda/llama-server', 'llama-prism-cuda/llama-server', 'cuda-runtime/libcudart.so.12', 'sd-cuda/sd-cli', 'whisper-cuda/whisper-cli', 'kev-runtime/python/bin/python3', 'onnx-cuda/libcublasLt.so.13', 'onnx-cuda/libcudnn.so.9'] +const folders = + platform === 'win32' + ? [ + 'llama-cuda', + 'llama-prism-cuda', + 'cuda-runtime', + 'sd-cuda', + 'whisper', + 'kev-runtime', + 'onnx-cuda' + ] + : [ + 'llama-cuda', + 'llama-prism-cuda', + 'cuda-runtime', + 'sd-cuda', + 'whisper-cuda', + 'kev-runtime', + 'onnx-cuda' + ] +const required = + platform === 'win32' + ? [ + 'llama-cuda/llama-server.exe', + 'llama-prism-cuda/llama-server.exe', + 'cuda-runtime/cudart64_12.dll', + 'sd-cuda/sd-cli.exe', + 'whisper/whisper-cli.exe', + 'kev-runtime/python/python.exe', + 'onnx-cuda/onnxruntime_binding.node', + 'onnx-cuda/onnxruntime.dll', + 'onnx-cuda/onnxruntime_providers_cuda.dll', + 'onnx-cuda/cublasLt64_12.dll', + 'onnx-cuda/cudnn64_9.dll' + ] + : [ + 'llama-cuda/llama-server', + 'llama-prism-cuda/llama-server', + 'cuda-runtime/libcudart.so.12', + 'sd-cuda/sd-cli', + 'whisper-cuda/whisper-cli', + 'kev-runtime/python/bin/python3', + 'onnx-cuda/libcublasLt.so.13', + 'onnx-cuda/libcudnn.so.9', + 'onnx-cuda/libcufft.so.12', + 'onnx-cuda/libcufftw.so.12', + 'onnx-cuda/libnvrtc.so.13', + 'onnx-cuda/libnvrtc-builtins.so.13.2' + ] for (const relative of required) { - if (!fs.statSync(path.join(binRoot, relative)).isFile()) throw new Error(`Missing CUDA input: ${relative}`) + if (!fs.statSync(path.join(binRoot, relative)).isFile()) + throw new Error(`Missing CUDA input: ${relative}`) } fs.mkdirSync(path.dirname(path.resolve(archive)), { recursive: true }) // Stream straight from the staged folders. A second full copy can exhaust // the runner disk before the archive is ready. -const tar = spawnSync('tar', [ - '-czf', path.resolve(archive), '--dereference', - '--transform', 's,^,bin/,', '-C', path.resolve(binRoot), ...folders -], { stdio: 'inherit' }) +const tar = spawnSync( + 'tar', + [ + '-czf', + path.resolve(archive), + '--dereference', + '--transform', + 's,^,bin/,', + '-C', + path.resolve(binRoot), + ...folders + ], + { stdio: 'inherit' } +) if (tar.status !== 0) throw new Error(`tar failed with exit code ${tar.status}`) const hash = createHash('sha256') for await (const chunk of fs.createReadStream(archive)) hash.update(chunk) diff --git a/scripts/patch-windows-onnx-cuda.mjs b/scripts/patch-windows-onnx-cuda.mjs index da40613a1..b0d3caaf6 100644 --- a/scripts/patch-windows-onnx-cuda.mjs +++ b/scripts/patch-windows-onnx-cuda.mjs @@ -1,7 +1,9 @@ #!/usr/bin/env node // ONNX Runtime's published Windows Node addon has no CUDA provider. The optional // performance pack supplies a CUDA-enabled addon; keep the stock addon for other -// Windows GPUs and for machines without the pack. +// Windows GPUs and for machines without the pack. The CUDA addon loads only in a +// voice or search helper whose environment says CUDA ran on this machine +// (src/main/onnx-host/onnx-runtime-env.ts), so DirectML and WebGPU stay available. import fs from 'node:fs' import path from 'node:path' @@ -42,12 +44,27 @@ if (ortInfo.version !== '1.30.0') { throw new Error(`Review the Windows CUDA binding for ONNX Runtime Node ${ortInfo.version}`) } const bindingFile = path.join(ortRoot, 'dist', 'binding.js') +// Installs patched before the CUDA addon was gated on a verified run get the gate added in place. +const bindingSource = fs.readFileSync(bindingFile, 'utf8') +if ( + bindingSource.includes('? process.env.OFFGRID_PERFORMANCE_PACK_BIN : undefined;') && + !bindingSource.includes('OFFGRID_ONNX_CUDA_VERIFIED') +) { + fs.writeFileSync( + bindingFile, + bindingSource.replace( + ' ? process.env.OFFGRID_PERFORMANCE_PACK_BIN : undefined;', + " && process.env.OFFGRID_ONNX_CUDA_VERIFIED === '1'\n ? process.env.OFFGRID_PERFORMANCE_PACK_BIN : undefined;" + ) + ) +} replaceOnce( bindingFile, 'require(`../bin/napi-v6/${process.platform}/${process.arch}/onnxruntime_binding.node`);', `(() => { const stock = \`../bin/napi-v6/\${process.platform}/\${process.arch}/onnxruntime_binding.node\`; const pack = process.platform === 'win32' && process.arch === 'x64' + && process.env.OFFGRID_ONNX_CUDA_VERIFIED === '1' ? process.env.OFFGRID_PERFORMANCE_PACK_BIN : undefined; const cuda = pack ? require('node:path').join(pack, 'onnx-cuda', 'onnxruntime_binding.node') : undefined; if (cuda && require('node:fs').existsSync(cuda)) { diff --git a/scripts/verify-electron-builder-artifact.mjs b/scripts/verify-electron-builder-artifact.mjs index 9b898e561..af69f0322 100644 --- a/scripts/verify-electron-builder-artifact.mjs +++ b/scripts/verify-electron-builder-artifact.mjs @@ -20,24 +20,28 @@ export default async function verifyElectronBuilderArtifact(event) { // nsis-web stores its payload in dist/nsis-web, while the staged app stays in // dist/win-unpacked. The target output directory is not the app directory. - const packageOutDir = event.target.name === 'nsis-web' - ? path.dirname(event.target.outDir) - : event.target.outDir + const packageOutDir = + event.target.name === 'nsis-web' ? path.dirname(event.target.outDir) : event.target.outDir const appOutDir = event.packager.computeAppOutDir(packageOutDir, event.arch) + const resourcesDir = asarOnlyArtifact + ? path.join(appOutDir, 'resources') + : path.join(appOutDir, `${event.packager.appInfo.productFilename}.app`, 'Contents', 'Resources') + if (fs.existsSync(path.join(resourcesDir, 'bin', 'onnx-cuda'))) { + throw new Error('installer input includes optional GPU runtime: bin/onnx-cuda') + } if (asarOnlyArtifact) { assertAsarArchiveInventory(path.join(appOutDir, 'resources', 'app.asar')) const executable = artifact.endsWith('.exe') ? 'llama-server.exe' : 'llama-server' const required = [ - ...[ - 'llama', - 'llama-cpu', - 'llama-prism', - 'llama-prism-cpu' - ].map((variant) => path.join('bin', variant, executable)) + ...['llama', 'llama-cpu', 'llama-prism', 'llama-prism-cpu'].map((variant) => + path.join('bin', variant, executable) + ) ] if (artifact.endsWith('.exe')) { required.push( path.join('bin', 'whisper-cpu', 'whisper-cli.exe'), + path.join('bin', 'whisper-vulkan', 'whisper-cli.exe'), + path.join('bin', 'whisper-vulkan', 'whisper-server.exe'), path.join('bin', 'sd', 'sd-cli.exe'), path.join('bin', 'sd-cpu', 'sd-cli.exe'), path.join('bin', 'ffmpeg.exe'), @@ -49,6 +53,9 @@ export default async function verifyElectronBuilderArtifact(event) { path.join('bin', 'whisper', 'whisper-cli'), path.join('bin', 'whisper-cpu', 'whisper-cli'), path.join('bin', 'whisper', 'LICENSE'), + ...['llama', 'llama-prism', 'whisper'].map((folder) => + path.join('bin', folder, 'libvulkan.so.1') + ), path.join('bin', 'ffmpeg'), path.join('bin', 'licenses', 'ffmpeg.txt'), path.join('bin', 'sd', 'sd-cli'), @@ -70,7 +77,11 @@ export default async function verifyElectronBuilderArtifact(event) { } } const optionalGpuFolders = [ - 'llama-cuda', 'llama-prism-cuda', 'cuda-runtime', 'sd-cuda', 'kev-runtime', + 'llama-cuda', + 'llama-prism-cuda', + 'cuda-runtime', + 'sd-cuda', + 'kev-runtime', ...(artifact.endsWith('.exe') ? ['whisper'] : ['whisper-cuda']) ] for (const folder of optionalGpuFolders) { diff --git a/scripts/windows-onnx-cuda-preload.cjs b/scripts/windows-onnx-cuda-preload.cjs deleted file mode 100644 index 5d4074aae..000000000 --- a/scripts/windows-onnx-cuda-preload.cjs +++ /dev/null @@ -1,33 +0,0 @@ -// This runs before Rollup's external requires in every main-process entry. -// The optional pack is trusted only after its manifest and install marker match. -if (process.platform === 'win32' && process.arch === 'x64') { - try { - const fs = require('node:fs') - const path = require('node:path') - const app = require('electron').app - const manifestPath = app.isPackaged - ? path.join(process.resourcesPath, 'performance-packs.json') - : path.join(app.getAppPath(), 'resources', 'performance-packs.json') - const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) - const asset = manifest.schemaVersion === 1 && manifest.cuda?.win32 - if (asset && /^[a-f0-9]{64}$/i.test(asset.sha256) && /^[a-zA-Z0-9._-]+$/.test(asset.version) && - new URL(asset.url).origin === 'https://runtime.getoffgridai.co' && - new URL(asset.url).pathname === `/desktop/cuda/win32/${asset.version}.tar.gz`) { - const data = process.env.OFFGRID_DATA_DIR || process.env.OFFGRID_USER_DATA || - path.join(app.getPath('appData'), 'Off Grid AI Desktop') - const root = path.join(data, 'performance-packs', `cuda-${asset.version}-${asset.sha256.slice(0, 12)}`) - const marker = fs.readFileSync(path.join(root, 'verified.sha256'), 'utf8').trim() - const bin = path.join(root, 'bin') - const libraries = path.join(bin, 'onnx-cuda') - if (marker === asset.sha256.toLowerCase() && - fs.existsSync(path.join(libraries, 'onnxruntime.dll')) && - fs.existsSync(path.join(libraries, 'onnxruntime_providers_shared.dll')) && - fs.existsSync(path.join(libraries, 'cudnn64_9.dll'))) { - process.env.OFFGRID_PERFORMANCE_PACK_BIN = bin - process.env.PATH = `${libraries};${process.env.PATH || ''}` - } - } - } catch { - // No verified pack yet. The stock ONNX Runtime remains available. - } -} diff --git a/src/main/__tests__/ai-request-embeddings.test.ts b/src/main/__tests__/ai-request-embeddings.test.ts index 328dbbf01..c644be3cc 100644 --- a/src/main/__tests__/ai-request-embeddings.test.ts +++ b/src/main/__tests__/ai-request-embeddings.test.ts @@ -1,6 +1,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const host = vi.hoisted(() => ({ embed: vi.fn() })) -vi.mock('../embeddings-core', () => ({ embedText: host.embed, embeddingDevice: () => 'coreml' })) +// The inference boundary: the ONNX pipeline reports its device once, then returns vectors. +vi.mock('../embeddings-core', () => ({ + embedTexts: async ( + texts: string[], + _model: string, + _modelsDir: string, + onReady?: (device: string) => void + ) => { + onReady?.('coreml') + return [await host.embed(texts[0])] + } +})) vi.mock('../backend-preferences', () => ({ getBackendPreference: () => 'auto' })) vi.mock('../runtime-residency', () => ({ getResidencyMode: () => 'resident' })) import { embeddings } from '../embeddings' @@ -19,10 +30,10 @@ describe('embedding request records', () => { await expect(embeddings.generateEmbedding('local text')).resolves.toEqual([0.1, 0.2]) expect(records.at(-1)).toMatchObject({ modality: 'embedding', - backend: 'coreml', + backend: 'Core ML', status: 'completed', metrics: { dimensions: 2 }, - response: [0.1, 0.2], + response: [[0.1, 0.2]], request: { text: 'local text' } }) }) diff --git a/src/main/__tests__/database-integration.dbtest.ts b/src/main/__tests__/database-integration.dbtest.ts index 2cedaf507..04e640569 100644 --- a/src/main/__tests__/database-integration.dbtest.ts +++ b/src/main/__tests__/database-integration.dbtest.ts @@ -237,7 +237,7 @@ describe('database.ts - RAG messages', () => { }) }) -describe('database.ts - entities and facts', () => { +describe('database.ts - entities', () => { it('EntityDomain creates an entity and returns a stable id on repeat (case-insensitive name)', () => { const id = resolveEntity('Ada Lovelace', 'Person') expect(id).toBeGreaterThan(0) @@ -246,60 +246,6 @@ describe('database.ts - entities and facts', () => { expect(idAgain).toBe(id) }) - it('EntityDomain defaults type to Unknown when blank', () => { - const id = resolveEntity('Mystery', ' ') - const { entity } = db.getEntityDetails(id) as { entity: { type: string } } - expect(entity.type).toBe('Unknown') - }) - - it('addEntityFact inserts once and dedupes on repeat (INSERT OR IGNORE)', () => { - const id = resolveEntity('Grace Hopper', 'Person') - expect(db.addEntityFact(id, 'Coined the term debugging', 'sess-1')).toBe(true) - // duplicate fact for the same entity is ignored -> no change. - expect(db.addEntityFact(id, 'Coined the term debugging', 'sess-1')).toBe(false) - // a distinct fact is inserted. - expect(db.addEntityFact(id, 'Rear admiral in the US Navy')).toBe(true) - }) - - it('updateEntitySummary persists a summary readable via getEntityDetails', () => { - const id = resolveEntity('Alan Turing', 'Person') - db.updateEntitySummary(id, 'Founder of theoretical computer science') - const { entity } = db.getEntityDetails(id) as { entity: { summary: string } } - expect(entity.summary).toBe('Founder of theoretical computer science') - }) - - it('getEntities returns fact_count aggregated per entity', () => { - const id = resolveEntity('Katherine Johnson', 'Person') - db.addEntityFact(id, 'Calculated trajectories for Mercury and Apollo') - db.addEntityFact(id, 'Received the Presidential Medal of Freedom') - const list = db.getEntities() as { id: number; fact_count: number }[] - const row = list.find((e) => e.id === id) - expect(row?.fact_count).toBe(2) - }) - - it('getEntityDetails returns the entity plus its facts (newest first)', () => { - const id = resolveEntity('Margaret Hamilton', 'Person') - db.addEntityFact(id, 'Led Apollo flight software') - const { entity, facts } = db.getEntityDetails(id) as { - entity: { id: number; name: string } - facts: { fact: string }[] - } - expect(entity.id).toBe(id) - expect(entity.name).toBe('Margaret Hamilton') - expect(facts.some((f) => f.fact === 'Led Apollo flight software')).toBe(true) - }) - - it('EntityDomain deletes the entity and its facts; false on a missing id', () => { - const id = resolveEntity('Temporary Person', 'Person') - db.addEntityFact(id, 'to be deleted') - expect(entityDomain.deleteEntityById(id)).toBe(true) - const { entity, facts } = db.getEntityDetails(id) as { entity: unknown; facts: unknown[] } - expect(entity).toBeUndefined() - expect(facts).toHaveLength(0) - // deleting again affects no rows. - expect(entityDomain.deleteEntityById(id)).toBe(false) - }) - it('upsertEntitySession links entities to a session and getEntitiesForSession returns them', () => { // entity_sessions.session_id is a FK to conversations(id), so the session // must be a real conversation row first. @@ -317,30 +263,10 @@ describe('database.ts - entities and facts', () => { const ids = forSession.map((e) => e.id) expect(ids).toContain(a) expect(ids).toContain(b) - expect( - forSession.every( - (entity) => typeof (entity as { fact_count?: unknown }).fact_count === 'number' - ) - ).toBe(true) expect(forSession.every((entity) => !('session_count' in entity))).toBe(true) }) }) -describe('database.ts - master memory', () => { - it('getMasterMemory returns an object with content and updated_at keys', () => { - const before = db.getMasterMemory() - expect(before).toHaveProperty('content') - expect(before).toHaveProperty('updated_at') - }) - - it('updateMasterMemory upserts the single row and getMasterMemory reads it back', () => { - db.updateMasterMemory('cumulative profile v1') - expect(db.getMasterMemory().content).toBe('cumulative profile v1') - db.updateMasterMemory('cumulative profile v2') - expect(db.getMasterMemory().content).toBe('cumulative profile v2') - }) -}) - describe('database.ts - chat summaries', () => { it('upsertChatSummary inserts then updates for the same session, getAllChatSummaries lists them', () => { db.upsertChatSummary('sum-sess', 'first summary') @@ -410,40 +336,7 @@ describe('database.ts - app settings (typed round-trip + defaults)', () => { }) }) -describe('database.ts - memories, dashboard stats and legacy purge', () => { - it('deleteMemory returns false for a non-existent memory id', () => { - expect(db.deleteMemory(999999)).toBe(false) - }) - - it('deleteMemory returns true after inserting a memory row directly', () => { - const handle = db.getDB() - const info = handle - .prepare('INSERT INTO memories (content, source_app, session_id) VALUES (?, ?, ?)') - .run('a captured memory', 'TestApp', 'sess-mem') - const memId = Number(info.lastInsertRowid) - expect(db.deleteMemory(memId)).toBe(true) - }) - - it('getMemoryRecordsForSession returns rows scoped to a session', () => { - const handle = db.getDB() - handle - .prepare('INSERT INTO memories (content, source_app, session_id) VALUES (?, ?, ?)') - .run('scoped memory', 'TestApp', 'sess-scope') - const rows = db.getMemoryRecordsForSession('sess-scope') as { content: string }[] - expect(rows.some((r) => r.content === 'scoped memory')).toBe(true) - }) - - it('getDashboardStats returns the full stat shape with numeric totals', () => { - const stats = db.getDashboardStats() - expect(typeof stats.totalMemories).toBe('number') - expect(typeof stats.totalEntities).toBe('number') - expect(Array.isArray(stats.recentMemories)).toBe(true) - expect(Array.isArray(stats.topEntities)).toBe(true) - expect(Array.isArray(stats.activityByDay)).toBe(true) - // 14-day activity window. - expect(stats.activityByDay.length).toBe(14) - }) - +describe('database.ts - legacy purge and chat sessions', () => { it('purgeLegacyChatImports returns null when there is nothing legacy to purge', () => { // No Claude/ChatGPT/Gemini conversations were seeded, so this is a no-op. expect(db.purgeLegacyChatImports()).toBeNull() diff --git a/src/main/__tests__/document-index-upgrade.dbtest.ts b/src/main/__tests__/document-index-upgrade.dbtest.ts new file mode 100644 index 000000000..04eb10628 --- /dev/null +++ b/src/main/__tests__/document-index-upgrade.dbtest.ts @@ -0,0 +1,128 @@ +/** + * Document passages in scripts written without spaces, and the passage indexes on an upgraded + * profile, against the real profile database and document engine. A Japanese passage is found by + * a word of three or more characters (the trigram twin) and by a two-character one. When a profile + * from an older schema opens, its passage indexes are rebuilt in the background after start-up + * and every passage is found again. Electron and the embedding model are the only controlled + * boundaries; the model is unavailable, so this is the keyword path alone. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-document-index-')) +const originalDataDir = process.env.OFFGRID_DATA_DIR +process.env.OFFGRID_DATA_DIR = PROFILE_DIR + +vi.mock('electron', () => ({ + app: { getPath: () => PROFILE_DIR, isPackaged: false, getAppPath: () => process.cwd() }, + BrowserWindow: { getAllWindows: () => [] }, + safeStorage: { + isEncryptionAvailable: () => false, + encryptString: (value: string) => Buffer.from(value), + decryptString: (value: Buffer) => value.toString() + } +})) + +vi.mock('@huggingface/transformers', () => ({ + env: {}, + pipeline: async () => { + throw new Error('no embedding model in this journey') + } +})) + +const JAPANESE = + '東京オフィスの移転は来月の予定です。新しい会議室の予約は総務部が担当します。引っ越しの日程は後日連絡します。' +const ENGLISH = + 'The harbourfront lease renews in March. Parking permits are issued by the building office.' + +let japaneseId = '' +let englishId = '' + +type Documents = typeof import('../documents') + +async function found(documents: Documents, query: string): Promise { + const passages = await documents.documentTurns.searchPassages( + { documentIds: [japaneseId, englishId] }, + query + ) + return [...new Set(passages.map((passage) => passage.documentId))] +} + +beforeAll(async () => { + const { ingestTextDocument } = await import('../files') + japaneseId = (await ingestTextDocument('移転計画', 'audio', JAPANESE)).documentId! + englishId = (await ingestTextDocument('Lease notes', 'audio', ENGLISH)).documentId! +}) + +afterAll(async () => { + const { stopIndexRebuilds, getDB } = await import('../database') + stopIndexRebuilds() + getDB().close() + if (originalDataDir === undefined) delete process.env.OFFGRID_DATA_DIR + else process.env.OFFGRID_DATA_DIR = originalDataDir + fs.rmSync(PROFILE_DIR, { recursive: true, force: true }) +}) + +describe('document passage indexes', () => { + it('finds a Japanese passage by a long word and by a two-character one', async () => { + const documents = await import('../documents') + await vi.waitFor( + async () => expect(await found(documents, 'harbourfront lease')).toEqual([englishId]), + { + timeout: 15_000, + interval: 100 + } + ) + await vi.waitFor(async () => expect(await found(documents, 'オフィス')).toEqual([japaneseId]), { + timeout: 15_000, + interval: 100 + }) + expect(await found(documents, '会議')).toEqual([japaneseId]) + // Both scripts in one question: each finds its own document. + expect((await found(documents, 'parking 総務部')).sort()).toEqual( + [englishId, japaneseId].sort() + ) + }) + + it('rebuilds the passage indexes of an older profile in the background after start-up', async () => { + // The profile as an older schema left it: the indexes are not current, the twin is gone. + const database = await import('../database') + database + .getDB() + .prepare('DELETE FROM fts_schema WHERE name IN (?, ?)') + .run('attached_document_fts', 'attached_document_fts_tri') + database.getDB().exec('DROP TABLE IF EXISTS attached_document_fts_tri') + database.getDB().close() + vi.resetModules() + + const reopened = await import('../database') + const documents = await import('../documents') + // Opening the store queues the rebuilds; start-up then runs them. + await documents.documentStore.getDocument(japaneseId) + reopened.startIndexRebuilds() + + await vi.waitFor( + async () => { + expect(await found(documents, 'オフィス')).toEqual([japaneseId]) + expect(await found(documents, 'harbourfront lease')).toEqual([englishId]) + }, + { timeout: 20_000, interval: 100 } + ) + expect(await found(documents, '会議')).toEqual([japaneseId]) + // Both indexes end current, so the next launch does not rebuild them again. + await vi.waitFor( + () => + expect( + reopened + .getDB() + .prepare( + "SELECT name FROM fts_schema WHERE name LIKE 'attached_document_fts%' ORDER BY name" + ) + .all() + ).toEqual([{ name: 'attached_document_fts' }, { name: 'attached_document_fts_tri' }]), + { timeout: 10_000, interval: 100 } + ) + }) +}) diff --git a/src/main/__tests__/embeddings-core.test.ts b/src/main/__tests__/embeddings-core.test.ts index 860744ff7..9edfa1327 100644 --- a/src/main/__tests__/embeddings-core.test.ts +++ b/src/main/__tests__/embeddings-core.test.ts @@ -18,7 +18,7 @@ vi.mock('../onnx-device', () => ({ }) })) -import { disposeEmbeddingModel, embedText } from '../embeddings-core' +import { disposeEmbeddingModel } from '../embeddings-core' let profile: string let selected: string @@ -36,18 +36,6 @@ afterEach(async () => { fs.rmSync(profile, { recursive: true, force: true }) }) -it('retries an embedding model load after a temporary provider failure', async () => { - const runtime = Object.assign( - vi.fn(async () => ({ data: new Float32Array([0.25, 0.5]) })), - { dispose: vi.fn() } - ) - host.pipeline.mockRejectedValueOnce(new Error('provider unavailable')).mockResolvedValue(runtime) - - await expect(embedText('first', selected)).rejects.toThrow('provider unavailable') - await expect(embedText('second', selected)).resolves.toEqual([0.25, 0.5]) - expect(host.pipeline).toHaveBeenCalledTimes(2) -}) - it('loads embeddings from a scan folder while the chosen folder is offline', async () => { const scan = path.join(profile, 'scan') const packageRoot = path.join(scan, '.cache/Xenova/all-MiniLM-L6-v2') @@ -65,7 +53,8 @@ it('loads embeddings from a scan folder while the chosen folder is offline', asy fs.rmdirSync(selected) host.pipeline.mockResolvedValue( Object.assign( - vi.fn(async () => ({ data: new Float32Array([0.25, 0.5]) })), + // A transformers.js Tensor: one row per text. + vi.fn(async () => ({ data: new Float32Array([0.25, 0.5]), tolist: () => [[0.25, 0.5]] })), { dispose: vi.fn() } ) ) @@ -77,41 +66,3 @@ it('loads embeddings from a scan folder while the chosen folder is offline', asy }) expect(fs.existsSync(selected)).toBe(false) }) - -it('refuses an embedding download when no complete package exists and storage is offline', async () => { - fs.rmdirSync(selected) - await expect(embedText('query', selected)).rejects.toThrow( - 'Connect the drive or choose another folder.' - ) - expect(host.pipeline).not.toHaveBeenCalled() - expect(fs.existsSync(selected)).toBe(false) -}) - -it.each([undefined, '1'])( - 'loads from an uncreated default models path (offline=%s)', - async (offline) => { - configureRuntime({ modelsDir: undefined }) - fs.rmdirSync(selected) - vi.stubEnv('OFFGRID_EMBEDDINGS_OFFLINE', offline) - host.pipeline.mockResolvedValue( - Object.assign( - vi.fn(async () => ({ data: new Float32Array([0.25, 0.5]) })), - { dispose: vi.fn() } - ) - ) - await expect(embedText('query', selected)).resolves.toEqual([0.25, 0.5]) - expect(host.pipeline).toHaveBeenCalledWith('feature-extraction', 'Xenova/all-MiniLM-L6-v2', { - device: 'cpu' - }) - expect(fs.existsSync(selected)).toBe(false) - } -) - -it('attempts local-only resolution without a writable chosen folder', async () => { - fs.rmdirSync(selected) - vi.stubEnv('OFFGRID_EMBEDDINGS_OFFLINE', '1') - host.pipeline.mockRejectedValue(new Error('local model is missing')) - await expect(embedText('query', selected)).rejects.toThrow('local model is missing') - expect(host.pipeline).toHaveBeenCalledTimes(1) - expect(fs.existsSync(selected)).toBe(false) -}) diff --git a/src/main/__tests__/engine-resume-health.test.ts b/src/main/__tests__/engine-resume-health.test.ts new file mode 100644 index 000000000..bc8a5252f --- /dev/null +++ b/src/main/__tests__/engine-resume-health.test.ts @@ -0,0 +1,157 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', async () => { + const { EventEmitter: Emitter } = await import('node:events') + return { powerMonitor: new Emitter() } +}) + +import { + installEngineResumeHealth, + registerResumeHealthTarget, + waitForEngineResumeHealth +} from '../engine-resume-health' + +/** An engine as the wake check sees it: whether it should run, whether it answers, its restarts. */ +interface Engine { + target: Parameters[0] + restarts: number[] + state: { running: boolean; healthy: boolean; restartFails?: boolean } +} + +function engine(id: string, state: Engine['state']): Engine { + const restarts: number[] = [] + const target = { + id, + shouldBeRunning: () => state.running, + healthCheck: async () => { + if (state.healthy === undefined) throw new Error('no answer') + return state.healthy + }, + restart: async () => { + restarts.push(Date.now()) + if (state.restartFails) throw new Error(`${id} would not start`) + state.healthy = true + } + } + return { target, restarts, state } +} + +let monitor: EventEmitter +let invalidations: number +let dispose: () => void +const unregister: Array<() => void> = [] + +beforeEach(() => { + vi.useFakeTimers() + monitor = new EventEmitter() + invalidations = 0 + dispose = installEngineResumeHealth({ + // The wake events, as Electron's powerMonitor emits them. + powerMonitor: monitor as unknown as NonNullable< + Parameters[0] + >['powerMonitor'], + invalidateBackendFacts: () => { + invalidations++ + } + }) +}) + +afterEach(() => { + dispose() + for (const off of unregister.splice(0)) off() + vi.useRealTimers() +}) + +describe('engine health after wake', () => { + it('restarts only the engines that should run and stopped answering', async () => { + const stalled = engine('chat', { running: true, healthy: false }) + const fine = engine('image', { running: true, healthy: true }) + const idle = engine('voice', { running: false, healthy: false }) + for (const { target } of [stalled, fine, idle]) + unregister.push(registerResumeHealthTarget(target)) + + monitor.emit('resume') + expect(invalidations).toBe(1) + // Requests that arrive during the wake debounce wait for the check. + let waited = false + const waiting = waitForEngineResumeHealth().then(() => { + waited = true + }) + await vi.advanceTimersByTimeAsync(1_999) + expect(stalled.restarts).toHaveLength(0) + expect(waited).toBe(false) + + await vi.advanceTimersByTimeAsync(1) + await waiting + expect(waited).toBe(true) + expect(stalled.restarts).toHaveLength(1) + expect(fine.restarts).toHaveLength(0) + expect(idle.restarts).toHaveLength(0) + }) + + it('checks once for a burst of wake and unlock events', async () => { + const stalled = engine('chat', { running: true, healthy: false }) + unregister.push(registerResumeHealthTarget(stalled.target)) + monitor.emit('resume') + await vi.advanceTimersByTimeAsync(1_000) + monitor.emit('unlock-screen') + await vi.advanceTimersByTimeAsync(2_000) + await waitForEngineResumeHealth() + expect(stalled.restarts).toHaveLength(1) + // Every event refreshes what the GPU can run (a driver may have changed while asleep). + expect(invalidations).toBe(2) + }) + + it('keeps checking the other engines when one cannot be recovered', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const broken = engine('grounding', { running: true, healthy: false, restartFails: true }) + const stalled = engine('chat', { running: true, healthy: false }) + const silent = engine('decision', { running: true, healthy: undefined as unknown as boolean }) + for (const { target } of [broken, stalled, silent]) + unregister.push(registerResumeHealthTarget(target)) + + monitor.emit('resume') + await vi.advanceTimersByTimeAsync(2_000) + await waitForEngineResumeHealth() + expect(broken.restarts).toHaveLength(1) + expect(stalled.restarts).toHaveLength(1) + // A health check that throws counts as not answering. + expect(silent.restarts).toHaveLength(1) + expect(error).toHaveBeenCalledWith( + '[engine-resume-health] grounding recovery failed:', + expect.objectContaining({ message: 'grounding would not start' }) + ) + error.mockRestore() + }) + + it('stops listening once disposed and releases anyone waiting', async () => { + const stalled = engine('chat', { running: true, healthy: false }) + unregister.push(registerResumeHealthTarget(stalled.target)) + monitor.emit('resume') + const waiting = waitForEngineResumeHealth() + dispose() + await waiting + await vi.advanceTimersByTimeAsync(5_000) + expect(stalled.restarts).toHaveLength(0) + monitor.emit('resume') + await vi.advanceTimersByTimeAsync(5_000) + expect(stalled.restarts).toHaveLength(0) + expect(monitor.listenerCount('resume')).toBe(0) + expect(monitor.listenerCount('unlock-screen')).toBe(0) + }) + + it('forgets an engine once it unregisters, but not a newer one under the same id', async () => { + const old = engine('chat', { running: true, healthy: false }) + const offOld = registerResumeHealthTarget(old.target) + const replacement = engine('chat', { running: true, healthy: false }) + unregister.push(registerResumeHealthTarget(replacement.target)) + offOld() + + monitor.emit('resume') + await vi.advanceTimersByTimeAsync(2_000) + await waitForEngineResumeHealth() + expect(old.restarts).toHaveLength(0) + expect(replacement.restarts).toHaveLength(1) + }) +}) diff --git a/src/main/__tests__/entity-domain.dbtest.ts b/src/main/__tests__/entity-domain.dbtest.ts index 12bae122f..34034d0cc 100644 --- a/src/main/__tests__/entity-domain.dbtest.ts +++ b/src/main/__tests__/entity-domain.dbtest.ts @@ -14,7 +14,7 @@ vi.mock('electron', () => ({ } })) -import { addEntityFact, getDB, upsertEntitySession } from '../database' +import { getDB, upsertEntitySession } from '../database' import { deleteEntityById, registerEntityDomain, @@ -114,7 +114,9 @@ describe('EntityDomain SQLite lifecycle', () => { ) const doomed = resolve('Delete With Dependants', 'Person') const survivor = resolve('Lifecycle Survivor', 'Person') - addEntityFact(doomed, 'Fact that must not become orphaned', 'entity-delete-session') + db.prepare( + 'INSERT INTO entity_facts (entity_id, fact, source_session_id) VALUES (?, ?, ?)' + ).run(doomed, 'Fact that must not become orphaned', 'entity-delete-session') upsertEntitySession(doomed, 'entity-delete-session') db.prepare( `INSERT INTO entity_edges @@ -149,7 +151,9 @@ describe('EntityDomain SQLite lifecycle', () => { 'Notes' ) const doomed = resolve('Rollback Entity', 'Person') - addEntityFact(doomed, 'This fact must survive rollback', 'entity-rollback-session') + db.prepare( + 'INSERT INTO entity_facts (entity_id, fact, source_session_id) VALUES (?, ?, ?)' + ).run(doomed, 'This fact must survive rollback', 'entity-rollback-session') upsertEntitySession(doomed, 'entity-rollback-session') db.exec(`CREATE TRIGGER reject_entity_delete BEFORE DELETE ON entities WHEN old.id = ${doomed} diff --git a/src/main/__tests__/fetch-win-binaries.test.ts b/src/main/__tests__/fetch-win-binaries.test.ts index 056fbd756..4727333bb 100644 --- a/src/main/__tests__/fetch-win-binaries.test.ts +++ b/src/main/__tests__/fetch-win-binaries.test.ts @@ -31,12 +31,15 @@ describe.skipIf(!SRC)('fetch-win-binaries.ps1 — stable-diffusion.cpp asset', ( expect(activeAvx2).toBe(false) }) - it('stages CUDA, Vulkan, and CPU image runtimes with one shared CUDA runtime', () => { + it('stages CUDA, Vulkan, and CPU image runtimes, with the image engine’s own CUDA 12.8 runtime', () => { expect(SRC).toMatch(/leejet\/stable-diffusion\.cpp'\s+'bin-win-cuda12-x64\\\.zip\$'/) expect(SRC).toMatch(/leejet\/stable-diffusion\.cpp'\s+'bin-win-vulkan-x64\\\.zip\$'/) expect(SRC).toMatch(/leejet\/stable-diffusion\.cpp'\s+'bin-win-cpu-x64\\\.zip\$'/) expect(SRC).toMatch(/Copy-Runtime \$x 'sd-cuda'/) - expect(SRC).not.toMatch(/cudart-sd-bin-win/) + // stable-diffusion.cpp builds against CUDA 12.8 and llama.cpp against 12.4: sd-cuda carries + // its own three runtime DLLs instead of borrowing llama's. + expect(SRC).toMatch(/cudart-sd-bin-win-cu12-x64/) + expect(SRC).toMatch(/'cudart64_12\.dll', 'cublas64_12\.dll', 'cublasLt64_12\.dll'/) }) it('still renames upstream sd.exe to the sd-cli.exe the app resolves', () => { @@ -54,7 +57,9 @@ describe.skipIf(!SRC)('fetch-win-binaries.ps1 — stable-diffusion.cpp asset', ( describe.skipIf(!SRC)('fetch-win-binaries.ps1 — Bonsai runtime', () => { it('stages pinned Prism CUDA, Vulkan, and CPU runtimes in the directories the model server resolves', () => { expect(SRC).toMatch(/offgrid\.prismLlamaRef/) - expect(SRC).toMatch(/PrismML-Eng\/llama\.cpp'\s+'\^llama-\.\+-bin-win-cuda-12\\\.4-x64\\\.zip\$'/) + expect(SRC).toMatch( + /PrismML-Eng\/llama\.cpp'\s+'\^llama-\.\+-bin-win-cuda-12\\\.4-x64\\\.zip\$'/ + ) expect(SRC).toMatch(/PrismML-Eng\/llama\.cpp'\s+'bin-win-vulkan-x64\\\.zip\$'/) expect(SRC).toMatch(/PrismML-Eng\/llama\.cpp'\s+'bin-win-cpu-x64\\\.zip\$'/) expect(SRC).toMatch(/Copy-Runtime \$x 'llama-prism-cuda'/) @@ -64,7 +69,9 @@ describe.skipIf(!SRC)('fetch-win-binaries.ps1 — Bonsai runtime', () => { it('stages the shared CUDA runtime and checks it before packaging', () => { expect(SRC).toMatch(/ggml-org\/llama\.cpp'\s+'\^llama-\.\+-bin-win-cuda-12\\\.4-x64\\\.zip\$'/) - expect(SRC).toMatch(/ggml-org\/llama\.cpp'\s+'\^cudart-llama-bin-win-cuda-12\\\.4-x64\\\.zip\$'/) + expect(SRC).toMatch( + /ggml-org\/llama\.cpp'\s+'\^cudart-llama-bin-win-cuda-12\\\.4-x64\\\.zip\$'/ + ) expect(SRC).toMatch(/Copy-Runtime \$x 'cuda-runtime'/) expect(SRC).toMatch(/cuda-runtime\\cudart64_12\.dll/) expect(SRC).toMatch(/cuda-runtime\\cublas64_12\.dll/) diff --git a/src/main/__tests__/fresh-profile.dbtest.ts b/src/main/__tests__/fresh-profile.dbtest.ts index 2a63454b8..edda3e5b9 100644 --- a/src/main/__tests__/fresh-profile.dbtest.ts +++ b/src/main/__tests__/fresh-profile.dbtest.ts @@ -22,10 +22,7 @@ vi.mock('electron', () => ({ import { getAllChatSummaries, getChatSessions, - getDashboardStats, getDB, - getEntities, - getMasterMemory, getRagConversations, getUserProfile } from '../database' @@ -37,35 +34,13 @@ afterAll(() => { }) describe('fresh profile integration', () => { - it('starts with no chats, projects, memories, entities, summaries, or identity (#3)', () => { + it('starts with no chats, projects, summaries, or identity (#3)', () => { expect(fs.readdirSync(profile)).toEqual([]) expect(getRagConversations()).toEqual([]) expect(listProjects()).toEqual([]) expect(getChatSessions()).toEqual([]) - expect(getEntities()).toEqual([]) expect(getAllChatSummaries()).toEqual([]) expect(getUserProfile()).toBeNull() - expect(getMasterMemory()).toEqual({ content: null, updated_at: null }) - const stats = getDashboardStats() - expect(stats).toEqual( - expect.objectContaining({ - totalChats: 0, - totalMemories: 0, - totalEntities: 0, - totalRelationships: 0, - totalMessages: 0, - totalFacts: 0, - recentChats: [], - recentMemories: [], - topEntities: [], - entityTypeCounts: [], - appDistribution: [] - }) - ) - expect(stats.activityByDay).toHaveLength(14) - expect(stats.activityByDay.every(({ chats, memories }) => chats === 0 && memories === 0)).toBe( - true - ) }) }) diff --git a/src/main/__tests__/fts-match.test.ts b/src/main/__tests__/fts-match.test.ts deleted file mode 100644 index ab250a92f..000000000 --- a/src/main/__tests__/fts-match.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { DatabaseSync } from 'node:sqlite' -import { describe, expect, it } from 'vitest' -import { escapeFtsMatch } from '../fts-match' - -describe('escapeFtsMatch', () => { - it.each([ - ['release?', '"release"'], - ['say "release"', '"say" OR "release"'], - ['title:release', '"title" OR "release"'], - ['(release)', '"release"'], - ['AND OR NOT NEAR', '"AND" OR "OR" OR "NOT" OR "NEAR"'], - ['release* ^code', '"release" OR "code"'], - ['', ''], - [' \t\n ', ''], - ['?:()*^"', ''], - ['café 日本語 नमस्ते', '"café" OR "日本語" OR "नमस्ते"'] - ])('escapes %j', (query, expected) => { - const match = escapeFtsMatch(query) - expect(match).toBe(expected) - if (!match) return - const db = new DatabaseSync(':memory:') - try { - db.exec('CREATE VIRTUAL TABLE terms USING fts5(content)') - db.prepare('INSERT INTO terms(content) VALUES (?)').run('release code café 日本語 नमस्ते AND') - expect(() => db.prepare('SELECT * FROM terms WHERE terms MATCH ?').all(match)).not.toThrow() - } finally { - db.close() - } - }) - - it('doubles internal quotes in phrase tokens', () => { - expect(escapeFtsMatch(['a"b'])).toBe('"a""b"') - }) - - it('keeps caller-supplied prefix and all-term search behavior', () => { - expect(escapeFtsMatch('release code', { prefix: true, join: 'AND', maxTokens: 1 })).toBe( - '"release"*' - ) - }) -}) diff --git a/src/main/__tests__/fts-rebuild-upgrade.dbtest.ts b/src/main/__tests__/fts-rebuild-upgrade.dbtest.ts new file mode 100644 index 000000000..fa24db901 --- /dev/null +++ b/src/main/__tests__/fts-rebuild-upgrade.dbtest.ts @@ -0,0 +1,134 @@ +/** + * The upgrade path of the keyword indexes, against the real profile database. A profile from + * before the retrieval engine already holds moments and meetings; their indexes are created empty + * and filled in the background after start-up, a slice at a time. Once that finishes every old row + * is found, and a row added or edited while it ran is found as it is now. Electron and the + * embedding model are the only controlled boundaries. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-fts-rebuild-')) +const originalDataDir = process.env.OFFGRID_DATA_DIR +process.env.OFFGRID_DATA_DIR = PROFILE_DIR + +vi.mock('electron', () => ({ + app: { getPath: () => PROFILE_DIR, isPackaged: false, getAppPath: () => process.cwd() }, + BrowserWindow: { getAllWindows: () => [] }, + safeStorage: { + isEncryptionAvailable: () => false, + encryptString: (value: string) => Buffer.from(value), + decryptString: (value: Buffer) => value.toString() + } +})) + +vi.mock('@huggingface/transformers', () => ({ + env: {}, + pipeline: async () => { + throw new Error('no embedding model in this journey') + } +})) + +import { getDB, startIndexRebuilds, stopIndexRebuilds } from '../database' +import { ensureRecordSchema, recordSearch } from '../records' + +/** Enough rows that the background rebuild takes several slices. */ +const OLD_MEETINGS = 4_000 + +const keysFor = async (question: string, kinds: string[]): Promise => + (await recordSearch(question, { semantic: false, window: null, kinds, limit: 50 })).hits.map( + (hit) => hit.key + ) + +beforeAll(() => { + // A profile written before the retrieval engine: Pro's tables, already full. + const db = getDB() + db.exec(` + CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, surface_app TEXT, + url TEXT, source_session_id TEXT, source_key TEXT, memory_id INTEGER, + category TEXT NOT NULL DEFAULT 'work', engagement REAL NOT NULL DEFAULT 0, + salience REAL NOT NULL DEFAULT 0, ts DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS frames (id INTEGER PRIMARY KEY AUTOINCREMENT, image_path TEXT, text TEXT, surface TEXT, url TEXT, ts DATETIME, source TEXT); + CREATE TABLE IF NOT EXISTS observation_frames (observation_id INTEGER, frame_id INTEGER); + CREATE TABLE IF NOT EXISTS meetings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, transcript TEXT, summary TEXT, + started_at INTEGER, ended_at INTEGER, duration_sec INTEGER, created_at INTEGER NOT NULL DEFAULT 0 + ); + `) + const meeting = db.prepare('INSERT INTO meetings (title, summary, started_at) VALUES (?, ?, ?)') + db.transaction(() => { + for (let i = 1; i <= OLD_MEETINGS; i++) + meeting.run(`Standup ${i}`, `Daily standup notes number ${i}`, 1_700_000_000_000 + i) + meeting.run( + 'Harbourfront lease review', + 'Lease renewal for the harbourfront office', + 1_700_100_000_000 + ) + db.prepare("INSERT INTO observations (summary, surface) VALUES (?, 'Notes')").run( + 'Read the harbourfront parking rules' + ) + db.prepare("INSERT INTO observations (summary, surface) VALUES (?, 'Notes')").run( + '東京オフィスの移転計画' + ) + })() +}) + +afterAll(() => { + stopIndexRebuilds() + getDB().close() + if (originalDataDir === undefined) delete process.env.OFFGRID_DATA_DIR + else process.env.OFFGRID_DATA_DIR = originalDataDir + fs.rmSync(PROFILE_DIR, { recursive: true, force: true }) +}) + +describe('keyword indexes on an upgraded profile', () => { + it('fills in the background after start-up and keeps edits made while it runs', async () => { + // Start-up: the indexes exist but are not filled yet, so old rows are not found. + ensureRecordSchema() + expect(await keysFor('harbourfront lease', ['meeting'])).toEqual([]) + + startIndexRebuilds() + // While the rebuild runs: a new meeting, and an old one renamed. + const added = Number( + getDB() + .prepare('INSERT INTO meetings (title, summary, started_at) VALUES (?, ?, ?)') + .run('Quayside budget call', 'Budget for the quayside fitout', Date.now()).lastInsertRowid + ) + getDB() + .prepare('UPDATE meetings SET title = ?, summary = ? WHERE id = ?') + .run('Lighthouse retrospective', 'What went well at the lighthouse event', 2) + + await vi.waitFor( + async () => { + expect(await keysFor('harbourfront lease', ['meeting'])).toEqual( + expect.arrayContaining([expect.stringMatching(/^mtg:\d+$/)]) + ) + expect(await keysFor(`standup number ${OLD_MEETINGS}`, ['meeting'])).toContain( + `mtg:${OLD_MEETINGS}` + ) + }, + { timeout: 20_000, interval: 100 } + ) + expect(await keysFor('quayside budget', ['meeting'])).toEqual([`mtg:${added}`]) + expect(await keysFor('lighthouse retrospective', ['meeting'])).toEqual(['mtg:2']) + // The renamed meeting is found by what it says now, not by its old words. + expect(await keysFor('standup number 2', ['meeting'])).not.toContain('mtg:2') + + // Moments too, including text in a script written without spaces (the trigram twin). + await vi.waitFor( + async () => { + expect(await keysFor('harbourfront parking', ['screen'])).toHaveLength(1) + expect(await keysFor('オフィス', ['screen'])).toHaveLength(1) + }, + { timeout: 20_000, interval: 100 } + ) + + // Once swapped in, the filled index follows edits live. + getDB().prepare('UPDATE meetings SET summary = ? WHERE id = ?').run('Marina berth fees', 3) + expect(await keysFor('marina berth', ['meeting'])).toEqual(['mtg:3']) + }) +}) diff --git a/src/main/__tests__/harness/fake-llama-server.ts b/src/main/__tests__/harness/fake-llama-server.ts index 861b2edf5..8128eded6 100644 --- a/src/main/__tests__/harness/fake-llama-server.ts +++ b/src/main/__tests__/harness/fake-llama-server.ts @@ -36,7 +36,7 @@ interface FakeTurn { requirePriorReasoningDetails?: boolean /** Tool calls emitted on this turn (the agentic loop then runs them and calls back). */ toolCalls?: FakeToolCall[] - /** Force a non-200 to exercise the error path (body is surfaced by describeServerError). */ + /** Force a non-200 to exercise the error path (body is surfaced by serverResponseError). */ errorStatus?: number errorBody?: string /** Delay the native-engine response after the request is admitted. Lifecycle tests use diff --git a/src/main/__tests__/ipc-fts-fallback.dbtest.ts b/src/main/__tests__/ipc-fts-fallback.dbtest.ts index 7a2295ccb..11b5a3f14 100644 --- a/src/main/__tests__/ipc-fts-fallback.dbtest.ts +++ b/src/main/__tests__/ipc-fts-fallback.dbtest.ts @@ -1,15 +1,13 @@ -// Real SQLite and IPC retrieval, with model transport controlled without a local server. +// User text that is FTS5 syntax (quotes, operators, column filters, wildcards) or no words at all +// never breaks a search: the real retrieval engine over the real profile database finds the record +// by its words, and an empty query matches nothing. Electron and the embedding model are the only +// controlled boundaries; the model is unavailable, so this is the keyword path alone. import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -type IpcHandler = (event: IpcEvent, ...args: unknown[]) => unknown -interface IpcEvent { - sender: { send: (channel: string, payload: unknown) => void } -} const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-fts-fallback-')) -const handlers = new Map() vi.mock('electron', () => ({ app: { @@ -18,24 +16,12 @@ vi.mock('electron', () => ({ getAppPath: () => process.cwd(), on: () => undefined }, + BrowserWindow: { getAllWindows: () => [] }, safeStorage: { isEncryptionAvailable: () => false, encryptString: (value: string) => Buffer.from(value), decryptString: (value: Buffer) => value.toString() - }, - ipcMain: { - handle: (channel: string, handler: IpcHandler) => handlers.set(channel, handler), - on: () => undefined - }, - BrowserWindow: { fromWebContents: () => undefined }, - clipboard: { readText: () => '', writeText: () => undefined }, - systemPreferences: { - isTrustedAccessibilityClient: () => true, - getMediaAccessStatus: () => 'granted' - }, - shell: { openExternal: async () => undefined, openPath: async () => '' }, - desktopCapturer: { getSources: async () => [] }, - dialog: {} + } })) vi.mock('@huggingface/transformers', () => ({ @@ -45,129 +31,47 @@ vi.mock('@huggingface/transformers', () => ({ } })) -vi.mock('@lancedb/lancedb', () => ({ - connect: async () => ({ - tableNames: async () => [], - openTable: async () => { - throw new Error('no vector table in a fresh profile') - } - }) -})) - import { getDB } from '../database' import { configureRuntime } from '../runtime-env' -import { setupIPC } from '../ipc' -import { llm } from '../llm' -import { listProjects } from '../rag/store' +import { ensureRagStoreSchema } from '../rag/store' +import { universalSearch } from '../search' + +const CHAT_KEY = 'chat:aurora-chat' beforeAll(() => { configureRuntime({ dataDir: TMP_DIR }) - setupIPC() - listProjects() - vi.spyOn(llm, 'chat').mockResolvedValue('{"intent":"chat","urls":[]}') - vi.spyOn(llm, 'chatStream').mockResolvedValue({ - content: 'Grounded answer.', - toolCalls: [], - finishReason: 'stop', - maxTokens: 200 - }) - getDB() - .prepare('INSERT INTO memories(content, source_app) VALUES (?, ?)') - .run('Aurora release code obsidian café 日本語', 'synthetic') - getDB().exec(` - CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - summary TEXT NOT NULL, - surface TEXT, - url TEXT, - ts DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE VIRTUAL TABLE IF NOT EXISTS observation_fts USING fts5( - summary, content='observations', content_rowid='id' - ); - CREATE TABLE IF NOT EXISTS frames ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - image_path TEXT, - text TEXT, - surface TEXT, - url TEXT, - ts DATETIME - ); - CREATE TABLE IF NOT EXISTS observation_frames (observation_id INTEGER, frame_id INTEGER); - CREATE TABLE IF NOT EXISTS meetings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - title TEXT, - summary TEXT, - transcript TEXT, - started_at INTEGER - ); - `) - try { - getDB().exec('ALTER TABLE entities ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0') - } catch { - /* already present */ - } + ensureRagStoreSchema() + const db = getDB() + db.prepare('INSERT INTO rag_conversations (id, title) VALUES (?, ?)').run( + 'aurora-chat', + 'Release' + ) + db.prepare('INSERT INTO rag_messages (conversation_id, role, content) VALUES (?, ?, ?)').run( + 'aurora-chat', + 'user', + 'Aurora release code obsidian café 日本語' + ) }) afterAll(() => { - vi.restoreAllMocks() getDB().close() configureRuntime({ dataDir: undefined }) fs.rmSync(TMP_DIR, { recursive: true, force: true }) }) -const event: IpcEvent = { sender: { send: () => undefined } } -const queries = [ - 'Aurora?', - '"Aurora"', - 'title:Aurora', - '(Aurora)', - 'Aurora AND OR NOT NEAR', - 'Aurora* ^release', - 'café 日本語' -] +/** FTS5 syntax around words the record holds: each still finds it. */ +const wordQueries = ['Aurora?', '"Aurora"', '(Aurora)', 'Aurora* ^release', 'café 日本語'] +/** Column filters, operators and no words at all: each is plain text, never a syntax error. Every + * typed word is required, and a query with no words opens on the newest records. */ +const syntaxQueries = ['title:Aurora', 'Aurora AND OR NOT NEAR', '', ' ', '?:()*^"', 'NEAR('] -describe('user text in IPC FTS fallbacks', () => { - it.each(queries)('retrieves memory for %j without FTS syntax errors', async (query) => { - const search = handlers.get('db:search-memories')! - await expect(search(event, query)).resolves.toEqual( - expect.arrayContaining([ - expect.objectContaining({ content: expect.stringContaining('Aurora') }) - ]) - ) - const chat = handlers.get('rag:chat')! - const reply = (await chat(event, query, 'All', [], null, 'fts-fallback-chat')) as { - answer: string - context: { memories: Array<{ content: string }> } - } - expect(reply.answer).toBe('Grounded answer.') - expect(reply.context.memories).toEqual( - expect.arrayContaining([ - expect.objectContaining({ content: expect.stringContaining('Aurora') }) - ]) - ) +describe('user text in search', () => { + it.each(wordQueries)('finds the chat for %j', async (query) => { + const results = await universalSearch(query, { semantic: false }) + expect(results.map((result) => result.key)).toContain(CHAT_KEY) }) - it.each(['', ' ', '?:()*^"'])('skips MATCH for empty terms in %j', async (query) => { - const prepared = vi.spyOn(getDB(), 'prepare') - try { - await expect(handlers.get('db:search-memories')!(event, query)).resolves.toEqual([]) - const reply = (await handlers.get('rag:chat')!( - event, - query, - 'All', - [], - null, - 'fts-empty-chat' - )) as { - answer: string - context: { memories: unknown[]; messages: unknown[] } - } - expect(reply.answer).toBe('Grounded answer.') - expect(reply.context).toMatchObject({ memories: [], messages: [] }) - expect(prepared.mock.calls.some(([sql]) => /\bMATCH\b/.test(sql))).toBe(false) - } finally { - prepared.mockRestore() - } + it.each(syntaxQueries)('searches %j without an FTS syntax error', async (query) => { + await expect(universalSearch(query, { semantic: false })).resolves.toBeInstanceOf(Array) }) }) diff --git a/src/main/__tests__/ipc-query-logic.test.ts b/src/main/__tests__/ipc-query-logic.test.ts deleted file mode 100644 index 5062c7c60..000000000 --- a/src/main/__tests__/ipc-query-logic.test.ts +++ /dev/null @@ -1,257 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { readFileSync } from 'fs' -import { join } from 'path' -import { DatabaseSync } from 'node:sqlite' -import { - tokenizeQuery, - ftsMatchExpression, - isGenerativeRequest, - clipText, - safeParseJson, - isTrivialMessage, - appNameLikeClause, - STOPWORDS -} from '../ipc-query-logic' - -describe('tokenizeQuery', () => { - it('lowercases and splits on punctuation and whitespace', () => { - expect(tokenizeQuery('Hello, World! Foobar')).toEqual(['hello', 'world', 'foobar']) - }) - - it('separates words at FTS punctuation and retains Unicode words', () => { - expect(tokenizeQuery('title:Aurora café 日本語 नमस्ते')).toEqual([ - 'title', - 'aurora', - 'café', - '日本語', - 'नमस्ते' - ]) - expect(ftsMatchExpression('?:()*^"')).toBe('') - expect(ftsMatchExpression(' ')).toBe('') - }) - - it('drops tokens shorter than 3 chars', () => { - // 'to' (2) and 'go' (2) are dropped; 'now' (3) kept - expect(tokenizeQuery('go to now')).toEqual(['now']) - }) - - it('drops stopwords', () => { - // every token here is a stopword → empty - expect(tokenizeQuery('what do you know about the project', 6)).toEqual(['project']) - // sanity: the words we dropped really are in STOPWORDS - expect(STOPWORDS.has('what')).toBe(true) - expect(STOPWORDS.has('about')).toBe(true) - }) - - it('de-duplicates tokens preserving first-seen order', () => { - expect(tokenizeQuery('alpha beta alpha gamma beta')).toEqual(['alpha', 'beta', 'gamma']) - }) - - it('caps the result at maxTokens (default 6)', () => { - const q = 'one2 two3 four5 six7 nine0 tenx elevenq twelvz' - expect(tokenizeQuery(q).length).toBe(6) - expect(tokenizeQuery(q, 2).length).toBe(2) - }) - - it('keeps underscores and hyphens inside a token', () => { - expect(tokenizeQuery('foo_bar baz-qux')).toEqual(['foo_bar', 'baz-qux']) - }) -}) - -describe('ftsMatchExpression', () => { - it('quotes each token as a phrase and OR-joins them (any-term recall)', () => { - expect(ftsMatchExpression('cafe food near')).toBe('"cafe" OR "food" OR "near"') - }) - - it('quotes a hyphenated token instead of leaving it bare in the MATCH', () => { - // tokenizeQuery keeps the hyphen, so a bare join produced `best-reviewed`, which FTS5 rejects. - const expr = ftsMatchExpression('best-reviewed places') - expect(expr).toContain('"best-reviewed"') - // never a bare hyphenated bareword adjacent to whitespace/operators - expect(expr).not.toMatch(/(^|\s)best-reviewed(\s|$)/) - }) - - it('uses quoted words when the stopword filter removes all terms', () => { - // all stopwords / too short → no tokens; the fallback must still be a quoted phrase, not raw text - expect(ftsMatchExpression('to of a')).toBe('"to" OR "of" OR "a"') - }) - - it('escapes an embedded double-quote so the phrase literal stays well-formed', () => { - expect(ftsMatchExpression('say "hey" now')).toBe('"say" OR "hey" OR "now"') - // punctuation in the fallback is removed - expect(ftsMatchExpression('a "b')).toBe('"a" OR "b"') - }) - - // The real regression: run the built expression against a REAL FTS5 table (node:sqlite ships FTS5) - // and prove SQLite accepts it. Before the fix the bare `best-reviewed` threw `no such column: - // reviewed`, which failed the entire rag:chat retrieval and surfaced as "something went wrong". - describe('against a real FTS5 table', () => { - const withFts = (fn: (db: InstanceType) => void): void => { - const db = new DatabaseSync(':memory:') - try { - db.exec('CREATE VIRTUAL TABLE entity_fts USING fts5(name, summary)') - db.prepare('INSERT INTO entity_fts(name, summary) VALUES (?, ?)').run( - 'Cafe Roma', - 'best reviewed place near good food' - ) - fn(db) - } finally { - db.close() - } - } - - const failingPrompt = - 'Find the three best-reviewed places near me for a specific kind of food, open right now.' - - it('reproduces the original failure with the old bare-join form', () => { - withFts((db) => { - const bare = tokenizeQuery(failingPrompt).join(' OR ') // the pre-fix expression - expect(bare).toContain('best-reviewed') - expect(() => - db.prepare('SELECT name FROM entity_fts WHERE entity_fts MATCH ?').all(bare) - ).toThrow(/no such column/) - }) - }) - - it('accepts the fixed expression and still matches the row', () => { - withFts((db) => { - const expr = ftsMatchExpression(failingPrompt) - const rows = db - .prepare('SELECT name FROM entity_fts WHERE entity_fts MATCH ?') - .all(expr) as { name: string }[] - expect(rows.map((r) => r.name)).toContain('Cafe Roma') - }) - }) - - it('never throws on hyphenated or punctuation-heavy input', () => { - withFts((db) => { - for (const q of [ - 'e-commerce back-end co-founder', - 'twenty-first', - 'a---b', - 'x: y', - '"quoted"' - ]) { - expect(() => - db - .prepare('SELECT name FROM entity_fts WHERE entity_fts MATCH ?') - .all(ftsMatchExpression(q)) - ).not.toThrow() - } - }) - }) - }) -}) - -describe('isGenerativeRequest', () => { - it('is true when a build verb and a code/UI noun co-occur', () => { - expect(isGenerativeRequest('build a react app')).toBe(true) - expect(isGenerativeRequest('write an svg')).toBe(true) - expect(isGenerativeRequest('make a landing page')).toBe(true) - expect(isGenerativeRequest('create a dashboard component')).toBe(true) - }) - - it('is false without a build verb', () => { - // has the noun "app" but no verb - expect(isGenerativeRequest('what is a react app')).toBe(false) - }) - - it('is false without a code/UI noun', () => { - expect(isGenerativeRequest('write a poem about the sea')).toBe(false) - }) - - it('is false for empty / whitespace input', () => { - expect(isGenerativeRequest('')).toBe(false) - expect(isGenerativeRequest(' ')).toBe(false) - }) -}) - -describe('clipText', () => { - it('returns text unchanged when under the limit', () => { - expect(clipText('hello', 10)).toBe('hello') - }) - - it('returns text unchanged when exactly at the limit', () => { - expect(clipText('hello', 5)).toBe('hello') - }) - - it('clips and appends an ellipsis when over the limit', () => { - // limit 5 → keep 4 chars + ellipsis, length stays 5 - expect(clipText('hello world', 5)).toBe('hell…') - }) - - it('returns empty string for empty/undefined input', () => { - expect(clipText('', 10)).toBe('') - expect(clipText(undefined as unknown as string, 10)).toBe('') - }) -}) - -describe('safeParseJson', () => { - it('parses valid JSON', () => { - expect(safeParseJson('{"a":1}', { a: 0 })).toEqual({ a: 1 }) - }) - - it('strips ```json fences before parsing', () => { - expect(safeParseJson('```json\n{"store":true}\n```', { store: false })).toEqual({ store: true }) - }) - - it('returns the fallback on invalid JSON', () => { - const fallback = { store: false } - expect(safeParseJson('not json at all', fallback)).toBe(fallback) - }) -}) - -describe('isTrivialMessage', () => { - it('treats empty / whitespace as trivial', () => { - expect(isTrivialMessage('')).toBe(true) - expect(isTrivialMessage(' ')).toBe(true) - }) - - it('treats short pleasantries as trivial (case-insensitive, optional punctuation)', () => { - expect(isTrivialMessage('hi')).toBe(true) - expect(isTrivialMessage('OK')).toBe(true) - expect(isTrivialMessage('Thanks!')).toBe(true) - expect(isTrivialMessage('thank you.')).toBe(true) - }) - - it('treats a real question as non-trivial', () => { - expect(isTrivialMessage('what did I work on yesterday?')).toBe(false) - }) - - it('a long message is never trivial even if it starts like a pleasantry', () => { - expect(isTrivialMessage('hello there, can you help me draft an email')).toBe(false) - }) -}) - -describe('appNameLikeClause — single source for the optional app-name filter', () => { - it('returns null for "All" or an empty/absent app (no filter = every app)', () => { - expect(appNameLikeClause('All', 'source_app')).toBeNull() - expect(appNameLikeClause('', 'source_app')).toBeNull() - expect(appNameLikeClause(undefined, 'source_app')).toBeNull() - }) - - it('builds a LIKE clause + wildcarded param for a specific app', () => { - expect(appNameLikeClause('Slack', 'source_app')).toEqual({ - clause: 'source_app LIKE ?', - param: '%Slack%' - }) - }) - - it('uses the column the caller passes (the 4 rag:chat sites differ)', () => { - expect(appNameLikeClause('Gmail', 'memories.source_app')).toEqual({ - clause: 'memories.source_app LIKE ?', - param: '%Gmail%' - }) - expect(appNameLikeClause('Gmail', 'c.app_name')).toEqual({ - clause: 'c.app_name LIKE ?', - param: '%Gmail%' - }) - }) - - it('ipc.ts builds every app-name filter from this helper, not an inline guard', () => { - // ipc.ts is a coverage-excluded IPC shell; guard the contract by reading source. - const src = readFileSync(join(__dirname, '../ipc.ts'), 'utf8') - expect(src).not.toMatch(/appName !== 'All'\)\s*\{[^}]*LIKE \?/s) // no inline appName+LIKE guard - expect(src).toContain('appNameLikeClause') - }) -}) diff --git a/src/main/__tests__/macos-artifact-integrity.integration.test.ts b/src/main/__tests__/macos-artifact-integrity.integration.test.ts index c82ad1e0b..6c636a954 100644 --- a/src/main/__tests__/macos-artifact-integrity.integration.test.ts +++ b/src/main/__tests__/macos-artifact-integrity.integration.test.ts @@ -306,12 +306,7 @@ describe('macOS artifact integrity', () => { const appOutDir = path.join(root, 'linux-unpacked') const resources = path.join(appOutDir, 'resources') writeAsarArchive(path.join(resources, 'app.asar'), ['out/main/index.js']) - for (const variant of [ - 'llama', - 'llama-cpu', - 'llama-prism', - 'llama-prism-cpu' - ]) { + for (const variant of ['llama', 'llama-cpu', 'llama-prism', 'llama-prism-cpu']) { const server = path.join(resources, 'bin', variant, 'llama-server') fs.mkdirSync(path.dirname(server), { recursive: true }) fs.writeFileSync(server, 'fixture') @@ -339,6 +334,13 @@ describe('macOS artifact integrity', () => { fs.mkdirSync(path.dirname(whisperCpu), { recursive: true }) fs.writeFileSync(whisperCpu, 'fixture') fs.writeFileSync(path.join(resources, 'bin', 'whisper', 'LICENSE'), 'fixture') + // Each Vulkan engine carries its loader, so a desktop without libvulkan1 still starts it. + await expect(verifyElectronBuilderArtifact(event)).rejects.toThrow( + 'installer input is missing required runtime: bin/llama/libvulkan.so.1' + ) + for (const folder of ['llama', 'llama-prism', 'whisper']) { + fs.writeFileSync(path.join(resources, 'bin', folder, 'libvulkan.so.1'), 'fixture') + } await expect(verifyElectronBuilderArtifact(event)).rejects.toThrow( 'installer input is missing required runtime: bin/ffmpeg' ) diff --git a/src/main/__tests__/mcp-server-action-gate.test.ts b/src/main/__tests__/mcp-server-action-gate.test.ts index ddbd4b5d7..8986fad16 100644 --- a/src/main/__tests__/mcp-server-action-gate.test.ts +++ b/src/main/__tests__/mcp-server-action-gate.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from 'vitest' // it can't be exercised in-process. This is a source-level regression guard for // the SECURITY contract instead: the action tools must be exposed ONLY to an // authorized request. If someone un-gates them (registers unconditionally, or -// stops threading isActionAuthorized into the build), this test fails. +// stops threading authorizeActionRequest into the build), this test fails. const SRC = readFileSync(path.join(__dirname, '..', 'mcp-server.ts'), 'utf8') describe('mcp-server action-tool gate', () => { diff --git a/src/main/__tests__/memory-rag-chat-lifecycle.integration.dbtest.ts b/src/main/__tests__/memory-rag-chat-lifecycle.integration.dbtest.ts index b125672df..7f4591c59 100644 --- a/src/main/__tests__/memory-rag-chat-lifecycle.integration.dbtest.ts +++ b/src/main/__tests__/memory-rag-chat-lifecycle.integration.dbtest.ts @@ -4,8 +4,7 @@ * This drives the same IPC handlers as the renderer through real file reading, the * document engine, SQLite persistence, the project search tool, prompt assembly, * deletion, and profile reopen. Only process boundaries are controlled: Electron - * registration, the MiniLM/LanceDB runtimes, and llama-server tokens over a real - * loopback socket. + * registration, the embedding model, and llama-server tokens over a real loopback socket. */ import fs from 'node:fs' import os from 'node:os' @@ -66,15 +65,6 @@ vi.mock('@huggingface/transformers', () => ({ } })) -vi.mock('@lancedb/lancedb', () => ({ - connect: async () => ({ - tableNames: async () => [], - openTable: async () => { - throw new Error('no external vector table in this profile') - } - }) -})) - const event: IpcEvent = { sender: { send: () => undefined } } let fake: FakeLlamaServer @@ -109,8 +99,10 @@ async function createCaptureTablesNow(): Promise { const { getDB } = await import('../database') getDB().exec(` CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, url TEXT, - category TEXT NOT NULL DEFAULT 'work', ts DATETIME DEFAULT CURRENT_TIMESTAMP, source_key TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, surface_app TEXT, + url TEXT, source_session_id TEXT, source_key TEXT, memory_id INTEGER, + category TEXT NOT NULL DEFAULT 'work', engagement REAL NOT NULL DEFAULT 0, + salience REAL NOT NULL DEFAULT 0, ts DATETIME DEFAULT CURRENT_TIMESTAMP ); CREATE VIRTUAL TABLE IF NOT EXISTS observation_fts USING fts5(summary, content='observations', content_rowid='id'); CREATE TABLE IF NOT EXISTS frames (id INTEGER PRIMARY KEY AUTOINCREMENT, image_path TEXT, text TEXT, surface TEXT, url TEXT, ts DATETIME, source TEXT); @@ -125,6 +117,26 @@ async function createCaptureTablesNow(): Promise { } const createCaptureTables = createCaptureTablesNow +/** A captured screen moment, as Pro's capture pipeline stores it. */ +async function captureMoment(summary: string): Promise { + const { getDB } = await import('../database') + const id = Number( + getDB().prepare("INSERT INTO observations (summary, surface) VALUES (?, 'Notes')").run(summary) + .lastInsertRowid + ) + getDB().prepare('INSERT INTO observation_fts(rowid, summary) VALUES (?, ?)').run(id, summary) + return id +} + +/** The user deletes a captured moment (Pro's capture deletion removes the row and its text). */ +async function deleteMoment(id: number, summary: string): Promise { + const { getDB } = await import('../database') + getDB() + .prepare("INSERT INTO observation_fts(observation_fts, rowid, summary) VALUES ('delete', ?, ?)") + .run(id, summary) + getDB().prepare('DELETE FROM observations WHERE id = ?').run(id) +} + async function invoke(channel: string, ...args: unknown[]): Promise { return (await handler(channel)(event, ...args)) as T } @@ -157,13 +169,18 @@ async function askProject( } const projectFiles = (answer: ProjectAnswer): string[] => [ - ...new Set(answer.unified.filter(({ kind }) => kind === 'attachment').map(({ surface }) => surface)) + ...new Set( + answer.unified.filter(({ kind }) => kind === 'attachment').map(({ surface }) => surface) + ) ] +/** Whether the answer drew on captured memory (a screen moment). */ const usedMemory = (answer: ProjectAnswer): boolean => - answer.unified.some(({ kind }) => kind === 'memory') + answer.unified.some(({ kind }) => kind === 'screen') /** The knowledge-base list once every file has been read and indexed. */ -async function settled(projectId: string): Promise> { +async function settled( + projectId: string +): Promise> { let rows: Array<{ id: number; name: string; enabled: boolean; state?: string }> = [] await vi.waitFor( async () => { @@ -215,8 +232,8 @@ describe('memory -> RAG -> scoped chat lifecycle', () => { boundary.selectedPaths = [otherDocument] expect(await invoke('projects:add-documents', otherProjectId)).toEqual({ added: 1 }) - const memory = await invoke<{ id: number }>('db:add-memory', capturedMemory, 'integration-test') - expect(Number(memory.id)).toBeGreaterThan(0) + const memoryId = await captureMoment(capturedMemory) + expect(memoryId).toBeGreaterThan(0) const currentConversation = 'cedar-current' const siblingConversation = 'cedar-sibling' @@ -261,7 +278,7 @@ describe('memory -> RAG -> scoped chat lifecycle', () => { expect(lastModelPrompt()).not.toContain(capturedMemory) expect(lastModelPrompt()).toContain(siblingContext) - expect(await invoke('db:delete-memory', Number(memory.id))).toBe(true) + await deleteMoment(memoryId, capturedMemory) const [oldDocument] = await invoke>( 'projects:list-documents', selectedProjectId @@ -302,7 +319,7 @@ describe('memory -> RAG -> scoped chat lifecycle', () => { const reopenedDb = (await import('../database')).getDB() expect( - reopenedDb.prepare('SELECT COUNT(*) AS count FROM memories WHERE id = ?').get(memory.id) + reopenedDb.prepare('SELECT COUNT(*) AS count FROM observations WHERE id = ?').get(memoryId) ).toEqual({ count: 0 }) // The replaced file's text left the document index: nothing holds it any more. expect( @@ -328,22 +345,24 @@ describe('memory -> RAG -> scoped chat lifecycle', () => { expect(lastModelPrompt()).not.toContain(otherChatContext) }) - it('includes saved memory in an All-memory answer through the public chat handler', async () => { + it('includes captured memory in an All-memory answer through the public chat handler', async () => { const fact = 'The Aurora release owner is Priya and the release code is obsidian.' - const saved = await invoke<{ id: number }>('db:add-memory', fact, 'integration-test') - expect(saved.id).toBeGreaterThan(0) + const id = await captureMoment(fact) fake.enqueue( - { content: '{"intent":"chat","urls":[]}' }, + { toolCalls: [{ name: 'search_memory', args: { query: 'Aurora release owner' } }] }, { content: 'Priya owns the Aurora release, code obsidian.' } ) - const reply = await invoke<{ - answer: string - context: { memories: Array<{ content: string }> } - }>('rag:chat', 'Who owns the Aurora release?', 'All', [], null, 'aurora-memory-chat', false, 'aurora-memory-turn', false, []) + const reply = await invoke('tools:chat', 'Who owns the Aurora release?', [], { + conversationId: 'aurora-memory-chat', + allMemory: true + }) expect(reply.answer).toBe('Priya owns the Aurora release, code obsidian.') - expect(reply.context.memories).toEqual(expect.arrayContaining([expect.objectContaining({ content: fact })])) + expect(reply.unified).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'screen', snippet: expect.stringContaining(fact) }) + ]) + ) expect(lastModelPrompt()).toContain(fact) - await invoke('db:delete-memory', saved.id) + await deleteMoment(id, fact) }) - }) diff --git a/src/main/__tests__/native-library-env.test.ts b/src/main/__tests__/native-library-env.test.ts index 25fcf9128..d73002e8c 100644 --- a/src/main/__tests__/native-library-env.test.ts +++ b/src/main/__tests__/native-library-env.test.ts @@ -15,6 +15,7 @@ describe('nativeLibraryEnv', () => { it('uses the Windows executable path only on Windows', () => { expect(nativeLibraryEnv('win32', 'C:\\app\\bin', { PATH: 'C:\\Windows' })).toEqual({ + GGML_VK_DISABLE_BFLOAT16: '1', PATH: 'C:\\app\\bin;C:\\Windows' }) }) diff --git a/src/main/__tests__/nvidia-driver.test.ts b/src/main/__tests__/nvidia-driver.test.ts new file mode 100644 index 000000000..cfb9be06a --- /dev/null +++ b/src/main/__tests__/nvidia-driver.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { parseLinuxDriverVersion } from '../nvidia-driver' + +describe('NVIDIA driver version', () => { + it('reads the kernel module version from /proc/driver/nvidia/version', () => { + expect( + parseLinuxDriverVersion( + 'NVRM version: NVIDIA UNIX x86_64 Kernel Module 550.54.14 Thu Feb 22 01:44:30 UTC 2024\nGCC version: gcc version 12.3.0\n' + ) + ).toBe('550.54.14') + expect( + parseLinuxDriverVersion( + 'NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 580.82.07 Release Build (dvs-builder@U16)\n' + ) + ).toBe('580.82.07') + }) + + it('returns null for text without a version', () => { + expect(parseLinuxDriverVersion('no driver here')).toBeNull() + }) +}) diff --git a/src/main/__tests__/ocr.dbtest.ts b/src/main/__tests__/ocr.dbtest.ts index 5987c0bc7..3f8e686c5 100644 --- a/src/main/__tests__/ocr.dbtest.ts +++ b/src/main/__tests__/ocr.dbtest.ts @@ -8,7 +8,7 @@ vi.mock('electron', () => ({ app: { isPackaged: false, getAppPath: () => '/app' } })) -import { runBoxedOCR, runOCR } from '../ocr' +import { runBoxedOCR } from '../ocr' beforeEach(() => { vi.clearAllMocks() @@ -22,7 +22,6 @@ describe('OCR helper boundary', () => { available: false, degradedReason: 'ocr_helper_missing' }) - await expect(runOCR('/tmp/frame.png')).resolves.toBe('') expect(mocks.exec).not.toHaveBeenCalled() }) @@ -58,12 +57,7 @@ describe('OCR helper boundary', () => { }) }) - it('returns trimmed text and degrades cleanly after process failures', async () => { - mocks.exec.mockImplementationOnce((_bin, _args, _options, callback) => - callback(null, { stdout: ' visible text \n', stderr: '' }) - ) - await expect(runOCR('/tmp/frame.png')).resolves.toBe('visible text') - + it('degrades cleanly after process failures', async () => { mocks.exec.mockImplementationOnce((_bin, _args, _options, callback) => callback(new Error('helper stopped')) ) diff --git a/src/main/__tests__/onnx-device.test.ts b/src/main/__tests__/onnx-device.test.ts index d6f4e7c45..6231c1017 100644 --- a/src/main/__tests__/onnx-device.test.ts +++ b/src/main/__tests__/onnx-device.test.ts @@ -1,15 +1,7 @@ import { describe, expect, it, vi } from 'vitest' -import { loadWithOnnxFallback, onnxDeviceCandidates } from '../onnx-device' +import { loadWithOnnxFallback } from '../onnx-device' describe('ONNX accelerator selection', () => { - it('uses the native GPU provider first and always keeps a CPU fallback', () => { - expect(onnxDeviceCandidates('darwin')).toEqual(['coreml', 'webgpu', 'cpu']) - expect(onnxDeviceCandidates('win32')).toEqual(['cuda', 'dml', 'webgpu', 'cpu']) - expect(onnxDeviceCandidates('linux')).toEqual(['cuda', 'webgpu', 'cpu']) - expect(onnxDeviceCandidates('linux', 'webgpu')).toEqual(['webgpu', 'cuda', 'cpu']) - expect(onnxDeviceCandidates('darwin', 'cpu')).toEqual(['cpu']) - }) - it('starts a fresh load with the next provider after a provider fails', async () => { const load = vi.fn(async (device: string) => { if (device === 'coreml') throw new Error('provider unavailable') @@ -26,9 +18,12 @@ describe('ONNX accelerator selection', () => { it('reports every provider failure when no runtime can start', async () => { await expect( - loadWithOnnxFallback(async (device) => { - throw new Error(`${device} unavailable`) - }, ['dml', 'cpu']) + loadWithOnnxFallback( + async (device) => { + throw new Error(`${device} unavailable`) + }, + ['dml', 'cpu'] + ) ).rejects.toThrow('dml: dml unavailable | cpu: cpu unavailable') }) }) diff --git a/src/main/__tests__/parser.test.ts b/src/main/__tests__/parser.test.ts deleted file mode 100644 index 7d2f7f9fd..000000000 --- a/src/main/__tests__/parser.test.ts +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Tests for chat message parsers - * - * These tests verify that each platform's parser correctly extracts: - * - User messages - * - Assistant messages - * - Chat titles - * - Window titles - * - Browser URLs - */ - -import { describe, it, expect } from 'vitest' -import { - parseClaudeDesktopOutput, - parseClaudeWebOutput, - parseChatGPTOutput, - parseGeminiOutput -} from '../parsers' - -// ============== TEST CASES ============== - -describe('Claude Desktop Parser', () => { - const sampleOutput = `[WINDOW_TITLE] Claude -[CHAT_TITLE] Debugging TypeScript Issues -[USER] Can you help me debug this error? -[ASSISTANT] Of course! Please share the error message. -[USER] TypeError: Cannot read property 'foo' of undefined -[ASSISTANT] This error typically occurs when you try to access a property on a variable that is undefined. -[ASSISTANT] Here are some common causes: -[ASSISTANT] 1. The object wasn't initialized -[ASSISTANT] 2. An async operation hasn't completed` - - it('should extract window title', () => { - const result = parseClaudeDesktopOutput(sampleOutput) - expect(result.windowTitle).toBe('Claude') - }) - - it('should extract chat title', () => { - const result = parseClaudeDesktopOutput(sampleOutput) - expect(result.chatTitle).toBe('Debugging TypeScript Issues') - }) - - it('should extract user messages', () => { - const result = parseClaudeDesktopOutput(sampleOutput) - const userMessages = result.messages.filter((m) => m.role === 'user') - expect(userMessages.length).toBe(2) - expect(userMessages[0]!.content).toBe('Can you help me debug this error?') - expect(userMessages[1]!.content).toBe("TypeError: Cannot read property 'foo' of undefined") - }) - - it('should extract assistant messages and combine consecutive ones', () => { - const result = parseClaudeDesktopOutput(sampleOutput) - const assistantMessages = result.messages.filter((m) => m.role === 'assistant') - expect(assistantMessages.length).toBe(2) - expect(assistantMessages[0]!.content).toBe('Of course! Please share the error message.') - expect(assistantMessages[1]!.content).toContain('This error typically occurs') - expect(assistantMessages[1]!.content).toContain("1. The object wasn't initialized") - }) - - it('should handle empty input', () => { - const result = parseClaudeDesktopOutput('') - expect(result.messages.length).toBe(0) - }) -}) - -describe('Claude Web Parser', () => { - const sampleOutput = `[WINDOW_TITLE] Chat - Claude - Brave -[BROWSER_URL] claude.ai/chat/abc123 -[CHAT_TITLE] API Integration Help -[USER] How do I use the fetch API? -[ASSISTANT] The fetch API is a modern way to make HTTP requests. -[ASSISTANT] Here's a basic example: -[USER] Thanks! What about error handling? -[ASSISTANT] You can use try/catch or .catch() for error handling.` - - it('should extract browser URL', () => { - const result = parseClaudeWebOutput(sampleOutput) - expect(result.browserUrl).toBe('claude.ai/chat/abc123') - }) - - it('should extract chat title', () => { - const result = parseClaudeWebOutput(sampleOutput) - expect(result.chatTitle).toBe('API Integration Help') - }) - - it('should extract all messages in order', () => { - const result = parseClaudeWebOutput(sampleOutput) - expect(result.messages.length).toBe(4) - expect(result.messages[0]!.role).toBe('user') - expect(result.messages[1]!.role).toBe('assistant') - expect(result.messages[2]!.role).toBe('user') - expect(result.messages[3]!.role).toBe('assistant') - }) - - it('should combine consecutive assistant messages', () => { - const result = parseClaudeWebOutput(sampleOutput) - const firstAssistant = result.messages[1]! - expect(firstAssistant.content).toContain('The fetch API') - expect(firstAssistant.content).toContain('basic example') - }) -}) - -describe('ChatGPT Parser', () => { - const sampleOutput = `[WINDOW_TITLE] ChatGPT - Brave -[BROWSER_URL] chatgpt.com/c/abc123 -[CHAT_TITLE] Team Context Reminder -[USER] You said: Rameez and me work together at Wednesday Solutions -[ASSISTANT] ChatGPT said: Got it. I'll remember that. -[ASSISTANT] If you want, I can remember this and use it for context.` - - it('should extract chat title', () => { - const result = parseChatGPTOutput(sampleOutput) - expect(result.chatTitle).toBe('Team Context Reminder') - }) - - it('should handle "You said:" role labels', () => { - const result = parseChatGPTOutput(sampleOutput) - const userMessages = result.messages.filter((m) => m.role === 'user') - expect(userMessages.length).toBeGreaterThanOrEqual(1) - }) - - it('should handle "ChatGPT said:" role labels', () => { - const result = parseChatGPTOutput(sampleOutput) - const assistantMessages = result.messages.filter((m) => m.role === 'assistant') - expect(assistantMessages.length).toBeGreaterThanOrEqual(1) - }) - - it('should filter out noise like "Chat history"', () => { - const noiseOutput = `[USER] Chat history -[USER] Search chats -[USER] Actual question here` - const result = parseChatGPTOutput(noiseOutput) - const contents = result.messages.map((m) => m.content) - expect(contents).not.toContain('Chat history') - expect(contents).not.toContain('Search chats') - }) - - it('should extract browser URL', () => { - const result = parseChatGPTOutput(sampleOutput) - expect(result.browserUrl).toBe('chatgpt.com/c/abc123') - }) -}) - -describe('Gemini Parser', () => { - const sampleOutput = `[WINDOW_TITLE] Google Gemini - Brave - Work -[BROWSER_URL] gemini.google.com/u/3/app/880725c54adcc1b7 -[CHAT_TITLE] System Check and Assistance Offered -[USER] Test -[ASSISTANT] System Check: Operational -[ASSISTANT] I am reading you loud and clear. Everything is working perfectly on my end. -[ASSISTANT] How can I assist you today? -[ASSISTANT] Would you like to start a new draft, brainstorm ideas, or search for information?` - - it('should extract chat title', () => { - const result = parseGeminiOutput(sampleOutput) - expect(result.chatTitle).toBe('System Check and Assistance Offered') - }) - - it('should extract browser URL', () => { - const result = parseGeminiOutput(sampleOutput) - expect(result.browserUrl).toBe('gemini.google.com/u/3/app/880725c54adcc1b7') - }) - - it('should extract user message', () => { - const result = parseGeminiOutput(sampleOutput) - const userMessages = result.messages.filter((m) => m.role === 'user') - expect(userMessages.length).toBe(1) - expect(userMessages[0]!.content).toBe('Test') - }) - - it('should combine consecutive assistant messages', () => { - const result = parseGeminiOutput(sampleOutput) - const assistantMessages = result.messages.filter((m) => m.role === 'assistant') - expect(assistantMessages.length).toBe(1) - expect(assistantMessages[0]!.content).toContain('System Check: Operational') - expect(assistantMessages[0]!.content).toContain('How can I assist you today?') - }) - - it('should filter out Gemini UI noise', () => { - const noiseOutput = `[USER] Gemini -[USER] Chats -[USER] New chat -[USER] Actual user question` - const result = parseGeminiOutput(noiseOutput) - expect(result.messages.length).toBe(1) - expect(result.messages[0]!.content).toBe('Actual user question') - }) - - it('should handle window title', () => { - const result = parseGeminiOutput(sampleOutput) - expect(result.windowTitle).toBe('Google Gemini - Brave - Work') - }) -}) - -describe('Cross-platform isolation', () => { - it('Claude Desktop parser should not use ChatGPT noise filtering', () => { - // "Chat history" (exact match) should be filtered by ChatGPT but not Claude - const input = `[USER] Chat history` - const claudeResult = parseClaudeDesktopOutput(input) - const chatgptResult = parseChatGPTOutput(input) - - // Claude should keep it, ChatGPT should filter it - expect(claudeResult.messages.length).toBe(1) - expect(chatgptResult.messages.length).toBe(0) - }) - - it('Gemini parser should filter Gemini-specific noise that ChatGPT does not', () => { - // "Show thinking" is Gemini-specific noise, not filtered by ChatGPT - const input = `[USER] Show thinking` - const geminiResult = parseGeminiOutput(input) - const chatgptResult = parseChatGPTOutput(input) - - // Gemini should filter it, ChatGPT should keep it - expect(geminiResult.messages.length).toBe(0) - expect(chatgptResult.messages.length).toBe(1) - }) - - it('Each parser should have independent state', () => { - // Run parsers in sequence and verify no state leakage - const input1 = `[USER] Message 1` - const input2 = `[ASSISTANT] Message 2` - - parseClaudeDesktopOutput(input1) - parseClaudeWebOutput(input1) - parseChatGPTOutput(input1) - parseGeminiOutput(input1) - - // Each subsequent call should start fresh - const result = parseClaudeDesktopOutput(input2) - expect(result.messages.length).toBe(1) - expect(result.messages[0]!.role).toBe('assistant') - }) -}) diff --git a/src/main/__tests__/performance-pack.test.ts b/src/main/__tests__/performance-pack.test.ts index 0a8f12997..8bd071f31 100644 --- a/src/main/__tests__/performance-pack.test.ts +++ b/src/main/__tests__/performance-pack.test.ts @@ -5,13 +5,14 @@ import path from 'node:path' import { createHash } from 'node:crypto' import { spawnSync } from 'node:child_process' -const fixture = vi.hoisted(() => ({ root: '', driver: true })) +const fixture = vi.hoisted(() => ({ root: '', driver: true, driverVersion: null as string | null })) vi.mock('../runtime-env', () => ({ dataDir: () => fixture.root, resourceFile: () => path.join(fixture.root, 'performance-packs.json') })) -vi.mock('../transcription/whisper-runtime', () => ({ +vi.mock('../nvidia-driver', () => ({ + cachedNvidiaDriverVersion: () => fixture.driverVersion, hasLinuxNvidiaDriver: () => fixture.driver, hasWindowsNvidiaDriver: () => fixture.driver })) @@ -19,12 +20,19 @@ vi.mock('../transcription/whisper-runtime', () => ({ const version = 'test-v1' const url = `https://runtime.getoffgridai.co/desktop/cuda/linux/${version}.tar.gz` let originalPlatform: PropertyDescriptor | undefined +let originalArch: PropertyDescriptor | undefined const originalLibraryPath = process.env.LD_LIBRARY_PATH -function archive(withSpeechLibraries = true): Buffer { +/** The pack's chat engine, as a script: `--list-devices` reports a CUDA GPU unless told not to. */ +function archive(withSpeechLibraries = true, withGpu = true): Buffer { const source = path.join(fixture.root, 'source', 'bin', 'llama-cuda') fs.mkdirSync(source, { recursive: true }) - fs.writeFileSync(path.join(source, 'llama-server'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }) + const devices = withGpu ? ' CUDA0: Test GPU (8192 MiB, 8000 MiB free)\\n' : '' + fs.writeFileSync( + path.join(source, 'llama-server'), + `#!/bin/sh\nprintf 'Available devices:\\n${devices}'\nexit 0\n`, + { mode: 0o755 } + ) if (withSpeechLibraries) { const onnx = path.join(fixture.root, 'source', 'bin', 'onnx-cuda') fs.mkdirSync(onnx, { recursive: true }) @@ -38,13 +46,18 @@ function archive(withSpeechLibraries = true): Buffer { } function manifest(bytes: Buffer, digest = createHash('sha256').update(bytes).digest('hex')): void { - fs.writeFileSync(path.join(fixture.root, 'performance-packs.json'), JSON.stringify({ - schemaVersion: 1, - cuda: { linux: { url, sha256: digest, bytes: bytes.length, version } } - })) + fs.writeFileSync( + path.join(fixture.root, 'performance-packs.json'), + JSON.stringify({ + schemaVersion: 1, + cuda: { linux: { url, sha256: digest, bytes: bytes.length, version } } + }) + ) } -async function settled(phase: 'installed' | 'failed'): Promise>> { +async function settled( + phase: 'installed' | 'failed' +): Promise>> { const { performancePackStatus } = await import('../performance-pack') for (let attempt = 0; attempt < 100; attempt++) { const status = performancePackStatus() @@ -58,8 +71,12 @@ beforeEach(() => { vi.resetModules() fixture.root = fs.mkdtempSync(path.join(os.tmpdir(), 'ogad-pack-test-')) fixture.driver = true + fixture.driverVersion = null originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + originalArch = Object.getOwnPropertyDescriptor(process, 'arch') Object.defineProperty(process, 'platform', { value: 'linux', configurable: true }) + // The pack carries x64 engines only. + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }) }) afterEach(() => { @@ -68,10 +85,18 @@ afterEach(() => { if (originalLibraryPath === undefined) delete process.env.LD_LIBRARY_PATH else process.env.LD_LIBRARY_PATH = originalLibraryPath if (originalPlatform) Object.defineProperty(process, 'platform', originalPlatform) + if (originalArch) Object.defineProperty(process, 'arch', originalArch) fs.rmSync(fixture.root, { recursive: true, force: true }) }) describe('optional NVIDIA performance pack', () => { + it('offers no download on an arm64 machine', async () => { + manifest(archive()) + Object.defineProperty(process, 'arch', { value: 'arm64', configurable: true }) + const pack = await import('../performance-pack') + expect(pack.performancePackStatus().phase).toBe('unavailable') + }) + it('offers no download without a pinned manifest or an NVIDIA driver', async () => { const pack = await import('../performance-pack') expect(pack.performancePackStatus().phase).toBe('unavailable') @@ -103,16 +128,103 @@ describe('optional NVIDIA performance pack', () => { expect(pack.startPerformancePack().phase).toBe('downloading') const status = await settled('installed') expect(status.downloadedBytes).toBe(bytes.length) - expect(status.restartRequired).toBe(true) + expect(status.device).toBe('Test GPU') expect(fetcher).toHaveBeenCalledOnce() - expect(fs.existsSync(path.join(process.env.OFFGRID_PERFORMANCE_PACK_BIN!, 'llama-cuda', 'llama-server'))).toBe(true) - expect(pack.prepareInstalledOnnxCudaLibraries()).toBe(true) - expect(process.env.LD_LIBRARY_PATH).toContain('/bin/onnx-cuda') - expect(pack.prepareInstalledOnnxCudaLibraries()).toBe(false) + expect( + fs.existsSync( + path.join(process.env.OFFGRID_PERFORMANCE_PACK_BIN!, 'llama-cuda', 'llama-server') + ) + ).toBe(true) + // The app never relaunches itself for the pack's libraries. + expect(process.env.LD_LIBRARY_PATH ?? '').not.toContain('onnx-cuda') expect(pack.startPerformancePack().phase).toBe('installed') - vi.resetModules() - const afterRestart = await import('../performance-pack') - expect(afterRestart.performancePackStatus().restartRequired).toBe(false) + }) + + it('does not offer the pack on a driver older than its floor', async () => { + const bytes = archive() + manifest(bytes) + fixture.driverVersion = '515.65.01' + const pack = await import('../performance-pack') + expect(pack.performancePackStatus()).toMatchObject({ + phase: 'driver-too-old', + driver: '515.65.01', + minDriver: '525.60.13' + }) + expect(pack.startPerformancePack().phase).toBe('driver-too-old') + }) + + it('installs a complete archive already on disk without downloading it again', async () => { + const bytes = archive() + const hash = createHash('sha256').update(bytes).digest('hex') + manifest(bytes, hash) + const packRoot = path.join( + fixture.root, + 'performance-packs', + `cuda-${version}-${hash.slice(0, 12)}` + ) + fs.mkdirSync(path.dirname(packRoot), { recursive: true }) + fs.writeFileSync(`${packRoot}.tar.gz.part`, bytes) + const fetcher = vi.fn() + vi.stubGlobal('fetch', fetcher) + const pack = await import('../performance-pack') + pack.startPerformancePack() + expect((await settled('installed')).device).toBe('Test GPU') + expect(fetcher).not.toHaveBeenCalled() + }) + + it('starts over after HTTP 416 instead of failing for good', async () => { + const bytes = archive() + const hash = createHash('sha256').update(bytes).digest('hex') + manifest(bytes, hash) + const packRoot = path.join( + fixture.root, + 'performance-packs', + `cuda-${version}-${hash.slice(0, 12)}` + ) + fs.mkdirSync(path.dirname(packRoot), { recursive: true }) + fs.writeFileSync(`${packRoot}.tar.gz.part`, bytes.subarray(0, 10)) + const fetcher = vi.fn(async (_url: string, options: RequestInit) => + options.headers + ? new Response(null, { status: 416 }) + : new Response(new Uint8Array(bytes), { status: 200 }) + ) + vi.stubGlobal('fetch', fetcher) + const pack = await import('../performance-pack') + pack.startPerformancePack() + expect((await settled('installed')).bytes).toBe(bytes.length) + expect(fetcher).toHaveBeenCalledTimes(2) + }) + + it('replaces a partial file larger than the pack', async () => { + const bytes = archive() + const hash = createHash('sha256').update(bytes).digest('hex') + manifest(bytes, hash) + const packRoot = path.join( + fixture.root, + 'performance-packs', + `cuda-${version}-${hash.slice(0, 12)}` + ) + fs.mkdirSync(path.dirname(packRoot), { recursive: true }) + fs.writeFileSync(`${packRoot}.tar.gz.part`, Buffer.concat([bytes, Buffer.from('extra')])) + const fetcher = vi.fn( + async (_url: string, _options: RequestInit) => + new Response(new Uint8Array(bytes), { status: 200 }) + ) + vi.stubGlobal('fetch', fetcher) + const pack = await import('../performance-pack') + pack.startPerformancePack() + expect((await settled('installed')).bytes).toBe(bytes.length) + expect(fetcher.mock.calls[0]?.[1]).toMatchObject({ headers: undefined }) + }) + + it('keeps the files out of use when the GPU does not start with them', async () => { + const bytes = archive(true, false) + manifest(bytes) + vi.stubGlobal('fetch', async () => new Response(new Uint8Array(bytes), { status: 200 })) + const pack = await import('../performance-pack') + pack.startPerformancePack() + expect((await settled('failed')).error).toMatch(/GPU did not start/) + expect(process.env.OFFGRID_PERFORMANCE_PACK_BIN).toBeUndefined() }) it('rejects an archive with the wrong hash and removes the partial file', async () => { @@ -122,7 +234,7 @@ describe('optional NVIDIA performance pack', () => { const pack = await import('../performance-pack') pack.startPerformancePack() const status = await settled('failed') - expect(status.error).toMatch(/hash check/) + expect(status.error).toMatch(/damaged/) expect(process.env.OFFGRID_PERFORMANCE_PACK_BIN).toBeUndefined() expect(fs.readdirSync(path.join(fixture.root, 'performance-packs'))).toEqual([]) }) @@ -140,7 +252,11 @@ describe('optional NVIDIA performance pack', () => { const bytes = archive() const hash = createHash('sha256').update(bytes).digest('hex') manifest(bytes, hash) - const packRoot = path.join(fixture.root, 'performance-packs', `cuda-${version}-${hash.slice(0, 12)}`) + const packRoot = path.join( + fixture.root, + 'performance-packs', + `cuda-${version}-${hash.slice(0, 12)}` + ) fs.mkdirSync(path.dirname(packRoot), { recursive: true }) const prefix = Math.floor(bytes.length / 2) fs.writeFileSync(`${packRoot}.tar.gz.part`, bytes.subarray(0, prefix)) @@ -162,14 +278,22 @@ describe('optional NVIDIA performance pack', () => { const bytes = archive() const hash = createHash('sha256').update(bytes).digest('hex') manifest(bytes, hash) - const packRoot = path.join(fixture.root, 'performance-packs', `cuda-${version}-${hash.slice(0, 12)}`) + const packRoot = path.join( + fixture.root, + 'performance-packs', + `cuda-${version}-${hash.slice(0, 12)}` + ) fs.mkdirSync(path.dirname(packRoot), { recursive: true }) const prefix = Math.floor(bytes.length / 2) fs.writeFileSync(`${packRoot}.tar.gz.part`, bytes.subarray(0, prefix)) - vi.stubGlobal('fetch', async () => new Response(new Uint8Array(bytes.subarray(prefix)), { - status: 206, - headers: { 'Content-Range': `bytes 0-${bytes.length - 1}/${bytes.length}` } - })) + vi.stubGlobal( + 'fetch', + async () => + new Response(new Uint8Array(bytes.subarray(prefix)), { + status: 206, + headers: { 'Content-Range': `bytes 0-${bytes.length - 1}/${bytes.length}` } + }) + ) const pack = await import('../performance-pack') pack.startPerformancePack() expect((await settled('failed')).error).toMatch(/invalid resume range/) diff --git a/src/main/__tests__/pure-branch-fills.test.ts b/src/main/__tests__/pure-branch-fills.test.ts index 7384149bf..15e522326 100644 --- a/src/main/__tests__/pure-branch-fills.test.ts +++ b/src/main/__tests__/pure-branch-fills.test.ts @@ -1,12 +1,10 @@ -// Remaining single-branch fills for three already-well-covered pure modules. Each +// Remaining single-branch fills for two already-well-covered pure modules. Each // case targets one uncovered branch left by the module's main test file: // - llama-error: an "unknown architecture" stderr with NO quoted arch name. // - chat-health: "ready" when there is no active model (the ?? undefined branch). -// - search-ranking: matchScore ignoring an empty-string term (the `t ?` false arm). import { describe, it, expect } from 'vitest' import { classifyLlamaError } from '../llama-error' import { decideChatStatus } from '../chat-health' -import { matchScore } from '../search-ranking' describe('classifyLlamaError - architecture without a quoted name', () => { it('reports engine_outdated with the generic reason when no arch is quoted', () => { @@ -32,13 +30,3 @@ describe('decideChatStatus - ready with no active model', () => { }) }) }) - -describe('matchScore - empty term is ignored', () => { - it('skips an empty-string term (the falsy-term branch) but still counts real ones', () => { - expect(matchScore('grid grid grid', ['', 'grid'])).toBe(3) - }) - - it('an all-empty term list scores 0', () => { - expect(matchScore('grid', ['', ''])).toBe(0) - }) -}) diff --git a/src/main/__tests__/rag-empty-memory.dbtest.ts b/src/main/__tests__/rag-empty-memory.dbtest.ts index 9daeffcaf..51bbb2bff 100644 --- a/src/main/__tests__/rag-empty-memory.dbtest.ts +++ b/src/main/__tests__/rag-empty-memory.dbtest.ts @@ -1,9 +1,9 @@ -// Fresh-profile integration coverage for RELEASE_TEST_CHECKLIST #35. +// Fresh-profile integration coverage for RELEASE_TEST_CHECKLIST #34 and #35. // -// This invokes the REAL `rag:chat` handler registered by setupIPC, over the REAL -// SQLite schema, retrieval queries, prompt assembly, modality queue, and LLM -// transport. Only true process boundaries are faked: Electron registration, -// MiniLM embeddings, LanceDB, and llama-server (a real loopback HTTP socket). +// This invokes the REAL `rag:chat` handler registered by setupIPC and the REAL chat tools loop +// (toolChat → search_memory → the retrieval engine), over the REAL SQLite schema, prompt assembly, +// modality queue, and LLM transport. Only true process boundaries are faked: Electron +// registration, the embedding model, and llama-server (a real loopback HTTP socket). import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import fs from 'fs' import os from 'os' @@ -50,19 +50,11 @@ vi.mock('@huggingface/transformers', () => ({ pipeline: async () => async () => ({ data: new Float32Array(384).fill(0.01) }) })) -vi.mock('@lancedb/lancedb', () => ({ - connect: async () => ({ - tableNames: async () => [], - openTable: async () => { - throw new Error('no vector table in a fresh profile') - } - }) -})) - import { getDB, saveSetting } from '../database' import { setupIPC } from '../ipc' import { llm } from '../llm' import { listProjects } from '../rag/store' +import { toolChat } from '../tools' let fake: FakeLlamaServer @@ -80,11 +72,19 @@ beforeAll(async () => { // empty here so universalSearch runs its complete production query set against // a genuinely empty corpus instead of short-circuiting on missing optional DDL. getDB().exec(` + -- As Pro's crm/schema.ts creates it. CREATE TABLE IF NOT EXISTS observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, + surface_app TEXT, url TEXT, + source_session_id TEXT, + source_key TEXT, + memory_id INTEGER, + category TEXT NOT NULL DEFAULT 'work', + engagement REAL NOT NULL DEFAULT 0, + salience REAL NOT NULL DEFAULT 0, ts DATETIME DEFAULT CURRENT_TIMESTAMP ); CREATE VIRTUAL TABLE IF NOT EXISTS observation_fts USING fts5( @@ -142,7 +142,7 @@ describe('rag:chat on an empty memory corpus', () => { expect(getActiveTranscription().isAvailable()).toBeTypeOf('boolean') }) - it('answers without context and releases the chat seam for the next turn', async () => { + it('answers a plain turn on an empty profile and releases the chat seam for the next', async () => { const handler = handlers.get('rag:chat') expect(handler).toBeTypeOf('function') expect( @@ -158,20 +158,13 @@ describe('rag:chat on an empty memory corpus', () => { .get() ).toEqual({ count: 0 }) - const streamed: Array<{ channel: string; payload: unknown }> = [] - const event: IpcEvent = { - sender: { - send: (channel, payload) => streamed.push({ channel, payload }) - } - } - + const event: IpcEvent = { sender: { send: () => undefined } } fake.enqueue( - { content: '{"intent":"chat","urls":[]}' }, { content: 'I do not have any saved memory yet, but I can still help.' }, - { content: '{"intent":"chat","urls":[]}' }, { content: 'This second response is ready.' } ) + // rag:chat is the direct model call (no retrieval); memory is the chat tools loop's. const first = (await handler!( event, 'What do you remember about me?', @@ -183,44 +176,19 @@ describe('rag:chat on an empty memory corpus', () => { 'empty-memory-turn-1', false, [] - )) as { - answer: string - context: Record - } - + )) as { answer: string; context?: unknown } expect(first.answer).toBe('I do not have any saved memory yet, but I can still help.') expect(first.answer).not.toMatch(/something went wrong/i) - expect(first.context).toMatchObject({ - memories: [], - messages: [], - summaries: [], - entities: [], - entityFacts: [], - unified: [] - }) - const modelPrompt = JSON.stringify(fake.requests[1]?.messages ?? []) - expect(modelPrompt).toContain('RELEVANT MEMORIES:\\n(none)') - expect(modelPrompt).toContain('RELEVANT MESSAGES:\\n(none)') - expect(modelPrompt).toContain('RELEVANT SUMMARIES:\\n(none)') - expect(modelPrompt).toContain('RELEVANT ENTITIES:\\n(none)') - expect(modelPrompt).toContain('RELEVANT ENTITY FACTS:\\n(none)') - expect(streamed).toContainEqual({ - channel: 'rag:stream', - payload: { - streamId: 'empty-memory-turn-1', - type: 'step', - step: { - kind: 'memory', - counts: { memories: 0, messages: 0, summaries: 0, entities: 0, facts: 0, unified: 0 } - } - } - }) + expect(first.context).toBeUndefined() const second = (await handler!( event, 'Can I ask another question?', 'All', - [], + [ + { role: 'user', content: 'What do you remember about me?' }, + { role: 'assistant', content: first.answer } + ], null, 'fresh-chat', false, @@ -228,14 +196,13 @@ describe('rag:chat on an empty memory corpus', () => { false, [] )) as { answer: string } - expect(second.answer).toBe('This second response is ready.') - expect(fake.requests).toHaveLength(4) + expect(fake.requests).toHaveLength(2) + expect(JSON.stringify(fake.requests[1])).toContain('What do you remember about me?') }) it('retrieves seeded local memory and returns its citation in All memory mode (#34)', async () => { - const handler = handlers.get('rag:chat') - expect(handler).toBeTypeOf('function') + fake.requests.length = 0 const db = getDB() const inserted = db .prepare( @@ -251,55 +218,28 @@ describe('rag:chat on an empty memory corpus', () => { inserted.lastInsertRowid, 'Project Aurora uses the release code obsidian.' ) - - const streamed: Array<{ channel: string; payload: unknown }> = [] - const event: IpcEvent = { - sender: { - send: (channel, payload) => streamed.push({ channel, payload }) - } - } fake.enqueue( - { content: '{"intent":"chat","urls":[]}' }, + { toolCalls: [{ name: 'search_memory', args: { query: 'Aurora release code' } }] }, { content: 'The Aurora release code is obsidian [S1].' } ) - const result = (await handler!( - event, - 'Aurora release code obsidian', - 'All', - [], - null, - 'seeded-memory-chat', - false, - 'seeded-memory-turn', - false, - [] - )) as { - answer: string - context: { unified: Array<{ refId: number; snippet: string; surface: string }> } - } + const result = await toolChat('Aurora release code obsidian', [], { + conversationId: 'seeded-memory-chat', + allMemory: true + }) expect(result.answer).toBe('The Aurora release code is obsidian [S1].') - expect(result.context.unified).toEqual([ + expect(result.unified).toEqual([ expect.objectContaining({ + key: `obs:${Number(inserted.lastInsertRowid)}`, refId: Number(inserted.lastInsertRowid), - snippet: 'Project Aurora uses the release code obsidian.', surface: 'Synthetic release notes' }) ]) + expect(result.unified[0]!.snippet).toContain('Project Aurora uses the release code obsidian.') + // The answer's model call saw the cited source with its tag. const answerPrompt = JSON.stringify(fake.requests.at(-1)?.messages ?? []) expect(answerPrompt).toContain('[S1]') expect(answerPrompt).toContain('Project Aurora uses the release code obsidian.') - expect(streamed).toContainEqual({ - channel: 'rag:stream', - payload: { - streamId: 'seeded-memory-turn', - type: 'step', - step: { - kind: 'memory', - counts: expect.objectContaining({ unified: 1 }) - } - } - }) }) }) diff --git a/src/main/__tests__/rag-store-integration.dbtest.ts b/src/main/__tests__/rag-store-integration.dbtest.ts index 72d2ebe95..5caf3964b 100644 --- a/src/main/__tests__/rag-store-integration.dbtest.ts +++ b/src/main/__tests__/rag-store-integration.dbtest.ts @@ -96,8 +96,15 @@ describe('rag/store.ts - projects CRUD', () => { addRagMessage('chat-sibling', 'user', 'Quartz budget is 300') addRagMessage('chat-other', 'user', 'Quartz budget is 400') + // Each hit names its conversation and when it was said, so a citation can open it, dated. expect(searchProjectConversations('p-chat-search', 'Quartz budget', 'chat-current')).toEqual([ - { role: 'user', content: 'Quartz budget is 300', title: 'Sibling' } + { + role: 'user', + content: 'Quartz budget is 300', + title: 'Sibling', + conversation_id: 'chat-sibling', + created_at: expect.any(String) + } ]) const { runTool } = await import('../tools') const result = await runTool( diff --git a/src/main/__tests__/record-sources.dbtest.ts b/src/main/__tests__/record-sources.dbtest.ts new file mode 100644 index 000000000..83545ff5d --- /dev/null +++ b/src/main/__tests__/record-sources.dbtest.ts @@ -0,0 +1,184 @@ +/** + * What the retrieval engine reads from each record source, against the real profile database: + * a question that is only a time ("what did I do yesterday") lists that day's records of every + * kind, a to-do reads as the user sees it, and a short run in a script written without spaces + * (two Chinese characters) still finds its record. Electron and the embedding model are the only + * controlled boundaries; the model is unavailable, so this is the keyword and time paths alone. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-record-sources-')) +const originalDataDir = process.env.OFFGRID_DATA_DIR +process.env.OFFGRID_DATA_DIR = PROFILE_DIR + +vi.mock('electron', () => ({ + app: { getPath: () => PROFILE_DIR, isPackaged: false, getAppPath: () => process.cwd() }, + BrowserWindow: { getAllWindows: () => [] }, + safeStorage: { + isEncryptionAvailable: () => false, + encryptString: (value: string) => Buffer.from(value), + decryptString: (value: Buffer) => value.toString() + } +})) + +vi.mock('@huggingface/transformers', () => ({ + env: {}, + pipeline: async () => { + throw new Error('no embedding model in this journey') + } +})) + +import { getDB } from '../database' +import { ensureRagStoreSchema } from '../rag/store' +import { ensureRecordSchema, recordSearch } from '../records' + +/** Noon, local time, on 2026-10-14; "yesterday" is the 13th. */ +const NOW = new Date(2026, 9, 14, 12, 0, 0).getTime() +const YESTERDAY_NOON = new Date(2026, 9, 13, 12, 0, 0).getTime() +const LAST_WEEK = new Date(2026, 9, 6, 12, 0, 0).getTime() + +/** SQLite's UTC text timestamp, as the capture pipeline stores it. */ +const sqliteTime = (ms: number): string => new Date(ms).toISOString().slice(0, 19).replace('T', ' ') + +const ids = { moment: 0, oldMoment: 0, todo: 0, done: 0, dismissed: 0, meeting: 0, chinese: 0 } + +function moment(summary: string, ms: number): number { + return Number( + getDB() + .prepare("INSERT INTO observations (summary, surface, ts) VALUES (?, 'Notes', ?)") + .run(summary, sqliteTime(ms)).lastInsertRowid + ) +} + +function todo( + text: string, + status: string, + ms: number, + extra: { due?: string; dueAt?: number; entity?: string } = {} +): number { + return Number( + getDB() + .prepare( + `INSERT INTO action_items (text, raw_text, status, created_at, due, due_at, entity_name, confidence) + VALUES (?, ?, ?, ?, ?, ?, ?, 0.9)` + ) + .run(text, text, status, ms, extra.due ?? null, extra.dueAt ?? null, extra.entity ?? null) + .lastInsertRowid + ) +} + +beforeAll(() => { + // Pro's capture, to-do and meeting tables, as its schema creates them; a core DB test cannot + // import Pro. + getDB().exec(` + CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, surface_app TEXT, + url TEXT, source_session_id TEXT, source_key TEXT, memory_id INTEGER, + category TEXT NOT NULL DEFAULT 'work', engagement REAL NOT NULL DEFAULT 0, + salience REAL NOT NULL DEFAULT 0, ts DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS frames (id INTEGER PRIMARY KEY AUTOINCREMENT, image_path TEXT, text TEXT, surface TEXT, url TEXT, ts DATETIME, source TEXT); + CREATE TABLE IF NOT EXISTS observation_frames (observation_id INTEGER, frame_id INTEGER); + CREATE TABLE IF NOT EXISTS action_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, text TEXT NOT NULL, raw_text TEXT, due TEXT, + entity_name TEXT, source_app TEXT, source_summary TEXT, source_ts INTEGER, + confidence REAL NOT NULL DEFAULT 0.6, status TEXT NOT NULL DEFAULT 'open', + created_at INTEGER NOT NULL DEFAULT 0, dedup_key TEXT UNIQUE, due_at INTEGER + ); + CREATE TABLE IF NOT EXISTS meetings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, transcript TEXT, summary TEXT, + started_at INTEGER, ended_at INTEGER, duration_sec INTEGER, created_at INTEGER NOT NULL DEFAULT 0 + ); + `) + ensureRagStoreSchema() + // The index and its triggers exist before anything is recorded, as at app start-up. + ensureRecordSchema() + + ids.moment = moment('Reviewed the vendor contract renewal terms', YESTERDAY_NOON) + ids.oldMoment = moment('Sketched the garden layout', LAST_WEEK) + ids.chinese = moment('明天的会议改到下午三点', YESTERDAY_NOON) + ids.todo = todo('Send the signed contract to legal', 'suggested', YESTERDAY_NOON, { + dueAt: new Date(2026, 9, 16, 17, 0, 0).getTime(), + entity: 'Priya' + }) + ids.done = todo('Book the venue', 'done', YESTERDAY_NOON, { due: 'by Friday' }) + ids.dismissed = todo('Order more staples', 'dismissed', YESTERDAY_NOON) + ids.meeting = Number( + getDB() + .prepare('INSERT INTO meetings (title, summary, started_at) VALUES (?, ?, ?)') + .run('Vendor sync', 'Agreed the renewal price', YESTERDAY_NOON).lastInsertRowid + ) +}) + +afterAll(() => { + getDB().close() + if (originalDataDir === undefined) delete process.env.OFFGRID_DATA_DIR + else process.env.OFFGRID_DATA_DIR = originalDataDir + fs.rmSync(PROFILE_DIR, { recursive: true, force: true }) +}) + +describe('record sources', () => { + it('lists the records of a day when the question is only that day', async () => { + const { hits, window } = await recordSearch('yesterday', { semantic: false, now: NOW }) + expect(window).toMatchObject({ + start: new Date(2026, 9, 13).getTime(), + end: new Date(2026, 9, 14).getTime() + }) + const keys = hits.map((hit) => hit.key) + expect(keys).toEqual( + expect.arrayContaining([ + `obs:${ids.moment}`, + `obs:${ids.chinese}`, + `act:${ids.todo}`, + `act:${ids.done}`, + `mtg:${ids.meeting}` + ]) + ) + // Another day's moment and a dismissed to-do are not that day's records. + expect(keys).not.toContain(`obs:${ids.oldMoment}`) + expect(keys).not.toContain(`act:${ids.dismissed}`) + }) + + it('reads a to-do as the list shows it: its state, who it is for and when it is due', async () => { + const { hits } = await recordSearch('signed contract legal', { + semantic: false, + window: null, + kinds: ['action'] + }) + expect(hits[0]).toMatchObject({ + key: `act:${ids.todo}`, + kind: 'action', + title: 'Suggested to-do · Priya', + surface: 'To-do' + }) + // A resolved deadline reads as its day. + expect(hits[0]!.text).toMatch(/^Send the signed contract to legal \(due .+16.+\)$/) + + const done = await recordSearch('book venue', { + semantic: false, + window: null, + kinds: ['action'] + }) + expect(done.hits[0]).toMatchObject({ + key: `act:${ids.done}`, + title: 'Done', + text: 'Book the venue (due by Friday)' + }) + }) + + it('finds a two-character run in a script written without spaces', async () => { + const { hits } = await recordSearch('会议', { semantic: false, window: null }) + expect(hits.map((hit) => hit.key)).toContain(`obs:${ids.chinese}`) + }) + + it('keeps a time-filtered keyword search inside its day', async () => { + const yesterday = await recordSearch('vendor yesterday', { semantic: false, now: NOW }) + const keys = yesterday.hits.map((hit) => hit.key) + expect(keys).toEqual(expect.arrayContaining([`obs:${ids.moment}`, `mtg:${ids.meeting}`])) + const lastMonth = await recordSearch('vendor last month', { semantic: false, now: NOW }) + expect(lastMonth.hits.map((hit) => hit.key)).not.toContain(`obs:${ids.moment}`) + }) +}) diff --git a/src/main/__tests__/record-vectors.dbtest.ts b/src/main/__tests__/record-vectors.dbtest.ts new file mode 100644 index 000000000..d0e7dca0e --- /dev/null +++ b/src/main/__tests__/record-vectors.dbtest.ts @@ -0,0 +1,224 @@ +/** + * Meaning search over records, against the real profile database: the embed queue the record + * triggers fill, the background drain that writes vectors into record_vectors, the in-memory int8 + * index, and the hydrated hits a search returns. Electron and the embedding model are the only + * controlled boundaries. The model is a bag-of-words embedder, so texts that share words are near + * each other and the ranking can be read from the test data. + */ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-record-vectors-')) +const originalDataDir = process.env.OFFGRID_DATA_DIR +process.env.OFFGRID_DATA_DIR = PROFILE_DIR + +const model = vi.hoisted(() => ({ calls: 0, texts: [] as string[] })) + +vi.mock('electron', () => ({ + app: { getPath: () => PROFILE_DIR, isPackaged: false, getAppPath: () => process.cwd() }, + BrowserWindow: { getAllWindows: () => [] }, + safeStorage: { + isEncryptionAvailable: () => false, + encryptString: (value: string) => Buffer.from(value), + decryptString: (value: Buffer) => value.toString() + } +})) + +vi.mock('@huggingface/transformers', () => { + const DIMENSIONS = 64 + // One dimension per word (by a stable hash), normalized: shared words are what make texts near. + const embed = (text: string): number[] => { + const vector = new Array(DIMENSIONS).fill(0) + const words = text + .replace(/^(query|passage): /, '') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((word) => word.length > 2) + for (const word of words) { + let hash = 0 + for (const char of word) hash = (hash * 31 + char.charCodeAt(0)) >>> 0 + vector[hash % DIMENSIONS]! += 1 + } + const norm = Math.hypot(...vector) || 1 + return vector.map((value) => value / norm) + } + return { + env: { backends: { onnx: { wasm: {} } } }, + pipeline: async () => async (input: string | string[]) => { + const texts = Array.isArray(input) ? input : [input] + model.calls++ + model.texts.push(...texts) + const rows = texts.map(embed) + return { + data: Float32Array.from(rows.flat()), + dims: [rows.length, DIMENSIONS], + tolist: () => rows + } + } + } +}) + +import { getDB } from '../database' +import { + embedRecordBacklog, + ensureRecordSchema, + recordIndexStatus, + recordSearch, + releaseRecordIndex +} from '../records' + +function moment(summary: string, ts = '2026-10-10 10:00:00'): number { + return Number( + getDB() + .prepare("INSERT INTO observations (summary, surface, ts) VALUES (?, 'Notes', ?)") + .run(summary, ts).lastInsertRowid + ) +} + +function meeting(title: string, summary: string): number { + return Number( + getDB() + .prepare('INSERT INTO meetings (title, summary, started_at) VALUES (?, ?, ?)') + .run(title, summary, Date.parse('2026-10-11T09:00:00Z')).lastInsertRowid + ) +} + +/** Meaning search alone. The Search screen's keywords need every word (keywordTypeahead), and no + * record holds this one, so every hit here comes from the vectors. */ +const UNMATCHED_WORD = 'xylophonequartz' + +async function meaning(question: string, kinds?: string[]): Promise { + const { hits } = await recordSearch(`${question} ${UNMATCHED_WORD}`, { + semantic: true, + keywordTypeahead: true, + window: null, + ...(kinds ? { kinds } : {}) + }) + return hits.map((hit) => hit.key) +} + +beforeAll(() => { + // Pro's capture and meeting tables, as its schema creates them; a core DB test cannot import Pro. + getDB().exec(` + CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, surface_app TEXT, + url TEXT, source_session_id TEXT, source_key TEXT, memory_id INTEGER, + category TEXT NOT NULL DEFAULT 'work', engagement REAL NOT NULL DEFAULT 0, + salience REAL NOT NULL DEFAULT 0, ts DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS frames (id INTEGER PRIMARY KEY AUTOINCREMENT, image_path TEXT, text TEXT, surface TEXT, url TEXT, ts DATETIME, source TEXT); + CREATE TABLE IF NOT EXISTS observation_frames (observation_id INTEGER, frame_id INTEGER); + CREATE TABLE IF NOT EXISTS meetings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, transcript TEXT, summary TEXT, + started_at INTEGER, ended_at INTEGER, duration_sec INTEGER, created_at INTEGER NOT NULL DEFAULT 0 + ); + `) + ensureRecordSchema() +}) + +beforeEach(() => { + releaseRecordIndex() + model.calls = 0 + model.texts.length = 0 +}) + +afterAll(() => { + releaseRecordIndex() + getDB().close() + if (originalDataDir === undefined) delete process.env.OFFGRID_DATA_DIR + else process.env.OFFGRID_DATA_DIR = originalDataDir + fs.rmSync(PROFILE_DIR, { recursive: true, force: true }) +}) + +describe('record vectors', () => { + it('embeds every queued record in the background, then finds them by meaning', async () => { + const budget = moment('Quarterly budget review with the finance team') + moment('Dentist appointment moved to Tuesday morning') + const launch = meeting('Product launch planning', 'Launch checklist and press timing agreed') + + expect(recordIndexStatus().pending).toBe(3) + await embedRecordBacklog() + expect(recordIndexStatus()).toEqual({ vectors: 3, pending: 0 }) + // Records are embedded as passages, in batches. + expect(model.texts.every((text) => text.startsWith('passage: '))).toBe(true) + + expect((await meaning('finance budget quarterly'))[0]).toBe(`obs:${budget}`) + // The question was embedded as a query. + expect(model.texts).toContain(`query: finance budget quarterly ${UNMATCHED_WORD}`) + // Keyword search alone finds nothing for it: the hits above are meaning hits. + const keywordOnly = await recordSearch(`finance budget quarterly ${UNMATCHED_WORD}`, { + typeahead: true, + window: null + }) + expect(keywordOnly.hits).toEqual([]) + expect(await meaning('launch checklist', ['meeting'])).toEqual([`mtg:${launch}`]) + // A kind with no records matches nothing. + expect(await meaning('budget', ['attachment'])).toEqual([]) + }) + + it('embeds again only what changed, and forgets a record whose text is gone', async () => { + const edited = moment('Hiking trip packing list') + const unchanged = moment('Garden watering schedule') + await embedRecordBacklog() + model.texts.length = 0 + + getDB() + .prepare('UPDATE observations SET summary = ? WHERE id = ?') + .run('Hiking trip permits and maps', edited) + getDB().prepare('UPDATE observations SET summary = summary WHERE id = ?').run(unchanged) + expect(recordIndexStatus().pending).toBe(2) + await embedRecordBacklog() + + // The unchanged record leaves the queue without a forward pass; the edited one is embedded. + expect(model.texts).toEqual(['passage: Hiking trip permits and maps']) + expect(recordIndexStatus().pending).toBe(0) + expect((await meaning('hiking permits maps'))[0]).toBe(`obs:${edited}`) + + getDB().prepare("UPDATE observations SET summary = '' WHERE id = ?").run(unchanged) + await embedRecordBacklog() + const stored = getDB() + .prepare('SELECT COUNT(*) AS n FROM record_vectors WHERE key = ?') + .get(`obs:${unchanged}`) + expect(stored).toEqual({ n: 0 }) + }) + + it('keeps a loaded index in step with writes and deletes', async () => { + const first = moment('Violin lesson notes on bowing') + await embedRecordBacklog() + // Load the index with one search, then write and delete while it is in memory. + expect((await meaning('violin bowing lesson'))[0]).toBe(`obs:${first}`) + + const second = moment('Cello rehearsal schedule for the orchestra') + await embedRecordBacklog() + expect((await meaning('cello orchestra rehearsal'))[0]).toBe(`obs:${second}`) + + getDB().prepare('DELETE FROM observations WHERE id = ?').run(first) + expect(await meaning('violin bowing lesson')).not.toContain(`obs:${first}`) + + // Dropped (idle, or its data deleted), the index reloads from SQLite on the next search. + releaseRecordIndex() + expect((await meaning('cello orchestra rehearsal'))[0]).toBe(`obs:${second}`) + }) + + it('searches only the time window it is given', async () => { + const october = moment('Tax filing documents gathered', '2026-10-05 12:00:00') + const july = moment('Tax filing extension requested', '2026-07-05 12:00:00') + await embedRecordBacklog() + const window = { + start: Date.parse('2026-10-01T00:00:00Z'), + end: Date.parse('2026-11-01T00:00:00Z'), + phrase: 'October' + } + const { hits } = await recordSearch(`tax filing ${UNMATCHED_WORD}`, { + semantic: true, + keywordTypeahead: true, + window, + kinds: ['screen'] + }) + const keys = hits.map((hit) => hit.key) + expect(keys).toContain(`obs:${october}`) + expect(keys).not.toContain(`obs:${july}`) + }) +}) diff --git a/src/main/__tests__/sd-server-fallback.test.ts b/src/main/__tests__/sd-server-fallback.test.ts index 37e31fa31..f073fa608 100644 --- a/src/main/__tests__/sd-server-fallback.test.ts +++ b/src/main/__tests__/sd-server-fallback.test.ts @@ -3,7 +3,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { spawnMock, findBinariesMock } = vi.hoisted(() => ({ spawnMock: vi.fn(), - findBinariesMock: vi.fn(() => ['/bin/sd-cuda/sd-server', '/bin/sd/sd-server']) + // The image ladder: CUDA first, then Vulkan (the shared backend policy decides the order). + findBinariesMock: vi.fn(async () => [ + { path: '/bin/sd-cuda/sd-server', backend: 'cuda', cpuOnly: false }, + { path: '/bin/sd/sd-server', backend: 'vulkan', cpuOnly: false } + ]) })) vi.mock('child_process', () => ({ spawn: spawnMock, execSync: vi.fn() })) vi.mock('fs', () => ({ default: { existsSync: () => true } })) @@ -19,7 +23,11 @@ vi.mock('../runtime-backends', () => ({ })) import { sdServer } from '../sd-server' -function processStub(): EventEmitter & { stdout: EventEmitter; stderr: EventEmitter; kill: ReturnType } { +function processStub(): EventEmitter & { + stdout: EventEmitter + stderr: EventEmitter + kill: ReturnType +} { const proc = Object.assign(new EventEmitter(), { stdout: new EventEmitter(), stderr: new EventEmitter(), @@ -45,40 +53,55 @@ describe('resident image runtime fallback', () => { }) it('starts Vulkan when CUDA cannot load and reports the selected binary', async () => { - spawnMock.mockImplementationOnce(() => { - const proc = processStub() - queueMicrotask(() => proc.emit('error', new Error('CUDA driver missing'))) - return proc - }).mockImplementation(() => processStub()) - vi.stubGlobal('fetch', vi.fn(async () => { - if (spawnMock.mock.calls.length < 2) throw new Error('not ready') - return { ok: true } - })) + spawnMock + .mockImplementationOnce(() => { + const proc = processStub() + queueMicrotask(() => proc.emit('error', new Error('CUDA driver missing'))) + return proc + }) + .mockImplementation(() => processStub()) + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + if (spawnMock.mock.calls.length < 2) throw new Error('not ready') + return { ok: true } + }) + ) const starting = sdServer.ensureUp({ modelPath: '/model.gguf' }) await vi.advanceTimersByTimeAsync(1000) await starting expect(spawnMock.mock.calls.map(([binary]) => binary)).toEqual([ - '/bin/sd-cuda/sd-server', '/bin/sd/sd-server' + '/bin/sd-cuda/sd-server', + '/bin/sd/sd-server' ]) expect(sdServer.getBinaryPath()).toBe('/bin/sd/sd-server') }) it('does not start a fallback after the user stops startup', async () => { spawnMock.mockImplementation(() => processStub()) - vi.stubGlobal('fetch', vi.fn(async () => { throw new Error('not ready') })) + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('not ready') + }) + ) const starting = sdServer.ensureUp({ modelPath: '/model.gguf' }) const rejected = expect(starting).rejects.toThrow() sdServer.stop() await vi.advanceTimersByTimeAsync(1000) await rejected - expect(spawnMock).toHaveBeenCalledTimes(1) + // Stopping during startup ends the ladder: at most the first engine was started. + expect(spawnMock.mock.calls.length).toBeLessThanOrEqual(1) }) it('keeps the loaded CUDA server for the same model and clears it after a crash', async () => { const proc = processStub() spawnMock.mockReturnValue(proc) - vi.stubGlobal('fetch', vi.fn(async () => ({ ok: true }))) + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: true })) + ) await sdServer.ensureUp({ modelPath: '/model.gguf' }) await sdServer.ensureUp({ modelPath: '/model.gguf' }) @@ -96,14 +119,20 @@ describe('resident image runtime fallback', () => { queueMicrotask(() => proc.emit('error', new Error('runtime failed'))) return proc }) - vi.stubGlobal('fetch', vi.fn(async () => { throw new Error('not ready') })) + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('not ready') + }) + ) const starting = sdServer.ensureUp({ modelPath: '/model.gguf' }) const rejected = expect(starting).rejects.toThrow('runtime failed') await vi.advanceTimersByTimeAsync(1000) await rejected expect(spawnMock.mock.calls.map(([binary]) => binary)).toEqual([ - '/bin/sd-cuda/sd-server', '/bin/sd/sd-server' + '/bin/sd-cuda/sd-server', + '/bin/sd/sd-server' ]) expect(sdServer.getBinaryPath()).toBeNull() }) diff --git a/src/main/__tests__/search-backfill-short-frames.dbtest.ts b/src/main/__tests__/search-backfill-short-frames.dbtest.ts index 1c9f50012..0618f2ae8 100644 --- a/src/main/__tests__/search-backfill-short-frames.dbtest.ts +++ b/src/main/__tests__/search-backfill-short-frames.dbtest.ts @@ -1,11 +1,11 @@ /** - * Search indexing drains capture frames too short to embed (backlog item 33), against the real + * Short capture frames never hold the search backlog open (backlog item 33), against the real * profile database. * - * The pending scans no longer measure frame text in SQL. A short frame is selected, skipped - * without an embedding, and recorded as handled, so the backlog empties instead of keeping it. - * Electron, the embedding model and the native vector store are the only controlled boundaries: - * the model throws if asked, so this proves a short frame never reaches it. + * A frame is not a search record: its words belong to the moment it is linked to, which is what + * gets embedded. So frames never enter the embedding queue, and a run of short frames ("ok", + * "see you at 3") leaves nothing pending. Electron and the embedding model are the only controlled + * boundaries: the model throws if asked, so this proves a frame never reaches it. */ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import fs from 'node:fs' @@ -30,9 +30,6 @@ vi.mock('@huggingface/transformers', () => ({ }, env: {} })) -vi.mock('@lancedb/lancedb', () => ({ - connect: async () => ({ openTable: async () => ({}), tableNames: async () => [] }) -})) import { getDB } from '../database' import { runBackfill, searchStatus } from '../search' @@ -59,18 +56,16 @@ afterAll(() => { }) describe('search indexing backlog', () => { - it('records frames too short to embed as handled, so the backlog empties', async () => { + it('never queues short capture frames, so the backlog stays empty', async () => { + expect((await searchStatus()).pending).toBe(0) const insert = getDB().prepare("INSERT INTO frames (surface, text) VALUES ('Slack', ?)") insert.run('ok') insert.run('see you at 3') - expect((await searchStatus()).pending).toBe(2) + expect((await searchStatus()).pending).toBe(0) await runBackfill() expect((await searchStatus()).pending).toBe(0) - const handled = getDB() - .prepare("SELECT key FROM vec_indexed WHERE key LIKE 'frame:%' ORDER BY key") - .all() as { key: string }[] - expect(handled.map((row) => row.key)).toEqual(['frame:1', 'frame:2']) + expect(getDB().prepare('SELECT COUNT(*) AS n FROM record_vectors').get()).toEqual({ n: 0 }) }) }) diff --git a/src/main/__tests__/search-backlog-count.dbtest.ts b/src/main/__tests__/search-backlog-count.dbtest.ts deleted file mode 100644 index 4d37ad7b8..000000000 --- a/src/main/__tests__/search-backlog-count.dbtest.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * The search backfill no longer recounts all seven pending sources after every batch. These - * journeys prove the progress it reports still equals the full seven-source count, that the - * backlog reaches an exact zero, and that items added during or after a run are still found. - * Electron, the embedding model and the vector store are the only replaced boundaries. - */ -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' - -const PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-backlog-count-')) -const originalDataDir = process.env.OFFGRID_DATA_DIR -process.env.OFFGRID_DATA_DIR = PROFILE_DIR - -vi.mock('electron', () => ({ - app: { getPath: () => PROFILE_DIR, isPackaged: false, getAppPath: () => process.cwd() }, - safeStorage: { - isEncryptionAvailable: () => false, - encryptString: (value: string) => Buffer.from(value), - decryptString: (value: Buffer) => value.toString() - } -})) -vi.mock('../embeddings', () => ({ - embeddings: { generateEmbedding: async () => [0.1, 0.2, 0.3] } -})) -vi.mock('../vectors', () => ({ - addChunks: async () => {}, - searchVectors: async () => [], - vectorCount: async () => 0 -})) - -import { getDB } from '../database' -import { runBackfill, searchStatus } from '../search' -import { PENDING_COUNT_SQL } from '../search-backfill-queries' - -const TEXT = 'a captured line long enough to be embedded' - -function exactPending(): number { - return (getDB().prepare(PENDING_COUNT_SQL).get() as { c: number }).c -} - -/** Add `n` pending items to each of the seven sources. */ -function seed(n: number, tag: string): void { - const db = getDB() - db.transaction(() => { - for (let index = 0; index < n; index++) { - const text = `${TEXT} ${tag} ${index}` - db.prepare("INSERT INTO frames (surface, text) VALUES ('Slack', ?)").run(text) - db.prepare("INSERT INTO observations (summary, surface) VALUES (?, 'Slack')").run(text) - db.prepare('INSERT INTO chat_summaries (session_id, summary) VALUES (?, ?)').run( - `s-${tag}-${index}`, - text - ) - db.prepare('INSERT INTO meetings (title, summary) VALUES (?, ?)').run('Standup', text) - db.prepare('INSERT INTO memories (content) VALUES (?)').run(text) - const entity = db - .prepare("INSERT INTO entities (name, type, summary, hidden) VALUES (?, 'Person', ?, 0)") - .run(`Person ${tag} ${index}`, text) - db.prepare('INSERT INTO entity_facts (entity_id, fact) VALUES (?, ?)').run( - entity.lastInsertRowid, - text - ) - } - })() -} - -beforeAll(() => { - // Pro's capture tables, as its schema creates them; a core DB test cannot import Pro. - getDB().exec(` - CREATE TABLE IF NOT EXISTS frames (id INTEGER PRIMARY KEY AUTOINCREMENT, ts DATETIME DEFAULT CURRENT_TIMESTAMP, surface TEXT, url TEXT, text TEXT, source TEXT); - CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT, surface TEXT, url TEXT, ts DATETIME); - CREATE TABLE IF NOT EXISTS meetings (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, summary TEXT, transcript TEXT, started_at INTEGER); - `) - try { - getDB().exec('ALTER TABLE entities ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0') - } catch { - /* already present */ - } -}) - -beforeEach(() => { - getDB().exec(` - DELETE FROM vec_indexed; DELETE FROM frames; DELETE FROM observations; DELETE FROM chat_summaries; - DELETE FROM meetings; DELETE FROM memories; DELETE FROM entity_facts; DELETE FROM entities; - `) -}) - -afterAll(() => { - getDB().close() - if (originalDataDir === undefined) delete process.env.OFFGRID_DATA_DIR - else process.env.OFFGRID_DATA_DIR = originalDataDir - fs.rmSync(PROFILE_DIR, { recursive: true, force: true }) -}) - -describe('search backfill backlog count', () => { - beforeAll(() => { - getDB().exec('CREATE TABLE IF NOT EXISTS vec_indexed (key TEXT PRIMARY KEY)') - }) - - it('reports the full seven-source count after every batch and finishes at zero', async () => { - seed(20, 'a') - expect((await searchStatus()).pending).toBe(140) - - const reported: { remaining: number; exact: number }[] = [] - await runBackfill((progress) => - reported.push({ remaining: progress.remaining, exact: exactPending() }) - ) - - expect(reported.length).toBeGreaterThan(2) - for (const step of reported) expect(step.remaining).toBe(step.exact) - expect(reported.at(-1)!.remaining).toBe(0) - expect((await searchStatus()).pending).toBe(0) - const sources = getDB() - .prepare( - "SELECT substr(key, 1, instr(key, ':') - 1) AS source, COUNT(*) AS n FROM vec_indexed GROUP BY source ORDER BY source" - ) - .all() - expect(sources).toEqual([ - { source: 'ent', n: 20 }, - { source: 'fact', n: 20 }, - { source: 'frame', n: 20 }, - { source: 'mem', n: 20 }, - { source: 'mtg', n: 20 }, - { source: 'obs', n: 20 }, - { source: 'sum', n: 20 } - ]) - }) - - it('does not report done while items added during the run are still pending', async () => { - seed(10, 'b') - let added = false - const reported: number[] = [] - await runBackfill((progress) => { - reported.push(progress.remaining) - if (!added) { - added = true - seed(3, 'late') - } - }) - - expect(reported.at(-1)).toBe(0) - expect(exactPending()).toBe(0) - expect( - (getDB().prepare('SELECT COUNT(*) AS n FROM vec_indexed').get() as { n: number }).n - ).toBe(13 * 7) - }) - - it('notices items added after a completed run', async () => { - seed(2, 'c') - await runBackfill() - expect((await searchStatus()).pending).toBe(0) - - seed(1, 'd') - expect((await searchStatus()).pending).toBe(7) - await runBackfill() - expect((await searchStatus()).pending).toBe(0) - }) -}) diff --git a/src/main/__tests__/search-ranking.test.ts b/src/main/__tests__/search-ranking.test.ts deleted file mode 100644 index b8b41943f..000000000 --- a/src/main/__tests__/search-ranking.test.ts +++ /dev/null @@ -1,423 +0,0 @@ -/** - * Regression tests for unified-search RANKING behaviour added this session: - * - recency bias (this week > last month > last quarter > older) - * - own-content nudge so chats / KB docs aren't buried under ambient captures - * - "Match" sort = literal term overlap - * Pure math (no DB/Electron), mirroring model-sizing.test.ts. - */ -import { describe, it, expect } from 'vitest' -import { - recencyBoost, - kindBoost, - matchScore, - queryTerms, - ftsExpr, - rrf, - fuseHits, - applyBoosts, - rankResults, - likeMatch, - LIKE_COLUMNS, - epochMsSql, - type RawHit, - type SearchResult -} from '../search-ranking' - -const DAY = 86_400_000 -const NOW = 1_800_000_000_000 // fixed "now" so tests are deterministic - -// A minimal RawHit for fusion tests. Overrides win over the defaults. -function hit(over: Partial & { key: string }): RawHit { - return { - kind: 'screen', - refId: 1, - title: 'Title', - snippet: 'Snippet', - surface: 'Screen', - url: null, - ts: 0, - ...over - } -} - -describe('recencyBoost — recent first, tiered', () => { - it('this week beats last month beats last quarter beats older', () => { - const week = recencyBoost(NOW - 3 * DAY, NOW) - const month = recencyBoost(NOW - 20 * DAY, NOW) - const quarter = recencyBoost(NOW - 60 * DAY, NOW) - const older = recencyBoost(NOW - 200 * DAY, NOW) - expect(week).toBeGreaterThan(month) - expect(month).toBeGreaterThan(quarter) - expect(quarter).toBeGreaterThan(older) - expect(older).toBe(0) - }) - - it('no timestamp → no boost', () => { - expect(recencyBoost(0, NOW)).toBe(0) - }) - - it('boost stays within ~one RRF rank (does not dominate)', () => { - // RRF top rank ≈ 1/60 ≈ 0.0167; the freshest boost must not exceed it, so a - // strong older multi-list match can still outrank a weak recent one. - expect(recencyBoost(NOW, NOW)).toBeLessThanOrEqual(1 / 60) - }) -}) - -describe('kindBoost — own deliberate content surfaces', () => { - it('chats and KB docs get a nudge; ambient kinds do not', () => { - expect(kindBoost('chat')).toBeGreaterThan(0) - expect(kindBoost('doc')).toBeGreaterThan(0) - expect(kindBoost('screen')).toBe(0) - expect(kindBoost('meeting')).toBe(0) - expect(kindBoost('entity')).toBe(0) - }) - - it('a fresh chat outranks an equally-ranked ambient screen capture', () => { - const base = 1 / 60 // both matched once - const chat = base + recencyBoost(NOW - 1 * DAY, NOW) + kindBoost('chat') - const screen = base + recencyBoost(NOW - 1 * DAY, NOW) + kindBoost('screen') - expect(chat).toBeGreaterThan(screen) - }) -}) - -describe('matchScore — literal term overlap for the Match sort', () => { - it('counts occurrences of each term, case-insensitive', () => { - expect(matchScore('Praveen and Mac, Praveen again', ['praveen'])).toBe(2) - expect(matchScore('Off Grid AI sync plan', ['off', 'grid'])).toBe(2) - }) - - it('ranks a denser match above a sparse one', () => { - const terms = ['off', 'grid'] - const dense = matchScore('Off Grid AI — Off Grid AI roadmap', terms) - const sparse = matchScore('a grid of icons', terms) - expect(dense).toBeGreaterThan(sparse) - }) - - it('no terms / empty haystack → 0', () => { - expect(matchScore('anything', [])).toBe(0) - expect(matchScore('', ['x'])).toBe(0) - }) -}) - -describe('queryTerms — the single tokeniser', () => { - it('lowercases, splits on punctuation/whitespace, drops symbols', () => { - expect(queryTerms('Off-Grid SYNC, plan!', 12)).toEqual(['off', 'grid', 'sync', 'plan']) - }) - - it('keeps unicode letters and digits', () => { - expect(queryTerms('café 2024 déjà', 12)).toEqual(['café', '2024', 'déjà']) - }) - - it('caps at max terms', () => { - expect(queryTerms('a b c d e', 3)).toEqual(['a', 'b', 'c']) - }) - - it('a punctuation-only / empty query yields no terms', () => { - expect(queryTerms(' --- ', 12)).toEqual([]) - expect(queryTerms('', 6)).toEqual([]) - }) -}) - -describe('ftsExpr — FTS5 prefix-match expression', () => { - it('wraps each of up to 12 tokens as a quoted prefix term', () => { - expect(ftsExpr('Off Grid AI')).toBe('"off"* "grid"* "ai"*') - }) - - it('drops punctuation so user input can never be an FTS syntax error', () => { - // A bare quote / paren would break MATCH; tokenising strips it. - expect(ftsExpr('a"b (c)')).toBe('"a"* "b"* "c"*') - }) - - it('caps at 12 terms', () => { - const q = Array.from({ length: 20 }, (_, i) => `t${i}`).join(' ') - expect(ftsExpr(q).split(' ').length).toBe(12) - }) - - it('empty / symbol-only query → empty match (callers treat as no keyword pass)', () => { - expect(ftsExpr('')).toBe('') - expect(ftsExpr('***')).toBe('') - }) -}) - -describe('rrf — reciprocal-rank fusion weight', () => { - it('rank 0 is the strongest and weights decay with rank', () => { - expect(rrf(0)).toBeCloseTo(1 / 60) - expect(rrf(0)).toBeGreaterThan(rrf(1)) - expect(rrf(1)).toBeGreaterThan(rrf(9)) - }) -}) - -describe('fuseHits — RRF fusion + dedupe by key', () => { - it('a key in two lists accumulates both lists’ rrf weights', () => { - const a: RawHit[] = [hit({ key: 'obs:1' }), hit({ key: 'obs:2' })] - const b: RawHit[] = [hit({ key: 'obs:2' }), hit({ key: 'obs:3' })] - const fused = fuseHits([a, b]) - // obs:2 = rank1 in a (rrf 1) + rank0 in b (rrf 0) -> two contributions. - expect(fused.get('obs:2')!.score).toBeCloseTo(rrf(1) + rrf(0)) - expect(fused.get('obs:1')!.score).toBeCloseTo(rrf(0)) - expect(fused.get('obs:3')!.score).toBeCloseTo(rrf(1)) - expect(fused.size).toBe(3) - }) - - it('an item ranked high in two lists beats one ranked high in only one', () => { - const a: RawHit[] = [hit({ key: 'both' }), hit({ key: 'solo' })] - const b: RawHit[] = [hit({ key: 'both' })] - const fused = fuseHits([a, b]) - expect(fused.get('both')!.score).toBeGreaterThan(fused.get('solo')!.score) - }) - - it('the first list a key appears in seeds its shape; later lists only add score', () => { - const a: RawHit[] = [hit({ key: 'k', title: 'FIRST', snippet: 'first snippet', surface: 'A' })] - const b: RawHit[] = [ - hit({ key: 'k', title: 'SECOND', snippet: 'second snippet', surface: 'B' }) - ] - const fused = fuseHits([a, b]) - const r = fused.get('k')! - expect(r.title).toBe('FIRST') - expect(r.snippet).toBe('first snippet') - expect(r.surface).toBe('A') - }) - - it('caps snippet at 280 chars and falls back title/surface/ts when blank', () => { - const long = 'x'.repeat(400) - const fused = fuseHits([ - [hit({ key: 'k', kind: 'memory', title: '', snippet: long, surface: '', ts: 0 })] - ]) - const r = fused.get('k')! - expect(r.snippet.length).toBe(280) - expect(r.title).toBe('memory') // empty title → kind - expect(r.surface).toBe('') - expect(r.ts).toBe(0) - expect(r.imagePath).toBeNull() - }) - - it('empty input → empty map; disjoint lists keep every key once', () => { - expect(fuseHits([]).size).toBe(0) - expect(fuseHits([[], []]).size).toBe(0) - const fused = fuseHits([[hit({ key: 'a' })], [hit({ key: 'b' })]]) - expect([...fused.keys()].sort()).toEqual(['a', 'b']) - }) - - it('a single source still fuses (scores follow rank order)', () => { - const fused = fuseHits([[hit({ key: 'a' }), hit({ key: 'b' }), hit({ key: 'c' })]]) - expect(fused.get('a')!.score).toBeGreaterThan(fused.get('b')!.score) - expect(fused.get('b')!.score).toBeGreaterThan(fused.get('c')!.score) - }) -}) - -describe('fuseHits — snippets read as prose', () => { - it('drops markdown, fences and tags from a stored chat or note', () => { - const fused = fuseHits([ - [ - hit({ - key: 'chat:1', - snippet: - '## Plan\n**Ship** the `beta` on [Friday](https://x.test).\n```html\n
Hi
\n```' - }) - ] - ]) - expect(fused.get('chat:1')!.snippet).toBe('Plan Ship the beta on Friday. Hi') - }) - - it('keeps snake_case words and caps the snippet at 280 characters', () => { - const fused = fuseHits([[hit({ key: 'a', snippet: `use search_messages ${'x'.repeat(400)}` })]]) - const snippet = fused.get('a')!.snippet - expect(snippet.startsWith('use search_messages ')).toBe(true) - expect(snippet).toHaveLength(280) - }) -}) - -describe('applyBoosts — recency + own-content nudge in place', () => { - it('adds the recency and kind boosts to each result score', () => { - const fused = fuseHits([[hit({ key: 'chat:1', kind: 'chat', ts: NOW - 1 * DAY })]]) - const before = fused.get('chat:1')!.score - applyBoosts(fused.values(), NOW) - expect(fused.get('chat:1')!.score).toBeCloseTo( - before + recencyBoost(NOW - 1 * DAY, NOW) + kindBoost('chat') - ) - }) - - it('a screen capture with no timestamp gets no boost', () => { - const fused = fuseHits([[hit({ key: 'obs:1', kind: 'screen', ts: 0 })]]) - const before = fused.get('obs:1')!.score - applyBoosts(fused.values(), NOW) - expect(fused.get('obs:1')!.score).toBe(before) - }) -}) - -describe('rankResults — filter then sort', () => { - // Build three fused results with known scores/timestamps. - const results = (): SearchResult[] => [ - { - key: 'obs:1', - kind: 'screen' as const, - refId: 1, - title: 'Off Grid AI plan', - snippet: 'grid grid', - surface: 'Slack', - url: null, - ts: NOW - 100 * DAY, - imagePath: null, - score: 0.1 - }, - { - key: 'chat:1', - kind: 'chat' as const, - refId: 0, - title: 'A chat', - snippet: 'off topic', - surface: 'Chat', - url: 'c1', - ts: NOW - 1 * DAY, - imagePath: null, - score: 0.3 - }, - { - key: 'mem:1', - kind: 'memory' as const, - refId: 2, - title: 'note', - snippet: 'nothing', - surface: 'Memory', - url: null, - ts: NOW - 5 * DAY, - imagePath: null, - score: 0.2 - } - ] - - it('relevance (default) sorts by descending score', () => { - const out = rankResults(results(), { query: 'Off Grid AI' }) - expect(out.map((r) => r.key)).toEqual(['chat:1', 'mem:1', 'obs:1']) - }) - - it('recency sorts newest first, score breaks ties', () => { - const out = rankResults(results(), { query: 'x', sort: 'recency' }) - expect(out.map((r) => r.key)).toEqual(['chat:1', 'mem:1', 'obs:1']) - }) - - it('match sorts by literal term overlap in title+snippet, score breaks ties', () => { - // "grid" appears twice in obs:1 snippet + once in its title -> densest match. - const out = rankResults(results(), { query: 'Off Grid AI', sort: 'match' }) - expect(out[0]!.key).toBe('obs:1') - }) - - it('match sort is NOT capped at 12 terms (uses the full query)', () => { - // 13th term is the only one that matches -> it must still count. - const many = [ - 'a1', - 'a2', - 'a3', - 'a4', - 'a5', - 'a6', - 'a7', - 'a8', - 'a9', - 'a10', - 'a11', - 'a12', - 'zebra' - ].join(' ') - const rs = [ - { - key: 'a', - kind: 'screen' as const, - refId: 1, - title: 'nothing', - snippet: 'here', - surface: 'X', - url: null, - ts: 0, - imagePath: null, - score: 0.5 - }, - { - key: 'b', - kind: 'screen' as const, - refId: 2, - title: 'zebra', - snippet: 'zebra', - surface: 'X', - url: null, - ts: 0, - imagePath: null, - score: 0.1 - } - ] - const out = rankResults(rs, { query: many, sort: 'match' }) - expect(out[0]!.key).toBe('b') // matched the 13th term despite lower score - }) - - it('source filter keeps only matching surfaces, case-insensitive', () => { - const out = rankResults(results(), { query: 'x', sources: ['slack'] }) - expect(out.map((r) => r.key)).toEqual(['obs:1']) - }) - - it('kind filter gives Replay only captured screen results', () => { - const out = rankResults(results(), { query: 'x', kinds: ['screen'] }) - expect(out.map((r) => r.key)).toEqual(['obs:1']) - }) - - it('excludeChatId drops that exact chat and nothing else', () => { - const out = rankResults(results(), { query: 'x', excludeChatId: '1' }) - expect(out.map((r) => r.key)).not.toContain('chat:1') - expect(out.map((r) => r.key)).toContain('obs:1') - }) - - it('empty input → empty output', () => { - expect(rankResults([], { query: 'x' })).toEqual([]) - }) -}) - -describe('likeMatch — shared LIKE WHERE + params (facet counts == results)', () => { - it('builds one OR-group per term over the columns, AND-ed together', () => { - const { where } = likeMatch(['a', 'b'], ['x', 'y']) - expect(where).toBe( - '(lower(a) LIKE ? OR lower(b) LIKE ?) AND (lower(a) LIKE ? OR lower(b) LIKE ?)' - ) - }) - - it('binds columns.length lowercased-wildcard params per term, in order', () => { - const { args } = likeMatch(['a', 'b'], ['x', 'y']) - expect(args).toEqual(['%x%', '%x%', '%y%', '%y%']) - }) - - it('clause placeholder count always equals the params length (can never disagree)', () => { - for (const cols of Object.values(LIKE_COLUMNS)) { - const { where, args } = likeMatch(cols, ['foo', 'bar', 'baz']) - expect((where.match(/\?/g) || []).length).toBe(args.length) - expect(args.length).toBe(cols.length * 3) - } - }) - - it('handles a single-column source (frame) as a bare OR-group of one', () => { - expect(likeMatch(LIKE_COLUMNS.frame, ['q'])).toEqual({ - where: '(lower(text) LIKE ?)', - args: ['%q%'] - }) - }) - - it('empty terms yield an empty clause and no params', () => { - expect(likeMatch(['a'], [])).toEqual({ where: '', args: [] }) - }) - - it('the column sets match the search.ts queries (facet + hit both build from these)', () => { - // Guard the single source: these are the columns the two divergent copies used. - expect(LIKE_COLUMNS.chat).toEqual(['rm.content', 'rc.title']) - expect(LIKE_COLUMNS.doc).toEqual(['c.content', 'd.name']) - expect(LIKE_COLUMNS.meeting).toEqual(['title', 'summary', 'transcript']) - }) -}) - -describe('epochMsSql — the shared SQLite datetime → epoch-ms fragment', () => { - it('produces the exact fragment the 6 search.ts SELECT sites used', () => { - expect(epochMsSql('ts')).toBe("CAST(strftime('%s', ts) AS INTEGER)*1000") - }) - - it('interpolates a qualified column identifier verbatim', () => { - expect(epochMsSql('rc.updated_at')).toBe("CAST(strftime('%s', rc.updated_at) AS INTEGER)*1000") - expect(epochMsSql('d.created_at')).toBe("CAST(strftime('%s', d.created_at) AS INTEGER)*1000") - expect(epochMsSql('o.ts')).toBe("CAST(strftime('%s', o.ts) AS INTEGER)*1000") - }) -}) diff --git a/src/main/__tests__/settings-consumers.dbtest.ts b/src/main/__tests__/settings-consumers.dbtest.ts index f9d883ca1..e15476cb7 100644 --- a/src/main/__tests__/settings-consumers.dbtest.ts +++ b/src/main/__tests__/settings-consumers.dbtest.ts @@ -45,15 +45,15 @@ describe('prompt persistence', () => { }) it('reads, fills, and resets a persisted custom template', () => { - db.saveSetting('prompt:ragChat', 'Question: {{QUERY}}') + db.saveSetting('prompt:sessionSummary', 'Summarize: {{CONVERSATION_TEXT}}') - expect(getPrompt('ragChat', { QUERY: 'where is the data?' })).toBe( - 'Question: where is the data?' + expect(getPrompt('sessionSummary', { CONVERSATION_TEXT: 'we met about the budget' })).toBe( + 'Summarize: we met about the budget' ) - resetPrompt('ragChat') - expect(getPromptTemplate('ragChat')).toBe( - PROMPT_REGISTRY.find((prompt) => prompt.key === 'ragChat')?.defaultTemplate + resetPrompt('sessionSummary') + expect(getPromptTemplate('sessionSummary')).toBe( + PROMPT_REGISTRY.find((prompt) => prompt.key === 'sessionSummary')?.defaultTemplate ) }) }) diff --git a/src/main/__tests__/tools-search.dbtest.ts b/src/main/__tests__/tools-search.dbtest.ts index 646cb9fdd..a8185ae8c 100644 --- a/src/main/__tests__/tools-search.dbtest.ts +++ b/src/main/__tests__/tools-search.dbtest.ts @@ -64,7 +64,8 @@ beforeAll(async () => { db.exec(` CREATE TABLE IF NOT EXISTS observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, surface TEXT, url TEXT, - category TEXT NOT NULL DEFAULT 'work', ts DATETIME DEFAULT CURRENT_TIMESTAMP, source_key TEXT + category TEXT NOT NULL DEFAULT 'work', ts DATETIME DEFAULT CURRENT_TIMESTAMP, source_key TEXT, + salience REAL NOT NULL DEFAULT 0 ); CREATE VIRTUAL TABLE IF NOT EXISTS observation_fts USING fts5(summary, content='observations', content_rowid='id'); CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN diff --git a/src/main/__tests__/tts-onnx-worker.test.ts b/src/main/__tests__/tts-onnx-worker.test.ts index 53fa500a2..14447abcb 100644 --- a/src/main/__tests__/tts-onnx-worker.test.ts +++ b/src/main/__tests__/tts-onnx-worker.test.ts @@ -11,21 +11,14 @@ const host = vi.hoisted(() => ({ readDirectories: [] as string[], device: 'coreml' as string })) -vi.mock('node:worker_threads', () => ({ - parentPort: { on: host.on, postMessage: host.post }, - workerData: { - get modelsDir() { - return host.modelsDir - }, - get modelReadDirectories() { - return host.readDirectories - } - } -})) +// The process boundary: a helper process's parent port (utility-host.ts sends init, then requests). +Object.defineProperty(process, 'parentPort', { + value: { on: host.on, postMessage: host.post }, + configurable: true +}) vi.mock('kokoro-js', () => ({ KokoroTTS: { from_pretrained: host.load } })) vi.mock('../embeddings-env', () => ({ configureTransformersEnv: vi.fn() })) vi.mock('../onnx-device', () => ({ - onnxDeviceCandidates: () => [host.device, 'cpu'], loadWithOnnxFallback: async (load: (device: string) => Promise) => ({ runtime: await load(host.device), device: host.device @@ -33,6 +26,23 @@ vi.mock('../onnx-device', () => ({ })) let profile: string let output: string + +/** Load the helper, send it the host's init message, and return its request channel. */ +async function startWorker(): Promise<(request: unknown) => void> { + await import('../tts-onnx-worker') + const listener = host.on.mock.calls[0]![1] as (event: { data: unknown }) => void + listener({ + data: { + type: 'init', + data: { + modelsDir: host.modelsDir, + modelReadDirectories: host.readDirectories, + devices: [host.device, 'cpu'] + } + } + }) + return (request) => listener({ data: request }) +} beforeEach(() => { profile = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-onnx-speech-'))) host.modelsDir = path.join(profile, 'models') @@ -55,7 +65,7 @@ beforeEach(() => { host.generate.mockResolvedValue({ data: new Float32Array([-1, 0, 1]), sampling_rate: 24000 }) }) afterEach(() => fs.rmSync(profile, { recursive: true, force: true })) -describe('ONNX speech worker response evidence', () => { +describe('voice helper response evidence', () => { it('loads a complete scan-folder package while the chosen folder is offline', async () => { const scan = path.join(profile, 'scan') const packageRoot = path.join(scan, '.cache/onnx-community/Kokoro-82M-v1.0-ONNX') @@ -72,8 +82,7 @@ describe('ONNX speech worker response evidence', () => { } host.readDirectories.push(scan) fs.rmdirSync(host.modelsDir) - await import('../tts-onnx-worker') - host.on.mock.calls[0]![1]({ id: 8, type: 'prepare', voice: 'af_heart' }) + ;(await startWorker())({ id: 8, type: 'prepare', voice: 'af_heart' }) await vi.waitFor(() => expect(host.post).toHaveBeenCalledWith({ id: 8, type: 'complete', device: 'coreml' }) ) @@ -86,8 +95,7 @@ describe('ONNX speech worker response evidence', () => { it('refuses a speech download when storage is offline and the package is incomplete', async () => { fs.rmdirSync(host.modelsDir) - await import('../tts-onnx-worker') - host.on.mock.calls[0]![1]({ id: 9, type: 'prepare', voice: 'af_heart' }) + ;(await startWorker())({ id: 9, type: 'prepare', voice: 'af_heart' }) await vi.waitFor(() => expect(host.post).toHaveBeenCalledWith( expect.objectContaining({ @@ -103,14 +111,14 @@ describe('ONNX speech worker response evidence', () => { it('uses FP32 for CUDA speech so the T4 does not return NaN audio', async () => { host.device = 'cuda' - await import('../tts-onnx-worker') - host.on.mock.calls[0]![1]({ id: 7, type: 'prepare', voice: 'af_heart' }) - await vi.waitFor(() => expect(host.post).toHaveBeenCalledWith({ id: 7, type: 'complete', device: 'cuda' })) + ;(await startWorker())({ id: 7, type: 'prepare', voice: 'af_heart' }) + await vi.waitFor(() => + expect(host.post).toHaveBeenCalledWith({ id: 7, type: 'complete', device: 'cuda' }) + ) expect(host.load.mock.calls[0]![1]).toMatchObject({ dtype: 'fp32', device: 'cuda' }) }) it('reports the actual provider, emits progress and reuses its loaded runtime', async () => { - await import('../tts-onnx-worker') - const receive = host.on.mock.calls[0]![1] + const receive = await startWorker() receive({ id: 1, type: 'prepare', voice: 'af_heart' }) await vi.waitFor(() => expect(host.post).toHaveBeenCalledWith({ id: 1, type: 'complete', device: 'coreml' }) @@ -139,8 +147,7 @@ describe('ONNX speech worker response evidence', () => { ]) }) it('returns an error for incomplete synthesis without claiming completion', async () => { - await import('../tts-onnx-worker') - host.on.mock.calls[0]![1]({ id: 3, type: 'synthesize', voice: 'af_heart' }) + ;(await startWorker())({ id: 3, type: 'synthesize', voice: 'af_heart' }) await vi.waitFor(() => expect(host.post).toHaveBeenCalledWith({ id: 3, @@ -153,8 +160,7 @@ describe('ONNX speech worker response evidence', () => { it('does not write a silent voice reply', async () => { host.generate.mockResolvedValue({ data: new Float32Array(2400), sampling_rate: 24000 }) - await import('../tts-onnx-worker') - host.on.mock.calls[0]![1]({ + ;(await startWorker())({ id: 6, type: 'synthesize', text: 'Hello', @@ -175,11 +181,14 @@ describe('ONNX speech worker response evidence', () => { it('retries model loading after a provider failure', async () => { host.load.mockRejectedValueOnce(new Error('provider unavailable')) - await import('../tts-onnx-worker') - const receive = host.on.mock.calls[0]![1] + const receive = await startWorker() receive({ id: 4, type: 'prepare', voice: 'af_heart' }) await vi.waitFor(() => - expect(host.post).toHaveBeenCalledWith({ id: 4, type: 'error', error: 'provider unavailable' }) + expect(host.post).toHaveBeenCalledWith({ + id: 4, + type: 'error', + error: 'provider unavailable' + }) ) receive({ id: 5, type: 'prepare', voice: 'af_heart' }) await vi.waitFor(() => diff --git a/src/main/__tests__/tts-onnx.test.ts b/src/main/__tests__/tts-onnx.test.ts index ab1df00bc..f77719203 100644 --- a/src/main/__tests__/tts-onnx.test.ts +++ b/src/main/__tests__/tts-onnx.test.ts @@ -1,8 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { EventEmitter } from 'node:events' +// The process boundary: Electron's utilityProcess. The host, its request map, crash handling and +// the helper's init message are production code. interface TestWorker extends EventEmitter { postMessage: ReturnType - terminate: ReturnType + kill: ReturnType } const host = vi.hoisted(() => ({ workers: [] as TestWorker[] })) vi.mock('node:fs', async (original) => ({ @@ -10,16 +12,21 @@ vi.mock('node:fs', async (original) => ({ existsSync: () => true })) vi.mock('../backend-preferences', () => ({ getBackendPreference: () => 'auto' })) -vi.mock('node:worker_threads', async () => { +vi.mock('electron', async () => { const { EventEmitter } = await import('node:events') + class FakeUtilityProcess extends EventEmitter { + postMessage = vi.fn() + kill = vi.fn(() => { + this.emit('exit', 0) + return true + }) + } return { - Worker: class extends EventEmitter { - postMessage = vi.fn() - terminate = vi.fn().mockResolvedValue(0) - unref = vi.fn() - constructor() { - super() - host.workers.push(this) + utilityProcess: { + fork: () => { + const child = new FakeUtilityProcess() + host.workers.push(child as unknown as TestWorker) + return child } } } @@ -28,7 +35,7 @@ import { OnnxSpeechRuntime } from '../tts-onnx' beforeEach(() => { host.workers.length = 0 }) -describe('speech worker lifecycle', () => { +describe('voice helper process lifecycle', () => { it('uses the actual completion device and forwards progress for each request', async () => { const runtime = new OnnxSpeechRuntime() const progress = vi.fn() @@ -61,8 +68,10 @@ describe('speech worker lifecycle', () => { ) worker.emit('message', { id: 2, type: 'complete', device: 'cuda' }) await expect(second).resolves.toBe('cuda') + // The helper's first message is its init data; requests follow. + expect(worker.postMessage.mock.calls[0]?.[0]).toMatchObject({ type: 'init' }) await runtime.close() - expect(worker.terminate).toHaveBeenCalledOnce() + expect(worker.kill).toHaveBeenCalledOnce() }) it('rejects worker errors and starts a new worker after a crash', async () => { const runtime = new OnnxSpeechRuntime() @@ -71,7 +80,7 @@ describe('speech worker lifecycle', () => { host.workers[0]!.emit('message', { id: 1, type: 'error', error: 'provider failed' }) await failed const second = runtime.prepare('af_heart') - const crashed = expect(second).rejects.toThrow('exited with code 9') + const crashed = expect(second).rejects.toThrow('exit code 9') host.workers[0]!.emit('exit', 9) await crashed const third = runtime.prepare('af_heart') diff --git a/src/main/__tests__/tts-spawn-contract.test.ts b/src/main/__tests__/tts-spawn-contract.test.ts index 038e286dd..d292525c1 100644 --- a/src/main/__tests__/tts-spawn-contract.test.ts +++ b/src/main/__tests__/tts-spawn-contract.test.ts @@ -2,16 +2,19 @@ import { describe, expect, it } from 'vitest' import { readFileSync } from 'node:fs' import { join } from 'node:path' -const src = readFileSync(join(__dirname, '..', 'tts.ts'), 'utf8') - .replace(/\/\*[\s\S]*?\*\//g, '') - .replace(/\/\/.*$/gm, '') +const strip = (text: string): string => + text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '') +const src = strip(readFileSync(join(__dirname, '..', 'tts.ts'), 'utf8')) +// The helper's location is resolved once, for tts.ts and the voice catalogue alike. +const availability = strip(readFileSync(join(__dirname, '..', 'runtime-availability.ts'), 'utf8')) describe('tts.ts ExecuTorch runtime contract', () => { it('resolves the packaged native helper before the sibling development checkout', () => { - expect(src).toMatch( + expect(src).toContain('executorchSpeechBinary()') + expect(availability).toMatch( /resourceDirs\(\)\.map\(\(root\) => path\.join\(root, 'bin', 'executorch-speech'\)\)/ ) - expect(src).toContain( + expect(availability).toContain( "path.resolve(process.cwd(), '../executorch-speech/native/bin/executorch-speech')" ) }) diff --git a/src/main/__tests__/vectors-predicates.test.ts b/src/main/__tests__/vectors-predicates.test.ts deleted file mode 100644 index af9204c70..000000000 --- a/src/main/__tests__/vectors-predicates.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Unit tests for the pure SQL-predicate builders extracted from vectors.ts: - * kindsPredicate (quote-escaping) + olderThanPredicate (floors the cutoff). - * No LanceDB — pure import-and-assert. - */ -import { describe, it, expect } from 'vitest' -import { kindsPredicate, olderThanPredicate } from '../vectors-predicates' - -describe('kindsPredicate — kind IN (...) with SQL-quote escaping', () => { - it('builds a single-kind predicate', () => { - expect(kindsPredicate(['screen'])).toBe("kind IN ('screen')") - }) - - it('builds a multi-kind predicate, comma-separated', () => { - expect(kindsPredicate(['screen', 'meeting', 'memory'])).toBe( - "kind IN ('screen', 'meeting', 'memory')" - ) - }) - - it("escapes a single quote in a kind by doubling it (' -> '')", () => { - expect(kindsPredicate(["o'brien"])).toBe("kind IN ('o''brien')") - }) - - it('escapes every quote in a value', () => { - expect(kindsPredicate(["a'b'c"])).toBe("kind IN ('a''b''c')") - }) - - it('an empty list yields an empty IN clause', () => { - expect(kindsPredicate([])).toBe('kind IN ()') - }) -}) - -describe('olderThanPredicate — kinds AND floored cutoff', () => { - it('appends the age filter with the cutoff floored to an integer', () => { - expect(olderThanPredicate(['screen'], 1_700_000_000_123.9)).toBe( - "kind IN ('screen') AND ts < 1700000000123" - ) - }) - - it('an integer cutoff is unchanged', () => { - expect(olderThanPredicate(['meeting', 'memory'], 42)).toBe( - "kind IN ('meeting', 'memory') AND ts < 42" - ) - }) - - it('reuses the same escaping as kindsPredicate', () => { - expect(olderThanPredicate(["o'brien"], 10.99)).toBe("kind IN ('o''brien') AND ts < 10") - }) -}) diff --git a/src/main/accessibility/__tests__/decision-runtime.test.ts b/src/main/accessibility/__tests__/decision-runtime.test.ts index bf6c2aabf..15e122ad6 100644 --- a/src/main/accessibility/__tests__/decision-runtime.test.ts +++ b/src/main/accessibility/__tests__/decision-runtime.test.ts @@ -29,7 +29,13 @@ vi.mock('../../llm', () => ({ vi.mock('../../llm/http-post', () => ({ postCompletionOnce: mocks.post })) vi.mock('../../llm/settings-math', () => ({ buildLaunchArgs: vi.fn(() => ['--serve']) })) vi.mock('../../llm/spawn-env', () => ({ engineSpawnEnv: vi.fn(() => ({ TEST_ENGINE: '1' })) })) -vi.mock('../../runtime-env', () => ({ binRoots: () => ['/bundle'], exe: (name: string) => name })) +vi.mock('../../runtime-env', () => ({ + binRoots: () => ['/bundle'], + exe: (name: string) => name, + // No CUDA pack manifest in this build. + resourceFile: () => null, + dataDir: () => '/data' +})) vi.mock('../../backend-preferences', () => ({ getBackendPreference: () => 'auto' })) vi.mock('../../free-port', () => ({ isPortFree: vi.fn(), pickFreePort: mocks.pickPort })) vi.mock('../../kill-orphan-port', () => ({ reapOrphanProcessesOnPort: mocks.reap })) @@ -64,7 +70,13 @@ beforeEach(() => { python: '/runtime/python', script: '/runtime/server.py' }) - mocks.exists.mockImplementation((candidate: string) => !candidate.includes('llama-cuda')) + // A macOS bundle: one Metal engine folder (no CUDA, CPU-only or legacy root engine). + mocks.exists.mockImplementation( + (candidate: string) => + !candidate.includes('llama-cuda') && + !candidate.includes('llama-cpu') && + candidate !== '/bundle/llama-server' + ) mocks.pickPort.mockResolvedValue(8460) mocks.parse.mockReturnValue({ choice: 1, confidence: 0.8, probabilities: [0.2, 0.8] }) mocks.post.mockResolvedValue('{"choice":"B"}') @@ -211,7 +223,7 @@ describe('DecisionRuntime', () => { await runtime.shutdown() const failing = new DecisionRuntime() - mocks.spawn.mockImplementationOnce(() => { + const outOfMemory = (): FakeProcess => { const child = new FakeProcess() queueMicrotask(() => { child.stderr.emit('data', 'CUDA out of memory') @@ -219,8 +231,12 @@ describe('DecisionRuntime', () => { child.emit('close', 1) }) return child - }) + } + // The GPU engine runs out of memory, then the same engine CPU-only does too. + mocks.spawn.mockClear() + mocks.spawn.mockImplementationOnce(outOfMemory).mockImplementationOnce(outOfMemory) vi.mocked(fetch).mockRejectedValue(new Error('not ready')) await expect(failing.start('offgrid/decider')).rejects.toMatchObject({ code: 'out_of_memory' }) + expect(mocks.spawn).toHaveBeenCalledTimes(2) }) }) diff --git a/src/main/accessibility/ax-agent.ts b/src/main/accessibility/ax-agent.ts index 3d30d8815..b1fa525b6 100644 --- a/src/main/accessibility/ax-agent.ts +++ b/src/main/accessibility/ax-agent.ts @@ -110,7 +110,7 @@ export interface ElementTaskDeps { control?: ElementTaskControl } -export interface ElementTaskControl { +interface ElementTaskControl { snapshot(): Pick waitUntilRunnable(signal?: AbortSignal): Promise> markObservationReady(): boolean @@ -439,8 +439,8 @@ export function buildElementDecisionContext(input: { const MAX_CONSECUTIVE_PARSE_FAILURES = 3 const MAX_CONSECUTIVE_NO_PROGRESS = 3 -export const AX_INVALID_REPLY_SUMMARY = `The action model returned an invalid reply ${MAX_CONSECUTIVE_PARSE_FAILURES} times in a row.` -export const AX_NO_PROGRESS_SUMMARY = 'The action did not produce its required result.' +const AX_INVALID_REPLY_SUMMARY = `The action model returned an invalid reply ${MAX_CONSECUTIVE_PARSE_FAILURES} times in a row.` +const AX_NO_PROGRESS_SUMMARY = 'The action did not produce its required result.' /** A stable signature of an actuating step, used to detect a runaway loop. Two * consecutive identical signatures mean the model is repeating itself (it sent diff --git a/src/main/accessibility/ax-decision.ts b/src/main/accessibility/ax-decision.ts index b207f9c5c..a43c3a295 100644 --- a/src/main/accessibility/ax-decision.ts +++ b/src/main/accessibility/ax-decision.ts @@ -743,19 +743,3 @@ export async function chooseFactorizedElementStep( ...(writerTargetIndex !== undefined ? { writerTargetIndex } : {}) } } - -/** Keep every model call within the model's trained A-J option head. Large AX - * trees are narrowed through bounded groups, then scored again inside the - * selected group. */ -export async function chooseElementStep( - context: string, - snapshot: AxSnapshot, - score: ScoreOptions -): Promise<{ step: ElementStep; confidence: number }> { - const decision = await chooseFactorizedElementStep(context, snapshot, score) - return { step: decision.step, confidence: decision.result.combinedConfidence } -} - -export function serializeElementStep(step: ElementStep): string { - return JSON.stringify(step) -} diff --git a/src/main/accessibility/ax-ranking.ts b/src/main/accessibility/ax-ranking.ts index 3e9c4fa23..fd93910f0 100644 --- a/src/main/accessibility/ax-ranking.ts +++ b/src/main/accessibility/ax-ranking.ts @@ -21,7 +21,7 @@ function intersection(a: AxRect, b: AxRect): number { return width * height } -export function overlapRatio(a: AxRect, b: AxRect): number { +function overlapRatio(a: AxRect, b: AxRect): number { const overlap = intersection(a, b) const smaller = Math.min(a.width * a.height, b.width * b.height) return smaller > 0 ? overlap / smaller : 0 diff --git a/src/main/accessibility/ax-state.ts b/src/main/accessibility/ax-state.ts index ec431df1d..bd11692f6 100644 --- a/src/main/accessibility/ax-state.ts +++ b/src/main/accessibility/ax-state.ts @@ -7,7 +7,7 @@ export interface AxRect { height: number } -export interface AxCaptureGeometry { +interface AxCaptureGeometry { sourceBounds: AxRect encodedWidth: number encodedHeight: number @@ -16,7 +16,7 @@ export interface AxCaptureGeometry { } export type ObservationSource = 'ax' | 'ocr' | 'ax+ocr' -export type ObservationModality = 'ax' | 'uia' | 'ocr' | 'capture' +type ObservationModality = 'ax' | 'uia' | 'ocr' | 'capture' export interface WindowIdentity { platform: NodeJS.Platform @@ -27,7 +27,7 @@ export interface WindowIdentity { bounds: AxRect } -export interface SourceStatus { +interface SourceStatus { available: boolean degradedReason?: string } @@ -40,9 +40,9 @@ export interface WindowBoundObservation { sources: Record } -export type CandidateRisk = 'reversible' | 'private' | 'authentication' | 'payment' | 'destructive' +type CandidateRisk = 'reversible' | 'private' | 'authentication' | 'payment' | 'destructive' -export type CandidateAction = +type CandidateAction = | { type: 'activate'; method: 'press' | 'click' | 'hover'; x: number; y: number } | { type: 'type'; text: string; submit: boolean } | { type: 'key'; keys: string } @@ -111,7 +111,7 @@ export interface ComputerUseDecisionResult { latencyMs: number } -export type VerificationStatus = 'satisfied' | 'unsatisfied' | 'unknown' | 'timeout' +type VerificationStatus = 'satisfied' | 'unsatisfied' | 'unknown' | 'timeout' export interface ComputerUseVerificationResult { status: VerificationStatus diff --git a/src/main/accessibility/decision-model-loader.ts b/src/main/accessibility/decision-model-loader.ts index e3f33b713..d8ace0c16 100644 --- a/src/main/accessibility/decision-model-loader.ts +++ b/src/main/accessibility/decision-model-loader.ts @@ -34,13 +34,19 @@ async function ensureDedicatedRuntime( session: NonNullable ): Promise { if (decisionRuntime.running) return - if (!decisionRuntime.memoryEvicted && session.restartsRemaining <= 0) { + if ( + isKevModel(session.modelId) && + !decisionRuntime.memoryEvicted && + session.restartsRemaining <= 0 + ) { throw new DecisionRuntimeError( 'The dedicated Decision runtime stopped before selection.', 'startup' ) } - if (!decisionRuntime.memoryEvicted) session.restartsRemaining -= 1 + // The shared llama runtime owns its restart budget. Keep Kev unchanged. + if (isKevModel(session.modelId) && !decisionRuntime.memoryEvicted) + session.restartsRemaining -= 1 console.log('[computer-use] loading Decision runtime after eviction or recovery') await decisionRuntime.start(session.modelId) } diff --git a/src/main/accessibility/decision-runtime.ts b/src/main/accessibility/decision-runtime.ts index d653e035c..e124b6507 100644 --- a/src/main/accessibility/decision-runtime.ts +++ b/src/main/accessibility/decision-runtime.ts @@ -1,6 +1,7 @@ -import { beginRuntimeBackend, parseNativeBackend } from '../runtime-backends' +import { DedicatedLlamaRuntime } from '../llm/dedicated-llama-runtime' +import { getBackendPreference } from '../backend-preferences' +import { beginRuntimeBackend } from '../runtime-backends' import { spawn, type ChildProcess } from 'node:child_process' -import path from 'node:path' import { currentAIRequest } from '../ai-request-log' import { Mutex } from 'async-mutex' import { prepareModelMemory, registerModelEvictor } from '../model-memory' @@ -11,7 +12,6 @@ import { type OptionDecision } from '../llm' import { postCompletionOnce } from '../llm/http-post' -import { buildLaunchArgs } from '../llm/settings-math' import { totalmem } from 'node:os' import type { GatewayMode } from '@offgrid/models' @@ -20,11 +20,7 @@ let gatewayModeSource: () => GatewayMode = () => 'single' export function setDecisionGatewayModeSource(source: () => GatewayMode): void { gatewayModeSource = source } -import { engineSpawnEnv } from '../llm/spawn-env' -import { selectLocalEngine } from '../llm/select-local-engine' -import { getBackendPreference } from '../backend-preferences' import { isPortFree, pickFreePort } from '../free-port' -import { reapOrphanProcessesOnPort } from '../kill-orphan-port' import { KEV_9B_ID, isKevModel, @@ -58,13 +54,43 @@ export class DecisionRuntime { private startPromise: Promise | null = null private modelId: string | null = null private stderr = '' - private backendOutput = '' private backend: string | undefined private backendState?: ReturnType + private readonly runtime = new DedicatedLlamaRuntime({ + id: 'decision', + label: 'Decision', + modality: 'decision', + port: DECIDER_PORT, + readyTimeoutMs: 120_000, + modelPaths: async () => { + const artifact = await resolveComputerUseModelArtifact(this.modelId!) + if (!artifact) + throw new DecisionRuntimeError('The selected Decision model is not installed.', 'missing') + return { + model: this.modelId!, + modelPath: artifact.primaryPath, + mmProjPath: artifact.projectorPath ?? '' + } + }, + extraArgs: () => ({ + effectiveCtxSize: DECIDER_CONTEXT_SIZE, + flashAttn: true, + kvCacheType: 'q8_0', + speculativeDecoding: 'off', + threads: undefined, + batchSize: 128, + reportModelPlacement: true, + // Team gateway mode opens slots here too, so several clients' decisions run at once. + // (Single mode passes nothing new, so its argv is unchanged.) + ...(gatewayModeSource() === 'team' + ? { gatewayMode: 'team' as const, totalRamBytes: totalmem() } + : {}) + }) + }) readonly timing: DecisionRuntimeTiming = { coldStartMs: 0, warmDecisionMs: [] } get activePort(): number { - return this.port + return isKevModel(this.modelId) ? this.port : this.runtime.port } get activeModelId(): string | null { @@ -72,7 +98,9 @@ export class DecisionRuntime { } get running(): boolean { - return this.process !== null && this.process.exitCode === null + return isKevModel(this.modelId) + ? this.process !== null && this.process.exitCode === null + : this.runtime.isRunning } async start(modelId: string): Promise { @@ -87,93 +115,26 @@ export class DecisionRuntime { private async startInternal(modelId: string): Promise { await prepareModelMemory('decision') this.memoryEvicted = false + if (this.modelId === modelId && !isKevModel(modelId)) { + await this.startDedicated() + return + } await this.shutdown() if (isKevModel(modelId)) return this.startKev(modelId) - const artifact = await resolveComputerUseModelArtifact(modelId) - if (!artifact) - throw new DecisionRuntimeError('The selected Decision model is not installed.', 'missing') - reapOrphanProcessesOnPort( - DECIDER_PORT, - (command) => /llama-server/i.test(command), - 'Decision llama-server' - ) - const port = await pickFreePort(DECIDER_PORT, isPortFree, 20) - if (port === null) - throw new DecisionRuntimeError( - 'No private port is available for the Decision runtime.', - 'startup' - ) - const serverPath = await selectLocalEngine(getBackendPreference('decision')) - if (!serverPath) - throw new DecisionRuntimeError('The bundled Decision engine is missing.', 'startup') - this.port = port this.modelId = modelId - this.stderr = '' - this.backendOutput = '' - this.backend = undefined - const backendState = beginRuntimeBackend('decision', this.modelId!) - this.backendState = backendState - const args = buildLaunchArgs({ - modelPath: artifact.primaryPath, - mmProjPath: artifact.projectorPath ?? '', - port, - // Mapika Decider supports 32k tokens. Keep enough room for structured - // Computer Use state without reserving the model's full context window. - effectiveCtxSize: DECIDER_CONTEXT_SIZE, - gpuLayers: getBackendPreference('decision') === 'cpu' ? 0 : 99, - flashAttn: true, - kvCacheType: 'q8_0', - speculativeDecoding: 'off', - threads: undefined, - batchSize: 128, - reportModelPlacement: true, - // Team gateway mode opens slots here too, so several clients' decisions run at once. - // (Single mode passes nothing new, so its argv is unchanged.) - ...(gatewayModeSource() === 'team' - ? { gatewayMode: 'team' as const, totalRamBytes: totalmem() } - : {}) - }) + await this.startDedicated() + } + + private async startDedicated(): Promise { const startedAt = Date.now() - const process = spawn(serverPath, args, { - env: { - ...globalThis.process.env, - ...engineSpawnEnv({ - platform: globalThis.process.platform, - binDir: path.dirname(serverPath), - currentEnv: globalThis.process.env - }) - }, - stdio: ['ignore', 'ignore', 'pipe'] - }) - this.process = process - let loading = true - process.stderr!.on('data', (chunk) => { - if (loading) { - const output = String(chunk) - this.stderr = `${this.stderr}${output}`.slice(-16_384) - this.backendOutput = `${this.backendOutput}${output}`.slice(-65_536) - backendState.observe(output) - console.log(`[Decision runtime] ${output}`) - } - }) - process.once('close', () => { - backendState.stop() - if (this.process === process) this.process = null - }) try { - await this.waitUntilReady(startedAt) - loading = false // Level 4 can contain request data after startup. - this.backend = parseNativeBackend(this.backendOutput)?.backend - backendState.ready(this.backend) + await this.runtime.start() this.timing.coldStartMs = Date.now() - startedAt } catch (error) { - await this.shutdown() - const outOfMemory = /out of memory|cannot allocate|metal.*alloc|cuda.*memory/i.test( - this.stderr - ) + if (error instanceof DecisionRuntimeError) throw error throw new DecisionRuntimeError( error instanceof Error ? error.message : 'The Decision runtime did not start.', - outOfMemory ? 'out_of_memory' : 'startup' + this.runtime.failureCode === 'out_of_memory' ? 'out_of_memory' : 'startup' ) } } @@ -270,11 +231,12 @@ export class DecisionRuntime { if (!this.running) throw new DecisionRuntimeError('The Decision runtime is not running.', 'startup') const run = async (): Promise => { - this.backendState?.recordRequest() + if (isKevModel(this.modelId)) this.backendState?.recordRequest() + else this.runtime.recordRequest() const startedAt = Date.now() currentAIRequest()?.update({ model: this.modelId ?? undefined, - backend: this.backend ?? 'Unknown', + backend: (isKevModel(this.modelId) ? this.backend : this.runtime.backend) ?? 'Unknown', effectiveRequest: { context, question, options } }) if (isKevModel(this.modelId)) { @@ -321,7 +283,7 @@ export class DecisionRuntime { } const prompt = buildDecisionPrompt(context, question, options) const body = JSON.stringify(buildDecisionRequest(prompt, options.length)) - const raw = await postCompletionOnce(this.port, body, undefined, signal, '/completion') + const raw = await postCompletionOnce(this.activePort, body, undefined, signal, '/completion') currentAIRequest()?.update({ effectiveRequest: JSON.parse(body), response: JSON.parse(raw) }) this.timing.warmDecisionMs.push(Date.now() - startedAt) if (this.timing.warmDecisionMs.length > 200) this.timing.warmDecisionMs.shift() @@ -331,16 +293,31 @@ export class DecisionRuntime { } async health(): Promise { + if (!isKevModel(this.modelId)) return this.runtime.health() if (!this.running) return false try { - const endpoint = isKevModel(this.modelId) ? '/v1/models' : '/health' - return (await fetch(`http://127.0.0.1:${this.port}${endpoint}`)).ok + return (await fetch(`http://127.0.0.1:${this.port}/v1/models`)).ok } catch { return false } } + shouldBeRunning(): boolean { + return isKevModel(this.modelId) ? this.running : this.runtime.shouldBeRunning() + } + + async restart(): Promise { + const modelId = this.modelId + if (modelId && isKevModel(modelId)) { + await this.shutdown() + await this.start(modelId) + return + } + await this.runtime.restart() + } + async shutdown(): Promise { + await this.runtime.stop() this.backendState?.stop() const process = this.process this.process = null @@ -363,7 +340,7 @@ export class DecisionRuntime { export const decisionRuntime = new DecisionRuntime() registerModelEvictor('decision', async () => { - if (decisionRuntime.running) { + if (decisionRuntime.running || decisionRuntime.shouldBeRunning()) { decisionRuntime.memoryEvicted = true await decisionRuntime.shutdown() } diff --git a/src/main/actions/connector-rail.ts b/src/main/actions/connector-rail.ts index 722583252..46c9e4b64 100644 --- a/src/main/actions/connector-rail.ts +++ b/src/main/actions/connector-rail.ts @@ -7,7 +7,7 @@ */ import type { ActionRecord, ExecuteResult } from '@offgrid/use' -export interface ConnectorCallResult { +interface ConnectorCallResult { ok: boolean result?: unknown error?: string diff --git a/src/main/actions/emit.ts b/src/main/actions/emit.ts index ad5e13da0..d50df49c9 100644 --- a/src/main/actions/emit.ts +++ b/src/main/actions/emit.ts @@ -94,7 +94,7 @@ const quoteBareKeys = (text: string): string => * Repair candidates in trust order: the raw text first, then progressively * repaired variants. Deduped; each is tried against the fail-closed schema. */ -export function extractCandidates(raw: string): string[] { +function extractCandidates(raw: string): string[] { const candidates: string[] = [raw.trim()] const unfenced = stripFences(raw).trim() candidates.push(unfenced) diff --git a/src/main/actions/gate-host.ts b/src/main/actions/gate-host.ts index f99eefa95..30e63d277 100644 --- a/src/main/actions/gate-host.ts +++ b/src/main/actions/gate-host.ts @@ -169,7 +169,7 @@ export function needsApproval(action: Pick): boolean { /** A routine runs with nobody watching: anything it does beyond reading waits for approval, * including the native actions a person in Chat runs straight away. */ -export function unattended(action: Pick): boolean { +function unattended(action: Pick): boolean { return action.source === 'routine' && action.risk !== 'read' } diff --git a/src/main/active-models.ts b/src/main/active-models.ts index b8f7beb07..84dd06aa3 100644 --- a/src/main/active-models.ts +++ b/src/main/active-models.ts @@ -105,7 +105,3 @@ export function setRemoteModelSelected(modality: RemoteModelModality, selected: export function remoteVoiceSelected(): boolean { return remoteModelSelected('voice') } - -export function setRemoteVoiceSelected(selected: boolean): void { - setRemoteModelSelected('voice', selected) -} diff --git a/src/main/backend-facts-cache.ts b/src/main/backend-facts-cache.ts new file mode 100644 index 000000000..9e23de56e --- /dev/null +++ b/src/main/backend-facts-cache.ts @@ -0,0 +1,24 @@ +// What backend-facts remembers between reads: cached device answers and the backends a fault +// demoted. A leaf, so the backend choice store can clear it when a choice changes. + +import { + BACKEND_MODALITIES, + type BackendFacts, + type BackendModality, + type NativeBackend +} from '@offgrid/models' + +export const demoted: BackendFacts['demoted'] = {} +export const probes = new Map>() + +export function demoteBackend(modality: BackendModality, backend: NativeBackend): void { + const backends = (demoted[modality] ??= []) + if (!backends.includes(backend)) backends.push(backend) +} +export function clearDemotions(modality?: BackendModality): void { + for (const value of modality ? [modality] : BACKEND_MODALITIES) delete demoted[value] +} +/** Forget cached device answers after a pack install, a driver change or wake. */ +export function invalidateBackendFacts(): void { + probes.clear() +} diff --git a/src/main/backend-facts.ts b/src/main/backend-facts.ts new file mode 100644 index 000000000..8e161c606 --- /dev/null +++ b/src/main/backend-facts.ts @@ -0,0 +1,239 @@ +import fs from 'node:fs' +import path from 'node:path' +import { + BACKEND_MODALITIES, + NVIDIA_DRIVER_FLOORS, + backendChoicesFor, + backendLadder, + nvidiaDriverMeets, + type BackendFacts, + type BackendReason, + type BackendModality, + type BackendChoice, + type EngineCandidate, + type NativeBackend, + type OnnxDevice +} from '@offgrid/models' +import { binRoots, exe } from './runtime-env' +import { gpuDeviceAvailable } from './llm/gpu-device-probe' +import { performancePackStatus, installedPerformancePackBin } from './performance-pack' +import { getBackendPreference } from './backend-preferences' +import { demoted, probes } from './backend-facts-cache' +import { + cachedNvidiaDriverVersion, + hasLinuxNvidiaDriver, + hasWindowsNvidiaDriver +} from './nvidia-driver' +import { onnxCudaVerified } from './onnx-host/onnx-cuda-verified' + +/** The installed NVIDIA driver version, as last read (nvidia-driver.ts). */ +export function nvidiaDriverVersion(): string | null { + return cachedNvidiaDriverVersion() +} + +function hasLinuxGpuDevice(): boolean { + if (hasLinuxNvidiaDriver()) return true + try { + return fs.readdirSync('/dev/dri').some((name) => name.startsWith('renderD')) + } catch { + return false + } +} + +/** Scan paths here; ordering and fallback belong to the shared policy. */ +function scanEngineCandidates( + modality: BackendModality, + binaryName?: string, + prism = false +): EngineCandidate[] { + if (modality === 'tts' || modality === 'embeddings') return [] + const platform = process.platform + const native = platform === 'darwin' ? 'metal' : 'vulkan' + const llama = ['llm', 'grounding', 'decision'].includes(modality) + const prefix = prism ? 'llama-prism' : 'llama' + const folders: [string, NativeBackend][] = [] + if (llama) { + if (platform === 'win32' || platform === 'linux') folders.push([`${prefix}-cuda`, 'cuda']) + folders.push([prefix, native], [`${prefix}-cpu`, 'cpu']) + if (!prism) folders.push(['', native]) + } else if (modality === 'image') { + folders.push(['sd-cuda', 'cuda'], ['sd', native], ['sd-cpu', 'cpu']) + } else { + folders.push( + ['whisper-cuda', 'cuda'], + ['whisper', platform === 'win32' ? 'cuda' : native], + ['whisper-vulkan', 'vulkan'], + ['whisper-cpu', 'cpu'] + ) + if (platform === 'darwin' && (!binaryName || binaryName === 'whisper-server')) { + folders.push(['whisper-server', 'metal']) + } + } + const names = binaryName + ? [binaryName] + : llama + ? ['llama-server'] + : modality === 'image' + ? ['sd-server', 'sd-cli'] + : ['whisper-cli', 'whisper-server'] + const candidates: EngineCandidate[] = [] + const driver = nvidiaDriverVersion() + for (const [folder, backend] of folders) { + for (const root of binRoots()) { + for (const name of names) { + const id = path.resolve(root, folder, exe(name)) + if (!fs.existsSync(id) || candidates.some((candidate) => candidate.id === id)) continue + let reason: EngineCandidate['reason'] + if (backend === 'cuda') { + const present = + platform === 'win32' + ? hasWindowsNvidiaDriver() + : platform === 'linux' && hasLinuxNvidiaDriver() + const floor = NVIDIA_DRIVER_FLOORS.cuda12[platform as 'win32' | 'linux'] + if (!present) reason = { code: 'no-device' } + else if (driver && floor && !nvidiaDriverMeets(driver, floor)) + reason = { code: 'driver-too-old', have: driver, need: floor } + } + if ( + modality === 'image' && + backend === 'vulkan' && + platform === 'win32' && + !fs.existsSync( + path.join( + process.env.WINDIR ?? process.env.SystemRoot ?? 'C:\\Windows', + 'System32', + 'vulkan-1.dll' + ) + ) + ) + reason = { code: 'no-device' } + if (modality === 'stt' && backend !== 'cpu' && platform === 'linux' && !hasLinuxGpuDevice()) + reason = { code: 'no-device' } + candidates.push({ id, backend, usable: !reason, ...(reason ? { reason } : {}) }) + } + } + } + return candidates +} + +export async function engineCandidates( + modality: BackendModality, + binaryName?: string, + prism = false +): Promise { + return probeCandidates(modality, scanEngineCandidates(modality, binaryName, prism)) +} + +async function probeCandidates( + modality: BackendModality, + candidates: EngineCandidate[] +): Promise { + if ( + !['llm', 'grounding', 'decision'].includes(modality) || + !['win32', 'linux'].includes(process.platform) + ) + return candidates + return Promise.all( + candidates.map(async (candidate) => { + if (!candidate.usable || candidate.backend === 'cpu') return candidate + let probe = probes.get(candidate.id) + if (!probe) { + probe = gpuDeviceAvailable( + candidate.id, + candidate.backend === 'cuda' ? 'CUDA' : 'Vulkan', + process.platform + ) + probes.set(candidate.id, probe) + } + return (await probe) + ? candidate + : { ...candidate, usable: false, reason: { code: 'no-device' as const } } + }) + ) +} + +/** Availability checks only inspect installed files; launches use nativeLadder. */ +export function nativeBinaryInstalled(modality: BackendModality, binaryName: string): boolean { + return scanEngineCandidates(modality, binaryName).length > 0 +} + +function runtimeFacts(): BackendFacts { + const status = performancePackStatus() + const packBin = installedPerformancePackBin() + const installed = packBin !== null + const driver = nvidiaDriverVersion() + const floor = NVIDIA_DRIVER_FLOORS.cuda13.linux + const libraries = !!packBin && fs.existsSync(path.join(packBin, 'onnx-cuda', 'libcublasLt.so.13')) + const onnxCuda: BackendFacts['onnxCuda'] = + process.platform === 'win32' + ? onnxCudaVerified() + : process.platform === 'linux' + ? !installed || !libraries + ? { usable: false, reason: { code: 'needs-pack' } } + : !nvidiaDriverMeets(driver, floor) + ? { + usable: false, + reason: { code: 'driver-too-old', have: driver ?? 'unknown', need: floor } + } + : { usable: true } + : { usable: false, reason: { code: 'no-device' } } + return { + platform: process.platform, + arch: process.arch, + engines: {}, + nvidiaDriver: driver, + pack: { offered: status.phase !== 'unavailable' && status.phase !== 'not-needed', installed }, + onnxCuda, + demoted + } +} + +export async function backendFacts(modality?: BackendModality): Promise { + const facts = runtimeFacts() + await Promise.all( + (modality ? [modality] : BACKEND_MODALITIES).map(async (value) => { + facts.engines[value] = await engineCandidates(value) + }) + ) + return facts +} + +export { clearDemotions, demoteBackend, invalidateBackendFacts } from './backend-facts-cache' +/** One rung of a native engine ladder: the binary, what it was built for, and whether it must run + * CPU-only (llama `--device none`, whisper `-ng`, sd `--backend cpu`). */ +export interface NativeStep { + path: string + backend: NativeBackend + cpuOnly: boolean + reason?: BackendReason +} + +export async function nativeLadder( + modality: BackendModality, + binaryName?: string, + preference = getBackendPreference(modality), + prism = false +): Promise { + const facts = runtimeFacts() + facts.engines[modality] = + preference === 'cpu' + ? scanEngineCandidates(modality, binaryName, prism) + : await engineCandidates(modality, binaryName, prism) + return backendLadder(modality, preference, facts).map((step) => ({ + path: step.engine, + backend: step.backend as NativeBackend, + cpuOnly: step.cpuOnly, + reason: step.reason + })) +} +export function onnxLadder(modality: 'tts' | 'embeddings'): OnnxDevice[] { + return backendLadder(modality, getBackendPreference(modality), runtimeFacts()).map( + (step) => step.engine as OnnxDevice + ) +} +export async function backendChoicesForUi(): Promise> { + const facts = await backendFacts() + return Object.fromEntries( + BACKEND_MODALITIES.map((modality) => [modality, backendChoicesFor(modality, facts)]) + ) as Record +} diff --git a/src/main/backend-preferences.ts b/src/main/backend-preferences.ts index fd8d01e8a..810ce02cd 100644 --- a/src/main/backend-preferences.ts +++ b/src/main/backend-preferences.ts @@ -1,3 +1,5 @@ +import { normalizeBackendPreference } from '@offgrid/models' +import { clearDemotions, invalidateBackendFacts } from './backend-facts-cache' import { getSetting, saveSetting } from './database' import { BACKEND_MODALITIES, @@ -22,11 +24,19 @@ export function setBackendPreference( modality: BackendModality, preference: BackendPreference ): BackendPreferences { - if (!BACKEND_MODALITIES.includes(modality) || - !backendChoices(modality, process.platform).includes(preference)) { + if ( + !BACKEND_MODALITIES.includes(modality) || + !backendChoices(modality, process.platform).includes(preference) + ) { throw new Error('Invalid model backend setting') } + preference = normalizeBackendPreference(modality, preference) + const previous = getBackendPreference(modality) const next = { ...getBackendPreferences(), [modality]: preference } saveSetting(KEY, next) + if (previous !== preference) { + invalidateBackendFacts() + clearDemotions(modality) + } return next } diff --git a/src/main/backup/index.ts b/src/main/backup/index.ts index 88f2b9f57..df1a8c596 100644 --- a/src/main/backup/index.ts +++ b/src/main/backup/index.ts @@ -29,4 +29,4 @@ export function createDesktopBackupEngine(): DesktopBackupEngine { ) } -export type { DesktopBackupData, DesktopRestoreSummary, DesktopBackupDelivery } +export type { DesktopBackupDelivery } diff --git a/src/main/backup/types.ts b/src/main/backup/types.ts index a129cb694..c606cab61 100644 --- a/src/main/backup/types.ts +++ b/src/main/backup/types.ts @@ -1,6 +1,6 @@ import { BundleError } from '@offgrid/sync/portable' -export interface DesktopBackupChunk { +interface DesktopBackupChunk { content: string position: number } diff --git a/src/main/best-settings.ts b/src/main/best-settings.ts index e873a2a21..cc5a7a4d8 100644 --- a/src/main/best-settings.ts +++ b/src/main/best-settings.ts @@ -3,11 +3,10 @@ // hardware facts and writes the chosen settings through each owner's own setter. import os from 'node:os' -import path from 'node:path' import { execFile } from 'node:child_process' import { promisify } from 'node:util' import { llm } from './llm' -import { selectLocalEngine } from './llm/select-local-engine' +import { nativeLadder } from './backend-facts' import { getBackendPreferences, setBackendPreference } from './backend-preferences' import { getResidency, setResidencyMode } from './runtime-residency' import { getSetting, saveSetting } from './database' @@ -38,12 +37,16 @@ async function applePerformanceCores(): Promise { /** The GPU backend of the engine llama.cpp would load with, after its own device probe. */ async function nativeGpu(): Promise { - if (process.platform === 'darwin') return 'Metal' - const engine = await selectLocalEngine('auto') - const directory = engine ? path.basename(path.dirname(engine)) : '' - if (directory === 'llama-cuda') return 'CUDA' - if (directory === 'llama') return 'Vulkan' - return null + const step = (await nativeLadder('llm', undefined, 'auto')).find( + (candidate) => !candidate.cpuOnly + ) + return step?.backend === 'cuda' + ? 'CUDA' + : step?.backend === 'vulkan' + ? 'Vulkan' + : step?.backend === 'metal' + ? 'Metal' + : null } export async function detectMachine(): Promise { diff --git a/src/main/browser/__tests__/browser-playwright-task-support.test.ts b/src/main/browser/__tests__/browser-playwright-task-support.test.ts index bf02a070b..266cf8d07 100644 --- a/src/main/browser/__tests__/browser-playwright-task-support.test.ts +++ b/src/main/browser/__tests__/browser-playwright-task-support.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest' import type { SemanticDecision } from '../browser-playwright-policy' -import { isUncertainActionTimeout, uncertainActionKey } from '../browser-playwright-task-support' +import { VisionGuard } from '../../vision/vision-guard' +import type { BrowserPlaywrightTaskInput } from '../browser-playwright-task-contract' +import { + observeWithCurrentLease, + isUncertainActionTimeout, + uncertainActionKey +} from '../browser-playwright-task-support' function decision(overrides: Partial = {}): SemanticDecision { return { @@ -55,3 +61,69 @@ describe('uncertain browser actions', () => { ) }) }) + +describe('semantic image observations', () => { + function input(): BrowserPlaywrightTaskInput { + return { + goal: 'Read the saved result', + plan: { version: 1, phases: [{ id: 'phase-1', title: 'Read the result' }] }, + guard: new VisionGuard({ taskId: 'semantic-image', kind: 'web_use' }), + session: { + snapshot: async () => ({ text: 'status "Saved" [ref=s1]', isError: false }), + recoverPage: async () => { + throw new Error('Unexpected recovery') + }, + call: async () => { + throw new Error('Unexpected action') + } + }, + activeDriver: () => null, + activeUrl: () => 'https://example.test/result', + waitForUser: async () => undefined, + takeGuidance: () => [], + onStep: () => undefined, + onPhase: () => undefined, + onProgress: () => undefined + } + } + + it('returns the captured frame with its semantic observation', async () => { + const task = input() + const screenshot = { image: '/frames/current.png', bounds: { width: 1200, height: 750 } } + task.captureObservationImage = async () => screenshot + expect(await observeWithCurrentLease(task)).toEqual({ + text: 'status "Saved" [ref=s1]', + isError: false, + screenshot + }) + }) + + it('discards both the snapshot and frame when input control changes during capture', async () => { + const task = input() + let captures = 0 + task.captureObservationImage = async () => { + captures += 1 + if (captures === 1) { + task.guard.pause() + task.guard.resume() + } + return { image: `/frames/${captures}.png`, bounds: { width: 1200, height: 750 } } + } + expect((await observeWithCurrentLease(task)).screenshot?.image).toBe('/frames/2.png') + }) + + it('keeps text-only sessions usable without a capture provider', async () => { + expect(await observeWithCurrentLease(input())).toEqual({ + text: 'status "Saved" [ref=s1]', + isError: false + }) + }) + + it('does not silently remove an image when its capture fails', async () => { + const task = input() + task.captureObservationImage = async () => { + throw new Error('Capture unavailable') + } + await expect(observeWithCurrentLease(task)).rejects.toThrow('Capture unavailable') + }) +}) diff --git a/src/main/browser/browser-host.ts b/src/main/browser/browser-host.ts index d28cf014b..4c15814ad 100644 --- a/src/main/browser/browser-host.ts +++ b/src/main/browser/browser-host.ts @@ -15,6 +15,11 @@ * on a real display in the e2e tour and the real-machine pass, not here. */ import { BrowserWindow, WebContentsView, ipcMain } from 'electron' +import { createBrowserVisionScreen } from './browser-vision-screen' +import { + getActiveRemoteVisionServer, + getRemoteVisionServerSettings +} from '../vision/remote-vision-server' import { randomUUID } from 'node:crypto' import { BrowserDriver, @@ -939,6 +944,13 @@ class BrowserHost implements BrowserRailHost { const page = this.sessions.findJourney(journeyId) ?? record return { view: page.resource, driver: this.driverFor(page) } } + const semanticScreen = createBrowserVisionScreen({ + goal, + taskId, + journeyId, + activePage, + settings: getWebUseSettings() + }) const recordSemanticEvidence = createBrowserSemanticEvidenceRecorder({ goal, taskId, @@ -1038,6 +1050,28 @@ class BrowserHost implements BrowserRailHost { currentAction: action }) }, + captureObservationImage: async () => { + // The semantic route has selected its model before entering this loop. + // Keep existing text-only model and extension behavior unchanged. + const { llm } = await import('../llm') + const session = currentRemoteScreenTaskSession() + const remote = session ? session.activeServer : getActiveRemoteVisionServer() + if (remote) { + const saved = getRemoteVisionServerSettings().servers.find( + (server) => + server.id === remote.id && + server.enabled !== false && + server.endpoint === remote.endpoint + ) + const supportsImages = saved?.modelCatalog?.some( + (model) => model.id === remote.model && model.inputModalities?.includes('image') + ) + return saved?.screenFramesAllowed && supportsImages + ? semanticScreen.capture() + : undefined + } + return llm.hasVision() ? semanticScreen.capture() : undefined + }, onObservation: recordSemanticEvidence, signal: owner.controller.signal } diff --git a/src/main/browser/browser-playwright-policy.ts b/src/main/browser/browser-playwright-policy.ts index c004dbf26..89ca848bf 100644 --- a/src/main/browser/browser-playwright-policy.ts +++ b/src/main/browser/browser-playwright-policy.ts @@ -130,6 +130,7 @@ export interface BrowserSemanticDecisionRequest { goal: string plan: TaskExecutionPlan snapshot: string + image?: string step: number maxSteps: number recoveryNote: string @@ -192,7 +193,8 @@ Rules: ${boundedSnapshot(request.snapshot, request.goal)} ` - const raw = await llm.chat(prompt, [], undefined, undefined, { + const raw = await llm.chat(prompt, request.image ? [request.image] : [], undefined, undefined, { + requireImages: Boolean(request.image), disableThinking: true, temperature: 0.1, responseFormat: useDecider ? PLAYWRIGHT_CANDIDATES_FORMAT : PLAYWRIGHT_STEP_FORMAT, diff --git a/src/main/browser/browser-playwright-task-contract.ts b/src/main/browser/browser-playwright-task-contract.ts index 8a5423413..c2bb728cc 100644 --- a/src/main/browser/browser-playwright-task-contract.ts +++ b/src/main/browser/browser-playwright-task-contract.ts @@ -1,4 +1,5 @@ import type { TaskExecutionPlan } from '../../shared/task-execution-plan' +import type { VisionScreen } from '../vision/vision-agent' import type { VisionGuard } from '../vision/vision-guard' import type { BrowserDriver } from './browser-driver' import type { SemanticPageSession } from './playwright-mcp-session' @@ -14,6 +15,7 @@ export interface BrowserSemanticObservation { step: number phase: 'observing' | 'checking' | 'waiting' | 'complete' summary: string + screenshot?: Awaited> } export interface BrowserPlaywrightTaskInput { @@ -29,6 +31,8 @@ export interface BrowserPlaywrightTaskInput { onStep: (note: string) => void onPhase: (phaseId: string) => void onProgress: (step: number, phase: 'observing' | 'thinking' | 'acting', action: string) => void + /** Capture a model frame within the same lease as the semantic snapshot. */ + captureObservationImage?: () => Promise> | undefined> onObservation?: (observation: BrowserSemanticObservation) => Promise signal?: AbortSignal } diff --git a/src/main/browser/browser-playwright-task-support.ts b/src/main/browser/browser-playwright-task-support.ts index 9105aad12..65fe5c747 100644 --- a/src/main/browser/browser-playwright-task-support.ts +++ b/src/main/browser/browser-playwright-task-support.ts @@ -109,6 +109,10 @@ export async function observeWithCurrentLease( let observation: PlaywrightToolResult try { observation = await input.session.snapshot(combinedSignal(input.signal, lease.signal)) + if (!observation.isError && input.captureObservationImage) { + const screenshot = await input.captureObservationImage() + observation = { ...observation, screenshot } + } } catch (error) { if (!lease.signal.aborted) throw error continue diff --git a/src/main/browser/browser-playwright-task.ts b/src/main/browser/browser-playwright-task.ts index 8944b5b8d..1b6fd3d67 100644 --- a/src/main/browser/browser-playwright-task.ts +++ b/src/main/browser/browser-playwright-task.ts @@ -149,7 +149,12 @@ async function runSemanticLoop( : 'The page did not expose usable semantic controls.' ) } - await publishObservation(input, { step: 0, phase: 'observing', summary: 'Initial page' }) + await publishObservation(input, { + step: 0, + phase: 'observing', + summary: 'Initial page', + screenshot: observation.screenshot + }) const state: SemanticLoopState = { observation, @@ -205,6 +210,9 @@ async function runSemanticStep( input.signal?.throwIfAborted() await input.guard.waitUntilRunnable(input.signal) if (input.guard.isHalted) input.signal?.throwIfAborted() + if (input.guard.snapshot().observationRequired) { + state.observation = await observeWithCurrentLease(input) + } input.onProgress(step, 'thinking', 'Deciding the next step') const modelLease = input.guard.currentActionLease() let decision: SemanticDecision @@ -213,6 +221,7 @@ async function runSemanticStep( goal: input.goal, plan: input.plan, snapshot: state.observation.text, + image: state.observation.screenshot?.image, step, maxSteps: MAX_STEPS, recoveryNote: state.recoveryNote, @@ -354,7 +363,12 @@ async function terminalDecision( input.onStep(`verification failed: ${failure}`) return { ok: false, fallback: false, summary: failure, handoffs: state.handoffs } } - await publishObservation(input, { step, phase: 'complete', summary }) + await publishObservation(input, { + step, + phase: 'complete', + summary, + screenshot: verification.screenshot + }) if (!input.guard.complete()) return undefined input.onStep(`verified: ${summary}`) return { ok: true, fallback: false, summary, handoffs: state.handoffs } @@ -379,7 +393,8 @@ async function performHumanHandoff( await publishObservation(input, { step, phase: 'waiting', - summary: 'Continued after user input' + summary: 'Continued after user input', + screenshot: state.observation.screenshot }) state.lastFingerprint = fingerprint(state.observation.text) state.lastActionKey = '' @@ -468,7 +483,12 @@ async function performSemanticAction( const recovery = await recoverObservationError(input, state) if (recovery === 'recovered') return undefined if (recovery) return recovery - await publishObservation(input, { step, phase: 'checking', summary: actionLabel(decision) }) + await publishObservation(input, { + step, + phase: 'checking', + summary: actionLabel(decision), + screenshot: state.observation.screenshot + }) updateProgressEvidence(context, { decision, actionKey, before }) remember( state, @@ -498,7 +518,12 @@ async function handleUncertainAction( `The page could not be checked after an uncertain action: ${compact(state.observation.text)}` ) } - await publishObservation(input, { step, phase: 'checking', summary: UNCERTAIN_ACTION_NOTE }) + await publishObservation(input, { + step, + phase: 'checking', + summary: UNCERTAIN_ACTION_NOTE, + screenshot: state.observation.screenshot + }) if (state.uncertainCount >= 2) { await performHumanHandoff( context, diff --git a/src/main/browser/browser-semantic-evidence.ts b/src/main/browser/browser-semantic-evidence.ts index 25e203930..e0f328741 100644 --- a/src/main/browser/browser-semantic-evidence.ts +++ b/src/main/browser/browser-semantic-evidence.ts @@ -35,7 +35,7 @@ export function createBrowserSemanticEvidenceRecorder( }) const executionDevice = getTaskExecutionDevice() return async (observation) => { - const screenshot = await screen.capture() + const screenshot = observation.screenshot ?? (await screen.capture()) appendTaskStepDetail( input.taskId, 'web_use', diff --git a/src/main/browser/playwright-mcp-session.ts b/src/main/browser/playwright-mcp-session.ts index 62d08e2e5..2b012ed8e 100644 --- a/src/main/browser/playwright-mcp-session.ts +++ b/src/main/browser/playwright-mcp-session.ts @@ -1,3 +1,4 @@ +import type { VisionScreen } from '../vision/vision-agent' import { setTimeout as delay } from 'node:timers/promises' import { createConnection } from '@playwright/mcp' import { Client } from '@modelcontextprotocol/sdk/client/index.js' @@ -34,6 +35,8 @@ export interface PlaywrightToolResult { text: string isError: boolean uncertain?: true + /** Local model frame paired with this observation; never an MCP action argument. */ + screenshot?: Awaited> } /** diff --git a/src/main/chat-fallback.ts b/src/main/chat-fallback.ts index bd06265d0..897292844 100644 --- a/src/main/chat-fallback.ts +++ b/src/main/chat-fallback.ts @@ -1,12 +1,15 @@ // Applies the shared chat fallback policy (@offgrid/models nextChatFallbackStep): a request that // does not fit or was refused fails once; a server that could not answer is retried on the same // model, then on at most one other local model, with one notice. It no longer walks every -// installed model and remote server. +// installed model and remote server. GPU faults restart the same model, then use its CPU engine. import { nextChatFallbackStep, type ChatFailureKind } from '@offgrid/models' +import { demoteBackend } from './backend-facts' import { ModelServerError } from './llm/http-post' import { parseRemoteVisionModelId, remoteVisionModelId } from '../shared/remote-vision-server' +let deviceFallbackNotified = false + function failureKind(error: unknown): ChatFailureKind { if (error instanceof ModelServerError) return error.kind if (error instanceof Error && error.name === 'AbortError') return 'aborted' @@ -40,11 +43,15 @@ export async function runChatWithFallback( options: { signal?: AbortSignal canRetry?: () => boolean + onDeviceFallback?: (message: string) => void + onRestarting?: (restarting: boolean) => void /** Both arguments are display names, never model ids. */ onModelChanged?: (failed: string, next: string) => void } = {} ): Promise<{ result: T; modelName?: string }> { let failures = 0 + let deviceFailures = 0 + let deviceRecovery = false let modelName: string | undefined let documentScale: number | undefined for (;;) { @@ -54,13 +61,47 @@ export async function runChatWithFallback( } catch (error) { if (options.signal?.aborted || options.canRetry?.() === false) throw error failures++ - const kind = failureKind(error) + const { llm } = await import('./llm') + const reportedKind = failureKind(error) + const kind = + reportedKind === 'device_fault' || + ((reportedKind === 'unavailable' || reportedKind === 'unknown') && + (llm.hasDeviceFault() || deviceRecovery)) + ? 'device_fault' + : reportedKind + if (kind === 'device_fault') { + deviceRecovery = true + deviceFailures++ + } const step = nextChatFallbackStep({ kind, - failures, - localAlternatives: failures === 2 && kind !== 'overflow' ? await localAlternatives() : [] + failures: kind === 'device_fault' ? deviceFailures : failures, + localAlternatives: + failures === 2 && kind !== 'overflow' && !deviceRecovery ? await localAlternatives() : [] }) if (step.action === 'fail') throw error + if (step.action === 'restart-same' || step.action === 'restart-cpu') { + if (step.action === 'restart-cpu') { + const backend = llm.gpuBackend() + if (backend) demoteBackend('llm', backend) + // A second GPU must not replace the failed GPU during CPU recovery. + for (const gpu of ['cuda', 'vulkan', 'metal'] as const) demoteBackend('llm', gpu) + } + options.onRestarting?.(true) + try { + await llm.restartSameModel() + } finally { + options.onRestarting?.(false) + } + if (options.signal?.aborted) throw error + if (step.action === 'restart-cpu' && !deviceFallbackNotified && options.onDeviceFallback) { + deviceFallbackNotified = true + options.onDeviceFallback( + 'Chat moved to the CPU. The GPU stopped responding. Replies will be slower.' + ) + } + continue + } if (step.action === 'retry-same') continue if (step.action === 'retry-smaller') { documentScale = step.scale diff --git a/src/main/data-privacy.ts b/src/main/data-privacy.ts index c1d419361..d682ebf64 100644 --- a/src/main/data-privacy.ts +++ b/src/main/data-privacy.ts @@ -7,7 +7,7 @@ import path from 'path' import { app } from 'electron' import { getDB } from './database' import { ATTACHED_DOCUMENT_TABLES } from './documents/sqlite-store' -import { deleteByKinds, deleteByKindsOlderThan, resetVectors } from './vectors' +import { releaseRecordIndex } from './records/store' export interface DataCategory { id: 'chats' | 'memories' | 'captures' | 'meetings' | 'images' @@ -123,9 +123,10 @@ const CHAT_TABLES = [ 'rag_conversations', 'rag_messages', 'chat_summaries', - // Documents attached to chats: their text, passages, vectors and per-chat exclusions. - ...ATTACHED_DOCUMENT_TABLES + 'conversation_document_exclusions' ] +// memories, master_memory and entity_facts are retired (nothing writes or reads them); they stay +// here so the rows a profile already has are still deleted with the rest. const MEMORY_TABLES = [ 'memories', 'master_memory', @@ -135,6 +136,31 @@ const MEMORY_TABLES = [ 'entity_sessions' ] +/** Who keeps documents outside chats (a meeting its transcript, a long dictation its text): the + * owner prefix (`meeting` for "meeting:12") and the table its id lives in. Registered by the + * owning feature, so core never names its tables. */ +const keptDocumentOwners = new Map() + +export function registerKeptDocumentOwner(prefix: string, table: string): void { + keptDocumentOwners.set(prefix, table) +} + +/** + * After rows were deleted by a category clear or retention: owners that are gone keep nothing, + * and every document nothing else uses (no chat, knowledge base or remaining owner) is deleted. + * Throws when a document cannot be deleted. + */ +async function forgetDocumentsOfDeletedOwners(): Promise { + if (!keptDocumentOwners.size) return + const { releaseDocumentsOfDeletedOwners, removeUnreferencedDocuments } = await import( + './documents' + ) + let released = 0 + for (const [prefix, table] of keptDocumentOwners) + released += releaseDocumentsOfDeletedOwners(prefix, table) + if (released) await removeUnreferencedDocuments() +} + // The SINGLE source of truth for a FULL erase ("Delete all my data"): every store // that holds personal data. deleteAllData iterates this — so a new personal table // or directory is erased by REGISTERING it here (or, for a pro feature, via @@ -152,15 +178,24 @@ interface PersonalStore { } const CORE_PERSONAL: PersonalStore[] = [ { tables: CHAT_TABLES, dirs: ['uploads'] }, + // Every indexed document (chat attachments, knowledge-base files, meeting transcripts, long + // dictations) and which owner keeps which. + { tables: ATTACHED_DOCUMENT_TABLES, dirs: [] }, { tables: MEMORY_TABLES, dirs: ['entity-photos'] }, // Project metadata + knowledge base. Children precede parents so deletion also // works when SQLite foreign-key enforcement is enabled. { tables: ['rag_documents', 'projects'], dirs: [] }, { tables: ['connectors', 'secrets'], dirs: [] }, // MCP integrations + their OAuth tokens (must not survive a wipe) + // Memory-search vectors and the queue of records waiting to be embedded. The in-memory index + // built from them is dropped first. + { tables: ['record_vectors', 'record_queue'], dirs: [], beforeDelete: releaseRecordIndex }, { tables: ['user_profile'], dirs: [] }, { tables: [], dirs: ['captures'] }, { tables: [], dirs: ['meetings'] }, - { tables: [], dirs: ['generated-images', 'artifacts-library', 'style-thumbs'] } + { tables: [], dirs: ['generated-images', 'artifacts-library', 'style-thumbs'] }, + // The retired LanceDB index held a plaintext copy of captured text. The record schema removes it + // at first launch without waiting; a wipe erases it too, in case that removal did not finish. + { tables: [], dirs: ['lancedb'] } ] const personalStores: PersonalStore[] = [...CORE_PERSONAL] @@ -336,43 +371,29 @@ export async function clearCategory( try { return await withDeletionGuards({ scope: id, olderThanDays }, async () => { switch (id) { - case 'chats': + case 'chats': { clearTables(...CHAT_TABLES) clearDirs(ud('uploads')) + // The documents attached to those chats go with them; a knowledge base's files and the + // transcripts a meeting or dictation keeps stay. + const { removeUnreferencedDocuments } = await import('./documents') + await removeUnreferencedDocuments() break + } case 'memories': + // Each row's delete trigger removes its record vector in the same statement. clearTables(...MEMORY_TABLES) clearDirs(ud('entity-photos')) - // Delete ONLY the memory-side vectors (not the shared lancedb dir — that - // would wipe capture/meeting/chat vectors and dangle the live handle). - await deleteByKinds(['memory', 'entity', 'fact']) break case 'captures': - if (olderThanDays && olderThanDays > 0) { - clearDirsOlderThan(olderThanDays, ud('captures')) - await deleteByKindsOlderThan(['screen'], Date.now() - olderThanDays * 86_400_000) // prune stale capture vectors too - } else { - clearDirs(ud('captures')) - await deleteByKinds(['screen']) // full clear → drop capture vectors too - // Registered cleaners below remove the semantic source rows. Drop their indexing - // receipts here as well so a future capture can never inherit a stale marker. - try { - getDB() - .prepare("DELETE FROM vec_indexed WHERE key LIKE 'obs:%' OR key LIKE 'frame:%'") - .run() - } catch { - /* search index has not been initialized */ - } - } + // Registered cleaners below remove the capture rows; their delete triggers remove the + // record vectors and full-text entries with them. + if (olderThanDays && olderThanDays > 0) clearDirsOlderThan(olderThanDays, ud('captures')) + else clearDirs(ud('captures')) break case 'meetings': - if (olderThanDays && olderThanDays > 0) { - clearDirsOlderThan(olderThanDays, ud('meetings')) - await deleteByKindsOlderThan(['meeting'], Date.now() - olderThanDays * 86_400_000) // prune stale meeting vectors too - } else { - clearDirs(ud('meetings')) - await deleteByKinds(['meeting']) // full clear → drop meeting vectors too - } + if (olderThanDays && olderThanDays > 0) clearDirsOlderThan(olderThanDays, ud('meetings')) + else clearDirs(ud('meetings')) pruneDanglingMeetings() // drop rows whose media we just deleted (no ghosts) break case 'images': @@ -382,6 +403,10 @@ export async function clearCategory( for (const cleaner of categoryCleaners.get(id)?.values() ?? []) { await cleaner({ olderThanDays }) } + // A meeting or recording deleted above (in full or by age) no longer keeps its transcript. + await forgetDocumentsOfDeletedOwners() + // Vectors of deleted rows went with them (delete triggers); the in-memory copy goes too. + releaseRecordIndex() return { success: true } }) } catch (e) { @@ -400,9 +425,8 @@ export async function deleteAllData(): Promise<{ success: boolean }> { return await withDeletionGuards({ scope: 'all' }, () => { for (const store of personalStores) store.beforeDelete?.() clearTables(...personalStores.flatMap((s) => s.tables ?? [])) - clearDirs(...personalStores.flatMap((s) => s.dirs ?? []).map((d) => ud(d)), ud('lancedb')) + clearDirs(...personalStores.flatMap((s) => s.dirs ?? []).map((d) => ud(d))) clearFiles(...personalStores.flatMap((s) => s.files ?? []).map((file) => ud(file))) - resetVectors() // the lancedb dir is gone — drop cached handles so it reopens clean pruneDanglingMeetings() // media is gone now → drop all meeting rows (no ghosts) return { success: true } }) diff --git a/src/main/database.ts b/src/main/database.ts index ccca31243..aeda528a8 100644 --- a/src/main/database.ts +++ b/src/main/database.ts @@ -5,7 +5,15 @@ import { selectedVersionRepairs, type StoredVersionRow } from '../shared/chat-ve import Database from 'better-sqlite3-multiple-ciphers' import { safeStorage } from 'electron' import path from 'path' -import { escapeFtsMatch } from './fts-match' +import { + FTS_SCHEMA_VERSION, + FTS_TOKENIZE, + FTS_TRIGRAM_TOKENIZE, + ftsQuery, + ftsTypeahead, + needsTrigram, + normaliseForIndex +} from '@offgrid/rag/records' import { dataDir } from './runtime-env' import fs from 'fs' import crypto from 'crypto' @@ -20,13 +28,6 @@ import type { UserProfileContract } from '../shared/ipc-contracts' -/** - * The full-text schema this build expects. Bump it when an FTS table, its columns or its triggers - * change: that is what makes every existing profile rebuild its indexes exactly once, on the next - * launch, instead of on every launch. - */ -const FTS_SCHEMA_VERSION = 1 - /** Conversation-list page size when a caller does not ask for one, and the ceiling it may ask for. */ const DEFAULT_CONVERSATION_PAGE = 500 const MAX_CONVERSATION_PAGE = 2_000 @@ -73,30 +74,633 @@ function loadOrCreateKey(dbPath: string): string | null { } } -// Cosine similarity function for vector search -// Returns similarity between 0 and 1 (1 = identical) -function cosineSimilarity(v1Str: string, v2Str: string): number { +// --- Full-text indexes -------------------------------------------------------------------------- +// Every FTS5 table uses one tokenizer (FTS_TOKENIZE: keeps Devanagari vowel signs inside words) and +// indexes text exactly as normaliseForIndex leaves it (NFC, no zero-width joiners), so a word +// matches the same way in every corpus. Triggers normalise through offgrid_fts_text(); a rebuild +// re-inserts through it too, never with FTS5's 'rebuild' command, which would read the raw text. +// Each table records the FTS_SCHEMA_VERSION it was built with in fts_schema, so a tokenizer or +// normalisation change rebuilds every profile's indexes exactly once, in the background after +// start-up (never inside getDB()). frame_fts is the exception: see frame-text-index.ts. +// +// Chinese, Japanese, Thai, Lao, Khmer and Burmese are written without spaces, so unicode61 makes a +// whole run one token and a word inside it never matches. Each index with `trigram` set has a twin, +// `${name}_tri`, on FTS5's trigram tokenizer (any substring of three characters or more), holding +// only the rows whose text has such a script (offgrid_needs_trigram). See ensureTrigramIndex. + +/** The SQLite functions the FTS triggers call. Every connection that writes indexed tables needs them. */ +export function registerFtsFunctions(database: Database.Database): void { + database.function('offgrid_fts_text', { deterministic: true }, (value: unknown) => + typeof value === 'string' ? normaliseForIndex(value) : (value as null) + ) + // 1 when any argument is text in a script written without spaces (the trigram twins' filter). + database.function( + 'offgrid_needs_trigram', + { deterministic: true, varargs: true }, + (...values: unknown[]) => + values.some((value) => typeof value === 'string' && needsTrigram(value)) ? 1 : 0 + ) +} + +/** An external-content FTS5 index over one table's text columns, kept in step by triggers. */ +export interface FtsIndexSpec { + /** The FTS5 table. */ + name: string + /** The table it indexes, and the column that is that table's rowid. */ + source: string + rowid: string + /** Searched columns; their text is normalised as it is indexed. */ + columns: string[] + /** Stored with each row but not searched (e.g. the owning conversation). */ + unindexed?: string[] + /** Its triggers are `${triggers}_ai`, `_ad` and `_au`. */ + triggers: string + /** Also keep a trigram twin (`${name}_tri`) of the rows in no-space scripts. */ + trigram?: boolean +} + +export const CORE_FTS_INDEXES: FtsIndexSpec[] = [ + { + name: 'rag_message_fts', + source: 'rag_messages', + rowid: 'id', + columns: ['content'], + unindexed: ['conversation_id'], + triggers: 'rag_messages', + trigram: true + }, + { + name: 'summary_fts', + source: 'chat_summaries', + rowid: 'rowid', + columns: ['summary'], + unindexed: ['session_id'], + triggers: 'summaries', + trigram: true + }, + { + name: 'entity_fts', + source: 'entities', + rowid: 'id', + columns: ['name', 'summary', 'type'], + triggers: 'entities', + trigram: true + } +] + +/** Screen moments. Pro creates `observations`; the index is rebuilt here once the table exists. */ +export const OBSERVATION_FTS: FtsIndexSpec = { + name: 'observation_fts', + source: 'observations', + rowid: 'id', + columns: ['summary'], + triggers: 'observations', + trigram: true +} + +/** + * The legacy `memories` and `entity_facts` stores are retired: nothing writes or searches them. Their + * rows stay (the user's data, cleared by "Delete all my data"); their full-text indexes, which were + * derived from those rows, and the triggers that kept them are removed so no stale copy of the text + * outlives the rows. Idempotent. + */ +function retireLegacyMemoryIndexes(database: Database.Database): void { + database.exec(` + DROP TRIGGER IF EXISTS memories_ai; + DROP TRIGGER IF EXISTS memories_ad; + DROP TRIGGER IF EXISTS memories_au; + DROP TRIGGER IF EXISTS memories_next_ai; + DROP TRIGGER IF EXISTS memories_next_ad; + DROP TRIGGER IF EXISTS memories_next_au; + DROP TRIGGER IF EXISTS entity_facts_ai; + DROP TRIGGER IF EXISTS entity_facts_ad; + DROP TRIGGER IF EXISTS entity_facts_au; + DROP TRIGGER IF EXISTS entity_facts_next_ai; + DROP TRIGGER IF EXISTS entity_facts_next_ad; + DROP TRIGGER IF EXISTS entity_facts_next_au; + DROP TABLE IF EXISTS memory_fts; + DROP TABLE IF EXISTS memory_fts_next; + DROP TABLE IF EXISTS entity_fact_fts; + DROP TABLE IF EXISTS entity_fact_fts_next; + `) + for (const name of ['memory_fts', 'entity_fact_fts']) clearRebuildCursor(database, name) +} + +/** Whether a table (or virtual table) exists. The one check every store uses: Pro creates many of + * the tables core reads, lazily. */ +export function tableExists(database: Database.Database, name: string): boolean { + return !!database + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(name) +} + +/** True when `name` exists and was built with this build's tokenizer and normalisation. */ +export function ftsIndexIsCurrent(database: Database.Database, name: string): boolean { + database.exec( + 'CREATE TABLE IF NOT EXISTS fts_schema (name TEXT PRIMARY KEY, version INTEGER NOT NULL)' + ) + const row = database.prepare('SELECT version FROM fts_schema WHERE name = ?').get(name) as + | { version: number } + | undefined + return !!row && row.version >= FTS_SCHEMA_VERSION && tableExists(database, name) +} + +export function markFtsIndexCurrent(database: Database.Database, name: string): void { + database.exec( + 'CREATE TABLE IF NOT EXISTS fts_schema (name TEXT PRIMARY KEY, version INTEGER NOT NULL)' + ) + database + .prepare( + 'INSERT INTO fts_schema (name, version) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET version = excluded.version' + ) + .run(name, FTS_SCHEMA_VERSION) +} + +/** Deleted terms are removed from the index itself, not only marked (FTS5 'secure-delete'). */ +export function enableFtsSecureDelete(database: Database.Database, name: string): void { + database.exec(`INSERT INTO ${name}(${name}, rank) VALUES('secure-delete', 1)`) +} + +function ftsRowValues(spec: FtsIndexSpec, row: 'new' | 'old'): string { + return [ + `${row}.${spec.rowid}`, + ...spec.columns.map((column) => `offgrid_fts_text(${row}.${column})`), + ...(spec.unindexed ?? []).map((column) => `${row}.${column}`) + ].join(', ') +} + +function ftsColumnNames(spec: FtsIndexSpec): string { + return ['rowid', ...spec.columns, ...(spec.unindexed ?? [])].join(', ') +} + +/** Create the three sync triggers if they are missing. */ +export function installFtsTriggers(database: Database.Database, spec: FtsIndexSpec): void { + const { name, source, triggers } = spec + const columns = ftsColumnNames(spec) + database.exec(` + CREATE TRIGGER IF NOT EXISTS ${triggers}_ai AFTER INSERT ON ${source} BEGIN + INSERT INTO ${name}(${columns}) VALUES (${ftsRowValues(spec, 'new')}); + END; + CREATE TRIGGER IF NOT EXISTS ${triggers}_ad AFTER DELETE ON ${source} BEGIN + INSERT INTO ${name}(${name}, ${columns}) VALUES ('delete', ${ftsRowValues(spec, 'old')}); + END; + CREATE TRIGGER IF NOT EXISTS ${triggers}_au AFTER UPDATE ON ${source} BEGIN + INSERT INTO ${name}(${name}, ${columns}) VALUES ('delete', ${ftsRowValues(spec, 'old')}); + INSERT INTO ${name}(${columns}) VALUES (${ftsRowValues(spec, 'new')}); + END; + `) +} + +/** Re-index every row of the source table (the index is emptied first). One statement: only for + * rare repairs (the legacy purge, "clear memories"); a schema upgrade rebuilds in the background + * (ensureFtsIndex queues it). */ +export function rebuildFtsIndex(database: Database.Database, spec: FtsIndexSpec): void { + database.exec(`INSERT INTO ${spec.name}(${spec.name}) VALUES('delete-all')`) + database.exec( + `INSERT INTO ${spec.name}(${ftsColumnNames(spec)}) + SELECT ${ftsSelectValues(spec)} FROM ${spec.source}` + ) + const twin = trigramName(spec) + // A twin still being filled in the background is left to finish (it reads the rows as they are). + if (spec.trigram && ftsIndexIsCurrent(database, twin)) { + database.exec(`INSERT INTO ${twin}(${twin}) VALUES('delete-all')`) + database.exec( + `INSERT INTO ${twin}(${trigramColumnNames(spec)}) + SELECT ${trigramSelectValues(spec)} FROM ${spec.source} WHERE ${trigramFilter(spec)}` + ) + } +} + +function ftsSelectValues(spec: FtsIndexSpec): string { + return [ + spec.rowid, + ...spec.columns.map((column) => `offgrid_fts_text(${column})`), + ...(spec.unindexed ?? []) + ].join(', ') +} + +function createFtsTable(database: Database.Database, spec: FtsIndexSpec, name: string): void { + const columns = [...spec.columns, ...(spec.unindexed ?? []).map((c) => `${c} UNINDEXED`)] + database.exec(` + CREATE VIRTUAL TABLE ${name} USING fts5( + ${columns.join(', ')}, + content='${spec.source}'${spec.rowid === 'rowid' ? '' : `, content_rowid='${spec.rowid}'`}, + tokenize = "${FTS_TOKENIZE}" + ); + `) + enableFtsSecureDelete(database, name) +} + +/** + * Make `spec`'s index usable without reading its rows, so it is safe inside getDB() and on any + * request path: + * - current: put back triggers something dropped; + * - missing: create it (empty) with its triggers; an empty source table makes it current; + * - built by an older schema: leave it serving, with the triggers it was built with (their + * deletes must match what it indexed), until the background rebuild replaces it. + * Returns true when the index still needs rebuilding (queued; startIndexRebuilds runs it). A no-op + * until the source table exists. + */ +/** Indexes small enough to verify at startup (to-dos, meetings): FTS5's own integrity check. */ +const VERIFIED_AT_STARTUP = new Set(['action_fts', 'meeting_fts']) +const verified = new Set() + +/** + * An index FTS5 reports inconsistent with its table (an entry deleted that was never added) is + * marked out of date, so the background rebuild replaces it. Once per index per launch. + */ +function verifySmallIndex(database: Database.Database, spec: FtsIndexSpec): void { + if (!VERIFIED_AT_STARTUP.has(spec.name) || verified.has(spec.name)) return + verified.add(spec.name) + if (!tableExists(database, spec.name) || !ftsIndexIsCurrent(database, spec.name)) return try { - const v1 = JSON.parse(v1Str) as number[] - const v2 = JSON.parse(v2Str) as number[] + database.exec(`INSERT INTO ${spec.name}(${spec.name}, rank) VALUES ('integrity-check', 1)`) + } catch (error) { + console.warn(`[fts] ${spec.name} is inconsistent; rebuilding it:`, (error as Error).message) + // Its triggers stop now, so edits cannot fail on it; the rebuild puts them back with the swap. + database.exec(` + DROP TRIGGER IF EXISTS ${spec.triggers}_ai; + DROP TRIGGER IF EXISTS ${spec.triggers}_ad; + DROP TRIGGER IF EXISTS ${spec.triggers}_au; + `) + database.prepare('DELETE FROM fts_schema WHERE name = ?').run(spec.name) + } +} + +export function ensureFtsIndex(database: Database.Database, spec: FtsIndexSpec): boolean { + verifySmallIndex(database, spec) + const pending = ensureMainFtsIndex(database, spec) + if (spec.trigram) ensureTrigramIndex(database, spec) + return pending +} + +function ensureMainFtsIndex(database: Database.Database, spec: FtsIndexSpec): boolean { + if (!tableExists(database, spec.source)) return false + if (ftsIndexIsCurrent(database, spec.name)) { + installFtsTriggers(database, spec) + return false + } + if (!tableExists(database, spec.name)) { + database.transaction(() => { + database.exec(` + DROP TRIGGER IF EXISTS ${spec.triggers}_ai; + DROP TRIGGER IF EXISTS ${spec.triggers}_ad; + DROP TRIGGER IF EXISTS ${spec.triggers}_au; + `) + createFtsTable(database, spec, spec.name) + // Triggers only once every existing row is indexed. On a table that already has rows, an + // update or delete would otherwise remove an entry that was never added, which FTS5 (with + // secure-delete) reports as a malformed index. The background rebuild indexes the rows into a + // twin with guarded triggers and installs these triggers when it swaps the twin in. + if (!database.prepare(`SELECT 1 FROM ${spec.source} LIMIT 1`).get()) { + installFtsTriggers(database, spec) + markFtsIndexCurrent(database, spec.name) + } + })() + if (ftsIndexIsCurrent(database, spec.name)) return false + } + queueIndexRebuild(spec.name, ftsRebuildStep(spec)) + return true +} + +// --- Trigram twins ------------------------------------------------------------------------------- +// `${name}_tri` is an external-content FTS5 table over the same source table and columns, on the +// trigram tokenizer, holding only the rows offgrid_needs_trigram picks (mostly Latin profiles hold +// next to nothing in it). Its triggers carry the same filter, so a delete removes exactly what an +// insert indexed; 'secure-delete' erases the deleted terms, as on every other index. A missing or +// outdated twin is created empty with its triggers and filled in the background in rowid order +// (fts_rebuild holds how far): until then its triggers index only rows already copied, and later +// rows are copied with their latest text, so no row is indexed twice or deleted unindexed. frame_fts +// has no twin: it is every word ever on screen, and observation summaries cover the moments. + +/** The trigram twin of an index. */ +export function trigramName(spec: { name: string }): string { + return `${spec.name}_tri` +} + +function trigramColumnNames(spec: FtsIndexSpec): string { + return ['rowid', ...spec.columns].join(', ') +} + +function trigramRowValues(spec: FtsIndexSpec, row: 'new' | 'old'): string { + return [ + `${row}.${spec.rowid}`, + ...spec.columns.map((column) => `offgrid_fts_text(${row}.${column})`) + ].join(', ') +} + +function trigramSelectValues(spec: FtsIndexSpec): string { + return [spec.rowid, ...spec.columns.map((column) => `offgrid_fts_text(${column})`)].join(', ') +} + +/** True for a row the twin indexes. `row` is 'new', 'old' or '' (the source table itself). */ +function trigramFilter(spec: FtsIndexSpec, row: 'new' | 'old' | '' = ''): string { + const prefix = row ? `${row}.` : '' + return `offgrid_needs_trigram(${spec.columns.map((column) => prefix + column).join(', ')})` +} - if (v1.length !== v2.length) return 0 +function createTrigramTable(database: Database.Database, spec: FtsIndexSpec): void { + const name = trigramName(spec) + database.exec(` + CREATE VIRTUAL TABLE ${name} USING fts5( + ${spec.columns.join(', ')}, + content='${spec.source}'${spec.rowid === 'rowid' ? '' : `, content_rowid='${spec.rowid}'`}, + tokenize = "${FTS_TRIGRAM_TOKENIZE}" + ); + `) + enableFtsSecureDelete(database, name) +} + +/** The twin's triggers. While it is being filled, only rows the fill has copied are kept in step. */ +function installTrigramTriggers(database: Database.Database, spec: FtsIndexSpec): void { + const name = trigramName(spec) + const columns = trigramColumnNames(spec) + rebuildState(database) // the triggers read fts_rebuild + const copied = (row: 'new' | 'old'): string => + `${row}.${spec.rowid} <= COALESCE((SELECT cursor FROM fts_rebuild WHERE name = '${name}'), ${Number.MAX_SAFE_INTEGER})` + const indexed = (row: 'new' | 'old'): string => `${trigramFilter(spec, row)} AND ${copied(row)}` + database.exec(` + CREATE TRIGGER IF NOT EXISTS ${spec.triggers}_tri_ai AFTER INSERT ON ${spec.source} + WHEN ${indexed('new')} BEGIN + INSERT INTO ${name}(${columns}) VALUES (${trigramRowValues(spec, 'new')}); + END; + CREATE TRIGGER IF NOT EXISTS ${spec.triggers}_tri_ad AFTER DELETE ON ${spec.source} + WHEN ${indexed('old')} BEGIN + INSERT INTO ${name}(${name}, ${columns}) VALUES ('delete', ${trigramRowValues(spec, 'old')}); + END; + CREATE TRIGGER IF NOT EXISTS ${spec.triggers}_tri_au AFTER UPDATE OF ${spec.columns.join(', ')} ON ${spec.source} + WHEN ${trigramFilter(spec, 'old')} OR ${trigramFilter(spec, 'new')} BEGIN + INSERT INTO ${name}(${name}, ${columns}) SELECT 'delete', ${trigramRowValues(spec, 'old')} WHERE ${indexed('old')}; + INSERT INTO ${name}(${columns}) SELECT ${trigramRowValues(spec, 'new')} WHERE ${indexed('new')}; + END; + `) +} - let dotProduct = 0 - let normA = 0 - let normB = 0 +function dropTrigramTriggers(database: Database.Database, spec: FtsIndexSpec): void { + database.exec(` + DROP TRIGGER IF EXISTS ${spec.triggers}_tri_ai; + DROP TRIGGER IF EXISTS ${spec.triggers}_tri_ad; + DROP TRIGGER IF EXISTS ${spec.triggers}_tri_au; + `) +} - for (let i = 0; i < v1.length; i++) { - // i < v1.length === v2.length (checked above), so both reads are in-bounds. - dotProduct += v1[i]! * v2[i]! - normA += v1[i]! * v1[i]! - normB += v2[i]! * v2[i]! +/** + * Make `spec`'s trigram twin usable without reading its rows (safe inside getDB()): a current twin + * gets back any trigger something dropped; a missing or outdated one is created empty and filled in + * the background after start-up (trigramFillStep); a fill under way resumes. A no-op until the + * source table exists. + */ +export function ensureTrigramIndex(database: Database.Database, spec: FtsIndexSpec): void { + if (!tableExists(database, spec.source)) return + const name = trigramName(spec) + if (ftsIndexIsCurrent(database, name)) { + installTrigramTriggers(database, spec) + return + } + if (rebuildCursor(database, name) === null || !tableExists(database, name)) { + database.transaction(() => { + dropTrigramTriggers(database, spec) + database.exec(`DROP TABLE IF EXISTS ${name}`) + createTrigramTable(database, spec) + if (database.prepare(`SELECT 1 FROM ${spec.source} LIMIT 1`).get()) { + setRebuildCursor(database, name, 0) + } else { + clearRebuildCursor(database, name) + markFtsIndexCurrent(database, name) + } + installTrigramTriggers(database, spec) + })() + if (ftsIndexIsCurrent(database, name)) return + } else { + installTrigramTriggers(database, spec) + } + queueIndexRebuild(name, trigramFillStep(spec)) +} + +/** One slice of filling a twin with the source rows that need it, in rowid order. The last slice + * ends the fill (its triggers then keep every row in step) in the same transaction. */ +function trigramFillStep(spec: FtsIndexSpec): () => boolean { + let batch = 500 + const name = trigramName(spec) + return () => { + const database = getDB() + if (!tableExists(database, spec.source) || !tableExists(database, name)) return false + const cursor = rebuildCursor(database, name) + if (cursor === null) return false + const started = Date.now() + const upper = ( + database + .prepare( + `SELECT MAX(r) AS r FROM (SELECT ${spec.rowid} AS r FROM ${spec.source} + WHERE ${spec.rowid} > ? ORDER BY ${spec.rowid} LIMIT ?)` + ) + .get(cursor, batch) as { r: number | null } + ).r + if (upper === null) { + database.transaction(() => { + clearRebuildCursor(database, name) + markFtsIndexCurrent(database, name) + })() + return false } + database.transaction(() => { + database + .prepare( + `INSERT INTO ${name}(${trigramColumnNames(spec)}) + SELECT ${trigramSelectValues(spec)} FROM ${spec.source} + WHERE ${spec.rowid} > ? AND ${spec.rowid} <= ? AND ${trigramFilter(spec)}` + ) + .run(cursor, upper) + setRebuildCursor(database, name, upper) + })() + batch = adaptBatch(batch, Date.now() - started) + return true + } +} - if (normA === 0 || normB === 0) return 0 - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)) - } catch { - return 0 +// --- Background index rebuilds ------------------------------------------------------------------ +// A tokenizer or normalisation change (FTS_SCHEMA_VERSION) rebuilds each index beside the old one, +// `${name}_next`, a slice at a time on the main thread's idle moments, while the old index keeps +// answering searches. Rows are copied in rowid order; fts_rebuild holds how far the copy got, and +// the _next triggers mirror only edits to rows already copied (later rows are copied with their +// latest text). When every row is in, the new index replaces the old in one transaction. Progress +// survives a restart. Nothing here runs until startIndexRebuilds() (after start-up). The same queue +// and cursors (fts_rebuild) serve the document index rebuild, the trigram twins' fills and the +// memory-search queue seeding. + +/** A slice indexes for about this long, then the thread rests. */ +const REBUILD_SLICE_MS = 20 +const REBUILD_PAUSE_MS = 60 + +const pendingRebuilds = new Map boolean>() +let rebuildsStarted = false +let rebuildTimer: ReturnType | null = null + +/** Queue background work (one slice per call; it returns whether more remains) under `name`. */ +export function queueIndexRebuild(name: string, step: () => boolean): void { + if (!pendingRebuilds.has(name)) pendingRebuilds.set(name, step) + if (rebuildsStarted) scheduleRebuildSlice() +} + +/** Start the queued index rebuilds, a slice at a time. Called once start-up has settled. */ +export function startIndexRebuilds(): void { + rebuildsStarted = true + scheduleRebuildSlice() +} + +/** Stop between slices (shutdown, profile switch). Progress is kept; the next start resumes. */ +export function stopIndexRebuilds(): void { + rebuildsStarted = false + if (rebuildTimer) clearTimeout(rebuildTimer) + rebuildTimer = null +} + +function scheduleRebuildSlice(): void { + if (rebuildTimer || !pendingRebuilds.size) return + rebuildTimer = setTimeout(runRebuildSlice, REBUILD_PAUSE_MS) + rebuildTimer.unref() +} + +function runRebuildSlice(): void { + rebuildTimer = null + // A closed connection (shutdown, profile switch) is not reopened for background work. + if (!rebuildsStarted || !db?.open) return + const next = pendingRebuilds.entries().next() + if (next.done) return + const [name, step] = next.value + let more = false + try { + more = step() + } catch (error) { + // Not marked current, so it is retried on the next launch; the old index keeps serving. + console.error(`[database] background rebuild of ${name} failed; will retry`, error) + } + if (!more) pendingRebuilds.delete(name) + scheduleRebuildSlice() +} + +/** Rows a slice copies; adjusted so a slice takes about REBUILD_SLICE_MS. */ +function adaptBatch(batch: number, elapsedMs: number): number { + if (elapsedMs > REBUILD_SLICE_MS) return Math.max(10, Math.floor(batch / 2)) + if (elapsedMs < REBUILD_SLICE_MS / 2) return Math.min(5_000, batch * 2) + return batch +} + +function rebuildState(database: Database.Database): void { + database.exec( + 'CREATE TABLE IF NOT EXISTS fts_rebuild (name TEXT PRIMARY KEY, cursor INTEGER NOT NULL)' + ) +} + +/** The highest rowid copied into `name`'s rebuild; null when no rebuild is under way. */ +export function rebuildCursor(database: Database.Database, name: string): number | null { + rebuildState(database) + const row = database.prepare('SELECT cursor FROM fts_rebuild WHERE name = ?').get(name) as + | { cursor: number } + | undefined + return row ? row.cursor : null +} + +export function setRebuildCursor(database: Database.Database, name: string, cursor: number): void { + rebuildState(database) + database + .prepare( + 'INSERT INTO fts_rebuild (name, cursor) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET cursor = excluded.cursor' + ) + .run(name, cursor) +} + +export function clearRebuildCursor(database: Database.Database, name: string): void { + rebuildState(database) + database.prepare('DELETE FROM fts_rebuild WHERE name = ?').run(name) +} + +/** Triggers that mirror edits to rows already copied into the rebuild. */ +function installRebuildTriggers(database: Database.Database, spec: FtsIndexSpec): void { + const next = `${spec.name}_next` + const columns = ftsColumnNames(spec) + const copied = (row: 'new' | 'old'): string => + `${row}.${spec.rowid} <= (SELECT cursor FROM fts_rebuild WHERE name = '${spec.name}')` + database.exec(` + CREATE TRIGGER IF NOT EXISTS ${spec.triggers}_next_ai AFTER INSERT ON ${spec.source} + WHEN ${copied('new')} BEGIN + INSERT INTO ${next}(${columns}) VALUES (${ftsRowValues(spec, 'new')}); + END; + CREATE TRIGGER IF NOT EXISTS ${spec.triggers}_next_ad AFTER DELETE ON ${spec.source} + WHEN ${copied('old')} BEGIN + INSERT INTO ${next}(${next}, ${columns}) VALUES ('delete', ${ftsRowValues(spec, 'old')}); + END; + CREATE TRIGGER IF NOT EXISTS ${spec.triggers}_next_au AFTER UPDATE ON ${spec.source} BEGIN + INSERT INTO ${next}(${next}, ${columns}) SELECT 'delete', ${ftsRowValues(spec, 'old')} WHERE ${copied('old')}; + INSERT INTO ${next}(${columns}) SELECT ${ftsRowValues(spec, 'new')} WHERE ${copied('new')}; + END; + `) +} + +function dropRebuildTriggers(database: Database.Database, spec: FtsIndexSpec): void { + database.exec(` + DROP TRIGGER IF EXISTS ${spec.triggers}_next_ai; + DROP TRIGGER IF EXISTS ${spec.triggers}_next_ad; + DROP TRIGGER IF EXISTS ${spec.triggers}_next_au; + `) +} + +/** One slice of `spec`'s background rebuild; the last one swaps the new index in. */ +function ftsRebuildStep(spec: FtsIndexSpec): () => boolean { + let batch = 200 + const next = `${spec.name}_next` + return () => { + const database = getDB() + if (!tableExists(database, spec.source) || ftsIndexIsCurrent(database, spec.name)) return false + let cursor = rebuildCursor(database, spec.name) + if (cursor === null || !tableExists(database, next)) { + database.transaction(() => { + dropRebuildTriggers(database, spec) + database.exec(`DROP TABLE IF EXISTS ${next}`) + createFtsTable(database, spec, next) + setRebuildCursor(database, spec.name, 0) + installRebuildTriggers(database, spec) + })() + cursor = 0 + } + const started = Date.now() + const upper = ( + database + .prepare( + `SELECT MAX(r) AS r FROM (SELECT ${spec.rowid} AS r FROM ${spec.source} + WHERE ${spec.rowid} > ? ORDER BY ${spec.rowid} LIMIT ?)` + ) + .get(cursor, batch) as { r: number | null } + ).r + const copy = database.prepare( + `INSERT INTO ${next}(${ftsColumnNames(spec)}) + SELECT ${ftsSelectValues(spec)} FROM ${spec.source} + WHERE ${spec.rowid} > ? AND ${spec.rowid} <= ?` + ) + if (upper === null) { + // Every row is in: the new index replaces the old, with the normalising triggers. + database.transaction(() => { + copy.run(cursor, Number.MAX_SAFE_INTEGER) + dropRebuildTriggers(database, spec) + database.exec(` + DROP TRIGGER IF EXISTS ${spec.triggers}_ai; + DROP TRIGGER IF EXISTS ${spec.triggers}_ad; + DROP TRIGGER IF EXISTS ${spec.triggers}_au; + DROP TABLE IF EXISTS ${spec.name}; + ALTER TABLE ${next} RENAME TO ${spec.name}; + `) + installFtsTriggers(database, spec) + markFtsIndexCurrent(database, spec.name) + clearRebuildCursor(database, spec.name) + })() + return false + } + database.transaction(() => { + copy.run(cursor, upper) + setRebuildCursor(database, spec.name, upper) + })() + batch = adaptBatch(batch, Date.now() - started) + return true } } @@ -128,15 +732,11 @@ export function getDB(): Database.Database { db.pragma('busy_timeout = 5000') db.pragma('temp_store = MEMORY') db.pragma('cache_size = -65536') - - // Register custom function for vector search. Declare the two params EXPLICITLY: - // better-sqlite3 derives the SQL arity from fn.length, and a rest param - // (...args) has length 0 — so it registered as a 0-arg function and every - // 2-arg call ("SELECT cosine_similarity(embedding, ?)") threw "wrong number of - // arguments to function cosine_similarity()". - db.function('cosine_similarity', (a: unknown, b: unknown) => - cosineSimilarity(a as string, b as string) - ) + // Deleted content is overwritten inside the pages it lived in (screenpipe leaves it; a deleted + // capture must not be readable from the file). FAST adds no I/O: pages already being written are + // zeroed, whole freed pages are not. The FTS5 indexes also use their own 'secure-delete'. + db.pragma('secure_delete = FAST') + registerFtsFunctions(db) // Initialize Schema db.exec(` @@ -159,60 +759,17 @@ export function getDB(): Database.Database { FOREIGN KEY(conversation_id) REFERENCES conversations(id) ON DELETE CASCADE ); - -- Legacy 'memories' for vector search (optional link to message_id later) + -- Legacy memories, retired: nothing writes, indexes, embeds or searches them. Existing rows stay + -- until the user deletes their data ("Delete all my data" clears them). CREATE TABLE IF NOT EXISTS memories ( id INTEGER PRIMARY KEY AUTOINCREMENT, content TEXT NOT NULL, - raw_text TEXT, + raw_text TEXT, source_app TEXT, - session_id TEXT, + session_id TEXT, message_id INTEGER, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - embedding TEXT - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5( - content, - content='memories' - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5( - content, - conversation_id UNINDEXED, - content='messages', - content_rowid='id' - ); - - -- Chat message content, searchable. The chat-list content search used to be a - -- lower(content) LIKE match per term over rag_messages, which no index can serve: every - -- keystroke's search read every message ever stored. External-content FTS5, same shape as - -- message_fts above, so the index holds terms and the rows stay in rag_messages. - CREATE VIRTUAL TABLE IF NOT EXISTS rag_message_fts USING fts5( - content, - conversation_id UNINDEXED, - content='rag_messages', - content_rowid='id' - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS summary_fts USING fts5( - summary, - session_id UNINDEXED, - content='chat_summaries' - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS entity_fts USING fts5( - name, - summary, - type, - content='entities', - content_rowid='id' - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS entity_fact_fts USING fts5( - fact, - entity_id UNINDEXED, - content='entity_facts', - content_rowid='id' + embedding TEXT ); CREATE TABLE IF NOT EXISTS entities ( @@ -225,6 +782,7 @@ export function getDB(): Database.Database { UNIQUE(name, type) ); + -- Legacy entity facts, retired like memories: rows stay, nothing writes or reads them. CREATE TABLE IF NOT EXISTS entity_facts ( id INTEGER PRIMARY KEY AUTOINCREMENT, entity_id INTEGER NOT NULL, @@ -270,7 +828,7 @@ export function getDB(): Database.Database { ); `) - // Create Master Memory Table - cumulative summary of all summaries + // Master Memory Table - the retired cumulative profile; nothing writes or reads it. db.exec(` CREATE TABLE IF NOT EXISTS master_memory ( id INTEGER PRIMARY KEY CHECK (id = 1), @@ -318,77 +876,27 @@ export function getDB(): Database.Database { initializeSettingsStore(db) - // Triggers to keep FTS in sync - const triggers = [ - `CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN - INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content); - END;`, - `CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN - INSERT INTO memory_fts(memory_fts, rowid, content) VALUES('delete', old.id, old.content); - END;`, - `CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN - INSERT INTO memory_fts(memory_fts, rowid, content) VALUES('delete', old.id, old.content); - INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content); - END;`, - - `CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN - INSERT INTO message_fts(rowid, content, conversation_id) VALUES (new.id, new.content, new.conversation_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN - INSERT INTO message_fts(message_fts, rowid, content, conversation_id) VALUES('delete', old.id, old.content, old.conversation_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN - INSERT INTO message_fts(message_fts, rowid, content, conversation_id) VALUES('delete', old.id, old.content, old.conversation_id); - INSERT INTO message_fts(rowid, content, conversation_id) VALUES (new.id, new.content, new.conversation_id); - END;`, - - `CREATE TRIGGER IF NOT EXISTS rag_messages_ai AFTER INSERT ON rag_messages BEGIN - INSERT INTO rag_message_fts(rowid, content, conversation_id) VALUES (new.id, new.content, new.conversation_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS rag_messages_ad AFTER DELETE ON rag_messages BEGIN - INSERT INTO rag_message_fts(rag_message_fts, rowid, content, conversation_id) VALUES('delete', old.id, old.content, old.conversation_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS rag_messages_au AFTER UPDATE ON rag_messages BEGIN - INSERT INTO rag_message_fts(rag_message_fts, rowid, content, conversation_id) VALUES('delete', old.id, old.content, old.conversation_id); - INSERT INTO rag_message_fts(rowid, content, conversation_id) VALUES (new.id, new.content, new.conversation_id); - END;`, - - `CREATE TRIGGER IF NOT EXISTS summaries_ai AFTER INSERT ON chat_summaries BEGIN - INSERT INTO summary_fts(rowid, summary, session_id) VALUES (new.rowid, new.summary, new.session_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS summaries_ad AFTER DELETE ON chat_summaries BEGIN - INSERT INTO summary_fts(summary_fts, rowid, summary, session_id) VALUES('delete', old.rowid, old.summary, old.session_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS summaries_au AFTER UPDATE ON chat_summaries BEGIN - INSERT INTO summary_fts(summary_fts, rowid, summary, session_id) VALUES('delete', old.rowid, old.summary, old.session_id); - INSERT INTO summary_fts(rowid, summary, session_id) VALUES (new.rowid, new.summary, new.session_id); - END;`, - - `CREATE TRIGGER IF NOT EXISTS entities_ai AFTER INSERT ON entities BEGIN - INSERT INTO entity_fts(rowid, name, summary, type) VALUES (new.id, new.name, new.summary, new.type); - END;`, - `CREATE TRIGGER IF NOT EXISTS entities_ad AFTER DELETE ON entities BEGIN - INSERT INTO entity_fts(entity_fts, rowid, name, summary, type) VALUES('delete', old.id, old.name, old.summary, old.type); - END;`, - `CREATE TRIGGER IF NOT EXISTS entities_au AFTER UPDATE ON entities BEGIN - INSERT INTO entity_fts(entity_fts, rowid, name, summary, type) VALUES('delete', old.id, old.name, old.summary, old.type); - INSERT INTO entity_fts(rowid, name, summary, type) VALUES (new.id, new.name, new.summary, new.type); - END;`, - - `CREATE TRIGGER IF NOT EXISTS entity_facts_ai AFTER INSERT ON entity_facts BEGIN - INSERT INTO entity_fact_fts(rowid, fact, entity_id) VALUES (new.id, new.fact, new.entity_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS entity_facts_ad AFTER DELETE ON entity_facts BEGIN - INSERT INTO entity_fact_fts(entity_fact_fts, rowid, fact, entity_id) VALUES('delete', old.id, old.fact, old.entity_id); - END;`, - `CREATE TRIGGER IF NOT EXISTS entity_facts_au AFTER UPDATE ON entity_facts BEGIN - INSERT INTO entity_fact_fts(entity_fact_fts, rowid, fact, entity_id) VALUES('delete', old.id, old.fact, old.entity_id); - INSERT INTO entity_fact_fts(rowid, fact, entity_id) VALUES (new.id, new.fact, new.entity_id); - END;` - ] - - for (const trigger of triggers) { - db.exec(trigger) + // Every full-text index over the core tables, on the shared tokenizer. message_fts indexed the + // legacy scraped `messages` table for the retired rag:chat retrieval; nothing reads it now. + db.exec(` + DROP TRIGGER IF EXISTS messages_ai; + DROP TRIGGER IF EXISTS messages_ad; + DROP TRIGGER IF EXISTS messages_au; + DROP TABLE IF EXISTS message_fts; + `) + // Cheap here: a missing index is created empty, and one built by an older schema keeps serving + // while it is rebuilt in the background after start-up (startIndexRebuilds). + for (const spec of [...CORE_FTS_INDEXES, OBSERVATION_FTS]) { + try { + ensureFtsIndex(db, spec) + } catch (error) { + console.error(`[database] full-text index ${spec.name} could not be prepared`, error) + } + } + try { + retireLegacyMemoryIndexes(db) + } catch (error) { + console.error('[database] legacy memory indexes could not be removed', error) } // The search index that makes the conversation list's content search an index lookup instead of @@ -404,34 +912,6 @@ export function getDB(): Database.Database { WHERE role = 'user' AND context LIKE '%"attachments"%'` ) - /** - * FTS backfill, versioned so it happens once per profile instead of once per launch. - * - * Rebuilding five full-text indexes reads every message, summary, entity and fact in the - * database, synchronously, on Electron's main thread. It ran on EVERY startup - so the bigger a - * user's history got, the longer their app took to become usable, for work that had already been - * done. `PRAGMA user_version` is the gate: bumping FTS_SCHEMA_VERSION is how a future change to - * an FTS table or trigger repairs every existing profile exactly once. - * - * Nothing is trimmed or dropped here. This only rebuilds a derived index from rows that stay - * exactly as they are. - */ - const installedFtsVersion = Number(db.pragma('user_version', { simple: true }) ?? 0) - if (installedFtsVersion < FTS_SCHEMA_VERSION) { - try { - db.exec("INSERT INTO message_fts(message_fts) VALUES('rebuild')") - db.exec("INSERT INTO rag_message_fts(rag_message_fts) VALUES('rebuild')") - db.exec("INSERT INTO summary_fts(summary_fts) VALUES('rebuild')") - db.exec("INSERT INTO entity_fts(entity_fts) VALUES('rebuild')") - db.exec("INSERT INTO entity_fact_fts(entity_fact_fts) VALUES('rebuild')") - db.pragma(`user_version = ${FTS_SCHEMA_VERSION}`) - } catch (error) { - // The version is deliberately NOT advanced: a failed rebuild must be retried on the next - // launch, not remembered as done. Search degrades to whatever the triggers have indexed. - console.error('[database] full-text index rebuild failed; will retry next launch', error) - } - } - // Migration: Add timestamp column to messages if it doesn't exist try { db.exec(`ALTER TABLE messages ADD COLUMN timestamp TEXT`) @@ -576,7 +1056,6 @@ export interface ChatSessionRecord { title: string | null source_app: string | null last_activity: string - memory_count: number entity_count: number summary: string | null } @@ -591,27 +1070,15 @@ export interface MessageRecord { created_at: string } -export interface MemoryRecord { - id: number - content: string - raw_text: string | null - source_app: string | null - session_id: string | null - message_id: number | null - created_at: string - name: string | null -} - export function getChatSessions(): ChatSessionRecord[] { const db = getDB() - // Query conversations with memory and entity counts instead of message count + // Query conversations with entity counts instead of message count const query = ` SELECT c.id as session_id, c.title, c.app_name as source_app, c.updated_at as last_activity, - (SELECT COUNT(*) FROM memories mem WHERE mem.session_id = c.id) as memory_count, (SELECT COUNT(DISTINCT es.entity_id) FROM entity_sessions es WHERE es.session_id = c.id) as entity_count, (SELECT summary FROM chat_summaries cs WHERE cs.session_id = c.id) as summary FROM conversations c @@ -646,14 +1113,6 @@ export function getMemoriesForSession(sessionId: string, limit: number = 200): M return messages } -export function getMemoryRecordsForSession(sessionId: string, limit: number = 200): MemoryRecord[] { - const db = getDB() - const stmt = db.prepare( - 'SELECT * FROM memories WHERE session_id = ? ORDER BY created_at ASC LIMIT ?' - ) - return stmt.all(sessionId, limit) as MemoryRecord[] -} - export function checkMessageExists(hash: string, conversationId: string): boolean { const db = getDB() const stmt = db.prepare('SELECT id FROM messages WHERE conversation_id = ? AND hash = ? LIMIT 1') @@ -661,31 +1120,10 @@ export function checkMessageExists(hash: string, conversationId: string): boolea return !!result } -// === MASTER MEMORY === - -export function getMasterMemory(): { content: string | null; updated_at: string | null } { - const db = getDB() - const result = db.prepare('SELECT content, updated_at FROM master_memory WHERE id = 1').get() as - { content: string; updated_at: string } | undefined - return result || { content: null, updated_at: null } -} - -export function updateMasterMemory(content: string): void { - const db = getDB() - const stmt = db.prepare(` - INSERT INTO master_memory (id, content, updated_at) - VALUES (1, ?, CURRENT_TIMESTAMP) - ON CONFLICT(id) DO UPDATE SET - content = excluded.content, - updated_at = excluded.updated_at - `) - stmt.run(content) -} - // One-shot, idempotent purge of the legacy "My Memories" data: the AI-chat // conversations the old watcher scraped (Claude / ChatGPT / Gemini, web + -// desktop) and everything derived from them — messages, vector memories, -// summaries, entity facts, and entities left orphaned afterwards. The shared +// desktop) and everything derived from them — messages, the retired memories +// and entity facts, summaries, and entities left orphaned afterwards. The shared // `entities` table is ALSO fed by the current screen-capture/observation // pipeline, so we only drop entities with no remaining link in entity_sessions, // entity_facts, OR observation_entities. Once the legacy conversations are gone @@ -742,10 +1180,10 @@ export function purgeLegacyChatImports(): Record | null { : 0 // without obs links we can't tell current from legacy — leave them // The stale consolidated profile is gone for good. counts.masterMemory = db.prepare(`DELETE FROM master_memory`).run().changes - // Rebuild external-content FTS indexes so they don't point at deleted rows. - for (const t of ['memory_fts', 'message_fts', 'summary_fts', 'entity_fts', 'entity_fact_fts']) { + // Re-index the external-content FTS tables so they don't point at deleted rows. + for (const spec of CORE_FTS_INDEXES) { try { - db.prepare(`INSERT INTO ${t}(${t}) VALUES('rebuild')`).run() + rebuildFtsIndex(db, spec) } catch { /* table may be absent */ } @@ -806,16 +1244,6 @@ export function updateEntitySummary(entityId: number, summary: string): void { stmt.run(summary, entityId) } -export function addEntityFact(entityId: number, fact: string, sessionId?: string): boolean { - const db = getDB() - const stmt = db.prepare(` - INSERT OR IGNORE INTO entity_facts (entity_id, fact, source_session_id) - VALUES (?, ?, ?) - `) - const info = stmt.run(entityId, fact.trim(), sessionId || null) - return info.changes > 0 -} - export function upsertEntitySession(entityId: number, sessionId: string): void { const db = getDB() const stmt = db.prepare(` @@ -833,85 +1261,14 @@ export interface EntityRecord { updated_at: string } -export interface EntityListRecord extends EntityRecord { - fact_count: number - session_count: number -} - -export interface SessionEntityRecord extends EntityRecord { - fact_count: number -} - -export interface EntityFactRecord { - id: number - fact: string - source_session_id: string | null - created_at: string -} - -export interface EntityDetailsRecord { - entity: (EntityRecord & Record) | undefined - facts: EntityFactRecord[] -} - -export function getEntities(appName?: string): EntityListRecord[] { - const db = getDB() - let query = ` - SELECT - e.id, - e.name, - e.type, - e.summary, - e.updated_at, - COUNT(DISTINCT f.id) as fact_count, - COUNT(DISTINCT es.session_id) as session_count - FROM entities e - LEFT JOIN entity_facts f ON f.entity_id = e.id - LEFT JOIN entity_sessions es ON es.entity_id = e.id - LEFT JOIN conversations c ON es.session_id = c.id - ` - - const params: unknown[] = [] - if (appName && appName !== 'All') { - query += ` WHERE c.app_name LIKE ? ` - params.push(`%${appName}%`) - } - - query += ` GROUP BY e.id ORDER BY e.updated_at DESC` - const stmt = db.prepare(query) - return stmt.all(...params) as EntityListRecord[] -} - -export function getEntityDetails(entityId: number, appName?: string): EntityDetailsRecord { - const db = getDB() - const entity = db.prepare('SELECT * FROM entities WHERE id = ?').get(entityId) as - (EntityRecord & Record) | undefined - - let factsQuery = ` - SELECT f.id, f.fact, f.source_session_id, f.created_at - FROM entity_facts f - LEFT JOIN conversations c ON f.source_session_id = c.id - WHERE f.entity_id = ? - ` - const params: unknown[] = [entityId] - if (appName && appName !== 'All') { - factsQuery += ` AND c.app_name LIKE ? ` - params.push(`%${appName}%`) - } - factsQuery += ` ORDER BY f.created_at DESC` - - const facts = db.prepare(factsQuery).all(...params) as EntityFactRecord[] - return { entity, facts } -} +export type SessionEntityRecord = EntityRecord export function getEntitiesForSession(sessionId: string): SessionEntityRecord[] { const db = getDB() const stmt = db.prepare(` - SELECT e.id, e.name, e.type, e.summary, e.updated_at, - COUNT(DISTINCT f.id) as fact_count + SELECT e.id, e.name, e.type, e.summary, e.updated_at FROM entities e JOIN entity_sessions es ON es.entity_id = e.id - LEFT JOIN entity_facts f ON f.entity_id = e.id WHERE es.session_id = ? GROUP BY e.id ORDER BY e.updated_at DESC @@ -933,6 +1290,7 @@ export function deleteEntityRecord(entityId: number): boolean { entityId, entityId ) + // Retired facts are not read any more, but an entity's delete still takes them with it. db.prepare('DELETE FROM entity_facts WHERE entity_id = ?').run(entityId) db.prepare('DELETE FROM entity_sessions WHERE entity_id = ?').run(entityId) return db.prepare('DELETE FROM entities WHERE id = ?').run(entityId).changes > 0 @@ -942,228 +1300,6 @@ export function deleteEntityRecord(entityId: number): boolean { return deleted } -export function deleteMemory(memoryId: number): boolean { - const db = getDB() - const stmt = db.prepare('DELETE FROM memories WHERE id = ?') - const info = stmt.run(memoryId) - console.log(`Deleted memory ${memoryId}, changes: ${info.changes}`) - return info.changes > 0 -} - -// === DASHBOARD STATS === - -export interface DashboardStats { - totalChats: number - totalMemories: number - totalEntities: number - totalRelationships: number - totalMessages: number - todayChats: number - todayMemories: number - todayEntities: number - totalFacts: number - recentChats: Array<{ - session_id: string - title: string | null - app_name: string - memory_count: number - entity_count: number - updated_at: string - }> - recentMemories: Array<{ - id: number - content: string - source_app: string - created_at: string - }> - topEntities: Array<{ - id: number - name: string - type: string - fact_count: number - session_count: number - }> - entityTypeCounts: Array<{ - type: string - count: number - }> - appDistribution: Array<{ - app_name: string - chat_count: number - memory_count: number - }> - activityByDay: Array<{ - date: string - chats: number - memories: number - }> -} - -export function getDashboardStats(): DashboardStats { - const db = getDB() - - // Total counts - const totalChats = ( - db.prepare('SELECT COUNT(*) as count FROM conversations').get() as { count: number } - ).count - const totalMemories = ( - db.prepare('SELECT COUNT(*) as count FROM memories').get() as { count: number } - ).count - const totalEntities = ( - db.prepare('SELECT COUNT(*) as count FROM entities').get() as { count: number } - ).count - const totalRelationships = ( - db.prepare('SELECT COUNT(*) as count FROM entity_edges').get() as { count: number } - ).count - const totalMessages = ( - db.prepare('SELECT COUNT(*) as count FROM messages').get() as { count: number } - ).count - const totalFacts = ( - db.prepare('SELECT COUNT(*) as count FROM entity_facts').get() as { count: number } - ).count - - // Today's activity (convert stored timestamps to localtime before comparing) - const todayChats = ( - db - .prepare( - `SELECT COUNT(*) as count FROM conversations WHERE date(updated_at, 'localtime') = date('now', 'localtime')` - ) - .get() as { count: number } - ).count - const todayMemories = ( - db - .prepare( - `SELECT COUNT(*) as count FROM memories WHERE date(created_at, 'localtime') = date('now', 'localtime')` - ) - .get() as { count: number } - ).count - const todayEntities = ( - db - .prepare( - `SELECT COUNT(*) as count FROM entities WHERE date(created_at, 'localtime') = date('now', 'localtime')` - ) - .get() as { count: number } - ).count - - // Recent chats (top 5) - const recentChats = db - .prepare( - ` - SELECT - c.id as session_id, - c.title, - c.app_name, - c.updated_at, - (SELECT COUNT(*) FROM memories m WHERE m.session_id = c.id) as memory_count, - (SELECT COUNT(DISTINCT es.entity_id) FROM entity_sessions es WHERE es.session_id = c.id) as entity_count - FROM conversations c - ORDER BY c.updated_at DESC - LIMIT 5 - ` - ) - .all() as DashboardStats['recentChats'] - - // Recent memories (top 5) - const recentMemories = db - .prepare( - ` - SELECT id, content, source_app, created_at - FROM memories - ORDER BY created_at DESC - LIMIT 5 - ` - ) - .all() as DashboardStats['recentMemories'] - - // Top entities by fact count - const topEntities = db - .prepare( - ` - SELECT - e.id, - e.name, - e.type, - COUNT(DISTINCT f.id) as fact_count, - COUNT(DISTINCT es.session_id) as session_count - FROM entities e - LEFT JOIN entity_facts f ON f.entity_id = e.id - LEFT JOIN entity_sessions es ON es.entity_id = e.id - GROUP BY e.id - ORDER BY fact_count DESC - LIMIT 6 - ` - ) - .all() as DashboardStats['topEntities'] - - // Entity type distribution - const entityTypeCounts = db - .prepare( - ` - SELECT type, COUNT(*) as count - FROM entities - GROUP BY type - ORDER BY count DESC - LIMIT 8 - ` - ) - .all() as DashboardStats['entityTypeCounts'] - - // App distribution - const appDistribution = db - .prepare( - ` - SELECT - COALESCE(c.app_name, 'Unknown') as app_name, - COUNT(DISTINCT c.id) as chat_count, - COUNT(DISTINCT m.id) as memory_count - FROM conversations c - LEFT JOIN memories m ON m.session_id = c.id - GROUP BY c.app_name - ORDER BY chat_count DESC - ` - ) - .all() as DashboardStats['appDistribution'] - - // Activity by day (last 14 days, using localtime) - const activityByDay = db - .prepare( - ` - WITH RECURSIVE dates(date) AS ( - SELECT date('now', 'localtime', '-13 days') - UNION ALL - SELECT date(date, '+1 day') - FROM dates - WHERE date < date('now', 'localtime') - ) - SELECT - dates.date, - (SELECT COUNT(*) FROM conversations WHERE date(updated_at, 'localtime') = dates.date) as chats, - (SELECT COUNT(*) FROM memories WHERE date(created_at, 'localtime') = dates.date) as memories - FROM dates - ORDER BY dates.date ASC - ` - ) - .all() as DashboardStats['activityByDay'] - - return { - totalChats, - totalMemories, - totalEntities, - totalRelationships, - totalMessages, - totalFacts, - todayChats, - todayMemories, - todayEntities, - recentChats, - recentMemories, - topEntities, - entityTypeCounts, - appDistribution, - activityByDay - } -} - // === USER PROFILE === export type UserProfile = UserProfileContract @@ -1360,25 +1496,7 @@ export function setRagConversationProject(id: string, projectId: string | null): } /** - * The FTS5 MATCH expression for a user's search box, or null when there is nothing to match. - * - * Pure and separately testable, because this is where a user's typing becomes query syntax. Terms - * are extracted as letters and digits only and each is quoted, so a stray quote, a hyphen or an - * FTS operator like NEAR or OR is data rather than syntax. The trailing `*` keeps the - * search-as-you-type behaviour the old LIKE had for the last, partial word. - */ -export function ragMessageMatchExpression(query: string): string | null { - return ( - escapeFtsMatch(query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [], { - join: 'AND', - prefix: true, - maxTokens: 6 - }) || null - ) -} - -/** - * Conversation ids whose MESSAGE CONTENT matches a query (all terms, AND) — so the chat-list + * Conversation ids whose MESSAGE CONTENT matches a query (all of its words) — so the chat-list * search can match what was said, not just the title. * * This was `lower(content) LIKE '%term%'` per term, with no LIMIT, over a `rag_messages` table @@ -1387,7 +1505,8 @@ export function ragMessageMatchExpression(query: string): string | null { * `rag_message_fts` now, and it returns a bounded page - the caller renders a list, not a report. */ export function searchRagConversationIds(query: string, limit?: number): string[] { - const match = ragMessageMatchExpression(query) + // Type-ahead: every word must match, each as a word start (the last may still be half typed). + const match = ftsTypeahead(query) if (!match) return [] const requested = limit ?? DEFAULT_SEARCH_LIMIT const bounded = Math.min( @@ -1422,7 +1541,7 @@ export function searchRagConversationPage( turn_count: number fallback_title: string | null }[] { - const match = ragMessageMatchExpression(query) + const match = ftsTypeahead(query) return getDB() .prepare( ` @@ -1494,62 +1613,41 @@ export function getProjectChatHistory( return rows.map(({ role, content, title }) => ({ role, content, title })).reverse() } -/** Relevant messages from other conversations in one project. Never crosses projects. */ +/** A sibling project chat's message, with the conversation it belongs to and when it was said. */ +export interface ProjectConversationMessage { + role: string + content: string + title: string | null + conversation_id: string + created_at: string +} + +/** Relevant messages from other conversations in one project, best match first (bm25 over + * rag_message_fts). Never crosses projects. */ export function searchProjectConversations( projectId: string, query: string, excludeConversationId = '', limit = 6 -): { role: string; content: string; title: string | null }[] { - const stopWords = new Set([ - 'about', - 'and', - 'are', - 'did', - 'for', - 'from', - 'how', - 'our', - 'project', - 'the', - 'this', - 'was', - 'what', - 'when', - 'where', - 'who', - 'with', - 'you' - ]) - const terms = [...new Set(query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])] - .filter((term) => term.length > 2 && !stopWords.has(term)) - .slice(0, 6) - if (!terms.length) return [] - const score = terms - .map(() => '(CASE WHEN lower(rm.content) LIKE ? THEN 1 ELSE 0 END)') - .join(' + ') - const patterns = terms.map((term) => `%${term}%`) +): ProjectConversationMessage[] { + const match = ftsQuery(query) + if (!match) return [] return getDB() .prepare( - `SELECT rm.role, rm.content, rc.title - FROM rag_messages rm + `SELECT rm.role, rm.content, rc.title, rm.conversation_id, rm.created_at + FROM rag_message_fts f + JOIN rag_messages rm ON rm.id = f.rowid JOIN rag_conversations rc ON rc.id = rm.conversation_id - WHERE rc.project_id = ? AND rm.conversation_id != ? - AND (${score}) > 0 - ORDER BY (${score}) DESC, rm.created_at DESC, rm.id DESC + WHERE rag_message_fts MATCH ? AND rc.project_id = ? AND rm.conversation_id != ? + ORDER BY bm25(rag_message_fts), rm.created_at DESC, rm.id DESC LIMIT ?` ) .all( + match, projectId, excludeConversationId, - ...patterns, - ...patterns, Math.min(12, Math.max(1, limit)) - ) as { - role: string - content: string - title: string | null - }[] + ) as ProjectConversationMessage[] } export function updateRagConversationTitle(id: string, title: string): RagConversation { diff --git a/src/main/diagnostics-log.ts b/src/main/diagnostics-log.ts index 066fde350..468430489 100644 --- a/src/main/diagnostics-log.ts +++ b/src/main/diagnostics-log.ts @@ -1,6 +1,6 @@ import path from 'node:path' import { dataDir } from './runtime-env' -import { createDiagnosticWriter, type DiagnosticWriterStats } from './diagnostics-writer' +import { createDiagnosticWriter } from './diagnostics-writer' export type DiagnosticLevel = 'info' | 'warn' | 'error' export type DiagnosticValue = string | number | boolean | null | undefined @@ -79,11 +79,6 @@ const writer = createDiagnosticWriter({ } }) -/** What this log has buffered, dropped and failed to write. Observable, not hidden. */ -export function diagnosticWriterStats(): DiagnosticWriterStats { - return writer.stats() -} - /** Drain buffered diagnostics on a controlled shutdown. Bounded: quit is never held hostage. */ export function flushDiagnosticLog(timeoutMs?: number): Promise { return writer.flush(timeoutMs) @@ -115,7 +110,7 @@ export function formatConsoleMessage(values: unknown[]): string { } /** Set to `1` to print every diagnostic line, including the per-request IPC trace. */ -export const VERBOSE_DIAGNOSTICS_ENV = 'OFFGRID_DIAGNOSTIC_VERBOSE' +const VERBOSE_DIAGNOSTICS_ENV = 'OFFGRID_DIAGNOSTIC_VERBOSE' function verboseDiagnosticsEnabled(): boolean { return process.env[VERBOSE_DIAGNOSTICS_ENV] === '1' diff --git a/src/main/diagnostics-writer.ts b/src/main/diagnostics-writer.ts index 36349bdb3..9e69b4592 100644 --- a/src/main/diagnostics-writer.ts +++ b/src/main/diagnostics-writer.ts @@ -16,7 +16,7 @@ import fs from 'node:fs' import path from 'node:path' -export type DiagnosticWriterStats = Readonly<{ +type DiagnosticWriterStats = Readonly<{ /** Lines accepted but not yet written. */ bufferedRecords: number bufferedBytes: number diff --git a/src/main/documents/chat-turn.ts b/src/main/documents/chat-turn.ts index 87dfe4e15..ccd126d49 100644 --- a/src/main/documents/chat-turn.ts +++ b/src/main/documents/chat-turn.ts @@ -46,7 +46,7 @@ function plural(n: number, word: string): string { } /** A passage as a turn source: the same key whether the turn or a search tool cited it. */ -export function passageSource(passage: { +function passageSource(passage: { documentId: string name: string page: number diff --git a/src/main/documents/extract-core.ts b/src/main/documents/extract-core.ts index 9e50e3c10..a304ce90e 100644 --- a/src/main/documents/extract-core.ts +++ b/src/main/documents/extract-core.ts @@ -35,7 +35,7 @@ function pageText(items: PdfTextItem[]): string { return text.replace(/\n{3,}/g, '\n\n') } -export async function extractPdfPages( +async function extractPdfPages( filePath: string, onPages: (pages: DocumentPage[]) => void, signal?: AbortSignal @@ -83,7 +83,7 @@ export async function extractPdfPages( } } -export async function readTextFile(filePath: string): Promise { +async function readTextFile(filePath: string): Promise { const bytes = await fs.promises.readFile(filePath) if (bytes[0] === 0xff && bytes[1] === 0xfe) return bytes.subarray(2).toString('utf16le') if (bytes[0] === 0xfe && bytes[1] === 0xff) { @@ -94,7 +94,7 @@ export async function readTextFile(filePath: string): Promise { return bytes.toString('utf8').replace(/^\uFEFF/, '') } -export async function extractDocxText(filePath: string): Promise { +async function extractDocxText(filePath: string): Promise { const { default: mammoth } = await import('mammoth') try { return (await mammoth.extractRawText({ path: filePath })).value diff --git a/src/main/documents/index.ts b/src/main/documents/index.ts index b709d7ae9..cc44eb777 100644 --- a/src/main/documents/index.ts +++ b/src/main/documents/index.ts @@ -16,6 +16,7 @@ import { documentEmbedder, documentReranker } from './models' import { conversationDocuments, referencedDocumentIds } from './conversation-documents' import { knowledgeBaseDocumentIds, projectDocuments } from './project-documents' import { pageOcr, shutdownOcr } from './ocr' +import { getDB, tableExists } from '../database' /** Foreground work (a chat reply, an image, speech) is running or waiting to run. */ async function foregroundBusy(): Promise { @@ -85,9 +86,63 @@ const CACHE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000 async function documentsInUse(): Promise> { const used = await referencedDocumentIds() for (const id of await knowledgeBaseDocumentIds()) used.add(id) + for (const { document_id } of keptDocumentsTable().prepare('SELECT document_id FROM kept_documents').all() as { + document_id: string + }[]) + used.add(document_id) return used } +let keptReady: object | undefined + +/** Documents kept by an owner outside chats (a meeting transcript, a long dictation). */ +function keptDocumentsTable(): ReturnType { + const db = getDB() + if (keptReady !== db) { + db.exec(`CREATE TABLE IF NOT EXISTS kept_documents ( + owner TEXT PRIMARY KEY, + document_id TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_kept_documents_document ON kept_documents(document_id);`) + keptReady = db + } + return db +} + +/** `owner` (e.g. "meeting:12") keeps `documentId`: cache cleanup never drops it while kept. */ +export function keepDocument(owner: string, documentId: string): void { + keptDocumentsTable() + .prepare('INSERT INTO kept_documents (owner, document_id) VALUES (?, ?) ON CONFLICT(owner) DO UPDATE SET document_id = excluded.document_id') + .run(owner, documentId) +} + +/** `owner` no longer keeps its document; cleanup may drop it when nothing else uses it. */ +export function releaseDocument(owner: string): void { + keptDocumentsTable().prepare('DELETE FROM kept_documents WHERE owner = ?').run(owner) +} + +/** + * Release every document kept by an owner `${prefix}:` whose row in `table` is gone (a + * privacy clear or retention deleted the rows directly, not through their owner's delete). All of + * them when the table does not exist. Returns how many were released. + */ +export function releaseDocumentsOfDeletedOwners(prefix: string, table: string): number { + const db = keptDocumentsTable() + const owner = `${prefix}:` + const ownedHere = 'substr(owner, 1, ?) = ?' + if (!tableExists(db, table)) { + return db + .prepare(`DELETE FROM kept_documents WHERE ${ownedHere}`) + .run(owner.length, owner).changes + } + return db + .prepare( + `DELETE FROM kept_documents WHERE ${ownedHere} + AND CAST(substr(owner, ?) AS INTEGER) NOT IN (SELECT id FROM ${table})` + ) + .run(owner.length, owner, owner.length + 1).changes +} + /** Drop document caches nothing references and that went unused for 30 days. */ export async function collectDocumentGarbage(): Promise { try { @@ -97,10 +152,16 @@ export async function collectDocumentGarbage(): Promise { } } +/** Drop documents nothing uses (no chat, knowledge base or owner keeps them), now. Throws on + * failure: a privacy delete must not report success while the text remains. */ +export async function removeUnreferencedDocuments(): Promise { + await documentIngest.collectGarbage(await documentsInUse(), 0) +} + /** A conversation or knowledge-base file was deleted: drop documents nothing else uses, now. */ export async function forgetUnreferencedDocuments(): Promise { try { - await documentIngest.collectGarbage(await documentsInUse(), 0) + await removeUnreferencedDocuments() } catch (error) { warn('Removing documents of a deleted chat failed', error) } diff --git a/src/main/documents/models.ts b/src/main/documents/models.ts index fe0cb175a..2f23c3703 100644 --- a/src/main/documents/models.ts +++ b/src/main/documents/models.ts @@ -1,9 +1,10 @@ -// EmbedPort and RerankPort adapters over the shared embedding worker: MiniLM for passages and -// questions (the model memory search keeps loaded), and the ms-marco cross-encoder for reranking. +// EmbedPort and RerankPort adapters over the shared embedding worker: the multilingual embedder for +// passages and questions (the one memory search uses, with the prefixes it was trained on), and the +// ms-marco cross-encoder for reranking. import type { EmbedPort, RerankPort } from '@offgrid/rag/documents' import { embeddings } from '../embeddings' -import { DOCUMENT_EMBEDDER } from '../embeddings-core' +import { DOCUMENT_EMBEDDER, DOCUMENT_EMBEDDER_SPEC } from '../embedding-models' /** How long the model stays loaded after the last indexing batch (on-demand residency). */ const HOLD_AFTER_MS = 30_000 @@ -27,10 +28,17 @@ export const documentEmbedder: EmbedPort = { embedder: DOCUMENT_EMBEDDER, async embedPassages(texts) { holdDuringIngest() - return embeddings.embedDocumentTexts(texts) + return embeddings.embedDocumentTexts( + texts.map((text) => `${DOCUMENT_EMBEDDER_SPEC.passagePrefix}${text}`) + ) }, async embedQuery(text) { - const [vector] = await embeddings.embedDocumentTexts([text], { interactive: true }) + const [vector] = await embeddings.embedDocumentTexts( + [`${DOCUMENT_EMBEDDER_SPEC.queryPrefix}${text}`], + { + interactive: true + } + ) return vector ?? [] } } diff --git a/src/main/documents/sqlite-store.ts b/src/main/documents/sqlite-store.ts index d48751a53..f821471b9 100644 --- a/src/main/documents/sqlite-store.ts +++ b/src/main/documents/sqlite-store.ts @@ -1,11 +1,30 @@ // DocumentIndexStore and KeywordSearchPort over the app's encrypted SQLite database (memories.db, // SQLCipher). Documents attached to chat live here, never as plain files, so their text is // protected like the rest of the user's data. Keyword search is SQLite FTS5 (GPT4All LocalDocs' -// approach); vectors are Float32 BLOBs beside each passage. +// approach); vectors are Float32 BLOBs beside each passage. Passages in scripts written without +// spaces (Chinese, Japanese, Thai…) are also kept in a trigram index, attached_document_fts_tri. -import { getDB } from '../database' import { - contentTerms, + clearRebuildCursor, + enableFtsSecureDelete, + ftsIndexIsCurrent, + getDB, + markFtsIndexCurrent, + queueIndexRebuild, + rebuildCursor, + setRebuildCursor, + tableExists +} from '../database' +import { + FTS_TOKENIZE, + FTS_TRIGRAM_TOKENIZE, + ftsQuery, + ftsTrigramQuery, + fuseLists, + needsTrigram, + normaliseForIndex +} from '@offgrid/rag/records' +import { indexedText, type DocumentChunk, type DocumentIndexStore, @@ -15,20 +34,6 @@ import { type StoredDocument } from '@offgrid/rag/documents' -/** - * FTS5's unicode61 tokenizer treats combining marks as separators, which cuts Hindi words at every - * vowel sign ("बदलाव" → "बदल" + "ा" + "व"). The Indic blocks are declared token characters, with - * zero-width joiners, so words stay whole; the danda (।, ॥) stays a separator. Verified against - * the bundled SQLite 3.53 with Devanagari text. - */ -function indicTokenChars(): string { - const chars: number[] = [] - for (let code = 0x0900; code <= 0x0d7f; code++) - if (code !== 0x0964 && code !== 0x0965) chars.push(code) - chars.push(0x200c, 0x200d) - return String.fromCodePoint(...chars).replace(/'/g, "''") -} - let schemaFor: object | undefined function db(): ReturnType { @@ -83,17 +88,225 @@ export function ensureAttachedDocumentSchema(database: ReturnType) PRIMARY KEY (conversation_id, document_id) ); `) - database.exec( - `CREATE VIRTUAL TABLE IF NOT EXISTS attached_document_fts USING fts5( + prepareDocumentIndex(database) + prepareDocumentTrigramIndex(database) +} + +const INDEX = 'attached_document_fts' +const NEXT = 'attached_document_fts_next' +/** The trigram twin: only passages whose text has a script written without spaces. */ +const TRIGRAM = 'attached_document_fts_tri' + +function createDocumentIndex(database: ReturnType, name: string): void { + database.exec(` + CREATE VIRTUAL TABLE ${name} USING fts5( body, document_id UNINDEXED, position UNINDEXED, - tokenize = "unicode61 remove_diacritics 0 tokenchars '${indicTokenChars()}'" - )` - ) + tokenize = "${FTS_TOKENIZE}" + ); + `) + enableFtsSecureDelete(database, name) +} + +/** + * The passage index on the shared tokenizer (it keeps Hindi vowel signs inside words), without + * reading any passage here: a missing index is created (current when there are no passages yet); + * one built by an older schema keeps answering while it is rebuilt in the background after + * start-up (rebuildDocumentIndexSlice), so documents attached before a tokenizer change still match. + */ +function prepareDocumentIndex(database: ReturnType): void { + if (ftsIndexIsCurrent(database, INDEX)) return + if (!tableExists(database, INDEX)) { + createDocumentIndex(database, INDEX) + if (!database.prepare('SELECT 1 FROM attached_document_chunks LIMIT 1').get()) { + markFtsIndexCurrent(database, INDEX) + return + } + } + queueIndexRebuild(INDEX, rebuildDocumentIndexSlice()) +} + +type ChunkRow = { + rowid: number + document_id: string + position: number + page: number + label: string | null + heading: string | null + text: string + tokens: number + name: string } -/** Every table that holds attached-document data (for "Delete all my data"). */ +/** Passages after `cursor` in rowid order, with their document's name (`limit` -1: all). */ +function readChunkRows( + database: ReturnType, + cursor: number, + limit: number +): ChunkRow[] { + return database + .prepare( + `SELECT c.rowid, c.document_id, c.position, c.page, c.label, c.heading, c.text, c.tokens, + COALESCE(d.name, '') AS name + FROM attached_document_chunks c + LEFT JOIN attached_documents d ON d.id = c.document_id + WHERE c.rowid > ? ORDER BY c.rowid LIMIT ?` + ) + .all(cursor, limit) as ChunkRow[] +} + +/** A passage's text exactly as both indexes hold it. */ +function chunkBody(row: ChunkRow): string { + const chunk: DocumentChunk = { + documentId: row.document_id, + position: row.position, + page: row.page, + ...(row.label ? { label: row.label } : {}), + ...(row.heading ? { heading: row.heading } : {}), + text: row.text, + tokens: row.tokens + } + return normaliseForIndex(indexedText(row.name, chunk)) +} + +/** + * One slice of the background rebuild: passages are copied into attached_document_fts_next in + * rowid order (fts_rebuild holds how far). Passages written later have higher rowids and are + * copied by a later slice; deletes reach both indexes (clearChunks, deleteDocument). The last + * slice replaces the old index in one transaction. Returns whether more remain. + */ +function rebuildDocumentIndexSlice(): () => boolean { + let batch = 200 + return () => { + const database = db() + if (ftsIndexIsCurrent(database, INDEX)) return false + let cursor = rebuildCursor(database, INDEX) + if (cursor === null || !tableExists(database, NEXT)) { + database.transaction(() => { + database.exec(`DROP TABLE IF EXISTS ${NEXT}`) + createDocumentIndex(database, NEXT) + setRebuildCursor(database, INDEX, 0) + })() + cursor = 0 + } + const started = Date.now() + const insert = database.prepare( + `INSERT INTO ${NEXT} (body, document_id, position) VALUES (?, ?, ?)` + ) + const copy = (rows: ChunkRow[]): void => { + for (const row of rows) insert.run(chunkBody(row), row.document_id, row.position) + } + const rows = readChunkRows(database, cursor, batch) + if (!rows.length) { + database.transaction(() => { + copy(readChunkRows(database, cursor, -1)) + database.exec(`DROP TABLE IF EXISTS ${INDEX}; ALTER TABLE ${NEXT} RENAME TO ${INDEX};`) + markFtsIndexCurrent(database, INDEX) + clearRebuildCursor(database, INDEX) + })() + return false + } + database.transaction(() => { + copy(rows) + setRebuildCursor(database, INDEX, rows[rows.length - 1]!.rowid) + })() + const elapsed = Date.now() - started + if (elapsed > 20) batch = Math.max(10, Math.floor(batch / 2)) + else if (elapsed < 10) batch = Math.min(2_000, batch * 2) + return true + } +} + +function createTrigramIndex(database: ReturnType): void { + database.exec(` + CREATE VIRTUAL TABLE ${TRIGRAM} USING fts5( + body, document_id UNINDEXED, position UNINDEXED, + tokenize = "${FTS_TRIGRAM_TOKENIZE}" + ); + `) + enableFtsSecureDelete(database, TRIGRAM) +} + +/** + * The trigram twin, without reading any passage here: a missing or outdated one is created empty + * (current when there are no passages) and filled in the background after start-up + * (fillTrigramIndexSlice); a fill under way resumes. + */ +function prepareDocumentTrigramIndex(database: ReturnType): void { + if (ftsIndexIsCurrent(database, TRIGRAM)) return + if (rebuildCursor(database, TRIGRAM) === null || !tableExists(database, TRIGRAM)) { + database.transaction(() => { + database.exec(`DROP TABLE IF EXISTS ${TRIGRAM}`) + createTrigramIndex(database) + if (database.prepare('SELECT 1 FROM attached_document_chunks LIMIT 1').get()) { + setRebuildCursor(database, TRIGRAM, 0) + } else { + clearRebuildCursor(database, TRIGRAM) + markFtsIndexCurrent(database, TRIGRAM) + } + })() + if (ftsIndexIsCurrent(database, TRIGRAM)) return + } + queueIndexRebuild(TRIGRAM, fillTrigramIndexSlice()) +} + +/** Whether new passages go straight into the trigram twin. While it is being filled they are left + * to the fill: they have higher rowids than any passage it has read. */ +function trigramIndexIsLive(database: ReturnType): boolean { + return tableExists(database, TRIGRAM) && rebuildCursor(database, TRIGRAM) === null +} + +/** One slice of filling the trigram twin with the passages that need it, in rowid order. The last + * slice ends the fill in the same transaction. Returns whether more remain. */ +function fillTrigramIndexSlice(): () => boolean { + let batch = 200 + return () => { + const database = db() + if (!tableExists(database, TRIGRAM)) return false + const cursor = rebuildCursor(database, TRIGRAM) + if (cursor === null) return false + const started = Date.now() + const rows = readChunkRows(database, cursor, batch) + if (!rows.length) { + database.transaction(() => { + clearRebuildCursor(database, TRIGRAM) + markFtsIndexCurrent(database, TRIGRAM) + })() + return false + } + const insert = database.prepare( + `INSERT INTO ${TRIGRAM} (body, document_id, position) VALUES (?, ?, ?)` + ) + database.transaction(() => { + for (const row of rows) { + const body = chunkBody(row) + if (needsTrigram(body)) insert.run(body, row.document_id, row.position) + } + setRebuildCursor(database, TRIGRAM, rows[rows.length - 1]!.rowid) + })() + const elapsed = Date.now() - started + if (elapsed > 20) batch = Math.max(10, Math.floor(batch / 2)) + else if (elapsed < 10) batch = Math.min(2_000, batch * 2) + return true + } +} + +/** Delete passages from the index, from its rebuild while one is under way, and from the trigram + * twin. */ +function unindexDocument(database: ReturnType, id: string): void { + database.prepare(`DELETE FROM ${INDEX} WHERE document_id = ?`).run(id) + if (tableExists(database, NEXT)) + database.prepare(`DELETE FROM ${NEXT} WHERE document_id = ?`).run(id) + if (tableExists(database, TRIGRAM)) + database.prepare(`DELETE FROM ${TRIGRAM} WHERE document_id = ?`).run(id) +} + +/** Every table that holds document data, and which owner keeps which document (for "Delete all + * my data"). */ export const ATTACHED_DOCUMENT_TABLES = [ + 'kept_documents', 'attached_document_fts', + 'attached_document_fts_next', + 'attached_document_fts_tri', 'attached_document_chunks', 'attached_document_pages', 'conversation_document_exclusions', @@ -227,7 +440,7 @@ export const documentStore: DocumentIndexStore = { const database = db() database.transaction(() => { database.prepare('DELETE FROM attached_document_chunks WHERE document_id = ?').run(id) - database.prepare('DELETE FROM attached_document_fts WHERE document_id = ?').run(id) + unindexDocument(database, id) database.prepare('UPDATE attached_documents SET map = NULL WHERE id = ?').run(id) })() }, @@ -246,6 +459,9 @@ export const documentStore: DocumentIndexStore = { const index = database.prepare( 'INSERT INTO attached_document_fts (body, document_id, position) VALUES (?, ?, ?)' ) + const trigram = trigramIndexIsLive(database) + ? database.prepare(`INSERT INTO ${TRIGRAM} (body, document_id, position) VALUES (?, ?, ?)`) + : null database.transaction((rows: DocumentChunk[]) => { for (const chunk of rows) { insert.run( @@ -257,7 +473,9 @@ export const documentStore: DocumentIndexStore = { chunk.text, chunk.tokens ) - index.run(indexedText(name, chunk), id, chunk.position) + const body = normaliseForIndex(indexedText(name, chunk)) + index.run(body, id, chunk.position) + if (trigram && needsTrigram(body)) trigram.run(body, id, chunk.position) } })(chunks) }, @@ -329,7 +547,7 @@ export const documentStore: DocumentIndexStore = { async deleteDocument(id) { const database = db() database.transaction(() => { - database.prepare('DELETE FROM attached_document_fts WHERE document_id = ?').run(id) + unindexDocument(database, id) database.prepare('DELETE FROM attached_document_chunks WHERE document_id = ?').run(id) database.prepare('DELETE FROM attached_document_pages WHERE document_id = ?').run(id) database.prepare('DELETE FROM conversation_document_exclusions WHERE document_id = ?').run(id) @@ -347,33 +565,77 @@ export const documentStore: DocumentIndexStore = { } } -/** An FTS5 MATCH expression from free text: quoted phrases kept, every other term OR-ed. */ -export function ftsQuery(text: string): string | null { - const quote = (term: string): string => `"${term.replace(/"/g, '""')}"` - const phrases = [...text.matchAll(/"([^"]{3,})"/g)].map((match) => quote(match[1]!.trim())) - const terms = contentTerms(text.replace(/"[^"]*"/g, ' ')).map(quote) - const all = [...phrases, ...terms] - return all.length ? all.join(' OR ') : null -} +type PassageRow = { documentId: string; position: number; rank: number } + +const toKeywordHit = (row: PassageRow): KeywordHit => ({ + documentId: row.documentId, + position: Number(row.position), + score: -row.rank +}) +/** + * Keyword passages: bm25 over the passage index, and for a question in a script written without + * spaces also the trigram twin (runs of three characters or more by MATCH, shorter ones by + * substring among the twin's passages of these documents), fused by rank. + */ export const documentKeywordSearch: KeywordSearchPort = { async search(documentIds, query, limit) { - const match = ftsQuery(query) - if (!match || !documentIds.length) return [] + if (!documentIds.length) return [] + const database = db() const placeholders = documentIds.map(() => '?').join(', ') - const rows = db() - .prepare( - `SELECT document_id AS documentId, position, bm25(attached_document_fts) AS rank - FROM attached_document_fts - WHERE attached_document_fts MATCH ? AND document_id IN (${placeholders}) - ORDER BY rank LIMIT ?` + const lists: KeywordHit[][] = [] + const match = ftsQuery(query) + if (match) { + lists.push( + ( + database + .prepare( + `SELECT document_id AS documentId, position, bm25(${INDEX}) AS rank + FROM ${INDEX} + WHERE ${INDEX} MATCH ? AND document_id IN (${placeholders}) + ORDER BY rank LIMIT ?` + ) + .all(match, ...documentIds, limit) as PassageRow[] + ).map(toKeywordHit) ) - .all(match, ...documentIds, limit) as { documentId: string; position: number; rank: number }[] - return rows.map((row): KeywordHit => ({ - documentId: row.documentId, - position: Number(row.position), - score: -row.rank - })) + } + const trigram = needsTrigram(query) ? ftsTrigramQuery(query) : null + if (trigram && tableExists(database, TRIGRAM)) { + if (trigram.match) + lists.push( + ( + database + .prepare( + `SELECT document_id AS documentId, position, bm25(${TRIGRAM}) AS rank + FROM ${TRIGRAM} + WHERE ${TRIGRAM} MATCH ? AND document_id IN (${placeholders}) + ORDER BY rank LIMIT ?` + ) + .all(trigram.match, ...documentIds, limit) as PassageRow[] + ).map(toKeywordHit) + ) + if (trigram.short.length) + lists.push( + ( + database + .prepare( + `SELECT document_id AS documentId, position, 0 AS rank + FROM ${TRIGRAM} + WHERE document_id IN (${placeholders}) + AND (${trigram.short.map(() => 'instr(body, ?) > 0').join(' OR ')}) + LIMIT ?` + ) + .all(...documentIds, ...trigram.short, limit) as PassageRow[] + ).map(toKeywordHit) + ) + } + if (lists.length <= 1) return lists[0] ?? [] + return fuseLists( + lists.map((items) => ({ items })), + (hit) => `${hit.documentId}:${hit.position}` + ) + .slice(0, limit) + .map(({ item, score }) => ({ ...item, score })) } } diff --git a/src/main/embedding-models.ts b/src/main/embedding-models.ts new file mode 100644 index 000000000..562ef958d --- /dev/null +++ b/src/main/embedding-models.ts @@ -0,0 +1,43 @@ +// The embedding and reranking models, as data. Kept apart from embeddings-core so main-process +// modules can name a model without loading @huggingface/transformers and ONNX Runtime: inference +// runs in the helper process (embeddings-worker), never beside the window. + +/** A sentence embedder and the prefixes its training expects on questions and passages. */ +export interface EmbedderSpec { + model: string + queryPrefix: string + passagePrefix: string +} + +const EMBEDDERS = { + /** English. Measured 3.3 ms a passage fp32 on CPU here; q8 was slower (30.6 ms). */ + miniLM: { model: 'Xenova/all-MiniLM-L6-v2', queryPrefix: '', passagePrefix: '' }, + /** Hindi and English in one space (a Hindi question finds the English passage). 7.9 ms a passage + * fp32; a ~470 MB download against MiniLM's ~90 MB. Trained with these prefixes. */ + multilingualE5Small: { + model: 'Xenova/multilingual-e5-small', + queryPrefix: 'query: ', + passagePrefix: 'passage: ' + } +} as const satisfies Record + +/** The model behind embeddings that have no corpus of their own (/v1/embeddings, MCP, tool routing). */ +export const DEFAULT_EMBEDDER: EmbedderSpec = EMBEDDERS.miniLM + +/** + * The model memory search embeds records and questions with. Changing it is this one line: each + * stored vector records its model, so every record is embedded again with the new one. + */ +export const RECORD_EMBEDDER: EmbedderSpec = EMBEDDERS.multilingualE5Small + +/** + * Models for documents (chat attachments, knowledge bases, transcripts): the same multilingual + * embedder as memory search, so one model is loaded and a Hindi question finds an English passage. + * fp32 is 7.9 ms a passage on CPU here (the ~1.7 s once measured was its q8 build). Documents + * indexed with MiniLM re-embed on their next use: the ingest compares each document's embedder. Indian-language documents still match by + * keyword (FTS5 keeps their words whole). ms-marco-MiniLM-L-6-v2 is the cross-encoder AnythingLLM's + * native reranker uses. + */ +export const DOCUMENT_EMBEDDER_SPEC: EmbedderSpec = EMBEDDERS.multilingualE5Small +export const DOCUMENT_EMBEDDER = DOCUMENT_EMBEDDER_SPEC.model +export const DOCUMENT_RERANKER = 'Xenova/ms-marco-MiniLM-L-6-v2' diff --git a/src/main/embeddings-core.ts b/src/main/embeddings-core.ts index 4ab78f898..4b66edca9 100644 --- a/src/main/embeddings-core.ts +++ b/src/main/embeddings-core.ts @@ -10,8 +10,17 @@ import { import { modelReadDirectories, resolveTransformersPackage } from './model-paths' import { requireModelStorage } from './runtime-env' import { configureTransformersEnv } from './embeddings-env' -import { loadWithOnnxFallback, onnxDeviceCandidates } from './onnx-device' -import type { BackendPreference } from '../shared/backend-preferences' +import { loadWithOnnxFallback } from './onnx-device' +import type { OnnxDevice } from '@offgrid/models' +import { DOCUMENT_RERANKER } from './embedding-models' +export { + DEFAULT_EMBEDDER, + DOCUMENT_EMBEDDER, + DOCUMENT_EMBEDDER_SPEC, + DOCUMENT_RERANKER, + RECORD_EMBEDDER, + type EmbedderSpec +} from './embedding-models' /** * Text -> vector. The actual inference, owned in ONE place. @@ -21,10 +30,9 @@ import type { BackendPreference } from '../shared/backend-preferences' * in-process, where there is no built worker to spawn. Sharing the implementation is what keeps the * second path honest: the alternative was a second copy that could drift from the one users run. * - * The pipeline is cached per process, because loading it is the expensive part. + * Pipelines are cached per process and per model, because loading one is the expensive part. */ -let pipe: FeatureExtractionPipeline | null = null -let loading: Promise | null = null +const pipes = new Map>() let activeDevice: DeviceType | null = null export function embeddingDevice(): DeviceType | null { @@ -33,26 +41,15 @@ export function embeddingDevice(): DeviceType | null { /** Called after serialized inference when the model is configured on demand. */ export async function disposeEmbeddingModel(): Promise { - const previous = pipe - pipe = null - loading = null + const loaded = [...pipes.values()] + pipes.clear() activeDevice = null - await previous?.dispose() + for (const pipe of loaded) await (await pipe.catch(() => null))?.dispose() const reranker = rerankerLoading rerankerLoading = null await (await reranker?.catch(() => null))?.model.dispose() } -/** - * Models for documents attached to chat. Passages are embedded with MiniLM, the model memory search - * already keeps loaded: measured ~15 ms a passage on CPU here, where multilingual-e5-small (q8) took - * ~1.7 s, which made a long book index for over an hour. Indian-language documents still match by - * keyword (FTS5 keeps their words whole). ms-marco-MiniLM-L-6-v2 is the cross-encoder AnythingLLM's - * native reranker uses. - */ -export const DOCUMENT_EMBEDDER = 'Xenova/all-MiniLM-L6-v2' -export const DOCUMENT_RERANKER = 'Xenova/ms-marco-MiniLM-L-6-v2' - let rerankerLoading: Promise<{ tokenizer: PreTrainedTokenizer; model: PreTrainedModel }> | null = null @@ -65,28 +62,58 @@ function prepareModel(id: string, modelsDir: string, readDirectories: string[]): } /** - * Vectors for several texts in one forward pass. Document passages are close in length, so padding - * costs little: measured 81 ms a passage batched against 186 ms one by one (MiniLM fp32 on CPU, - * 1,000-character passages, app under load), with the same vectors (max difference 1.5e-7). + * Vectors for several texts with one model, in one forward pass. Document passages are close in + * length, so padding costs little: measured 81 ms a passage batched against 186 ms one by one + * (MiniLM fp32 on CPU, 1,000-character passages, app under load), with the same vectors (max + * difference 1.5e-7). `onReady` hears the device once, when the model first loads. */ export async function embedTexts( texts: string[], + model: string, modelsDir: string, - backendPreference: BackendPreference = 'auto', + onReady?: (device: string, reason?: string) => void, + devices: OnnxDevice[] = ['cpu'], readDirectories = modelReadDirectories() ): Promise { - const out: number[][] = [] - let rest = texts - // The first text loads the model. - if (!pipe && rest.length) { - out.push(await embedText(rest[0]!, modelsDir, undefined, backendPreference, readDirectories)) - rest = rest.slice(1) - } - if (rest.length) { - const output = await pipe!(rest, { pooling: 'mean', normalize: true }) - out.push(...(output.tolist() as number[][])) - } - return out + if (!texts.length) return [] + const pipe = await loadPipeline(model, modelsDir, onReady, devices, readDirectories) + const output = await pipe(texts, { pooling: 'mean', normalize: true }) + return output.tolist() as number[][] +} + +/** The model's pipeline: full-precision weights when present, else quantized; downloads on first use. */ +function loadPipeline( + model: string, + modelsDir: string, + onReady: ((device: string, reason?: string) => void) | undefined, + devices: OnnxDevice[], + readDirectories: string[] +): Promise { + const loaded = pipes.get(model) + if (loaded) return loaded + const fp32 = resolveTransformersPackage(model, 'model.onnx', readDirectories) + const cached = fp32 ?? resolveTransformersPackage(model, 'model_quantized.onnx', readDirectories) + // Local-only reads must not require writable storage. Guard only an explicit + // folder choice when the pipeline can download; default storage is created lazily. + if (!cached && process.env.OFFGRID_EMBEDDINGS_OFFLINE !== '1') requireModelStorage() + configureTransformersEnv(modelsDir, !!cached) + const loading = loadWithOnnxFallback( + (device) => + pipeline('feature-extraction', cached ?? model, { + device, + ...(cached ? { local_files_only: true, ...(fp32 ? {} : { dtype: 'q8' as const }) } : {}) + }), + embeddingDevices(devices) + ).then(({ runtime, device, fallbackReason }) => { + onReady?.(device, fallbackReason) + activeDevice = device + return runtime + }) + loading.catch(() => { + if (pipes.get(model) === loading) pipes.delete(model) + }) + pipes.set(model, loading) + return loading } /** Relevance of each passage to the query (sigmoid of the cross-encoder logit), same order. */ @@ -94,7 +121,7 @@ export async function rerankPassages( query: string, passages: string[], modelsDir: string, - backendPreference: BackendPreference = 'auto', + devices: OnnxDevice[] = ['cpu'], readDirectories = modelReadDirectories() ): Promise { if (!passages.length) return [] @@ -110,7 +137,7 @@ export async function rerankPassages( dtype: 'q8', ...options }), - embeddingDevices(backendPreference) + embeddingDevices(devices) ).then(({ runtime }) => runtime) ]).then(([tokenizer, model]) => ({ tokenizer, model })) rerankerLoading.catch(() => { @@ -134,52 +161,7 @@ export async function rerankPassages( return scores } -/** - * Devices for the MiniLM embedder and the reranker. On macOS, CoreML ran MiniLM at about 450 ms a - * passage against 50 ms on the CPU (measured 2026-10-15, app under load), so CPU is used unless the - * user chose a backend. - */ -function embeddingDevices(backendPreference: BackendPreference): ReturnType { - const devices = onnxDeviceCandidates(process.platform, backendPreference) - return backendPreference === 'auto' && process.platform === 'darwin' - ? devices.filter((device) => device !== 'coreml') - : devices -} - -export async function embedText( - text: string, - modelsDir: string, - onReady?: (device: string, reason?: string) => void, - backendPreference: BackendPreference = 'auto', - readDirectories = modelReadDirectories() -): Promise { - if (!pipe) { - const id = 'Xenova/all-MiniLM-L6-v2' - const fp32 = resolveTransformersPackage(id, 'model.onnx', readDirectories) - const cached = fp32 ?? resolveTransformersPackage(id, 'model_quantized.onnx', readDirectories) - // Local-only reads must not require writable storage. Guard only an explicit - // folder choice when the pipeline can download; default storage is created lazily. - if (!cached && process.env.OFFGRID_EMBEDDINGS_OFFLINE !== '1') requireModelStorage() - configureTransformersEnv(modelsDir, !!cached) - loading ??= loadWithOnnxFallback( - (device) => - pipeline('feature-extraction', cached ?? 'Xenova/all-MiniLM-L6-v2', { - device, - ...(cached ? { local_files_only: true, ...(fp32 ? {} : { dtype: 'q8' as const }) } : {}) - }), - embeddingDevices(backendPreference) - ) - .then(({ runtime, device, fallbackReason }) => { - onReady?.(device, fallbackReason) - activeDevice = device - return runtime - }) - .catch((error) => { - loading = null - throw error - }) - pipe = await loading - } - const output = await pipe(text, { pooling: 'mean', normalize: true }) - return Array.from(output.data) +/** The host supplies the shared policy result to both execution paths. */ +function embeddingDevices(devices: OnnxDevice[]): OnnxDevice[] { + return devices } diff --git a/src/main/embeddings-worker.ts b/src/main/embeddings-worker.ts index 7fdc76140..8af0921c9 100644 --- a/src/main/embeddings-worker.ts +++ b/src/main/embeddings-worker.ts @@ -1,5 +1,5 @@ /** - * Embedding inference, off the main thread. + * Embedding inference, in its own utility process (onnx-host/utility-host.ts). * * all-MiniLM-L6-v2 runs on the ONNX/WASM runtime, and profiling the main process showed why that * cannot live beside the window: tokenization and mean pooling are JS (they block the event loop), @@ -7,46 +7,40 @@ * for cores. A startup backfill of a few dozen rows held the main process at 110% CPU with no chat * open and no task running. * - * This worker owns the pipeline and answers one request at a time. The host (embeddings.ts) keeps + * This helper process owns the pipeline and answers one request at a time. The host (embeddings.ts) keeps * the queue, so back-pressure and cancellation stay a host concern and this file stays a pure * text-in / vector-out service. */ -import { parentPort, workerData } from 'worker_threads' -import { embedText, embedTexts, embeddingDevice, rerankPassages } from './embeddings-core' +import { DEFAULT_EMBEDDER, embedTexts, embeddingDevice, rerankPassages } from './embeddings-core' import { configureRuntime } from './runtime-env' -import type { BackendPreference } from '../shared/backend-preferences' +import type { OnnxDevice } from '@offgrid/models' +import { connectHelper, type HelperPort } from './onnx-host/helper-port' -if (!parentPort) throw new Error('embeddings-worker must be started as a worker thread') -const port = parentPort - -// modelsDir() reads Electron's app paths, which do not exist in a worker thread — the host -// resolves the directory and passes it in. -const { - modelsDir, - chosenModelsDir, - modelReadDirectories = [modelsDir], - backendPreference = 'auto' -} = workerData as { +// modelsDir() reads Electron's app paths, which a helper process does not have — the host +// resolves the directories and sends them as the first message. +export interface EmbeddingWorkerInit { modelsDir: string chosenModelsDir?: string modelReadDirectories?: string[] - backendPreference?: BackendPreference + devices: OnnxDevice[] } -// Preserve the host's explicit choice without treating its default path as chosen. -configureRuntime({ modelsDir: chosenModelsDir }) +let port: HelperPort +let modelsDir = '' +let modelReadDirectories: string[] = [] +let devices: OnnxDevice[] = ['cpu'] export interface EmbeddingRequest { id: number - text: string - /** Several texts for one model (documents attached to chat). */ - texts?: string[] + /** The texts to embed, or to score against `rerankQuery`. */ + texts: string[] + /** The sentence embedder (embeddings-core EMBEDDERS); DEFAULT_EMBEDDER when absent. */ + model?: string /** Score `texts` against this query with the cross-encoder instead of embedding them. */ rerankQuery?: string } export interface EmbeddingResponse { id: number - vector?: number[] vectors?: number[][] scores?: number[] ready?: boolean @@ -55,54 +49,55 @@ export interface EmbeddingResponse { error?: string } -port.on('message', (request: EmbeddingRequest) => { +function handle(request: EmbeddingRequest): void { void (async () => { try { if (request.rerankQuery !== undefined) { const scores = await rerankPassages( request.rerankQuery, - request.texts ?? [], - modelsDir, - backendPreference, - modelReadDirectories - ) - port.postMessage({ id: request.id, scores } satisfies EmbeddingResponse) - return - } - if (request.texts) { - const vectors = await embedTexts( request.texts, modelsDir, - backendPreference, + devices, modelReadDirectories ) - port.postMessage({ id: request.id, vectors } satisfies EmbeddingResponse) + port.post({ id: request.id, scores } satisfies EmbeddingResponse) return } - const vector = await embedText( - request.text, + const vectors = await embedTexts( + request.texts, + request.model ?? DEFAULT_EMBEDDER.model, modelsDir, (device, fallbackReason) => { - port.postMessage({ + port.post({ id: request.id, ready: true, device, fallbackReason } satisfies EmbeddingResponse) }, - backendPreference, + devices, modelReadDirectories ) - port.postMessage({ + port.post({ id: request.id, - vector, + vectors, device: embeddingDevice() ?? undefined - } as EmbeddingResponse) + } satisfies EmbeddingResponse) } catch (error) { - port.postMessage({ + port.post({ id: request.id, error: error instanceof Error ? error.message : String(error) - } as EmbeddingResponse) + } satisfies EmbeddingResponse) } })() +} + +void connectHelper().then(({ init, port: connected }) => { + port = connected + modelsDir = init.modelsDir + modelReadDirectories = init.modelReadDirectories ?? [init.modelsDir] + devices = init.devices + // Preserve the host's explicit choice without treating its default path as chosen. + configureRuntime({ modelsDir: init.chosenModelsDir }) + port.onRequest(handle) }) diff --git a/src/main/embeddings.ts b/src/main/embeddings.ts index 2d05c11ad..b5cf0df8f 100644 --- a/src/main/embeddings.ts +++ b/src/main/embeddings.ts @@ -2,23 +2,15 @@ import { withStorageDownload } from './model-storage-guard' import { beginRuntimeBackend, providerLabel } from './runtime-backends' import path from 'path' import { existsSync } from 'fs' -import { Worker } from 'worker_threads' +import { clearHelperCrashes, forkHelper, type HelperProcess } from './onnx-host/utility-host' import { chosenModelsDir, modelsDir } from './runtime-env' import { modelReadDirectories } from './model-paths' -import { - DOCUMENT_EMBEDDER, - DOCUMENT_RERANKER, - disposeEmbeddingModel, - embedText, - embedTexts, - embeddingDevice, - rerankPassages -} from './embeddings-core' +import { DEFAULT_EMBEDDER, DOCUMENT_EMBEDDER, DOCUMENT_RERANKER } from './embedding-models' import { getResidencyMode } from './runtime-residency' import { recordAIRequest, type AIRequestHandle } from './ai-request-log' -import type { EmbeddingRequest, EmbeddingResponse } from './embeddings-worker' +import type { EmbeddingRequest, EmbeddingResponse, EmbeddingWorkerInit } from './embeddings-worker' import { writeDiagnosticLog } from './diagnostics-log' -import { getBackendPreference } from './backend-preferences' +import { onnxLadder } from './backend-facts' /** * The built worker, when there is one. @@ -28,6 +20,11 @@ import { getBackendPreference } from './backend-preferences' * no TypeScript transform, so its own imports fail to resolve. Returning null there is deliberate: * the caller embeds in-process instead, using the same implementation. */ +const SERVICE_NAME = 'offgrid-search' + +/** The in-process path loads transformers.js only when there is no helper to run it. */ +const core = (): Promise => import('./embeddings-core') + function builtWorkerEntry(): string | null { const built = path.join(__dirname, 'embeddings-worker.js') return existsSync(built) ? built : null @@ -36,14 +33,13 @@ function builtWorkerEntry(): string | null { /** * Text -> vector, executed in a worker thread (see embeddings-worker.ts for why). * - * The public surface is unchanged: callers still `await embeddings.generateEmbedding(text)`. What - * changed is where that runs. Requests are serialized onto one worker rather than issued - * concurrently, because the ONNX runtime is already multi-threaded internally — firing several at - * once multiplies its WASM threads and starves the UI, which is the behaviour this move exists to - * stop. + * Every embedding goes through embed(): memory records, document passages, /v1/embeddings, MCP and + * tool routing. Requests are serialized onto one worker rather than issued concurrently, because + * the ONNX runtime is already multi-threaded internally — firing several at once multiplies its + * WASM threads and starves the UI, which is the behaviour this move exists to stop. */ class EmbeddingService { - private worker: Worker | null = null + private worker: HelperProcess | null = null private nextId = 1 private readonly waiting = new Map< number, @@ -59,6 +55,8 @@ class EmbeddingService { * on) go ahead of background indexing, which can queue hundreds. */ private readonly pending: { interactive: boolean; start: () => void }[] = [] private running = false + /** When the last interactive request was scheduled or finished (ms epoch). */ + private lastInteractiveAt = 0 private reportedDevice: string | null = null private shuttingDown = false @@ -77,25 +75,33 @@ class EmbeddingService { // Queued requests see shuttingDown and cannot start another worker. The source-only // fallback has no worker to terminate, so let its active inference finish before disposal. await this.queue - if (!worker) await disposeEmbeddingModel() + if (!worker) await (await core()).disposeEmbeddingModel() this.reportedDevice = null } - private spawn(entry: string): Worker { + private spawn(entry: string): HelperProcess { if (this.worker) return this.worker // Built entries sit side by side in out/main (electron.vite.config.ts), but tests and any // run-from-source context have only the .ts next to this file. Resolve whichever EXISTS rather // than assuming the built layout: assuming it made every embedding fail outside a packaged // build, which silently demoted vector search to the FTS fallback instead of erroring. - const backendState = beginRuntimeBackend('embeddings', 'Xenova/all-MiniLM-L6-v2') - const worker = new Worker(entry, { - workerData: { - modelsDir: modelsDir(), - chosenModelsDir: chosenModelsDir(), - modelReadDirectories: modelReadDirectories(), - backendPreference: getBackendPreference('embeddings') - } - }) + const backendState = beginRuntimeBackend('embeddings', DEFAULT_EMBEDDER.model) + let worker: HelperProcess + try { + worker = forkHelper( + entry, + { + modelsDir: modelsDir(), + chosenModelsDir: chosenModelsDir(), + modelReadDirectories: modelReadDirectories(), + devices: onnxLadder('embeddings') + } satisfies EmbeddingWorkerInit, + { serviceName: SERVICE_NAME, label: 'Search' } + ) + } catch (error) { + backendState.fail(error) + throw error + } worker.on('message', (response: EmbeddingResponse) => { if (response.ready && response.device) { backendState.ready(providerLabel(response.device), undefined, response.fallbackReason) @@ -125,38 +131,111 @@ class EmbeddingService { for (const [, pending] of this.waiting) pending.reject(error) this.waiting.clear() } - worker.on('error', fail) worker.on('exit', (code) => { - backendState.stop() - if (this.worker !== worker) return + if (this.worker !== worker) { + backendState.stop() + return + } this.reportedDevice = null - if (code !== 0) fail(new Error(`Embedding worker exited with code ${code}`)) - else this.worker = null + if (code !== 0) fail(new Error(`The search helper stopped (exit code ${code}).`)) + else { + backendState.stop() + this.worker = null + } }) - worker.unref() // never hold the app open just for this this.worker = worker return worker } + /** True while a search helper process is running. */ + get isRunning(): boolean { + return this.worker !== null + } + + /** Start the next request on a fresh helper with the current backend choice and environment, + * after the requests already scheduled finish. */ + async restart(): Promise { + clearHelperCrashes(SERVICE_NAME) + await this.queue.catch(() => {}) + const worker = this.worker + this.worker = null + this.reportedDevice = null + if (worker) await worker.terminate() + } + /** Kept for callers that want to pay the model load cost up front. */ async init(): Promise { await this.generateEmbedding('') } - /** `interactive`: someone is waiting on this vector (a search), so it runs before background - * indexing that is already queued. */ + /** + * Vectors for `texts`: the one embedding path. `model` is a sentence embedder from + * embeddings-core's EMBEDDERS (DEFAULT_EMBEDDER when omitted). `interactive`: someone is waiting + * on it (a search), so it runs before background indexing that is already queued. + */ + async embed( + texts: string[], + opts: { model?: string; interactive?: boolean; source?: string } = {} + ): Promise { + if (!texts.length) return [] + const model = opts.model ?? DEFAULT_EMBEDDER.model + const source = + opts.source ?? + (texts.length > 1 ? 'Embedding batch' : texts[0] ? 'Embedding' : 'Embedding warm-up') + // A cold worker may download its model while handling its first request. + return withStorageDownload(() => + this.runModel( + texts.length === 1 + ? { text: texts[0], pooling: 'mean', normalize: true } + : { passages: texts.length, characters: texts.reduce((sum, t) => sum + t.length, 0) }, + (entry, log) => + // No built worker means we are running from source. Embed here rather than failing: a + // failed embedding silently demotes every search to keyword only, which is a far worse + // outcome than briefly holding this thread in a context that has no UI to block. + entry + ? this.post({ id: this.nextId++, texts, model }, entry, log).then( + (r) => r.vectors ?? [] + ) + : core().then(({ embedTexts }) => + embedTexts( + texts, + model, + modelsDir(), + (device, reason) => { + this.reportedDevice = device + beginRuntimeBackend('embeddings', model).ready( + providerLabel(device), + undefined, + reason + ) + }, + onnxLadder('embeddings'), + modelReadDirectories() + ) + ), + opts.interactive === true, + model, + source, + (vectors) => ({ dimensions: vectors[0]?.length ?? 0 }) + ) + ) + } + + /** One text's vector with DEFAULT_EMBEDDER: embed() for a single text. */ async generateEmbedding(text: string, opts: { interactive?: boolean } = {}): Promise { - // A cold worker may download its model while handling its first embedding. - return withStorageDownload(() => this.generateEmbeddingInner(text, opts.interactive === true)) + const [vector] = await this.embed([text], opts) + return vector ?? [] } /** Run `task` when the worker is free, interactive requests first, then in arrival order. */ private schedule(task: () => Promise, interactive: boolean): Promise { + if (interactive) this.lastInteractiveAt = Date.now() const result = new Promise((resolve, reject) => { const start = (): void => { void task() .then(resolve, reject) .finally(() => { + if (interactive) this.lastInteractiveAt = Date.now() this.running = false this.pump() }) @@ -179,48 +258,6 @@ class EmbeddingService { next.start() } - private async generateEmbeddingInner(text: string, interactive: boolean): Promise { - if (this.shuttingDown) throw new Error('Embedding runtime is shutting down.') - return recordAIRequest( - { - modality: 'embedding', - source: text ? 'Embedding' : 'Embedding warm-up', - model: 'Xenova/all-MiniLM-L6-v2', - request: { text, pooling: 'mean', normalize: true } - }, - async (log) => { - const run = (): Promise => { - if (this.shuttingDown) throw new Error('Embedding runtime is shutting down.') - const entry = builtWorkerEntry() - // No built worker means we are running from source. Embed here rather than failing: a failed - // embedding silently demotes every search to the FTS fallback, which is a far worse outcome - // than briefly holding this thread in a context that has no UI to block. - if (!entry) - return embedText( - text, - modelsDir(), - (device, reason) => { - beginRuntimeBackend('embeddings', 'Xenova/all-MiniLM-L6-v2').ready( - providerLabel(device), - undefined, - reason - ) - }, - getBackendPreference('embeddings'), - modelReadDirectories() - ) - return this.post({ id: this.nextId++, text }, entry, log).then((r) => r.vector ?? []) - } - const vector = await this.schedule(() => this.withResidency(run), interactive) - log.update({ - backend: this.reportedDevice ?? embeddingDevice() ?? 'Unknown', - metrics: { dimensions: vector.length } - }) - return vector - } - ) - } - private post( request: EmbeddingRequest, entry: string, @@ -237,6 +274,19 @@ class EmbeddingService { private holds = 0 /** Keep the worker and models loaded until the returned release is called, even on demand. */ + /** + * Resolve once no interactive request has run for `quietMs`. Background indexing waits on this + * before each batch, so a search never queues behind a batch that started between its query + * vector and its rerank (a search makes both requests, a moment apart). + */ + async whenQuiet(quietMs: number): Promise { + for (;;) { + const wait = this.lastInteractiveAt + quietMs - Date.now() + if (wait <= 0 && !this.pending.some((p) => p.interactive)) return + await new Promise((resolve) => setTimeout(resolve, Math.max(wait, 100))) + } + } + hold(): () => void { this.holds++ let released = false @@ -258,44 +308,25 @@ class EmbeddingService { const worker = this.worker this.worker = null if (worker) await worker.terminate() - else await disposeEmbeddingModel() + else await (await core()).disposeEmbeddingModel() this.reportedDevice = null } } } /** - * Vectors for passages of documents attached to chat, with the multilingual document model. - * Background work unless `interactive` (a question is waiting on a query vector). + * Vectors for passages of documents attached to chat, with the document model. Background work + * unless `interactive` (a question is waiting on a query vector). */ async embedDocumentTexts( texts: string[], opts: { interactive?: boolean; source?: string } = {} ): Promise { - if (!texts.length) return [] - return withStorageDownload(() => - this.runModel( - texts.length === 1 - ? { text: texts[0], model: DOCUMENT_EMBEDDER } - : { passages: texts.length, characters: texts.reduce((sum, t) => sum + t.length, 0) }, - (entry, log) => - entry - ? this.post( - { id: this.nextId++, text: '', texts }, - entry, - log - ).then((r) => r.vectors ?? []) - : embedTexts( - texts, - modelsDir(), - getBackendPreference('embeddings'), - modelReadDirectories() - ), - opts.interactive === true, - DOCUMENT_EMBEDDER, - opts.source ?? (texts.length === 1 ? 'Document question embedding' : 'Document indexing') - ) - ) + return this.embed(texts, { + ...opts, + model: DOCUMENT_EMBEDDER, + source: opts.source ?? (texts.length === 1 ? 'Document question embedding' : 'Document indexing') + }) } /** Cross-encoder relevance of each passage to the query, same order. Someone is waiting on it. */ @@ -307,16 +338,18 @@ class EmbeddingService { (entry, log) => entry ? this.post( - { id: this.nextId++, text: '', texts: passages, rerankQuery: query }, + { id: this.nextId++, texts: passages, rerankQuery: query }, entry, log ).then((r) => r.scores ?? []) - : rerankPassages( - query, - passages, - modelsDir(), - getBackendPreference('embeddings'), - modelReadDirectories() + : core().then(({ rerankPassages }) => + rerankPassages( + query, + passages, + modelsDir(), + onnxLadder('embeddings'), + modelReadDirectories() + ) ), true, DOCUMENT_RERANKER, @@ -330,7 +363,8 @@ class EmbeddingService { run: (entry: string | null, log: AIRequestHandle) => Promise, interactive: boolean, model: string, - source: string + source: string, + metrics?: (result: T) => Record ): Promise { if (this.shuttingDown) throw new Error('Embedding runtime is shutting down.') return recordAIRequest({ modality: 'embedding', source, model, request }, async (log) => { @@ -342,7 +376,10 @@ class EmbeddingService { }), interactive ) - log.update({ backend: this.reportedDevice ?? embeddingDevice() ?? 'Unknown' }) + log.update({ + backend: this.reportedDevice ?? 'Unknown', + ...(metrics ? { metrics: metrics(result) } : {}) + }) return result }) } diff --git a/src/main/engine-resume-health.ts b/src/main/engine-resume-health.ts new file mode 100644 index 000000000..d5b4e5788 --- /dev/null +++ b/src/main/engine-resume-health.ts @@ -0,0 +1,93 @@ +import { powerMonitor } from 'electron' +import { invalidateBackendFacts } from './backend-facts' +import { refreshNvidiaDriverVersion } from './nvidia-driver' + +/** A driver update while asleep changes which GPU engines may run. */ +function refreshBackendFacts(): void { + invalidateBackendFacts() + void refreshNvidiaDriverVersion() + .then(invalidateBackendFacts) + .catch(() => {}) +} + +interface ResumeHealthTarget { + id: string + shouldBeRunning(): boolean + healthCheck(): Promise + restart(): Promise +} + +const targets = new Map() +let pending: Promise | null = null + +export function registerResumeHealthTarget(target: ResumeHealthTarget): () => void { + targets.set(target.id, target) + return () => { + if (targets.get(target.id) === target) targets.delete(target.id) + } +} + +/** Requests arriving during the wake debounce wait for the engines to be checked. */ +export async function waitForEngineResumeHealth(): Promise { + while (pending) await pending +} + +export function installEngineResumeHealth( + deps: { + powerMonitor?: Pick + invalidateBackendFacts?: () => void + } = {} +): () => void { + const monitor = deps.powerMonitor ?? powerMonitor + let timer: ReturnType | undefined + let finish: (() => void) | undefined + let checking = false + let disposed = false + const check = async (): Promise => { + timer = undefined + checking = true + try { + await Promise.all( + [...targets.values()].map(async (target) => { + try { + if (!target.shouldBeRunning()) return + const healthy = await target.healthCheck().catch(() => false) + if (!disposed && !healthy && target.shouldBeRunning()) await target.restart() + } catch (error) { + console.error(`[engine-resume-health] ${target.id} recovery failed:`, error) + } + }) + ) + } finally { + checking = false + pending = null + finish?.() + finish = undefined + } + } + const onResume = (): void => { + ;(deps.invalidateBackendFacts ?? refreshBackendFacts)() + if (checking || disposed) return + if (!pending) + pending = new Promise((resolve) => { + finish = resolve + }) + clearTimeout(timer) + timer = setTimeout(() => { + void check() + }, 2000) + } + monitor.on('resume', onResume) + monitor.on('unlock-screen', onResume) + return () => { + disposed = true + monitor.removeListener('resume', onResume) + monitor.removeListener('unlock-screen', onResume) + clearTimeout(timer) + if (!checking) { + pending = null + finish?.() + finish = undefined + } + } +} diff --git a/src/main/engine-updates-service.ts b/src/main/engine-updates-service.ts index fed156b72..9d2071fe8 100644 --- a/src/main/engine-updates-service.ts +++ b/src/main/engine-updates-service.ts @@ -6,7 +6,7 @@ import { dataDir } from './runtime-env' import { hasLinuxNvidiaDriver, hasWindowsNvidiaDriver } from './transcription/whisper-runtime' /** Setting key; engines stay current unless the user turns this off. */ -export const KEEP_ENGINES_UPDATED = 'keepEnginesUpdated' +const KEEP_ENGINES_UPDATED = 'keepEnginesUpdated' export const engineUpdater = createEngineUpdater({ platform: process.platform, diff --git a/src/main/extension-bridge/bridge-conversations.ts b/src/main/extension-bridge/bridge-conversations.ts index 872ed09dc..39c24d2e2 100644 --- a/src/main/extension-bridge/bridge-conversations.ts +++ b/src/main/extension-bridge/bridge-conversations.ts @@ -55,7 +55,7 @@ export function setBrowserConversationOrigin( export const MAX_LISTED = 100 export const MAX_TURNS = 40 -export const MAX_TURN_CHARS = 20_000 +const MAX_TURN_CHARS = 20_000 /** SQLite CURRENT_TIMESTAMP ('YYYY-MM-DD HH:MM:SS', UTC) to epoch ms; 0 when unreadable. */ export function sqlTime(value: string | null | undefined): number { @@ -183,7 +183,7 @@ export const HISTORY_CONFLICT = export type BridgeListedConversation = Omit & { readonly turns: (BridgeTurn & { readonly image?: string })[] } -export type BridgeConversationSearchHit = Omit & { +type BridgeConversationSearchHit = Omit & { readonly snippet: string readonly turnCount: number } diff --git a/src/main/extension-bridge/bridge-settings.ts b/src/main/extension-bridge/bridge-settings.ts index b95ac5e44..b0a725723 100644 --- a/src/main/extension-bridge/bridge-settings.ts +++ b/src/main/extension-bridge/bridge-settings.ts @@ -50,7 +50,7 @@ export interface TextSettings { readonly systemPrompt?: string readonly kvCacheType?: 'f16' | 'q8_0' | 'q4_0' readonly flashAttn?: boolean - readonly gpuLayers?: number + readonly gpuLayers?: number | null readonly threads?: number readonly batchSize?: number readonly speculativeDecoding?: 'off' | 'ngram' | 'mtp' | 'draft' | 'dflash' @@ -294,7 +294,7 @@ const SECTION_RULES: Record> = { reasoningBudget: int(0, 1048576), kvCacheType: oneOf(['f16', 'q8_0', 'q4_0']), flashAttn: bool, - gpuLayers: int(-1, 1000), + gpuLayers: (value) => (value === null ? null : int(-1, 1000)(value)), threads: int(0, 1024), batchSize: int(1, 65536), speculativeDecoding: oneOf(['off', 'ngram', 'mtp', 'draft', 'dflash']), diff --git a/src/main/extension-bridge/bridge-task-progress.ts b/src/main/extension-bridge/bridge-task-progress.ts index 62d5b4019..d0bdaa68f 100644 --- a/src/main/extension-bridge/bridge-task-progress.ts +++ b/src/main/extension-bridge/bridge-task-progress.ts @@ -41,7 +41,7 @@ export function latestBrowserTask( /** A task this browser's chat was given an id for, that has not started yet: queued behind * another task, or waiting for approval. Its own id, so the chat follows and stops only it. */ -export function queuedBrowserTask(taskId: string): BrowserTaskProgress { +function queuedBrowserTask(taskId: string): BrowserTaskProgress { return { taskId, status: 'queued', diff --git a/src/main/files-classify.ts b/src/main/files-classify.ts index 1981473c6..c87a61b09 100644 --- a/src/main/files-classify.ts +++ b/src/main/files-classify.ts @@ -42,7 +42,7 @@ export function sanitizeUploadName(name: string): string { } /** Formats we cannot read yet. Read as text, their bytes would reach the model as garbage. */ -export const BINARY_EXT = [ +const BINARY_EXT = [ 'xlsx', 'xls', 'xlsm', diff --git a/src/main/files-image-probe.ts b/src/main/files-image-probe.ts index 858fd2fb8..38ab4be00 100644 --- a/src/main/files-image-probe.ts +++ b/src/main/files-image-probe.ts @@ -31,7 +31,7 @@ export type ImageDecodeVerdict = /** Load sharp on demand. Returns null when the module cannot load at all, so the * caller can distinguish a missing checker from a bad file. */ -export async function loadImageProbe(): Promise { +async function loadImageProbe(): Promise { try { const mod = (await import('sharp')) as unknown as { default?: (p: string, o?: unknown) => { metadata(): Promise } diff --git a/src/main/frame-text-index.ts b/src/main/frame-text-index.ts index 57cc12645..c98f874c8 100644 --- a/src/main/frame-text-index.ts +++ b/src/main/frame-text-index.ts @@ -3,8 +3,23 @@ // is contentless (the text is not stored twice) with contentless_delete, so a frame that changes // or goes away is simply dropped from it. New and edited frames are indexed by triggers; frames // captured before the index existed are indexed in the background, newest first, in small batches. +// +// frame_fts keeps the tokenizer it was created with. It is the largest index by far (every word +// ever on screen), and rebuilding it for the shared tokenizer (FTS_TOKENIZE, which keeps Devanagari +// vowel signs inside words) would mean indexing everything twice on the main thread. Frame text is +// mostly Latin, and a Hindi whole word still matches the old tokenizer through the phrase query +// ftsQuery builds. A new index is created on the shared tokenizer; an existing one is rebuilt only +// once a chunked, off-main-thread rebuild exists. Text is normalised (normaliseForIndex) on the way +// in either way, so queries and indexed text agree. -import { getDB } from './database' +import { + enableFtsSecureDelete, + getDB, + markFtsIndexCurrent, + registerFtsFunctions, + tableExists +} from './database' +import { FTS_TOKENIZE, normaliseForIndex } from '@offgrid/rag/records' /** The backfill runs on the main thread in short slices: an average frame carries ~6KB of OCR, and * 200-frame batches held the thread over half a second each, which made the whole app lag. Each @@ -14,38 +29,68 @@ const BACKFILL_PAUSE_MS = 80 /** Frames read per slice; a slice usually stops on its time budget first. */ const BACKFILL_BATCH = 50 -interface IndexDatabase { - exec(sql: string): void - prepare(sql: string): { get(...args: unknown[]): unknown } +type IndexDatabase = ReturnType + +/** Below this frame id, frames still wait for the backfill; it works down from here. */ +function startCursor(db: IndexDatabase): number { + return (db.prepare('SELECT COALESCE(MAX(id), 0) + 1 AS id FROM frames').get() as { id: number }) + .id +} + +function writeState(db: IndexDatabase, cursor: number): void { + // The cursor lives beside the index, so the index needs nothing but this database. + db.exec( + `CREATE TABLE IF NOT EXISTS frame_fts_state (id INTEGER PRIMARY KEY CHECK (id = 1), cursor INTEGER NOT NULL); + INSERT OR REPLACE INTO frame_fts_state (id, cursor) VALUES (1, ${Math.floor(cursor)});` + ) } +/** Synced mail and chat frames stay out, as search already left them out. */ +const TRIGGERS = ` + CREATE TRIGGER IF NOT EXISTS frames_fts_ai AFTER INSERT ON frames + WHEN new.text IS NOT NULL AND COALESCE(new.source, '') != 'connector' BEGIN + INSERT INTO frame_fts(rowid, text) VALUES (new.id, offgrid_fts_text(new.text)); + END; + CREATE TRIGGER IF NOT EXISTS frames_fts_ad AFTER DELETE ON frames BEGIN + DELETE FROM frame_fts WHERE rowid = old.id; + END; + CREATE TRIGGER IF NOT EXISTS frames_fts_au AFTER UPDATE OF text, source ON frames BEGIN + DELETE FROM frame_fts WHERE rowid = old.id; + INSERT INTO frame_fts(rowid, text) + SELECT new.id, offgrid_fts_text(new.text) + WHERE new.text IS NOT NULL AND COALESCE(new.source, '') != 'connector'; + END; +` + /** Create the index and its triggers on a database that has the frames table. Idempotent. */ export function ensureFrameTextIndex(db: IndexDatabase): void { - const exists = db - .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'frame_fts'") - .get() - if (!exists) { + // The app's connection has the triggers' function already; another handle may not. + try { + db.prepare("SELECT offgrid_fts_text('')").get() + } catch { + registerFtsFunctions(db) + } + // An earlier build of this branch rebuilt the index beside itself; that copy goes. + db.exec(` + DROP TRIGGER IF EXISTS frames_fts_next_ai; + DROP TRIGGER IF EXISTS frames_fts_next_ad; + DROP TRIGGER IF EXISTS frames_fts_next_au; + DROP TABLE IF EXISTS frame_fts_next; + DROP TABLE IF EXISTS frame_fts_next_state; + `) + if (!tableExists(db, 'frame_fts')) { db.exec( `CREATE VIRTUAL TABLE frame_fts USING fts5( - text, content='', contentless_delete=1, tokenize='unicode61 remove_diacritics 2' + text, content='', contentless_delete=1, tokenize = "${FTS_TOKENIZE}" )` ) - // Everything below this id predates the index; the backfill works down from here. The - // cursor lives beside the index, so the index needs nothing but this database. - const top = db.prepare('SELECT COALESCE(MAX(id), 0) + 1 AS id FROM frames').get() as { - id: number - } - db.exec( - `CREATE TABLE IF NOT EXISTS frame_fts_state (id INTEGER PRIMARY KEY CHECK (id = 1), cursor INTEGER NOT NULL); - INSERT OR REPLACE INTO frame_fts_state (id, cursor) VALUES (1, ${Math.floor(Number(top.id))});` - ) + enableFtsSecureDelete(db, 'frame_fts') + markFtsIndexCurrent(db, 'frame_fts') + writeState(db, startCursor(db)) } - // An index made before the cursor moved here kept it in app settings; carry it over once, or - // start again from the top (re-indexing a frame is safe). - const hasState = db - .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'frame_fts_state'") - .get() - if (!hasState) { + // An index made before the cursor moved beside it kept it in app settings; carry it over once, + // or start again from the top (re-indexing a frame is safe). + if (!tableExists(db, 'frame_fts_state')) { let cursor: number | null = null try { const row = db @@ -55,32 +100,20 @@ export function ensureFrameTextIndex(db: IndexDatabase): void { } catch { /* no settings table here */ } - if (cursor === null || !Number.isFinite(cursor)) { - cursor = (db.prepare('SELECT COALESCE(MAX(id), 0) + 1 AS id FROM frames').get() as { id: number }) - .id - } - db.exec( - `CREATE TABLE IF NOT EXISTS frame_fts_state (id INTEGER PRIMARY KEY CHECK (id = 1), cursor INTEGER NOT NULL); - INSERT OR REPLACE INTO frame_fts_state (id, cursor) VALUES (1, ${Math.floor(cursor)});` - ) + writeState(db, cursor === null || !Number.isFinite(cursor) ? startCursor(db) : cursor) } - - // Synced mail and chat frames stay out, as search already left them out. - db.exec(` - CREATE TRIGGER IF NOT EXISTS frames_fts_ai AFTER INSERT ON frames - WHEN new.text IS NOT NULL AND COALESCE(new.source, '') != 'connector' BEGIN - INSERT INTO frame_fts(rowid, text) VALUES (new.id, new.text); - END; - CREATE TRIGGER IF NOT EXISTS frames_fts_ad AFTER DELETE ON frames BEGIN - DELETE FROM frame_fts WHERE rowid = old.id; - END; - CREATE TRIGGER IF NOT EXISTS frames_fts_au AFTER UPDATE OF text, source ON frames BEGIN - DELETE FROM frame_fts WHERE rowid = old.id; - INSERT INTO frame_fts(rowid, text) - SELECT new.id, new.text - WHERE new.text IS NOT NULL AND COALESCE(new.source, '') != 'connector'; - END; - `) + // Older triggers indexed the raw text; these normalise it. Deletes are by rowid (contentless), + // so replacing them is safe on an index of either kind. + const insertTrigger = db + .prepare("SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = 'frames_fts_ai'") + .get() as { sql: string } | undefined + if (insertTrigger && !insertTrigger.sql.includes('offgrid_fts_text')) { + db.exec(` + DROP TRIGGER IF EXISTS frames_fts_ai; + DROP TRIGGER IF EXISTS frames_fts_au; + `) + } + db.exec(TRIGGERS) } /** The lowest frame id the backfill has reached; 0 once every older frame is indexed. */ @@ -106,10 +139,7 @@ export function frameTextIndexBackfilling(): boolean { /** Index a slice of older frames, newest first: up to `batch` frames, stopping early once * `budgetMs` is spent. Returns whether more remain. */ -export function indexFrameTextBatch( - batch = BACKFILL_BATCH, - budgetMs = SLICE_BUDGET_MS -): boolean { +export function indexFrameTextBatch(batch = BACKFILL_BATCH, budgetMs = SLICE_BUDGET_MS): boolean { const cursor = readCursor() if (cursor <= 1) return false const db = getDB() @@ -128,7 +158,7 @@ export function indexFrameTextBatch( for (const row of rows) { // An edit since the index existed may already have indexed this frame. drop.run(row.id) - if (row.text && row.source !== 'connector') add.run(row.id, row.text) + if (row.text && row.source !== 'connector') add.run(row.id, normaliseForIndex(row.text)) lowest = row.id if (Date.now() - started >= budgetMs) break } diff --git a/src/main/fts-match.ts b/src/main/fts-match.ts deleted file mode 100644 index b6522810a..000000000 --- a/src/main/fts-match.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** Build FTS5 syntax from text. Only the caller supplies the join and prefix operators. - * Quoted words make user-supplied AND/OR/NOT/NEAR ordinary search terms. - * Empty input returns an empty string: callers must skip MATCH in that case. - */ -export function escapeFtsMatch( - input: string | readonly string[], - options: { join?: 'AND' | 'OR' | ' '; prefix?: boolean; maxTokens?: number } = {} -): string { - const terms = - typeof input === 'string' ? (input.match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? []) : input - return terms - .filter((term) => /[\p{L}\p{N}]/u.test(term)) - .slice(0, options.maxTokens ?? 12) - .map((term) => `"${term.replace(/"/g, '""')}"${options.prefix ? '*' : ''}`) - .join(options.join === ' ' ? ' ' : ` ${options.join ?? 'OR'} `) -} diff --git a/src/main/imagegen.ts b/src/main/imagegen.ts index ce2f3da97..a4431a8fc 100644 --- a/src/main/imagegen.ts +++ b/src/main/imagegen.ts @@ -1,3 +1,4 @@ +import { nativeBinaryInstalled } from './backend-facts' import { recordManagedStorageFile } from './managed-storage-files' import { withStorageDownload } from './model-storage-guard' import { beginRuntimeBackend } from './runtime-backends' @@ -47,7 +48,6 @@ import { imageBackendForRuntime, sdRuntimeLibraryEnv } from './imagegen/sd-runtime' -import { getBackendPreference } from './backend-preferences' import { standardModelDefaults, taesdFilename } from '../shared/image-defaults' import { defaultImageModelFilename } from './image-default' import { @@ -85,7 +85,7 @@ import { type ImageGenerationRequestContract } from '../shared/image-generation-contract' -function findSdCli(): string | null { +function findSdCli(): Promise { return findSdBinary('sd-cli') } @@ -430,7 +430,7 @@ export function imageGenStatus(): { // used to land on the parked Core ML model). const active = activeImageModel() // Available if EITHER runtime is usable: sd-cli (with a model) or MLX/mflux. - if (!findSdCli() && !mfluxAvailable()) + if (!nativeBinaryInstalled('image', 'sd-cli') && !mfluxAvailable()) return { available: false, models, active, reason: 'no image runtime found' } if (!models.length) return { available: false, models, active, reason: 'no image model installed' } @@ -444,7 +444,7 @@ export type ImageGenOutput = ImageGenerationOutputContract /** Native runtimes receive the final prompt as input. The wrapper adds it to their output once. */ type NativeImageGenOutput = Omit -export interface ImageGenProgress { +interface ImageGenProgress { step: number total: number secPerStep: number @@ -650,7 +650,8 @@ export async function generateImage( throw new Error('An image is already generating — please wait for it to finish.') const stored = getSetting('imageParams', {}) const remoteParameters = resolveImageParameters( - { id: remoteId!, name: remote.selectedModel }, stored + { id: remoteId!, name: remote.selectedModel }, + stored ) const effectiveRemote = { ...params, @@ -886,7 +887,7 @@ async function runImageGen( `${coreml ? 'The selected Core ML image model' : 'The selected Z-Image model'} does not support image editing. Select a local image model that supports an init image.` ) } - const cli = coreml ? findCoreMLBin() : findSdCli() + const cli = coreml ? findCoreMLBin() : await findSdCli() if (!cli) { throw new Error( coreml @@ -945,13 +946,16 @@ async function runImageGen( // Progress observers must not force a fresh CLI process for every API request. const eligibleForServer = getResidencyMode('image') === 'resident' && - !coreml && !isZImage && !loras.length && + !coreml && + !isZImage && + !loras.length && (isQwenImage21 || (!params.initImage && ggufIsFullCheckpoint(model))) if (eligibleForServer) { const defaults = standardModelDefaults(base) const context: SdServerContext = { - modelPath: model, diffusionFa: true, - taesdPath: !isQwenImage21 && params.fastVae ? resolveTaesd(base) ?? undefined : undefined + modelPath: model, + diffusionFa: true, + taesdPath: !isQwenImage21 && params.fastVae ? (resolveTaesd(base) ?? undefined) : undefined } if (isQwenImage21) { context.diffusionModel = true @@ -959,7 +963,9 @@ async function runImageGen( context.llmVisionPath = findInModels(/^mmproj-Qwen3VL-8B-Instruct-.*\.gguf$/i) ?? undefined context.vaePath = findInModels(/^qwen_image_2\.1_vae.*\.(safetensors|sft)$/i) ?? undefined if (!context.llmPath || !context.vaePath || (params.initImage && !context.llmVisionPath)) { - throw new Error('Qwen-Image 2.1 companion files are missing. Download the complete model from Models.') + throw new Error( + 'Qwen-Image 2.1 companion files are missing. Download the complete model from Models.' + ) } } generationLifecycle.start(signal, cancelRuntime) @@ -967,23 +973,30 @@ async function runImageGen( await sdServer.ensureUp(context) const residentBinary = sdServer.getBinaryPath() ?? cli generationLifecycle.throwIfCancelled() - const { png, seed: usedSeed } = await sdServer.generate({ - prompt: params.prompt, - negativePrompt: params.negativePrompt?.trim() || (isQwenImage21 ? '' : DEFAULT_NEGATIVE), - width: params.width ?? defaults.defaultSize, - height: params.height ?? defaults.defaultSize, - steps: params.steps ?? defaults.defaultSteps, - cfgScale: params.cfgScale ?? defaults.defaultCfg, - sampleMethod: isQwenImage21 ? 'euler' : defaults.sampler, - scheduler: isQwenImage21 ? 'default' : defaults.scheduler, - seed, - refImages: params.initImage ? [(await fs.promises.readFile(params.initImage)).toString('base64')] : undefined - }, onProgress ? (progress) => onProgress({ ...progress, secPerStep: 0 }) : undefined) + const { png, seed: usedSeed } = await sdServer.generate( + { + prompt: params.prompt, + negativePrompt: params.negativePrompt?.trim() || (isQwenImage21 ? '' : DEFAULT_NEGATIVE), + width: params.width ?? defaults.defaultSize, + height: params.height ?? defaults.defaultSize, + steps: params.steps ?? defaults.defaultSteps, + cfgScale: params.cfgScale ?? defaults.defaultCfg, + sampleMethod: isQwenImage21 ? 'euler' : defaults.sampler, + scheduler: isQwenImage21 ? 'default' : defaults.scheduler, + seed, + refImages: params.initImage + ? [(await fs.promises.readFile(params.initImage)).toString('base64')] + : undefined + }, + onProgress ? (progress) => onProgress({ ...progress, secPerStep: 0 }) : undefined + ) generationLifecycle.throwIfCancelled() await fs.promises.writeFile(outPath, png) return { dataUrl: `data:image/png;base64,${png.toString('base64')}`, - path: outPath, seed: usedSeed, model: base, + path: outPath, + seed: usedSeed, + model: base, computeBackend: imageBackendForRuntime(process.platform, residentBinary) } } finally { @@ -1147,15 +1160,18 @@ async function runImageGen( // model's load spike — otherwise the brief overlap causes a short stutter. try { await generationLifecycle.waitForMemoryReclaim() - const runNativeCli = (runtime: string): Promise => + const runNativeCli = (runtime: string, cpuOnly = false, backend = 'cpu'): Promise => new Promise((resolve, reject) => { // cwd at the binary dir so @executable_path rpath resolves libstable-diffusion.dylib. const binDir = path.dirname(runtime) - // Qwen needs the model on the L40S for CUDA inference. Keep CPU offload - // for other runtimes, including a fallback after CUDA startup fails. + // Qwen keeps CPU offload unless the GPU is known to hold it (see qwenImageArgsForBackend). const runtimeArgs = isQwenImage21 - ? qwenImageArgsForBackend(args, imageBackendForRuntime(process.platform, runtime)) - : args + ? qwenImageArgsForBackend( + [...args], + cpuOnly ? 'CPU' : imageBackendForRuntime(process.platform, runtime) + ) + : [...args] + if (cpuOnly && backend !== 'cpu') runtimeArgs.push('--backend', 'cpu') const child = spawn(runtime, runtimeArgs, { cwd: binDir, env: { ...process.env, ...sdRuntimeLibraryEnv(process.platform, runtime, process.env) } @@ -1240,34 +1256,40 @@ async function runImageGen( }) }) - const runtimes = coreml ? [cli] : findSdBinaries('sd-cli', getBackendPreference('image')) + const runtimes = coreml + ? [{ path: cli, backend: 'metal', cpuOnly: false }] + : await findSdBinaries('sd-cli') let completedRuntime: string | undefined + let completedCpuOnly = false let lastError: unknown - for (const [index, runtime] of runtimes.entries()) { + for (const [index, step] of runtimes.entries()) { + const runtime = step.path try { await recordAIRequest( { modality: 'image', source: 'Image runtime attempt', model: path.basename(model), - backend: imageBackendForRuntime(process.platform, runtime), + backend: step.cpuOnly ? 'CPU' : imageBackendForRuntime(process.platform, runtime), request: params }, - async () => runNativeCli(runtime) + async () => runNativeCli(runtime, step.cpuOnly, step.backend) ) completedRuntime = runtime + completedCpuOnly = step.cpuOnly break } catch (error) { lastError = error const next = runtimes[index + 1] if (!next || generationLifecycle.isCancelled()) throw error console.warn( - `[imagegen] ${imageBackendForRuntime(process.platform, runtime)} runtime failed; retrying with ${imageBackendForRuntime(process.platform, next)}`, + `[imagegen] ${imageBackendForRuntime(process.platform, runtime)} runtime failed; retrying with ${imageBackendForRuntime(process.platform, next.path)}`, error ) } } - if (!completedRuntime) throw lastError + if (!completedRuntime) + throw lastError ?? new Error('No image engine is installed for this choice.') if (!fs.existsSync(outPath)) throw new Error('Image generation produced no output file.') const b64 = fs.readFileSync(outPath).toString('base64') @@ -1279,7 +1301,9 @@ async function runImageGen( model: path.basename(model), computeBackend: coreml ? 'Core ML (ANE)' - : imageBackendForRuntime(process.platform, completedRuntime) + : completedCpuOnly + ? 'CPU' + : imageBackendForRuntime(process.platform, completedRuntime) } } finally { generationLifecycle.finish() diff --git a/src/main/imagegen/__tests__/args.test.ts b/src/main/imagegen/__tests__/args.test.ts index cb5d613e2..d58e31a3b 100644 --- a/src/main/imagegen/__tests__/args.test.ts +++ b/src/main/imagegen/__tests__/args.test.ts @@ -139,11 +139,19 @@ describe('buildQwenImage21Args', () => { expect(flagVal(args, '-r')).toBe('/in.png') }) - it('keeps Qwen weights on CUDA and retains CPU offload for a fallback runtime', () => { + it('keeps Qwen CPU offload unless the CUDA device is known to hold the model', () => { const args = buildQwenImage21Args(base) - expect(qwenImageArgsForBackend(args, 'CUDA')).not.toContain('--offload-to-cpu') + expect(qwenImageArgsForBackend(args, 'CUDA')).toContain('--offload-to-cpu') + expect( + qwenImageArgsForBackend(args, 'CUDA', { freeVramBytes: 8 * 2 ** 30, needBytes: 20 * 2 ** 30 }) + ).toContain('--offload-to-cpu') + expect( + qwenImageArgsForBackend(args, 'CUDA', { + freeVramBytes: 40 * 2 ** 30, + needBytes: 20 * 2 ** 30 + }) + ).not.toContain('--offload-to-cpu') expect(qwenImageArgsForBackend(args, 'Vulkan')).toContain('--offload-to-cpu') - expect(args).toContain('--offload-to-cpu') }) }) diff --git a/src/main/imagegen/__tests__/sd-runtime.test.ts b/src/main/imagegen/__tests__/sd-runtime.test.ts index b28694220..554a619b4 100644 --- a/src/main/imagegen/__tests__/sd-runtime.test.ts +++ b/src/main/imagegen/__tests__/sd-runtime.test.ts @@ -17,7 +17,10 @@ describe('sdRuntimeLibraryEnv', () => { it('adds the shared CUDA runtime for the Linux CUDA image engine', () => { expect( sdRuntimeLibraryEnv('linux', '/app/bin/sd-cuda/sd-cli', { LD_LIBRARY_PATH: '/host/lib' }) - ).toEqual({ LD_LIBRARY_PATH: '/app/bin/sd-cuda:/app/bin/cuda-runtime:/host/lib' }) + ).toEqual({ + CUDA_CACHE_MAXSIZE: '4294967296', + LD_LIBRARY_PATH: '/app/bin/sd-cuda:/app/bin/cuda-runtime:/host/lib' + }) }) it('adds the shared CUDA runtime for the Windows CUDA image engine', () => { @@ -26,6 +29,8 @@ describe('sdRuntimeLibraryEnv', () => { PATH: 'C:\\Windows' }) ).toEqual({ + CUDA_CACHE_MAXSIZE: '4294967296', + GGML_VK_DISABLE_BFLOAT16: '1', PATH: 'C:\\app\\bin\\sd-cuda;C:\\app\\bin\\cuda-runtime;C:\\Windows' }) }) @@ -35,6 +40,6 @@ describe('sdRuntimeLibraryEnv', () => { sdRuntimeLibraryEnv('win32', 'C:\\app\\bin\\sd\\sd-cli.exe', { PATH: 'C:\\Windows' }) - ).toEqual({ PATH: 'C:\\app\\bin\\sd;C:\\Windows' }) + ).toEqual({ GGML_VK_DISABLE_BFLOAT16: '1', PATH: 'C:\\app\\bin\\sd;C:\\Windows' }) }) }) diff --git a/src/main/imagegen/args.ts b/src/main/imagegen/args.ts index 216fcb4ba..1e812e1c7 100644 --- a/src/main/imagegen/args.ts +++ b/src/main/imagegen/args.ts @@ -160,9 +160,20 @@ export function buildQwenImage21Args(i: QwenImage21ArgsInput): string[] { return args } -/** Keep Qwen weights on the CUDA device; other runtimes can still offload. */ -export function qwenImageArgsForBackend(args: string[], backend: string): string[] { - return backend === 'CUDA' ? args.filter((arg) => arg !== '--offload-to-cpu') : args +/** + * Qwen-Image keeps `--offload-to-cpu` on every backend: without a VRAM reading per device, dropping + * it on CUDA only fits data-centre cards (it was tuned on an L40S) and runs a consumer card out of + * memory mid-image. `freeVramBytes`/`needBytes`, when both are known, allow the weights to stay on + * the GPU. + */ +export function qwenImageArgsForBackend( + args: string[], + backend: string, + vram?: { freeVramBytes: number; needBytes: number } +): string[] { + return backend === 'CUDA' && vram && vram.freeVramBytes >= vram.needBytes + ? args.filter((arg) => arg !== '--offload-to-cpu') + : args } export interface StandardArgsInput { diff --git a/src/main/imagegen/generated-image-share.ts b/src/main/imagegen/generated-image-share.ts index f015e9097..17e038589 100644 --- a/src/main/imagegen/generated-image-share.ts +++ b/src/main/imagegen/generated-image-share.ts @@ -13,7 +13,7 @@ import { readGeneratedImageSidecar, writeGeneratedImageSidecar } from './gallery * a generated image looks like on the wire is not. That decision lived here AND on the phone, and the * two copies had already disagreed about the name of the model field. */ -export function describeOwnGeneratedImage( +function describeOwnGeneratedImage( imagePath: string, shownIn?: ChatHome ): SharedFileDescriptor | null { diff --git a/src/main/imagegen/sd-runtime.ts b/src/main/imagegen/sd-runtime.ts index aab3b8bbc..160e8834c 100644 --- a/src/main/imagegen/sd-runtime.ts +++ b/src/main/imagegen/sd-runtime.ts @@ -1,9 +1,6 @@ -import fs from 'fs' import path from 'path' -import { binRoots, exe } from '../runtime-env' import { nativeLibraryEnv } from '../native-library-env' -import { hasLinuxNvidiaDriver } from '../transcription/whisper-runtime' -import { prioritizeBackend, type BackendPreference } from '../../shared/backend-preferences' +import { nativeLadder, type NativeStep } from '../backend-facts' /** The backend selected by the native image binary that completed the run. */ export function imageBackendForRuntime( @@ -16,43 +13,14 @@ export function imageBackendForRuntime( return platform === 'darwin' ? 'Metal' : 'Vulkan' } -/** Windows needs the system Vulkan loader before the GPU image binary can start. */ -export function hasWindowsVulkanLoader( - windowsDir = process.env.WINDIR ?? process.env.SystemRoot ?? 'C:\\Windows' -): boolean { - return fs.existsSync(path.join(windowsDir, 'System32', 'vulkan-1.dll')) +export async function findSdBinaries(name: 'sd-cli' | 'sd-server'): Promise { + const steps = await nativeLadder('image', name) + if (!steps.length) throw new Error('No image engine is installed for this choice.') + return steps } -/** - * Resolve an image runtime. Windows uses the compact Vulkan build when its loader - * is installed, then uses the separately packaged CPU build. Linux and macOS keep - * their existing `sd` runtime; the Linux Vulkan build contains CPU kernels too. - */ -export function findSdBinaries(name: 'sd-cli' | 'sd-server', preference: BackendPreference = 'auto'): string[] { - let directories = - process.platform === 'linux' && hasLinuxNvidiaDriver() - ? ['sd-cuda', 'sd', 'sd-cpu'] - : ['sd', 'sd-cpu'] - if (process.platform === 'win32') { - directories = hasWindowsVulkanLoader() - ? ['sd-cuda', 'sd', 'sd-cpu'] - : ['sd-cuda', 'sd-cpu', 'sd'] - } - const matches: string[] = [] - for (const directory of directories) { - for (const root of binRoots()) { - const candidate = path.join(root, directory, exe(name)) - if (fs.existsSync(candidate) && !matches.includes(candidate)) matches.push(candidate) - } - } - return prioritizeBackend(matches, preference, (binary) => { - const backend = imageBackendForRuntime(process.platform, binary) - return backend.toLowerCase() as BackendPreference - }) -} - -export function findSdBinary(name: 'sd-cli' | 'sd-server'): string | null { - return findSdBinaries(name)[0] ?? null +export async function findSdBinary(name: 'sd-cli' | 'sd-server'): Promise { + return (await findSdBinaries(name))[0]?.path ?? null } /** CUDA image binaries reuse the CUDA libraries already packaged for llama.cpp. */ diff --git a/src/main/index.ts b/src/main/index.ts index 597412e41..ff5b7c0d4 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -70,7 +70,14 @@ import { import { PERSONAL_MESH_ENTITLEMENT_REVALIDATION_INTERVAL_MS } from '@offgrid/sync' import { setupLicenseIpc } from './license-ipc' import { nativeImage } from 'electron' -import { purgeLegacyChatImports, getSetting, saveSetting, optimizeDB } from './database' +import { + purgeLegacyChatImports, + getSetting, + saveSetting, + optimizeDB, + startIndexRebuilds, + stopIndexRebuilds +} from './database' import { modalityQueue } from './modality-queue/queue' import { applyQueueConfig, readQueueConfig } from './modality-queue/config' import { liveMemory, registerRuntime } from './runtime-manager' @@ -91,12 +98,10 @@ import { applicationShutdown, commitApplicationRelaunch, installApplicationShutdown, - requestApplicationRelaunch, registerCoreShutdownOwners } from './shutdown' import { shutdownRuntimes } from './runtime-manager' import { shutdownModelDownloads } from './models/download-queue' -import { prepareInstalledOnnxCudaLibraries } from './performance-pack' // Before anything logs: a broken stdout/stderr pipe (parent/e2e-harness exited, closed pipe) // must never crash main via an uncaught EPIPE. See stream-guards.ts. @@ -367,12 +372,13 @@ function startServerOnlyMode(): void { app.whenReady().then(async () => { // A Team-mode gateway keeps the machine awake while it serves clients. setGatewayPowerBlocker(powerSaveBlocker) - // glibc reads LD_LIBRARY_PATH when Electron starts, before an ONNX worker runs. - // Restart once with the verified optional libraries if this launch lacks them. - if (app.isPackaged && prepareInstalledOnnxCudaLibraries()) { - requestApplicationRelaunch(app) - return - } + // The NVIDIA driver version decides which CUDA engines may run; read it off the startup path. + void import('./nvidia-driver') + .then(async ({ refreshNvidiaDriverVersion }) => { + await refreshNvidiaDriverVersion() + ;(await import('./backend-facts')).invalidateBackendFacts() + }) + .catch((error: unknown) => console.warn('[nvidia-driver] read failed', error)) const { cleanupMovedModelStorage } = await import('./model-storage-move') const { modelsDir } = await import('./runtime-env') restoreCanonicalProductName() @@ -662,6 +668,15 @@ app.whenReady().then(async () => { registerRuntime(ttsRuntime) registerRuntime(imageRuntime) registerRuntime(sttRuntime) + // An image job frees the task engines' memory; they start again on their next use. + modalityQueue.registerEvictable('grounding', { + evict: async () => + (await import('./vision/grounder-runtime')).grounderRuntime.shutdown() + }) + modalityQueue.registerEvictable('decision', { + evict: async () => + (await import('./accessibility/decision-runtime')).decisionRuntime.shutdown() + }) }) } }, @@ -697,7 +712,56 @@ app.whenReady().then(async () => { lateEffect: 'keep', run: async () => { const { llm } = await import('./llm') - registerRuntime(llm.runtime, { readChatBackend: () => llm.runtimeBackend() }) + registerRuntime(llm.runtime) + const { installEngineResumeHealth, registerResumeHealthTarget } = + await import('./engine-resume-health') + registerResumeHealthTarget({ + id: 'chat', + shouldBeRunning: () => llm.shouldBeRunning(), + healthCheck: () => llm.healthCheck(), + restart: () => llm.restartSameModel() + }) + const [{ embeddings }, { ttsOnnxRuntime }, { grounderRuntime }, { decisionRuntime }] = + await Promise.all([ + import('./embeddings'), + import('./tts'), + import('./vision/grounder-runtime'), + import('./accessibility/decision-runtime') + ]) + for (const [id, runtime] of [ + ['grounding', grounderRuntime], + ['decision', decisionRuntime] + ] as const) { + registerResumeHealthTarget({ + id, + shouldBeRunning: () => runtime.shouldBeRunning(), + healthCheck: () => runtime.health(), + restart: () => runtime.restart() + }) + } + // A helper that died while asleep restarts on its next request; restart a running one + // so it does not hold a lost GPU context. + for (const [id, helper] of [ + ['search', embeddings], + ['voice', ttsOnnxRuntime] + ] as const) { + registerResumeHealthTarget({ + id, + shouldBeRunning: () => helper.isRunning, + healthCheck: async () => !helper.isRunning, + restart: () => helper.restart() + }) + } + const disposeResumeHealth = installEngineResumeHealth() + // Windows: test the pack's ONNX CUDA addon once per pack and driver, in a helper. + void import('./onnx-host/onnx-cuda-probe') + .then(async ({ probeOnnxCuda }) => { + if (!(await probeOnnxCuda())) return + ;(await import('./backend-facts')).invalidateBackendFacts() + await Promise.all([embeddings.restart(), ttsOnnxRuntime.restart()]) + }) + .catch((error: unknown) => console.warn('[ONNX] CUDA check failed to run', error)) + app.once('will-quit', disposeResumeHealth) applyQueueConfig(modalityQueue, readQueueConfig(getSetting)) // Models stay loaded while they fit live memory; the queue frees only what it must. modalityQueue.setMemoryProbe(liveMemory) @@ -731,6 +795,43 @@ app.whenReady().then(async () => { void import('./documents').then((documents) => documents.collectDocumentGarbage()) }, 120_000).unref() + // Background index work, once start-up has settled, a short slice at a time: full-text indexes + // built by an older schema are rebuilt beside the old ones (which keep answering until each + // rebuild finishes), and rows that predate the memory-search triggers are queued for embedding. + const indexWork = setTimeout(() => { + void import('./records') + .then((records) => records.ensureRecordSchema()) + .catch((error: unknown) => console.error('[search index] schema check failed', error)) + .finally(startIndexRebuilds) + }, 15_000) + indexWork.unref() + // Embed the memory-search backlog (records queued by their tables' triggers): once shortly after + // launch, then every few minutes for what was added since. Core owns this, so builds without + // Pro index chats and people too. + const embedBacklog = (): void => { + void import('./records') + .then((records) => + records.embedRecordBacklog((progress) => { + if (progress.remaining % 200 === 0) + console.log(`[search index] ${progress.done} embedded, ${progress.remaining} left`) + }) + ) + .catch((error: unknown) => console.error('[search index]', error)) + } + const firstEmbed = setTimeout(embedBacklog, 20_000) + firstEmbed.unref() + const embedTimer = setInterval(embedBacklog, 5 * 60_000) + embedTimer.unref() + applicationShutdown.register({ + name: 'core:background-indexing', + shutdown: () => { + clearTimeout(indexWork) + clearTimeout(firstEmbed) + clearInterval(embedTimer) + stopIndexRebuilds() + } + }) + app.on('activate', function () { if (BrowserWindow.getAllWindows().length === 0) void createWindow() }) diff --git a/src/main/input/actuation.ts b/src/main/input/actuation.ts index be8a51ee7..22fdd893f 100644 --- a/src/main/input/actuation.ts +++ b/src/main/input/actuation.ts @@ -171,7 +171,3 @@ function keyCodes(keys: readonly string[], keyMap: Record): numb return code }) } - -export function actuationAvailable(): boolean { - return loadActuation() !== null -} diff --git a/src/main/ipc-query-logic.ts b/src/main/ipc-query-logic.ts deleted file mode 100644 index 190134194..000000000 --- a/src/main/ipc-query-logic.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { escapeFtsMatch } from './fts-match' - -// Pure query/message helpers extracted from ipc.ts so the retrieval-gating logic -// is unit-testable without Electron / the DB (mirrors search-ranking.ts, -// model-sizing.ts). No side effects. ipc.ts re-imports these; the -// ipcMain.handle registrations stay in ipc.ts. Behaviour-neutral move. - -/** Parse a model/LLM JSON reply into T, tolerating ```json fences; falls back on - * any parse error so a malformed reply never throws into the caller. */ -export function safeParseJson(input: string, fallback: T): T { - try { - const clean = input.replace(/```json\n?|\n?```/g, '').trim() - return JSON.parse(clean) as T - } catch { - return fallback - } -} - -/** Stopwords dropped from a tokenised query (single source of truth). */ -export const STOPWORDS = new Set([ - 'the', - 'and', - 'for', - 'with', - 'from', - 'that', - 'this', - 'what', - 'know', - 'about', - 'your', - 'you', - 'me', - 'my', - 'all', - 'do', - 'are', - 'was', - 'were', - 'been', - 'being', - 'have', - 'has', - 'had', - 'will', - 'would', - 'should', - 'could', - 'can', - 'may', - 'might' -]) - -/** Extract lowercase Unicode words (keep internal hyphens/underscores), - * drop tokens < 3 chars and STOPWORDS, de-dup, cap at maxTokens. */ -export function tokenizeQuery(query: string, maxTokens: number = 6): string[] { - const tokens = (query.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? []) - .filter((t) => t.length >= 3) - .filter((t) => !STOPWORDS.has(t)) - return Array.from(new Set(tokens)).slice(0, maxTokens) -} - -/** Any-term recall with the existing stopword filter, using the shared FTS escape. */ -export function ftsMatchExpression(query: string, maxTokens: number = 6): string { - const tokens = tokenizeQuery(query, maxTokens) - return escapeFtsMatch(tokens.length ? tokens : query, { maxTokens }) -} - -/** Clip text to maxLength, replacing the final char with an ellipsis when it - * overflows. Empty/undefined text → ''. */ -export function clipText(text: string, maxLength: number): string { - if (!text) return '' - if (text.length <= maxLength) return text - return text.slice(0, Math.max(0, maxLength - 1)) + '…' -} - -// Build/generate requests ("build a react app", "write an svg", "make a landing -// page") don't benefit from memory retrieval — pulling in unrelated SOURCES makes -// the model cite junk and second-guess itself. Detect them so we can answer with -// the artifact instructions only and skip the search. -export function isGenerativeRequest(text: string): boolean { - const q = (text || '').trim().toLowerCase() - if (!q) return false - const hasNoun = - /\b(react|next\.?js|vue|svelte|html|css|svg|website|web ?app|web ?page|landing page|component|widget|diagram|chart|flowchart|mermaid|game|canvas|prototype|mock-?up|ui|app|script|function|snippet|webpage|playground|frontend|front-end|dashboard|form|interface|page|tool|visualization|visualisation|simulator|editor|viewer|demo|site)\b/.test( - q - ) - const hasVerb = - /\b(build|create|make|write|generate|code|implement|design|draw|render|scaffold|give me a|show me a)\b/.test( - q - ) - return hasNoun && hasVerb -} - -/** - * The ` LIKE ?` fragment + its bound param for an optional app-name - * filter, or null when no filter applies ('All' / empty = every app). Callers add - * their own connector (WHERE / AND). Single source for the appName gate that was - * inlined 4× across db:get-memories and the rag:chat vector / FTS-fallback / - * message queries (each with a different column, same guard + `%…%` wildcarding). - */ -export function appNameLikeClause( - appName: string | undefined, - column: string -): { clause: string; param: string } | null { - if (!appName || appName === 'All') { - return null - } - return { clause: `${column} LIKE ?`, param: `%${appName}%` } -} - -/** A short pleasantry/acknowledgement ("hi", "ok", "thanks") — or empty — that - * shouldn't trigger memory extraction. Real messages return false. */ -export function isTrivialMessage(text: string): boolean { - const normalized = (text || '').trim() - if (normalized.length === 0) return true - if (normalized.length < 20) { - if ( - /^(hi|hello|hey|thanks|thank you|ok|okay|sure|yes|no|cool|great|nice|good|fine|bye|see ya|yep|nope)[!.]?$/i.test( - normalized - ) - ) { - return true - } - } - return false -} diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 3bda77dbc..b0df66c14 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -1,3 +1,4 @@ +import { backendChoicesForUi } from './backend-facts' import { settingsChanged } from './settings-version' import { withStorageDownload, storageDownloadsActive } from './model-storage-guard' import { moveModelStorage } from './model-storage-move' @@ -12,16 +13,7 @@ import { getChatSessions, upsertChatSummary, getMemoriesForSession, - getMemoryRecordsForSession, - getMasterMemory, - addEntityFact, - updateEntitySummary, - getEntities, - getEntityDetails, - upsertEntitySession, - deleteMemory, getEntitiesForSession, - getDashboardStats, getUserProfile, saveUserProfile, UserProfile, @@ -37,12 +29,11 @@ import { saveSetting, getSetting } from './database' -import { deleteEntityById, resolveEntityCandidate } from './entity-domain' +import { deleteEntityById } from './entity-domain' import { setComputerUseSettings } from './computer-use-settings' import { COMPUTER_USE_SETTINGS_KEY } from '../shared/computer-use-settings' import { setWebUseSettings } from './web-use-settings' import { WEB_USE_SETTINGS_KEY } from '../shared/web-use-settings' -import { embeddings } from './embeddings' import { getResidency, setResidencyMode, @@ -66,18 +57,11 @@ import { getAllPromptDefs } from './prompts' import { getPrompt, getPromptTemplate, resetPrompt } from './prompt-store' import { setupTtsIpc } from './tts-ipc' import { setupVoiceTranscriptionIpc } from './voice-transcription-ipc' -import { - safeParseJson, - ftsMatchExpression, - clipText, - isGenerativeRequest, - isTrivialMessage, - appNameLikeClause -} from './ipc-query-logic' import { requestApplicationRelaunch } from './shutdown' import { activateInstalledPerformancePack, onPerformancePackChanged, + onPerformancePackInstalled, pausePerformancePack, performancePackStatus, startPerformancePack @@ -87,28 +71,6 @@ import { sampleProgressRate, type ProgressRateSample } from '@offgrid/ui' import { notifyRagConversationChanged } from './rag-conversation-events' // import { llm } from './llm'; // Moved to dynamic import to support ESM -// Incrementally update master memory with a new conversation summary -// This approach keeps context bounded by only processing current master + new summary -async function updateMasterMemoryIncremental(_newSummary: string): Promise { - // Master memory (the consolidated "profile") is a retired My Memories feature — - // no longer injected into chat. Don't regenerate it so it stays cleared. - return null -} - -// Full regeneration using map-reduce: -// Phase 1 (map): Split summaries into chunks, generate a partial summary for each -// Phase 2 (reduce): Merge all partial summaries into the final master memory -// This avoids the growing-prompt problem and minimizes LLM calls -async function regenerateMasterMemoryFull(): Promise { - // Retired feature — see updateMasterMemoryIncremental. Don't rebuild the profile. - return null -} - -// Main entry point - full regeneration -async function regenerateMasterMemory(): Promise { - return regenerateMasterMemoryFull() -} - // Generate the answer for a rag:chat turn. When a streamId + sender are present, // stream tokens/reasoning to the renderer over the 'rag:stream' channel as they // arrive (inline chain-of-thought); otherwise fall back to a single blocking call. @@ -133,6 +95,30 @@ import { runChatWithFallback } from './chat-fallback' const streamControllers = new Map() +function deviceFallbackNotices( + sender: { send: (channel: string, payload: unknown) => void } | undefined, + streamId: string | undefined +): Pick< + NonNullable[1]>, + 'onDeviceFallback' | 'onRestarting' +> { + if (!sender || !streamId) return {} + const send = (step: unknown): void => { + try { + sender.send('rag:stream', { streamId, type: 'step', step }) + } catch { + /* window gone */ + } + } + return { + onDeviceFallback: (message) => send({ kind: 'device_fallback', message }), + onRestarting: (restarting) => { + if (restarting) resetChatStreamPartial(streamId) + send({ kind: restarting ? 'restarting_model' : 'waiting' }) + } + } +} + async function streamAnswer( event: { sender?: { send: (channel: string, payload: unknown) => void } } | undefined, streamId: string | undefined, @@ -152,8 +138,9 @@ async function streamAnswer( // evicts nothing (evicting 'llm' would evict itself). run()'s finally releases // the slot even if fn throws, so we let errors propagate from inside. return modalityQueue.run(CHAT_JOB, async () => { - const { result, modelName } = await runChatWithFallback(() => - llm.chatStream(prompt, images, () => {}, { thinking }) + const { result, modelName } = await runChatWithFallback( + () => llm.chatStream(prompt, images, () => {}, { thinking }), + deviceFallbackNotices(event?.sender, streamId) ) const response = toResponseGenerationResult(result) return modelName ? { ...response, metrics: { ...response.metrics, modelName } } : response @@ -188,6 +175,7 @@ async function streamAnswer( ), { signal: controller.signal, + ...deviceFallbackNotices(sender, streamId), onModelChanged: (failed, next) => { resetChatStreamPartial(streamId) try { @@ -216,286 +204,11 @@ async function streamAnswer( } } -type ChatIntent = { intent: 'build' | 'image' | 'chat'; urls: string[] } - -const INTENT_SCHEMA = { - type: 'object', - properties: { - intent: { type: 'string', enum: ['build', 'image', 'chat'] }, - urls: { type: 'array', items: { type: 'string' } } - }, - required: ['intent', 'urls'], - additionalProperties: false -} - -// Decide the output format for a turn with the model itself (grammar-constrained -// JSON), instead of brittle keyword matching: build (runnable artifact), image -// (generate a picture), or chat. Also pulls out any URLs the user wants read. -// Falls back to the keyword heuristic if the classifier call fails. -async function classifyIntent( - query: string, - history?: { role: string; content: string }[], - streamId?: string -): Promise { - const regexUrls = (query.match(/https?:\/\/[^\s)<>"']+/g) || []).slice(0, 3) - // Register a controller for this pre-stream classify so a rag:cancel during the - // "Searching your memory…" window actually aborts the model call (D11) — without - // it the classify ran to completion after Stop, holding the model + the next turn. - const controller = streamId ? new AbortController() : undefined - if (streamId && controller) streamControllers.set(streamId, controller) - try { - const { llm } = await import('./llm') - const hist = (history ?? []) - .slice(-4) - .map((m) => `${m.role === 'assistant' ? 'Assistant' : 'User'}: ${clipText(m.content, 200)}`) - .join('\n') - const prompt = [ - 'You route a request for an on-device assistant. Decide the OUTPUT FORMAT:', - '- "build": the user wants runnable code / a UI created — an app, component, page, playground, dashboard, form, diagram, chart, visualization, game, etc. (rendered live in a canvas).', - '- "image": the user wants a picture/photo/logo/illustration/art generated.', - '- "chat": anything else — questions, explanations, writing, discussion.', - 'Also list any http(s) URLs the user wants you to read or build from.', - 'Reply with ONLY JSON: {"intent":"build|image|chat","urls":[]}.', - hist ? `Recent conversation:\n${hist}` : '', - `User: ${query}` - ] - .filter(Boolean) - .join('\n\n') - const raw = await llm.chat(prompt, [], undefined, 200, { - disableThinking: true, - responseFormat: { - type: 'json_schema', - json_schema: { name: 'intent', schema: INTENT_SCHEMA, strict: true } - }, - signal: controller?.signal - }) - const j = JSON.parse(raw) as Partial - const intent = j.intent === 'build' || j.intent === 'image' ? j.intent : 'chat' - const urls = Array.isArray(j.urls) ? j.urls.filter((u) => /^https?:\/\//i.test(u)) : [] - return { intent, urls: urls.length ? urls.slice(0, 3) : regexUrls } - } catch (e) { - console.warn('[intent] classifier failed, falling back to heuristic', (e as Error).message) - return { intent: isGenerativeRequest(query) ? 'build' : 'chat', urls: regexUrls } - } finally { - // Only remove OUR entry — streamAnswer re-registers its own controller under - // the same streamId for the streaming phase. - if (streamId && controller && streamControllers.get(streamId) === controller) - streamControllers.delete(streamId) - } -} - -async function insertMemoryRecord(params: { - content: string - name?: string | null - rawText?: string | null - sourceApp?: string | null - sessionId?: string | null - messageId?: number | null -}): Promise { - const db = getDB() - const content = (params.content || '').trim() - if (!content) return null - - if (params.messageId) { - const existing = db - .prepare('SELECT id FROM memories WHERE message_id = ? LIMIT 1') - .get(params.messageId) as { id: number } | undefined - if (existing?.id) return existing.id - } - - if (params.sessionId) { - const existing = db - .prepare('SELECT id FROM memories WHERE session_id = ? AND content = ? LIMIT 1') - .get(params.sessionId, content) as { id: number } | undefined - if (existing?.id) return existing.id - } - - let vectorJson = '[]' - try { - const vector = await embeddings.generateEmbedding(content) - vectorJson = JSON.stringify(vector) - } catch (e) { - console.error('Failed to generate embedding for memory record:', e) - } - - const stmt = db.prepare( - 'INSERT INTO memories (content, name, raw_text, source_app, session_id, embedding, message_id) VALUES (?, ?, ?, ?, ?, ?, ?)' - ) - const info = stmt.run( - content, - params.name || null, - params.rawText || null, - params.sourceApp || null, - params.sessionId || null, - vectorJson, - params.messageId || null - ) - const memoryId = Number(info.lastInsertRowid || 0) || null - - // Send notification about new memory - if (memoryId) { - BrowserWindow.getAllWindows().forEach((win) => { - win.webContents.send('notification:new-memory', { - sessionId: params.sessionId || null, - memoryContent: content.slice(0, 100) + (content.length > 100 ? '...' : '') - }) - }) - } - - return memoryId -} - -export async function evaluateAndStoreMemoryForMessage(params: { - sessionId: string - appName: string - role: string - content: string - messageId?: number | null -}): Promise { - const role = (params.role || 'unknown').toLowerCase() - const text = (params.content || '').trim() - if (!text || isTrivialMessage(text)) return - - // Get strictness setting - const strictness = getSetting<'lenient' | 'balanced' | 'strict'>('memoryStrictness', 'balanced') - - const prompt = getPrompt(`memoryFilter.${strictness}`, { ROLE: role, MESSAGE: text }) - - // Minimum content length filter: skip very short messages - if (role === 'user' && text.length < 30) return - if (role === 'assistant' && text.length < 50) return - - try { - const { llm } = await import('./llm') - const response = await llm.chat(prompt) - const parsed = safeParseJson<{ store: boolean; name?: string; memory?: string }>(response, { - store: false - }) - if (!parsed.store) return - const memoryText = (parsed.memory || '').trim() - const memoryName = (parsed.name || '').trim() || null - if (!memoryText) return - if (memoryText.split(/\s+/).length < 4) return - if (memoryText.length > 280) return - - // Post-LLM filter: skip memories matching generic / low-value patterns - const genericPatterns = [ - /^the user (asked|said|mentioned|wanted|is|was|has|had)\b/i, - /^(this|that|it) (is|was|seems|appears|looks)\b/i, - /^(a|an|the) (good|great|nice|common|typical|standard|normal)\b/i, - /\b(in general|generally speaking|as usual|as always)\b/i - ] - if (genericPatterns.some((p) => p.test(memoryText))) return - - // Post-LLM filter: skip near-duplicates via substring check against existing session memories - if (params.sessionId) { - const existingMemories = getMemoryRecordsForSession(params.sessionId) - const memLower = memoryText.toLowerCase() - const isDuplicate = existingMemories.some((m) => { - const existing = (m.content || '').toLowerCase() - return existing === memLower || existing.includes(memLower) || memLower.includes(existing) - }) - if (isDuplicate) return - } - - await insertMemoryRecord({ - content: memoryText, - name: memoryName, - rawText: text, - sourceApp: params.appName, - sessionId: params.sessionId, - messageId: params.messageId || null - }) - } catch (e) { - console.error('[IPC] Memory evaluation failed:', e) - } -} - -async function extractEntitiesForSession(sessionId: string): Promise { - const memories = getMemoryRecordsForSession(sessionId) - if (memories.length === 0) return - - // Get strictness setting - const strictness = getSetting<'lenient' | 'balanced' | 'strict'>('entityStrictness', 'balanced') - - const memoryText = memories.map((m) => `- ${m.content}`).join('\n') - - const prompt = getPrompt(`entityExtraction.${strictness}`, { MEMORY_TEXT: memoryText }) - - try { - const { llm } = await import('./llm') - const response = await llm.chat(prompt) - const parsed = safeParseJson<{ entities: { name: string; type?: string; facts?: string[] }[] }>( - response, - { entities: [] } - ) - - if (parsed.entities.length === 0) return - - for (const entity of parsed.entities) { - const name = (entity.name || '').trim() - if (!name) continue - const type = (entity.type || 'Unknown').trim() || 'Unknown' - const facts = Array.isArray(entity.facts) - ? entity.facts - .filter(Boolean) - .map((f) => f.trim()) - .filter(Boolean) - : [] - - if (facts.length === 0) continue - - const resolution = resolveEntityCandidate({ name, type }) - if (!resolution.admitted) continue - const entityId = resolution.entityId - if (!entityId) continue - upsertEntitySession(entityId, sessionId) - - const newFacts: string[] = [] - for (const fact of facts) { - const inserted = addEntityFact(entityId, fact, sessionId) - if (inserted) newFacts.push(fact) - } - - // Send notification about new entity with facts - if (newFacts.length > 0) { - BrowserWindow.getAllWindows().forEach((win) => { - win.webContents.send('notification:new-entity', { - entityId, - entityName: name, - entityType: type, - factsCount: newFacts.length - }) - }) - } - - if (newFacts.length === 0) continue - - const details = getEntityDetails(entityId) as { entity?: { summary?: string } } | null - const existingSummary = details?.entity?.summary || '' - - const summaryPrompt = getPrompt('entitySummary', { - NAME: name, - TYPE: type, - EXISTING_SUMMARY: existingSummary || '(none)', - NEW_FACTS: '- ' + newFacts.join('\n- ') - }) - - try { - const updatedSummary = await llm.chat(summaryPrompt) - if (updatedSummary && updatedSummary.trim()) { - updateEntitySummary(entityId, updatedSummary.trim()) - } - } catch (e) { - console.error('[IPC] Failed to update entity summary:', e) - } - } - } catch (e) { - console.error('[IPC] Entity extraction failed:', e) - } -} - -export async function summarizeSession(sessionId: string): Promise { +// A legacy chat session's summary (chat_summaries, searched as 'summary' records). The legacy +// memories and entity facts that used to be extracted alongside it are retired: nothing writes or +// reads them, and their rows stay until the user deletes their data. Live memory is screen moments, +// entities, to-dos, meetings and chats. +async function summarizeSession(sessionId: string): Promise { const memories = getMemoriesForSession(sessionId) if (memories.length === 0) return null @@ -506,21 +219,6 @@ export async function summarizeSession(sessionId: string): Promise { + // Voice and search helpers restart onto the pack (Windows tests its ONNX CUDA addon first), + // and auto-managed settings move onto the CUDA engine. + void (async () => { + const [{ invalidateBackendFacts }, { probeOnnxCuda }, { embeddings }, { ttsOnnxRuntime }] = + await Promise.all([ + import('./backend-facts'), + import('./onnx-host/onnx-cuda-probe'), + import('./embeddings'), + import('./tts') + ]) + invalidateBackendFacts() + await probeOnnxCuda() + invalidateBackendFacts() + await Promise.all([embeddings.restart(), ttsOnnxRuntime.restart()]) + })().catch((error: unknown) => console.warn('[performance-pack] helper restart failed', error)) + void import('./best-settings') + .then((m) => { + m.forgetDetectedMachine() + return m.ensureBestSettings() + }) + .catch((error: unknown) => console.warn('[best-settings] CUDA refresh failed', error)) + }) ipcMain.handle('performance-pack:status', performancePackStatus) ipcMain.handle('performance-pack:start', startPerformancePack) ipcMain.handle('performance-pack:pause', pausePerformancePack) - ipcMain.handle('performance-pack:restart', () => requestApplicationRelaunch(app)) // Engines published between releases install in the background unless the user turns it off. engineUpdater.onChanged((status) => { BrowserWindow.getAllWindows().forEach((window) => @@ -559,7 +279,7 @@ export function setupIPC(): void { setupAIRequestLogIPC((ref) => nameModel?.(ref) ?? modelDisplayName(ref) ?? ref) setupChatNextActionIpc() setupVoiceTranscriptionIpc() - const db = getDB() + getDB() setupTtsIpc() setupSystemStatusIpc(ipcMain, { publish: (health) => @@ -568,102 +288,6 @@ export function setupIPC(): void { ) }) - ipcMain.handle('db:get-memories', (_, limit: number = 50, appName?: string) => { - let query = 'SELECT * FROM memories ' - const params: unknown[] = [] - - const memFilter = appNameLikeClause(appName, 'source_app') - if (memFilter) { - query += `WHERE ${memFilter.clause} ` - params.push(memFilter.param) - } - - query += 'ORDER BY created_at DESC LIMIT ?' - params.push(limit) - - const stmt = db.prepare(query) - // SQLite stores timestamps as UTC strings "YYYY-MM-DD HH:MM:SS" by default with CURRENT_TIMESTAMP - // To ensure JS treats them as UTC, we might need to append 'Z' or standardise. - // However, simplest is to let frontend handle "UTC" assumption. - return stmt.all(...params) - }) - - ipcMain.handle( - 'db:add-memory', - async (_, content: string, source: string = 'user-input', sessionId?: string) => { - // Generate embedding - let vectorJson = '[]' - try { - const vector = await embeddings.generateEmbedding(content) - vectorJson = JSON.stringify(vector) - } catch (e) { - console.error('Failed to generate embedding:', e) - } - - // Check if we can update an existing session - if (sessionId) { - // Look for a recent memory (e.g. last 12 hours) with this session_id - const existing = db - .prepare( - 'SELECT id FROM memories WHERE session_id = ? AND created_at > datetime("now", "-12 hours") ORDER BY id DESC LIMIT 1' - ) - .get(sessionId) as { id: number } | undefined - - if (existing) { - console.log(`Updating existing memory session ${sessionId} (ID: ${existing.id})`) - const stmt = db.prepare( - 'UPDATE memories SET content = ?, embedding = ?, created_at = CURRENT_TIMESTAMP WHERE id = ?' - ) - stmt.run(content, vectorJson, existing.id) - return { id: existing.id, updated: true } - } - } - - const stmt = db.prepare( - 'INSERT INTO memories (content, source_app, session_id, embedding) VALUES (?, ?, ?, ?)' - ) - const info = stmt.run(content, source, sessionId || null, vectorJson) - return { id: info.lastInsertRowid } - } - ) - - ipcMain.handle('db:search-memories', async (_, query: string) => { - try { - const queryVector = await embeddings.generateEmbedding(query, { interactive: true }) - const vecStr = JSON.stringify(queryVector) - - const stmt = db.prepare(` - SELECT *, cosine_similarity(embedding, ?) as score - FROM memories - WHERE embedding IS NOT NULL AND embedding != '[]' - ORDER BY score DESC - LIMIT 20 - `) - return stmt.all(vecStr) - } catch (e) { - console.error('Vector search failed, falling back to FTS', e) - const match = ftsMatchExpression(query) - if (!match) return [] - const stmt = db.prepare(` - SELECT memories.* - FROM memories - JOIN memory_fts ON memories.id = memory_fts.rowid - WHERE memory_fts MATCH ? - LIMIT 20 - `) - return stmt.all(match) - } - }) - - ipcMain.handle('db:get-stats', () => { - const count = db.prepare('SELECT COUNT(*) as count FROM memories').get() - return count - }) - - ipcMain.handle('db:get-dashboard-stats', () => { - return getDashboardStats() - }) - ipcMain.handle('llm:extract', async (_, text: string) => { try { const { llm } = await import('./llm') @@ -692,403 +316,38 @@ export function setupIPC(): void { streamControllers.get(streamId)?.abort() }) + // A plain on-device model turn: no memory retrieval and no routing. Memory, tools and artifacts + // are the chat tools loop's (tools.ts); this channel is the direct call the preset setup form + // uses to fill a field. The memory arguments of the old contract are accepted and ignored. ipcMain.handle( 'rag:chat', async ( event, query: string, - appName?: string, + _appName?: string, conversationHistory?: { role: string; content: string }[], - projectId?: string | null, + _projectId?: string | null, conversationId?: string, - noMemory?: boolean, + _noMemory?: boolean, streamId?: string, thinking?: boolean, images?: string[] ) => { - const imgs = images || [] - // Before the classifier, so the whole turn - including its thinking - is attributable to the - // conversation it belongs to. bindChatStream(streamId, conversationId, thinking ? 'thinking' : 'waiting') - // Intelligence layer: a grammar-constrained classifier picks the output - // format (build / image / chat) and extracts URLs to read — replacing the - // brittle keyword gate. Skip it in project mode (that path is its own thing). - const { intent, urls: intentUrls } = projectId - ? { intent: 'chat' as const, urls: [] as string[] } - : await classifyIntent(query, conversationHistory, streamId) - - // Image request → have the model write a vivid prompt, then the renderer - // generates it (it already detects an ```image block). - if (intent === 'image') { - const imgPrompt = `Write ONE vivid, detailed image-generation prompt (visual description only, no preamble) for this request:\n${query}` - const desc = ( - await ( - await import('./llm') - ).llm.chat(imgPrompt, [], undefined, 200, { disableThinking: true }) - ) - .trim() - .replace(/^["']|["']$/g, '') - return { answer: '```image\n' + (desc || query) + '\n```', context: undefined } - } - - // Build request → artifact prompt (even in No-memory mode), with any URLs - // fetched for us so the small model never has to chain tools. - if (intent === 'build') { - let historyBlock = '' - if (conversationHistory && conversationHistory.length > 0) { - const historyLines = conversationHistory - .map( - (msg) => - `${msg.role === 'user' ? 'User' : 'Assistant'}: ${clipText(msg.content, 400)}` - ) - .join('\n') - historyBlock = `Conversation so far:\n${historyLines}` - } - // read_url → build: fetch the classifier's URLs (deterministic). - let referenceBlock = '' - const urls = intentUrls - if (urls.length) { - if (streamId) - event.sender.send('rag:stream', { - streamId, - type: 'step', - step: { kind: 'reading', counts: { urls: urls.length } } - }) - const { readUrlText } = await import('./tools') - const parts: string[] = [] - for (const u of urls) { - try { - parts.push( - `--- Content fetched from ${u} ---\n${clipText(await readUrlText(u), 5000)}` - ) - } catch (e) { - parts.push(`--- Could not fetch ${u}: ${(e as Error).message} ---`) - } - } - referenceBlock = `REFERENCE — the user pointed you at these page(s); BUILD using this content (e.g. if it's API docs, build a UI that actually calls those endpoints):\n${parts.join('\n\n')}` - } - const prompt = [ - 'You are Off Grid AI, an on-device assistant with a LIVE, sandboxed code canvas built in.', - 'The user wants you to BUILD something. Output the FINISHED, self-contained code as ONE fenced block — it runs immediately in the canvas beside the chat:', - '- React app/component -> ```jsx — write idiomatic React (you may `import React, { useState } from "react"` and `export default function App() {…}`; the sandbox handles imports/exports). Define the main component as `App` or a default export.', - '- a plain web page / interactive UI (no React) -> ```html — one complete document, inline all CSS and JS.', - '- a diagram -> ```mermaid. a static graphic -> ```svg.', - 'You DO have a real execution sandbox — do NOT say "since I am on-device" or "copy this into a new project", do NOT give npm/Vite/Create-React-App setup steps, and do NOT split it into src/App.js + src/App.css instructions. Just write ONE runnable code block. At most one short sentence before it.', - referenceBlock, - historyBlock, - `User: ${query}`, - 'Assistant:' - ] - .filter(Boolean) - .join('\n\n') - const completion = await streamAnswer(event, streamId, prompt, thinking, imgs) - return { ...completion, context: undefined } - } - - // No-memory mode: a plain on-device assistant — no retrieval at all. - if (noMemory) { - const { llm } = await import('./llm') - const hist = (conversationHistory ?? []) - .slice(-10) - .map((m) => `${m.role === 'assistant' ? 'Assistant' : 'User'}: ${m.content}`) - .join('\n') - const prompt = [ - 'You are Off Grid AI, a private, on-device assistant.', - 'You can generate images on-device. If (and only if) the user is asking for a picture/image/logo/art to be CREATED, respond with ONLY a fenced block ```image\\n\\n``` and nothing else. For everything else, answer normally in text.', - hist ? `Conversation so far:\n${hist}` : '', - `User: ${query}`, - 'Assistant:' - ] - .filter(Boolean) - .join('\n\n') - void llm // retained for non-stream fallback inside streamAnswer - const completion = await streamAnswer(event, streamId, prompt, thinking, imgs) - return { ...completion, context: undefined } - } - - if (streamId) - event.sender.send('rag:stream', { streamId, type: 'step', step: { kind: 'searching' } }) - const db = getDB() - // Quote each token as an FTS5 phrase (via the shared safe builder) so a hyphenated word like - // "best-reviewed" can't reach MATCH as invalid syntax and throw "no such column: reviewed", - // which failed the whole retrieval. Preserves the any-term (OR) recall the retrieval expects. - const ftsQuery = ftsMatchExpression(query) - - let memories: Array<{ - source_app: string | null - created_at: string - content: string - score?: number - }> = [] - try { - const queryVector = await embeddings.generateEmbedding(query, { interactive: true }) - const vecStr = JSON.stringify(queryVector) - const params: unknown[] = [vecStr] - let memoryQuery = ` - SELECT *, cosine_similarity(embedding, ?) as score - FROM memories - WHERE embedding IS NOT NULL AND embedding != '[]' - ` - const vecFilter = appNameLikeClause(appName, 'source_app') - if (vecFilter) { - memoryQuery += ` AND ${vecFilter.clause} ` - params.push(vecFilter.param) - } - memoryQuery += ` ORDER BY score DESC LIMIT 12` - memories = db.prepare(memoryQuery).all(...params) - memories = memories.filter((m) => typeof m.score !== 'number' || m.score >= 0.2) - } catch (e) { - console.error('[RAG] Vector search failed, falling back to FTS', e) - const params: unknown[] = [] - let fallbackQuery = ` - SELECT memories.* - FROM memories - JOIN memory_fts ON memories.id = memory_fts.rowid - WHERE memory_fts MATCH ? - ` - params.push(ftsQuery) - const ftsFilter = appNameLikeClause(appName, 'memories.source_app') - if (ftsFilter) { - fallbackQuery += ` AND ${ftsFilter.clause} ` - params.push(ftsFilter.param) - } - fallbackQuery += ` LIMIT 12` - memories = ftsQuery - ? db.prepare(fallbackQuery).all(...params) - : [] - } - - const messageParams: unknown[] = [ftsQuery] - let messageQuery = ` - SELECT m.id, m.conversation_id, m.role, m.content, m.created_at, c.title, c.app_name, - bm25(message_fts) as score - FROM message_fts - JOIN messages m ON message_fts.rowid = m.id - JOIN conversations c ON c.id = m.conversation_id - WHERE message_fts MATCH ? - ` - const msgFilter = appNameLikeClause(appName, 'c.app_name') - if (msgFilter) { - messageQuery += ` AND ${msgFilter.clause} ` - messageParams.push(msgFilter.param) - } - messageQuery += ` ORDER BY score ASC LIMIT 12` - const messages = ftsQuery - ? db - .prepare< - unknown[], - { - app_name: string | null - title: string | null - created_at: string - role: string - content: string - } - >(messageQuery) - .all(...messageParams) - : [] - - const summaryParams: unknown[] = [ftsQuery] - let summaryQuery = ` - SELECT cs.session_id, cs.summary, c.title, c.app_name, c.updated_at, - bm25(summary_fts) as score - FROM summary_fts - JOIN chat_summaries cs ON summary_fts.rowid = cs.rowid - JOIN conversations c ON c.id = cs.session_id - WHERE summary_fts MATCH ? - ` - const sumFilter = appNameLikeClause(appName, 'c.app_name') - if (sumFilter) { - summaryQuery += ` AND ${sumFilter.clause} ` - summaryParams.push(sumFilter.param) - } - summaryQuery += ` ORDER BY score ASC LIMIT 8` - const summaries = ftsQuery - ? db - .prepare( - summaryQuery - ) - .all(...summaryParams) - : [] - - const entityParams: unknown[] = [ftsQuery] - let entityQuery = ` - SELECT e.id, e.name, e.type, e.summary, e.updated_at, - bm25(entity_fts) as score - FROM entity_fts - JOIN entities e ON entity_fts.rowid = e.id - WHERE entity_fts MATCH ? - ` - const entFilter = appNameLikeClause(appName, 'c.app_name') - if (entFilter) { - entityQuery += ` - AND e.id IN ( - SELECT es.entity_id - FROM entity_sessions es - JOIN conversations c ON c.id = es.session_id - WHERE ${entFilter.clause} - ) - ` - entityParams.push(entFilter.param) - } - entityQuery += ` ORDER BY score ASC LIMIT 8` - const entities = ftsQuery - ? db - .prepare( - entityQuery - ) - .all(...entityParams) - : [] - - const factParams: unknown[] = [ftsQuery] - let factQuery = ` - SELECT f.fact, f.created_at, f.source_session_id, e.name, e.type, - bm25(entity_fact_fts) as score - FROM entity_fact_fts - JOIN entity_facts f ON entity_fact_fts.rowid = f.id - JOIN entities e ON e.id = f.entity_id - WHERE entity_fact_fts MATCH ? - ` - const factFilter = appNameLikeClause(appName, 'app_name') - if (factFilter) { - factQuery += ` AND f.source_session_id IN (SELECT id FROM conversations WHERE ${factFilter.clause}) ` - factParams.push(factFilter.param) - } - factQuery += ` ORDER BY score ASC LIMIT 8` - const entityFacts = ftsQuery - ? db - .prepare(factQuery) - .all(...factParams) - : [] - - // Supplementary context (no bracket labels — the ONLY citeable tags are the - // numbered [S#] SOURCES below, so the model can't invent uncited labels). - const memoryLines = memories - .slice(0, 6) - .map( - (m) => `- (${m.source_app || 'Unknown'} | ${m.created_at}): ${clipText(m.content, 500)}` - ) - .join('\n') - - const messageLines = messages - .slice(0, 6) - .map( - (m) => - `- (${m.app_name || 'Unknown'} | ${m.title || 'Untitled'} | ${m.created_at}) ${m.role}: ${clipText(m.content, 400)}` - ) + const history = (conversationHistory ?? []) + .slice(-10) + .map((m) => `${m.role === 'assistant' ? 'Assistant' : 'User'}: ${m.content}`) .join('\n') - - const summaryLines = summaries - .slice(0, 6) - .map( - (s) => - `- (${s.app_name || 'Unknown'} | ${s.title || 'Untitled'}): ${clipText(s.summary, 600)}` - ) - .join('\n') - - const entityLines = entities - .slice(0, 6) - .map((e) => `- (${e.type || 'Unknown'}) ${e.name}: ${clipText(e.summary || '', 400)}`) - .join('\n') - - const factLines = entityFacts - .slice(0, 6) - .map((f) => `- (${f.type || 'Unknown'}) ${f.name}: ${clipText(f.fact, 400)}`) - .join('\n') - - const contextBlock = `RELEVANT MEMORIES:\n${memoryLines || '(none)'}\n\nRELEVANT MESSAGES:\n${messageLines || '(none)'}\n\nRELEVANT SUMMARIES:\n${summaryLines || '(none)'}\n\nRELEVANT ENTITIES:\n${entityLines || '(none)'}\n\nRELEVANT ENTITY FACTS:\n${factLines || '(none)'}` - - // Unified search: fuse in the best-ranked hits across screens, meetings, - // memories, entities and facts (hybrid FTS + vectors with RRF) — the same - // engine as the search screen, so the chat gets the right context too. - let unifiedBlock = '' - let unifiedHits: { - kind: string - title: string - snippet: string - surface: string - ts: number - refId: number - imagePath: string | null - }[] = [] - try { - const { universalSearch } = await import('./search') - unifiedHits = await universalSearch(query, { limit: 12, semantic: true }) - if (unifiedHits.length) { - unifiedBlock = - '\n\nSOURCES — every factual claim must cite the source it came from using its tag in square brackets, e.g. [S2]. Cite ONLY sources you actually used; never invent a citation:\n' + - unifiedHits - .map((h, i) => { - const when = h.ts ? ` · ${new Date(h.ts).toISOString().slice(0, 10)}` : '' - return `[S${i + 1}] (${h.kind} · ${h.surface || 'unknown'}${when})${h.title ? ` ${h.title} —` : ''} ${clipText(h.snippet || '', 350)}` - }) - .join('\n') - } - } catch (e) { - console.error('[RAG] universalSearch failed', e) - } - - // Build conversation history block if provided - let historyBlock = '' - if (conversationHistory && conversationHistory.length > 0) { - const historyLines = conversationHistory - .map( - (msg) => `${msg.role === 'user' ? 'User' : 'Assistant'}: ${clipText(msg.content, 500)}` - ) - .join('\n\n') - historyBlock = `\nCONVERSATION HISTORY:\n${historyLines}\n` - } - - let skillsBlock = 'None installed.' - try { - const { listSkills } = await import('./skills') - const sk = listSkills() - if (sk.length) skillsBlock = sk.map((s) => `- /${s.name}: ${s.description}`).join('\n') - } catch { - /* skills optional */ - } - - const prompt = getPrompt('ragChat', { - HISTORY_BLOCK: historyBlock, - QUERY: query, - CONTEXT_BLOCK: contextBlock + unifiedBlock, - SKILLS_BLOCK: skillsBlock - }) - - try { - if (streamId) - event.sender.send('rag:stream', { - streamId, - type: 'step', - step: { - kind: 'memory', - counts: { - memories: memories.length, - messages: messages.length, - summaries: summaries.length, - entities: entities.length, - facts: entityFacts.length, - unified: unifiedHits.length - } - } - }) - const completion = await streamAnswer(event, streamId, prompt, thinking, imgs) - return { - ...completion, - context: { - masterMemory: null, - memories, - messages, - summaries, - entities, - entityFacts, - unified: unifiedHits - } - } - } catch (e) { - console.error('[RAG] LLM chat failed:', e) - throw e - } + const prompt = [ + 'You are Off Grid AI, a private, on-device assistant.', + history ? `Conversation so far:\n${history}` : '', + `User: ${query}`, + 'Assistant:' + ] + .filter(Boolean) + .join('\n\n') + const completion = await streamAnswer(event, streamId, prompt, thinking, images ?? []) + return { ...completion, context: undefined } } ) @@ -1099,31 +358,18 @@ export function setupIPC(): void { // Need to export this from database.ts first or import it return getMemoriesForSession(sessionId) }) - ipcMain.handle('db:get-entities', (_, appName?: string) => { - return getEntities(appName) - }) - ipcMain.handle('db:get-entity-details', (_, entityId: number, appName?: string) => { - return getEntityDetails(entityId, appName) - }) ipcMain.handle('db:get-entities-for-session', (_, sessionId: string) => { return getEntitiesForSession(sessionId) }) - ipcMain.handle('db:get-memory-records-for-session', (_, sessionId: string) => { - return getMemoryRecordsForSession(sessionId) - }) - ipcMain.handle('db:delete-session', async (_, sessionId: string) => { + ipcMain.handle('db:delete-session', (_, sessionId: string) => { const db = getDB() // Delete from new tables (messages will cascade due to foreign key) db.prepare('DELETE FROM conversations WHERE id = ?').run(sessionId) - // Also delete from legacy tables for cleanup + // Also delete from legacy tables for cleanup (retired memories stay deletable) db.prepare('DELETE FROM memories WHERE session_id = ?').run(sessionId) db.prepare('DELETE FROM chat_summaries WHERE session_id = ?').run(sessionId) console.log(`Deleted session: ${sessionId}`) - - // Regenerate master memory after deletion - await regenerateMasterMemory() - return true }) @@ -1131,26 +377,12 @@ export function setupIPC(): void { return await summarizeSession(sessionId) }) - ipcMain.handle('db:get-master-memory', () => { - return getMasterMemory() - }) - ipcMain.handle('db:delete-entity', (_, entityId: number) => { const result = deleteEntityById(entityId) console.log(`[IPC] Deleted entity ${entityId}: ${result}`) return result }) - ipcMain.handle('db:delete-memory', (_, memoryId: number) => { - const result = deleteMemory(memoryId) - console.log(`[IPC] Deleted memory ${memoryId}: ${result}`) - return result - }) - - ipcMain.handle('db:regenerate-master-memory', async () => { - return await regenerateMasterMemory() - }) - // User Profile handlers ipcMain.handle('db:get-user-profile', () => { return getUserProfile() @@ -1364,13 +596,38 @@ export function setupIPC(): void { ;(await import('./best-settings')).markSpeedSettingsManual() return setResidencyMode(modality, mode) }) + ipcMain.handle('runtime:backend:choices', () => backendChoicesForUi()) + // "Try the GPU again": forget this session's demotion and restart the engine on its ladder. + ipcMain.handle('runtime:backend:retry', async (_e, modality: BackendModality) => { + const { clearDemotions, invalidateBackendFacts } = await import('./backend-facts') + clearDemotions(modality) + invalidateBackendFacts() + if (modality === 'llm') { + const { llm } = await import('./llm') + if (llm.shouldBeRunning()) await llm.restartSameModel() + } else if (modality === 'grounding') { + const { grounderRuntime } = await import('./vision/grounder-runtime') + if (grounderRuntime.shouldBeRunning()) await grounderRuntime.restart() + } else if (modality === 'decision') { + const { decisionRuntime } = await import('./accessibility/decision-runtime') + if (decisionRuntime.shouldBeRunning()) await decisionRuntime.restart() + } else if (modality === 'tts') { + await (await import('./tts')).ttsOnnxRuntime.restart() + } else if (modality === 'embeddings') { + await (await import('./embeddings')).embeddings.restart() + } + // Image and transcription load per job; their next job uses the restored ladder. + }) ipcMain.handle('runtime:backend:get', () => getBackendPreferences()) ipcMain.handle( 'runtime:backend:set', async (_e, modality: BackendModality, preference: BackendPreference) => { ;(await import('./best-settings')).markSpeedSettingsManual() const next = setBackendPreference(modality, preference) - // A loaded engine keeps its present backend until its next load. The UI says so. + // Chat, image and the task engines take the new backend at their next load (the UI says + // so); voice and search helpers restart between requests onto it. + if (modality === 'embeddings') void (await import('./embeddings')).embeddings.restart() + if (modality === 'tts') void (await import('./tts')).ttsOnnxRuntime.restart() return next } ) @@ -1455,128 +712,6 @@ export function setupIPC(): void { return true }) - // === REPROCESS ALL SESSIONS === - - ipcMain.handle('db:reprocess-all-sessions', async (_, clean: boolean = false) => { - const db = getDB() - const sessions = db.prepare('SELECT id FROM conversations').all() as { id: string }[] - let processed = 0 - - if (clean) { - // Clean reprocess: delete all old data and rebuild from scratch - console.log('[IPC] Clean reprocess: clearing all entities, facts, edges, and memories...') - - // Drop FTS AFTER DELETE triggers first — if the FTS index is out of sync - // with the source tables, the delete triggers will error and silently - // prevent rows from being deleted. We recreate them after. - db.exec('DROP TRIGGER IF EXISTS memories_ad') - db.exec('DROP TRIGGER IF EXISTS entities_ad') - db.exec('DROP TRIGGER IF EXISTS entity_facts_ad') - - // Delete only strictness-dependent data (children first) - // Conversations, messages, chat_summaries, and master_memory are NOT touched - db.prepare('DELETE FROM entity_edges').run() - db.prepare('DELETE FROM entity_facts').run() - db.prepare('DELETE FROM entity_sessions').run() - db.prepare('DELETE FROM entities').run() - db.prepare('DELETE FROM memories').run() - - // Recreate the delete triggers - db.exec(`CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN - INSERT INTO memory_fts(memory_fts, rowid, content) VALUES('delete', old.id, old.content); - END;`) - db.exec(`CREATE TRIGGER IF NOT EXISTS entities_ad AFTER DELETE ON entities BEGIN - INSERT INTO entity_fts(entity_fts, rowid, name, summary, type) VALUES('delete', old.id, old.name, old.summary, old.type); - END;`) - db.exec(`CREATE TRIGGER IF NOT EXISTS entity_facts_ad AFTER DELETE ON entity_facts BEGIN - INSERT INTO entity_fact_fts(entity_fact_fts, rowid, fact, entity_id) VALUES('delete', old.id, old.fact, old.entity_id); - END;`) - - // Rebuild FTS indexes so they reflect the now-empty source tables - try { - db.exec("INSERT INTO memory_fts(memory_fts) VALUES('rebuild')") - db.exec("INSERT INTO entity_fts(entity_fts) VALUES('rebuild')") - db.exec("INSERT INTO entity_fact_fts(entity_fact_fts) VALUES('rebuild')") - } catch (e) { - console.error('[IPC] FTS rebuild during clean reprocess failed (non-fatal):', e) - } - - const deletedCounts = { - entities: (db.prepare('SELECT COUNT(*) as c FROM entities').get() as { c: number }).c, - memories: (db.prepare('SELECT COUNT(*) as c FROM memories').get() as { c: number }).c, - facts: (db.prepare('SELECT COUNT(*) as c FROM entity_facts').get() as { c: number }).c - } - console.log('[IPC] Post-delete counts (should all be 0):', deletedCounts) - - // Notify frontend to refresh immediately after clearing - BrowserWindow.getAllWindows().forEach((win) => { - win.webContents.send('reprocess:progress', { - phase: 'cleared', - processed: 0, - total: sessions.length - }) - }) - - for (const session of sessions) { - try { - // Re-evaluate memories for each message in the session - const msgs = db - .prepare( - 'SELECT id, role, content FROM messages WHERE conversation_id = ? ORDER BY created_at ASC' - ) - .all(session.id) as { id: number; role: string; content: string }[] - const conv = db - .prepare('SELECT app_name FROM conversations WHERE id = ?') - .get(session.id) as { app_name: string } | undefined - const appName = conv?.app_name || 'Unknown' - - for (const msg of msgs) { - await evaluateAndStoreMemoryForMessage({ - sessionId: session.id, - appName, - role: msg.role, - content: msg.content, - messageId: msg.id - }) - } - - // Re-extract entities from the newly created memories - await extractEntitiesForSession(session.id) - processed++ - - // Send progress updates - BrowserWindow.getAllWindows().forEach((win) => { - win.webContents.send('reprocess:progress', { - phase: 'processing', - processed, - total: sessions.length - }) - }) - } catch (e) { - console.error(`[IPC] Failed to reprocess session ${session.id}:`, e) - } - } - } else { - // Additive reprocess: keep existing data, just re-run entity extraction on top - console.log( - '[IPC] Additive reprocess: re-extracting entities with current settings (keeping existing data)...' - ) - for (const session of sessions) { - try { - await extractEntitiesForSession(session.id) - processed++ - } catch (e) { - console.error(`[IPC] Failed to reprocess session ${session.id}:`, e) - } - } - } - - console.log( - `[IPC] Reprocessed ${processed} sessions (clean=${clean}) with current strictness settings` - ) - return { processed, total: sessions.length } - }) - // === MODEL DOWNLOAD HANDLERS === ipcMain.handle('model:check-status', async () => { @@ -2189,8 +1324,9 @@ export function setupIPC(): void { // Non-stream fallback (no streamId): buffer, no live deltas (matches streamAnswer). if (!streamId) { return modalityQueue.run(CHAT_JOB, async () => { - const { result, modelName } = await runChatWithFallback((attempt) => - toolChat(query, history || [], { ...(opts || {}), ...attempt }) + const { result, modelName } = await runChatWithFallback( + (attempt) => toolChat(query, history || [], { ...(opts || {}), ...attempt }), + deviceFallbackNotices(sender, streamId) ) return modelName ? { ...result, metrics: { ...result.metrics, modelName } } : result }) @@ -2265,6 +1401,7 @@ export function setupIPC(): void { { signal: controller.signal, canRetry: () => !toolStarted, + ...deviceFallbackNotices(sender, streamId), onModelChanged: (failed, next) => { resetChatStreamPartial(streamId) try { diff --git a/src/main/licensing/license-service.ts b/src/main/licensing/license-service.ts index 6e6b1eaed..c76cd76d3 100644 --- a/src/main/licensing/license-service.ts +++ b/src/main/licensing/license-service.ts @@ -5,7 +5,7 @@ export const PRO_PAY_PAGE_URL = PRO_PURCHASE_URL export type ActivateResult = PersonalMeshActivationResult -export type ProTier = 'lifetime' | 'monthly' | 'annual' | 'subscription' +type ProTier = 'lifetime' | 'monthly' | 'annual' | 'subscription' export interface ProLicenseInfo { isPro: boolean diff --git a/src/main/llama-error.ts b/src/main/llama-error.ts index ee9b2114f..14af25a15 100644 --- a/src/main/llama-error.ts +++ b/src/main/llama-error.ts @@ -13,6 +13,7 @@ export interface LlamaFailure { | 'os_too_old' | 'out_of_memory' | 'gpu_unsupported' + | 'gpu_fault' | 'speculation_unsupported' | 'missing_library' | 'model_corrupt' @@ -22,6 +23,9 @@ export interface LlamaFailure { reason: string } +export const GPU_FAULT_PATTERN = + /compute error|cuda error|ggml_cuda|device lost|ErrorDeviceLost|vk::DeviceLostError|failed to allocate|out of memory|cudaMalloc/i + /** One source of truth for a live second-instance conflict, whether it is detected before spawn * from process ownership or reported by the native engine as EADDRINUSE. */ export function modelPortConflictReason(port: number): string { @@ -137,6 +141,10 @@ export function classifyLlamaError( } } + if (GPU_FAULT_PATTERN.test(stderr)) { + return { code: 'gpu_fault', reason: 'The GPU stopped responding.' } + } + // Corrupt / truncated weights. if ( /failed to load model|invalid magic|tensor.*not found|gguf.*(invalid|corrupt|truncat)|done_getting_tensors.*wrong/.test( diff --git a/src/main/llm.ts b/src/main/llm.ts index a598e5127..8c06586ec 100644 --- a/src/main/llm.ts +++ b/src/main/llm.ts @@ -11,17 +11,11 @@ import path from 'path' import { recordAIRequest, currentAIRequest } from './ai-request-log' import * as fs from 'fs' import { modelMetadataFile, resolveModelReference } from './model-paths' -import { - modelsDir as getModelsDir, - binRoots, - isPackaged, - exe, - requireModelStorage -} from './runtime-env' +import { modelsDir as getModelsDir, binRoots, isPackaged, requireModelStorage } from './runtime-env' import { reapOrphanProcessesOnPort, type PortReapResult } from './kill-orphan-port' import { loadAttempts, type KvCacheType, type PerformanceMode } from './model-sizing' import { resolveMaxTokens, maxTokensForWire, MAX_TOKENS_AUTO } from './llm/gen-params' -import { classifyLlamaError, modelPortConflictReason } from './llama-error' +import { classifyLlamaError, modelPortConflictReason, GPU_FAULT_PATTERN } from './llama-error' import type { ManagedRuntime } from './runtime-manager' import { LLAMA_SERVER_PORT } from '../shared/ports' import { @@ -32,6 +26,8 @@ import { import { REASONING_BUDGET_AUTO, reasoningBudgetPayload, + backendReasonText, + type NativeBackend, isGatewayMode, llmSlotPlan, type GatewayMode, @@ -68,15 +64,14 @@ import { type CatalogEntry } from './models/catalog-logic' import { pickFreePort, isPortFree } from './free-port' -import { postCompletionOnce } from './llm/http-post' +import { ModelServerError, postCompletionOnce } from './llm/http-post' import { modalityQueue } from './modality-queue/queue' import { engineSpawnEnv } from './llm/spawn-env' -import { gpuDeviceAvailable } from './llm/gpu-device-probe' import { shouldAutoRecover } from './llm/crash-policy' -import { enginePriority } from './llm/engine-priority' +import { engineCandidates, nativeLadder } from './backend-facts' import { modelStartupQuietLimit, startupStalled } from './llm/startup-timeout' +import { waitForEngineResumeHealth } from './engine-resume-health' import { getBackendPreference } from './backend-preferences' -import { prioritizeBackend, type BackendPreference } from '../shared/backend-preferences' import { streamCompletion, type StreamResult } from './llm/stream' import type { AssembledToolCall } from './llm/sse-stream' import { @@ -116,7 +111,7 @@ export interface LlmSettings { // Launch-time (require a server respawn to take effect): kvCacheType?: KvCacheType // quantize the KV cache to cut memory (needs flash-attn) flashAttn?: boolean // FlashAttention: faster + lower memory; required for quantized KV - gpuLayers?: number // -ngl: layers offloaded to the GPU. 99 = all. + gpuLayers?: number | null // null lets llama.cpp fit layers automatically. threads?: number // CPU threads for inference batchSize?: number // -b: prompt batch size speculativeDecoding?: SpeculativeDecodingMode @@ -272,6 +267,10 @@ export class LLMService { private backendOutput = '' private backendModelPath = '' private backendPlacement: ReturnType + /** Layers the engine put on the GPU, from its load log ("offloaded N/M layers to GPU"). */ + private offloadedLayers: { gpu: number; total: number } | undefined + /** When the current engine started loading, for the first-load notice. */ + private loadingSince: number | undefined private backendFallbackReason: string | undefined // A model selection is durable as soon as the manager writes active-model.json, but // replacing llama-server while it is answering destroys the user's in-flight turn. @@ -319,7 +318,7 @@ export class LLMService { // choice (the "q8_0 reverts to f16 on every restart" bug). Persisted alongside the // values so a plain restart keeps the pin. private readonly userExplicit = new Set() - private gpuLayers = 99 + private gpuLayers: number | null = null private threads: number | undefined private batchSize: number | undefined private speculativeDecoding: SpeculativeDecodingMode = 'off' @@ -348,6 +347,10 @@ export class LLMService { // a server that keeps dying (e.g. memory pressure on a too-large model) can NOT // thrash-respawn a multi-GB process forever. private restartTimes: number[] = [] + private deviceFaultAt: number | null = null + private recoveryPromise: Promise | null = null + private recoveryModel: { path: string; mmproj: string } | null = null + private currentBackend: NativeBackend | undefined // Launch-setting restarts are serialized and last-wins. A slider drag or a fast // sequence of settings writes used to call stop()+init() once per change, so several // spawns raced and an EARLIER one could win and leave the engine running arguments the @@ -413,7 +416,8 @@ export class LLMService { if (s.kvCacheType === 'f16' || s.kvCacheType === 'q8_0' || s.kvCacheType === 'q4_0') this.kvCacheType = s.kvCacheType if (typeof s.flashAttn === 'boolean') this.flashAttn = s.flashAttn - if (typeof s.gpuLayers === 'number') this.gpuLayers = s.gpuLayers + if (s.gpuLayers === null || typeof s.gpuLayers === 'number') + this.gpuLayers = s.gpuLayers === 99 ? null : s.gpuLayers if (typeof s.threads === 'number') this.threads = s.threads if (typeof s.batchSize === 'number') this.batchSize = s.batchSize if ( @@ -428,7 +432,11 @@ export class LLMService { if (isGatewayMode(s.gatewayMode)) this.gatewayModeSetting = s.gatewayMode if (Number.isInteger(s.gatewayClients) && s.gatewayClients >= 1) this.gatewayClientsSetting = s.gatewayClients - if (Number.isInteger(s.speculativeDraftMax) && s.speculativeDraftMax >= 0 && s.speculativeDraftMax <= 16) + if ( + Number.isInteger(s.speculativeDraftMax) && + s.speculativeDraftMax >= 0 && + s.speculativeDraftMax <= 16 + ) this.speculativeDraftMax = s.speculativeDraftMax if ( s.performanceMode === 'conservative' || @@ -596,11 +604,27 @@ export class LLMService { runtimeBackend(): RuntimeBackend { const placement = this.initialized ? this.backendPlacement : undefined return { - id: 'chat', model: this.backendModelPath, - state: this.initialized ? 'loaded' : this.isStarting() ? 'loading' : this.lastError() ? 'error' : 'stopped', + id: 'chat', + model: this.backendModelPath, + state: + this.deviceFaultAt !== null + ? 'error' + : this.initialized + ? 'loaded' + : this.isStarting() + ? 'loading' + : this.lastError() + ? 'error' + : 'stopped', backend: placement?.backend, device: placement?.device, - detail: this.lastError() ?? this.backendFallbackReason + detail: this.lastError() ?? this.backendFallbackReason, + fallbackReason: this.backendFallbackReason, + lastError: this.lastError() ?? undefined, + ...(placement && this.offloadedLayers + ? { gpuLayers: this.offloadedLayers.gpu, totalLayers: this.offloadedLayers.total } + : {}), + ...(this.isStarting() && this.loadingSince ? { loadingSince: this.loadingSince } : {}) } } @@ -664,7 +688,11 @@ export class LLMService { } /** Build argv for the selected context and a GPU-layer count. */ - private launchArgsFor(effectiveCtxSize: number, gpuLayers: number): string[] { + private launchArgsFor( + effectiveCtxSize: number, + gpuLayers: number | null, + cpuOnly = false + ): string[] { const useSelectedModelSpeculation = this.runtimeModelOverride === null && this.speculativeModeSupported(this.speculativeDecoding, this.draftModel) @@ -676,6 +704,7 @@ export class LLMService { port: this.port, effectiveCtxSize, gpuLayers, + cpuOnly, flashAttn: this.flashAttn, kvCacheType: this.kvCacheType, threads: this.threads, @@ -920,8 +949,12 @@ export class LLMService { /** Update inference settings; respawns the server if any launch-time arg changed * (context, KV-cache type, flash-attn, GPU layers, threads, batch). */ async setSettings(s: LlmSettings, options: LlmSettingsUpdateOptions = {}): Promise { - if (s.speculativeDraftMax !== undefined && - (!Number.isInteger(s.speculativeDraftMax) || s.speculativeDraftMax < 0 || s.speculativeDraftMax > 16)) { + if ( + s.speculativeDraftMax !== undefined && + (!Number.isInteger(s.speculativeDraftMax) || + s.speculativeDraftMax < 0 || + s.speculativeDraftMax > 16) + ) { throw new Error('Draft token limit must be Auto (0) or a whole number from 1 to 16.') } this.ensureLoaded() @@ -1007,7 +1040,7 @@ export class LLMService { this.kvCacheType = compatibleSettings.kvCacheType if (typeof compatibleSettings.flashAttn === 'boolean') this.flashAttn = compatibleSettings.flashAttn - if (typeof compatibleSettings.gpuLayers === 'number') + if (compatibleSettings.gpuLayers === null || typeof compatibleSettings.gpuLayers === 'number') this.gpuLayers = compatibleSettings.gpuLayers if (typeof compatibleSettings.threads === 'number') this.threads = compatibleSettings.threads if (typeof compatibleSettings.batchSize === 'number') @@ -1072,7 +1105,8 @@ export class LLMService { this.systemPrompt = priorSettings.systemPrompt ?? this.systemPrompt this.kvCacheType = priorSettings.kvCacheType ?? this.kvCacheType this.flashAttn = priorSettings.flashAttn ?? this.flashAttn - this.gpuLayers = priorSettings.gpuLayers ?? this.gpuLayers + this.gpuLayers = + priorSettings.gpuLayers === undefined ? this.gpuLayers : priorSettings.gpuLayers this.threads = priorSettings.threads this.batchSize = priorSettings.batchSize this.speculativeDecoding = priorSettings.speculativeDecoding ?? this.speculativeDecoding @@ -1169,6 +1203,11 @@ export class LLMService { // ({ id, primary, mmproj }) after resolving a catalog entry; default to the // bundled Qwen3-VL vision model when nothing is selected yet. private resolveModel(): void { + if (this.recoveryModel) { + this.modelPath = this.recoveryModel.path + this.mmProjPath = this.recoveryModel.mmproj + return + } if (this.runtimeModelOverride) { this.modelPath = resolveModelReference(this.runtimeModelOverride.primary) this.mmProjPath = this.runtimeModelOverride.mmproj @@ -1419,30 +1458,31 @@ export class LLMService { // Try CUDA on NVIDIA, Vulkan on other GPUs, then the CPU-only engine. // macOS has its own Metal engines, so the absent CUDA/Vulkan paths are skipped. const roots = binRoots() - const serverPaths = roots - .flatMap((r) => - requiresPrism - ? [ - path.join(r, 'llama-prism-cuda', exe('llama-server')), - path.join(r, 'llama-prism', exe('llama-server')), - path.join(r, 'llama-prism-cpu', exe('llama-server')) - ] - : [ - path.join(r, 'llama-cuda', exe('llama-server')), - path.join(r, 'llama', exe('llama-server')), - path.join(r, 'llama-cpu', exe('llama-server')), - path.join(r, exe('llama-server')) - ] + const serverPaths = await nativeLadder( + 'llm', + undefined, + getBackendPreference('llm'), + requiresPrism + ) + this.backendFallbackReason = + serverPaths[0]?.cpuOnly && this.recoveryModel && this.hasDeviceFault() + ? 'The GPU stopped responding.' + : undefined + const candidates = await engineCandidates('llm', undefined, requiresPrism) + const firstGpu = candidates.find((candidate) => candidate.backend !== 'cpu') + if (serverPaths[0]?.cpuOnly || (firstGpu && serverPaths[0]?.path !== firstGpu.id)) { + const skipped = candidates.find( + (candidate) => candidate.backend !== 'cpu' && !candidate.usable ) - .filter((p) => fs.existsSync(p)) + if (skipped?.reason) this.backendFallbackReason = backendReasonText(skipped.reason) + } if (!serverPaths.length) { if (requiresPrism) { throw new Error( 'Bonsai 2 requires the bundled Prism llama.cpp engine, which is missing from this build.' ) } - console.error(`[LLMService] llama-server binary not found under: ${roots.join(', ')}`) - return + throw new Error(`No chat engine is installed. Searched: ${roots.join(', ')}`) } console.log(`[LLMService] Model: ${this.modelPath}`) @@ -1476,54 +1516,16 @@ export class LLMService { /** Try all accelerated engines before CPU at the selected context. */ private async launchWithFallback( - serverPaths: string[], + serverPaths: Awaited>, generation = this.launchGeneration ): Promise { - this.backendFallbackReason = undefined - const preference = getBackendPreference('llm') - const ordered = prioritizeBackend( - [...serverPaths].sort((a, b) => enginePriority(a) - enginePriority(b)), - preference === 'cpu' ? 'auto' : preference, - (serverPath): BackendPreference => { - const directory = path.basename(path.dirname(serverPath)) - if (directory.endsWith('-cpu')) return 'cpu' - if (directory.endsWith('-cuda')) return 'cuda' - return process.platform === 'darwin' ? 'metal' : 'vulkan' - } - ) - const candidates = preference === 'cpu' - ? ordered.map((serverPath) => ({ serverPath, cpuOnly: true })) - : [ - ...ordered.map((serverPath) => ({ serverPath, cpuOnly: false })), - ...ordered - .filter((p) => !p.includes('llama-cpu') && !p.includes('llama-prism-cpu')) - .map((serverPath) => ({ serverPath, cpuOnly: true })) - ] - for (const { serverPath, cpuOnly } of candidates) { + for (const { path: serverPath, cpuOnly, backend } of serverPaths) { if (generation !== this.launchGeneration) return false - if (!serverPath) continue + this.currentBackend = cpuOnly ? 'cpu' : backend const engineDir = path.basename(path.dirname(serverPath)) - const backend = engineDir.endsWith('-cuda') - ? 'CUDA' - : engineDir === 'llama' || engineDir === 'llama-prism' - ? 'Vulkan' - : null - if (!cpuOnly && (process.platform === 'win32' || process.platform === 'linux') && backend) { - if (this.gpuLayers === 0) continue - if (!(await gpuDeviceAvailable(serverPath, backend, process.platform))) { - this.backendFallbackReason = `${backend} did not report a usable device. Another engine was selected.` - console.warn( - `[LLMService] no usable ${backend} device for ${serverPath}; trying next engine` - ) - continue - } - } // Read after the device probe's wait, with no await before the launch arguments, so a save made // during the probe cannot leave the checkpoint and the arguments describing different launches. - const attempts = loadAttempts( - this.ctxSize, - cpuOnly || enginePriority(serverPath) === 2 ? 0 : this.gpuLayers - ).slice(0, 1) + const attempts = loadAttempts(this.ctxSize, cpuOnly ? 0 : this.gpuLayers).slice(0, 1) for (let a = 0; a < attempts.length; a++) { if (generation !== this.launchGeneration) return false const at = attempts[a] @@ -1531,12 +1533,16 @@ export class LLMService { if (a > 0) { console.warn(`[LLMService] out of memory — retrying load at ${at.reason}`) } - const args = this.launchArgsFor(at.ctxSize, at.gpuLayers) + const args = this.launchArgsFor(at.ctxSize, at.gpuLayers, cpuOnly) // The six-token Qwen3.5 9B gain was measured with stock llama.cpp on // Metal. Preserve the existing limit for other engines and backends. const draftMaxIndex = args.indexOf('--spec-draft-n-max') - if (draftMaxIndex >= 0 && this.speculativeDecoding === 'mtp' && this.speculativeDraftMax === 0 && - (engineDir !== 'llama' || cpuOnly || process.platform !== 'darwin')) { + if ( + draftMaxIndex >= 0 && + this.speculativeDecoding === 'mtp' && + this.speculativeDraftMax === 0 && + (engineDir !== 'llama' || cpuOnly || process.platform !== 'darwin') + ) { args[draftMaxIndex + 1] = '2' } // Captured in the same step as the arguments: a save can land during init's earlier awaits, @@ -1551,8 +1557,9 @@ export class LLMService { } if (generation !== this.launchGeneration) return false const failure = classifyLlamaError(this.stderrTail.join('\n')) - this.backendFallbackReason = failure?.reason ?? 'The previous engine could not load the model.' - if (failure?.code === 'speculation_unsupported') { + this.backendFallbackReason = + failure?.reason ?? 'The previous engine could not load the model.' + if (failure?.code === 'speculation_unsupported' && !this.recoveryModel) { console.warn( `[LLMService] ${failure.reason} Retrying the selected model with speculative decoding off.` ) @@ -1564,10 +1571,7 @@ export class LLMService { } await this.prepareModelPort() if (generation !== this.launchGeneration) return false - const retryArgs = this.launchArgsFor( - this.ctxSize, - cpuOnly || enginePriority(serverPath) === 2 ? 0 : this.gpuLayers - ) + const retryArgs = this.launchArgsFor(this.ctxSize, cpuOnly ? 0 : this.gpuLayers, cpuOnly) const retriedWith = { settings: this.getSettings(), explicit: new Set(this.userExplicit) } if (await this.launchServer(serverPath, retryArgs)) { this.lastWorkingLaunch = retriedWith @@ -1628,6 +1632,8 @@ export class LLMService { this.backendOutput = '' this.backendModelPath = this.modelPath this.backendPlacement = undefined + this.offloadedLayers = undefined + this.loadingSince = Date.now() this.stderrTail = [] this.lastServerOutputAt = Date.now() this.invalidateHealth() @@ -1652,7 +1658,15 @@ export class LLMService { const text = String(data) // Verbosity 4 also emits request traces. Read and log it only during model // load so user prompts cannot enter the app log or its diagnostic tail. - if (!probing || this.server !== proc) return + if (this.server !== proc) return + if (!probing) { + if (GPU_FAULT_PATTERN.test(text)) { + this.lastErrorMsg = 'The GPU stopped responding.' + this.deviceFaultAt = Date.now() + this.invalidateHealth() + } + return + } console.log(`[llama-server] ${text}`) this.backendOutput = (this.backendOutput + text).slice(-65536) // Keep a rolling tail so we can classify a load failure after it exits. @@ -1687,7 +1701,10 @@ export class LLMService { this.invalidateHealth() // Recover unexpected exits of a previously healthy engine, including OS kills. // App stops and memory pauses stay stopped; launch failures use the engine ladder. - if (shouldAutoRecover({ probing, wasIntentional, paused: this.paused, signal })) { + if ( + this.deviceFaultAt === null && + shouldAutoRecover({ probing, wasIntentional, paused: this.paused, signal }) + ) { void this.handleCrash(code ?? -1).catch((error) => { console.error('[LLMService] automatic recovery failed:', error) }) @@ -1705,11 +1722,19 @@ export class LLMService { console.log('[LLMService] Vision server ready!') const engineDir = path.basename(binDir) this.backendPlacement = parseNativeBackend(this.backendOutput) + const layers = [ + ...this.backendOutput.matchAll(/\boffloaded\s+(\d+)\/(\d+)\s+layers to GPU\b/gi) + ].at(-1) + this.offloadedLayers = layers + ? { gpu: Number(layers[1]), total: Number(layers[2]) } + : undefined + this.loadingSince = undefined this.lastPlacedBackend = this.backendPlacement?.backend ?? this.lastPlacedBackend console.log( `[LLMService] model ready: engine=${engineDir}, backend=${this.backendPlacement?.backend ?? 'unconfirmed'}` ) this.initialized = true + this.deviceFaultAt = null this.lastErrorMsg = null // healthy again — clear any prior failure reason this.invalidateHealth() return true @@ -1737,6 +1762,11 @@ export class LLMService { * init, and fail loudly if init didn't take. Single source of truth so the * three chat methods don't each re-implement it. */ private async ensureReady(): Promise { + await waitForEngineResumeHealth() + if (this.recoveryPromise) await this.recoveryPromise + if (this.hasDeviceFault()) { + throw new ModelServerError('device_fault', 'The GPU stopped responding while answering.') + } await prepareModelMemory('chat') if (this.paused) throw new Error('LLM paused during image generation — deferred') if (!this.initialized) { @@ -1811,7 +1841,8 @@ export class LLMService { await this.init() } catch (error) { console.error('[LLMService] recovery startup failed:', error) - if (generation === this.launchGeneration && !(this.server as ChildProcess | null)) await this.handleCrash(code) + if (generation === this.launchGeneration && !(this.server as ChildProcess | null)) + await this.handleCrash(code) } } @@ -2021,6 +2052,7 @@ export class LLMService { timeoutMs?: number, maxTokens?: number, opts: { + requireImages?: boolean responseFormat?: unknown temperature?: number enableThinking?: boolean @@ -2031,7 +2063,13 @@ export class LLMService { shared?: boolean } = {} ): Promise { - const messages = buildMessages(message, readImages(images), this.systemPrompt) + const decodedImages = readImages(images) + if (opts.requireImages && (images.length === 0 || decodedImages.length !== images.length)) { + throw new Error( + 'Required model image could not be read. Take a fresh screenshot before retrying.' + ) + } + const messages = buildMessages(message, decodedImages, this.systemPrompt) return recordAIRequest( { modality: 'text', @@ -2429,6 +2467,7 @@ export class LLMService { stop(): void { this.launchGeneration++ this.initialized = false + this.intentionalStop = true if (this.server) { this.intentionalStop = true // deliberate shutdown — don't auto-restart this.server.kill() @@ -2536,7 +2575,9 @@ export class LLMService { private async pauseAndWait(): Promise { const { portFree } = await this.unload(true) if (!portFree) { - throw new Error('The chat model port is still occupied; image generation cannot start safely.') + throw new Error( + 'The chat model port is still occupied; image generation cannot start safely.' + ) } } @@ -2599,6 +2640,67 @@ export class LLMService { return this.lastErrorMsg } + hasDeviceFault(): boolean { + return this.deviceFaultAt !== null + } + + gpuBackend(): NativeBackend | undefined { + return this.currentBackend === 'cpu' ? undefined : this.currentBackend + } + + shouldBeRunning(): boolean { + return ( + !this.paused && !this.intentionalStop && this.lastWorkingLaunch !== null && !this.initPromise + ) + } + + async healthCheck(): Promise { + const proc = this.server + if ( + !proc || + proc.killed || + proc.exitCode !== null || + proc.signalCode !== null || + this.hasDeviceFault() + ) + return false + try { + const response = await fetch(`http://127.0.0.1:${this.port}/health`, { + signal: AbortSignal.timeout(3000) + }) + await response.body?.cancel() + // The process may have been replaced while the request was in flight. + return response.status === 200 && (this.server as ChildProcess | null) === proc + } catch { + return false + } + } + + /** Fault recovery shares the crash budget and never writes the user's settings. */ + async restartSameModel(): Promise { + if (this.recoveryPromise) return this.recoveryPromise + const now = Date.now() + this.restartTimes = this.restartTimes.filter((time) => now - time < 120_000) + if (this.restartTimes.length >= 3) { + this.lastErrorMsg = + 'Model server stopped repeatedly. Automatic restart limit reached. Check GPU memory and use Restart after fixing the cause.' + this.invalidateHealth() + throw new Error(this.lastErrorMsg) + } + this.restartTimes.push(now) + this.recoveryModel = { path: this.modelPath, mmproj: this.mmProjPath } + this.recoveryPromise = (async () => { + this.stop() + await this.init() + if (!this.isReady()) + throw new Error('Server did not come back up — check the model is downloaded') + })().finally(() => { + this.recoveryModel = null + this.recoveryPromise = null + }) + return this.recoveryPromise + } + /** Hard restart: kill the server and spawn it fresh (picks up a model swap or * recovers a crashed/hung instance). Used by "Configure for me", the Health * panel's restart action, and the Models screen dev control. Clears `paused` diff --git a/src/main/llm/__tests__/dedicated-llama-runtime.test.ts b/src/main/llm/__tests__/dedicated-llama-runtime.test.ts new file mode 100644 index 000000000..2ffa31348 --- /dev/null +++ b/src/main/llm/__tests__/dedicated-llama-runtime.test.ts @@ -0,0 +1,160 @@ +// A dedicated llama-server (grounding, decision) owned by DedicatedLlamaRuntime: it starts, comes +// back on its own when the engine dies, stops trying after repeated deaths, and stops cleanly. +// The native engine is the external boundary: a small process that implements llama-server's +// health contract and dies on request. Engine selection, ports, launch and recovery are real. +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterAll, afterEach, describe, expect, it } from 'vitest' +import { binRoots, configureRuntime, dataDir } from '../../runtime-env' +import { DedicatedLlamaRuntime } from '../dedicated-llama-runtime' + +const previous = { dataDir: dataDir(), binRoots: binRoots() } +const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-dedicated-runtime-')) +const binaries = path.join(profile, 'bin') +const engine = path.join(binaries, 'llama-cpu', 'llama-server') +/** What the engine does on its next start: serve, die once healthy, or never answer. */ +const modeFile = path.join(profile, 'mode') +/** One line per engine start. */ +const spawnLog = path.join(profile, 'spawns') +configureRuntime({ dataDir: profile, binRoots: [binaries] }) + +fs.mkdirSync(path.dirname(engine), { recursive: true }) +fs.writeFileSync( + engine, + `#!/usr/bin/env node +const fs = require('node:fs'); +const http = require('node:http'); +fs.appendFileSync(${JSON.stringify(spawnLog)}, process.argv.slice(2).join(' ') + '\\n'); +const mode = fs.existsSync(${JSON.stringify(modeFile)}) ? fs.readFileSync(${JSON.stringify(modeFile)}, 'utf8').trim() : 'serve'; +const args = process.argv.slice(2); +const port = Number(args[args.indexOf('--port') + 1]); +const server = http.createServer((req, res) => { + if (mode === 'silent') return; + res.setHeader('Content-Type', 'application/json'); + if (req.url === '/health') { + res.end(JSON.stringify({ status: 'ok' })); + if (mode === 'fault') setTimeout(() => { process.stderr.write('CUDA error: device lost\\n'); process.exit(1); }, 150); + return; + } + res.statusCode = 404; res.end('{}'); +}); +server.listen(port, '127.0.0.1'); +process.on('SIGTERM', () => server.close(() => process.exit(0))); +`, + { mode: 0o755 } +) + +const spawns = (): number => + fs.existsSync(spawnLog) ? fs.readFileSync(spawnLog, 'utf8').trim().split('\n').length : 0 + +let runtime: DedicatedLlamaRuntime | undefined + +function makeRuntime(readyTimeoutMs?: number): DedicatedLlamaRuntime { + runtime = new DedicatedLlamaRuntime({ + id: 'decision', + label: 'Decision', + modality: 'decision', + port: 30_000 + Math.floor(Math.random() * 20_000), + ...(readyTimeoutMs ? { readyTimeoutMs } : {}), + modelPaths: async () => ({ + model: 'test/decider', + modelPath: path.join(profile, 'decider.gguf'), + mmProjPath: '' + }), + extraArgs: () => ({ + effectiveCtxSize: 4_096, + flashAttn: true, + kvCacheType: 'q8_0', + speculativeDecoding: 'off', + threads: undefined, + batchSize: 128 + }) + }) + return runtime +} + +async function until( + condition: () => boolean | Promise, + timeoutMs = 15_000 +): Promise { + const deadline = Date.now() + timeoutMs + while (!(await condition())) { + if (Date.now() > deadline) throw new Error('timed out waiting') + await new Promise((resolve) => setTimeout(resolve, 50)) + } +} + +afterEach(async () => { + await runtime?.stop() + runtime = undefined + fs.rmSync(modeFile, { force: true }) + fs.rmSync(spawnLog, { force: true }) +}) + +afterAll(() => { + configureRuntime(previous) + fs.rmSync(profile, { recursive: true, force: true }) +}) + +describe.skipIf(process.platform === 'win32')('DedicatedLlamaRuntime', () => { + it('starts the engine on the CPU build and answers health', async () => { + const decision = makeRuntime() + await decision.start() + expect(decision.isRunning).toBe(true) + expect(decision.shouldBeRunning()).toBe(true) + expect(await decision.health()).toBe(true) + expect(decision.backend).toBe('CPU') + // The CPU build runs CPU only. + expect(fs.readFileSync(spawnLog, 'utf8')).toContain('--device none') + // A second start while running is the same engine. + await decision.start() + expect(spawns()).toBe(1) + }) + + it('brings the same model back on its own after the engine dies', async () => { + const decision = makeRuntime() + fs.writeFileSync(modeFile, 'fault') + await decision.start() + fs.writeFileSync(modeFile, 'serve') + await decision.health() + // The engine dies after answering; the runtime starts it again with the same model. + await until(async () => spawns() >= 2 && decision.isRunning && (await decision.health())) + const starts = fs.readFileSync(spawnLog, 'utf8').trim().split('\n') + expect(starts[1]).toContain(path.join(profile, 'decider.gguf')) + }) + + it('stops restarting after repeated deaths and says so', async () => { + const decision = makeRuntime() + fs.writeFileSync(modeFile, 'fault') + await decision.start() + // Each death is restarted with a growing pause, until the budget runs out. + await until(() => decision.failureCode === 'startup', 30_000) + expect(decision.isRunning).toBe(false) + const before = spawns() + await new Promise((resolve) => setTimeout(resolve, 1_500)) + expect(spawns()).toBe(before) + }, 40_000) + + it('gives up on an engine that never becomes ready', async () => { + const decision = makeRuntime(800) + fs.writeFileSync(modeFile, 'silent') + await expect(decision.start()).rejects.toThrow() + expect(decision.isRunning).toBe(false) + }) + + it('restarts on request and stops cleanly', async () => { + const decision = makeRuntime() + await decision.start() + await decision.restart() + expect(spawns()).toBe(2) + expect(await decision.health()).toBe(true) + await decision.stop() + expect(decision.isRunning).toBe(false) + expect(decision.shouldBeRunning()).toBe(false) + expect(await decision.health()).toBe(false) + // Restarting a stopped runtime does nothing. + await decision.restart() + expect(spawns()).toBe(2) + }) +}) diff --git a/src/main/llm/__tests__/engine-priority.test.ts b/src/main/llm/__tests__/engine-priority.test.ts deleted file mode 100644 index ec3fab92e..000000000 --- a/src/main/llm/__tests__/engine-priority.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { expect, it } from 'vitest' -import { enginePriority } from '../engine-priority' -it('tries CUDA and Metal/Vulkan across roots before CPU', () => { - const paths = ['/a/llama-cpu/server', '/b/llama/server', 'C:\\b\\llama-cuda\\server.exe'] - expect(paths.sort((a, b) => enginePriority(a) - enginePriority(b))).toEqual([ - 'C:\\b\\llama-cuda\\server.exe', - '/b/llama/server', - '/a/llama-cpu/server' - ]) - expect(enginePriority('/a/llama-prism-cpu/server')).toBe(2) -}) diff --git a/src/main/llm/__tests__/model-switch.integration.test.ts b/src/main/llm/__tests__/model-switch.integration.test.ts index 2053646a6..f8f8116ed 100644 --- a/src/main/llm/__tests__/model-switch.integration.test.ts +++ b/src/main/llm/__tests__/model-switch.integration.test.ts @@ -30,7 +30,9 @@ describe('model switch during startup', () => { it('does not try another engine after the current load is cancelled', async () => { const service = new LLMService() const internals = service as unknown as { - launchWithFallback(paths: string[]): Promise + launchWithFallback( + steps: { path: string; backend: string; cpuOnly: boolean }[] + ): Promise launchServer(path: string, args: string[], layers: number): Promise launchArgsFor(context: number, layers: number): string[] prepareModelPort(): Promise @@ -41,9 +43,12 @@ describe('model switch during startup', () => { return false }) const prepare = vi.spyOn(internals, 'prepareModelPort').mockResolvedValue(undefined) - await expect(internals.launchWithFallback(['/old/server', '/old/fallback'])).resolves.toBe( - false - ) + await expect( + internals.launchWithFallback([ + { path: '/old/server', backend: 'metal', cpuOnly: false }, + { path: '/old/fallback', backend: 'metal', cpuOnly: true } + ]) + ).resolves.toBe(false) expect(launch).toHaveBeenCalledTimes(1) expect(prepare).not.toHaveBeenCalled() }) diff --git a/src/main/llm/__tests__/mtp-launch.integration.test.ts b/src/main/llm/__tests__/mtp-launch.integration.test.ts index 4943bd0bb..5b217f664 100644 --- a/src/main/llm/__tests__/mtp-launch.integration.test.ts +++ b/src/main/llm/__tests__/mtp-launch.integration.test.ts @@ -25,11 +25,17 @@ function gguf(file: string, embeddedMtp: boolean, size = 1024, metadataPadding = const str = (value: string): Buffer => Buffer.concat([u64(Buffer.byteLength(value)), Buffer.from(value)]) const architecture = /gemma-4/i.test(file) - ? path.basename(file).startsWith('mtp-') ? 'gemma4-assistant' : 'gemma4' + ? path.basename(file).startsWith('mtp-') + ? 'gemma4-assistant' + : 'gemma4' : 'qwen35' const values = [Buffer.concat([str('general.architecture'), u32(8), str(architecture)])] - if (metadataPadding) values.push(Buffer.concat([str('tokenizer.chat_template'), u32(8), str('x'.repeat(metadataPadding))])) - if (embeddedMtp) values.push(Buffer.concat([str(`${architecture}.nextn_predict_layers`), u32(4), u32(1)])) + if (metadataPadding) + values.push( + Buffer.concat([str('tokenizer.chat_template'), u32(8), str('x'.repeat(metadataPadding))]) + ) + if (embeddedMtp) + values.push(Buffer.concat([str(`${architecture}.nextn_predict_layers`), u32(4), u32(1)])) fs.writeFileSync( file, Buffer.concat([Buffer.from('GGUF'), u32(3), u64(0), u64(values.length), ...values]) @@ -91,7 +97,10 @@ describe('MTP installation and persisted launch contract', () => { expect.arrayContaining(['--spec-type', 'draft-mtp', '--spec-draft-model', sidecar]) ) expect(args[args.indexOf('--spec-draft-n-max') + 1]).toBe('2') - expect(args[args.indexOf('--spec-draft-ngl') + 1]).toBe('99') + // A stored 99 (the old "all layers") is Auto now: llama.cpp fits the model and its draft to the + // GPU itself, which an explicit layer count would switch off. + expect(args).not.toContain('-ngl') + expect(args).not.toContain('--spec-draft-ngl') expect(args).not.toContain('unrelated.gguf') }) @@ -123,8 +132,9 @@ describe('MTP installation and persisted launch contract', () => { it.each([-1, 17, 2.5, NaN])('rejects an invalid draft token limit (%s)', async (limit) => { const service = await load(primary, true) await service.pause() - await expect(service.setSettings({ speculativeDraftMax: limit }, { emitSync: false })) - .rejects.toThrow('Draft token limit') + await expect( + service.setSettings({ speculativeDraftMax: limit }, { emitSync: false }) + ).rejects.toThrow('Draft token limit') expect(service.getSettings().speculativeDraftMax).toBe(0) }) @@ -175,35 +185,51 @@ describe('MTP installation and persisted launch contract', () => { it.each([ ['unsloth/gemma-4-E4B-it-GGUF', 'gemma-4-E4B-it-Q4_K_M.gguf'], ['unsloth/gemma-4-12b-it-GGUF', 'gemma-4-12b-it-Q4_K_M.gguf'] - ] as const)('offers %s MTP only after its matching companion is installed', async (id, selected) => { - const { desktopCatalog } = await import('../../models-manager') - const { LLMService } = await import('../../llm') - const service = await load(selected, false, 'off') - const file = MTP_COMPANIONS[id] - const entry = (await desktopCatalog()).find((model) => model.id === id) - expect(entry?.files).toContainEqual(file) - expect(visionStatus(entry!, (name) => fs.existsSync(path.join(profile, 'models', name)))) - .toMatchObject({ supportsMtp: true, mtpInstalled: false }) - expect(service.getSettings()).toMatchObject({ supportsMtp: false, speculativeDecoding: 'off' }) + ] as const)( + 'offers %s MTP only after its matching companion is installed', + async (id, selected) => { + const { desktopCatalog } = await import('../../models-manager') + const { LLMService } = await import('../../llm') + const service = await load(selected, false, 'off') + const file = MTP_COMPANIONS[id] + const entry = (await desktopCatalog()).find((model) => model.id === id) + expect(entry?.files).toContainEqual(file) + expect( + visionStatus(entry!, (name) => fs.existsSync(path.join(profile, 'models', name))) + ).toMatchObject({ supportsMtp: true, mtpInstalled: false }) + expect(service.getSettings()).toMatchObject({ + supportsMtp: false, + speculativeDecoding: 'off' + }) - // Real, sparse metadata files exercise installation readiness without downloading weights. - const wrong = MTP_COMPANIONS[id.includes('E4B') - ? 'unsloth/gemma-4-12b-it-GGUF' : 'unsloth/gemma-4-E4B-it-GGUF'] - gguf(path.join(profile, 'models', wrong.name), true, wrong.sizeBytes) - expect(service.getSettings().supportsMtp).toBe(false) - const sidecar = path.join(profile, 'models', file.name) - gguf(sidecar, true, file.sizeBytes - 1) - expect(service.getSettings().supportsMtp).toBe(false) - gguf(sidecar, true, file.sizeBytes) - expect(visionStatus(entry!, (name) => fs.existsSync(path.join(profile, 'models', name)))) - .toMatchObject({ mtpInstalled: true }) - await service.pause() - await service.setSettings({ speculativeDecoding: 'mtp' }, { emitSync: false }) - expect(new LLMService().getSettings()).toMatchObject({ supportsMtp: true, speculativeDecoding: 'mtp' }) + // Real, sparse metadata files exercise installation readiness without downloading weights. + const wrong = + MTP_COMPANIONS[ + id.includes('E4B') ? 'unsloth/gemma-4-12b-it-GGUF' : 'unsloth/gemma-4-E4B-it-GGUF' + ] + gguf(path.join(profile, 'models', wrong.name), true, wrong.sizeBytes) + expect(service.getSettings().supportsMtp).toBe(false) + const sidecar = path.join(profile, 'models', file.name) + gguf(sidecar, true, file.sizeBytes - 1) + expect(service.getSettings().supportsMtp).toBe(false) + gguf(sidecar, true, file.sizeBytes) + expect( + visionStatus(entry!, (name) => fs.existsSync(path.join(profile, 'models', name))) + ).toMatchObject({ mtpInstalled: true }) + await service.pause() + await service.setSettings({ speculativeDecoding: 'mtp' }, { emitSync: false }) + expect(new LLMService().getSettings()).toMatchObject({ + supportsMtp: true, + speculativeDecoding: 'mtp' + }) - fs.unlinkSync(sidecar) - expect(new LLMService().getSettings()).toMatchObject({ supportsMtp: false, speculativeDecoding: 'off' }) - }) + fs.unlinkSync(sidecar) + expect(new LLMService().getSettings()).toMatchObject({ + supportsMtp: false, + speculativeDecoding: 'off' + }) + } + ) it.each([primary, 'Ternary-Bonsai-2-27B-PQ2_0.gguf'])( 'preserves N-gram decoding for %s without adding an MTP companion', diff --git a/src/main/llm/__tests__/select-local-engine.test.ts b/src/main/llm/__tests__/select-local-engine.test.ts deleted file mode 100644 index 752fc37ce..000000000 --- a/src/main/llm/__tests__/select-local-engine.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { afterEach, expect, it, vi } from 'vitest' -const mocks = vi.hoisted(() => ({ probe: vi.fn(), exists: vi.fn(() => true) })) -vi.mock('node:fs', () => ({ default: { existsSync: mocks.exists } })) -vi.mock('../../runtime-env', () => ({ binRoots: () => ['/bundle'], exe: () => 'llama-server' })) -vi.mock('../gpu-device-probe', () => ({ gpuDeviceAvailable: mocks.probe })) -import { selectLocalEngine } from '../select-local-engine' -afterEach(() => { - vi.unstubAllGlobals() - vi.clearAllMocks() -}) - -it('prefers a usable CUDA engine on Windows', async () => { - vi.stubGlobal('process', { ...process, platform: 'win32' }) - mocks.probe.mockResolvedValue(true) - expect(await selectLocalEngine()).toBe('/bundle/llama-cuda/llama-server') -}) -it('uses Vulkan when CUDA has no device', async () => { - vi.stubGlobal('process', { ...process, platform: 'linux' }) - mocks.probe.mockImplementation(async (_path, backend) => backend === 'Vulkan') - expect(await selectLocalEngine()).toBe('/bundle/llama/llama-server') -}) -it('preserves the Metal engine on macOS', async () => { - vi.stubGlobal('process', { ...process, platform: 'darwin' }) - expect(await selectLocalEngine()).toBe('/bundle/llama/llama-server') - expect(mocks.probe).not.toHaveBeenCalled() -}) diff --git a/src/main/llm/dedicated-llama-runtime.ts b/src/main/llm/dedicated-llama-runtime.ts new file mode 100644 index 000000000..4ade1aafb --- /dev/null +++ b/src/main/llm/dedicated-llama-runtime.ts @@ -0,0 +1,322 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import path from 'node:path' +import { beginRuntimeBackend, parseNativeBackend } from '../runtime-backends' +import { demoteBackend } from '../backend-facts' +import { classifyLlamaError } from '../llama-error' +import { isPortFree, pickFreePort } from '../free-port' +import { reapOrphanProcessesOnPort } from '../kill-orphan-port' +import { selectLocalEngines } from './select-local-engine' +import { buildLaunchArgs } from './settings-math' +import { engineSpawnEnv } from './spawn-env' + +type Step = Awaited>[number] +type LaunchArgs = Parameters[0] +interface Options { + id: 'grounding' | 'decision' + label: string + modality: 'grounding' | 'decision' + port: number + modelPaths(): Promise<{ model: string; modelPath: string; mmProjPath: string }> + extraArgs(step: Step): Omit + readyTimeoutMs?: number +} + +const WINDOW_MS = 120_000 +const LIMIT_REASON = 'Stopped repeatedly. Automatic restart limit reached.' +const GPU_FAULT = /compute error|cuda error|ggml_cuda|device lost|ErrorDeviceLost/i +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +/** Owns the process, backend ladder and recovery budget for a dedicated server. */ +export class DedicatedLlamaRuntime { + private process: ChildProcess | null = null + private pending: Promise | null = null + private stopping: Promise | null = null + private wanted = false + private generation = 0 + private attempted = false + private ready = false + private model = '' + private healthyOnce = false + private restarts: number[] = [] + private faults: { backend: Step['backend']; at: number }[] = [] + private record?: ReturnType + private failure = '' + failureCode: string | undefined + backend: string | undefined + port: number + + constructor(private readonly options: Options) { + this.port = options.port + } + + get isRunning(): boolean { + return this.ready && this.processAlive + } + + /** Running until the process reports an exit code or signal (a sent kill is not an exit yet). */ + private get processAlive(): boolean { + return this.process !== null && this.process.exitCode == null && this.process.signalCode == null + } + + shouldBeRunning(): boolean { + return this.wanted + } + recordRequest(): void { + this.record?.recordRequest() + } + + async health(): Promise { + const child = this.process + if (!this.processAlive) return false + try { + const response = await fetch(`http://127.0.0.1:${this.port}/health`, { + signal: AbortSignal.timeout(3_000) + }) + return response.status === 200 && this.process === child && this.processAlive + } catch { + return false + } + } + + async start(): Promise { + if (this.stopping) await this.stopping + if (this.pending) return this.pending + if (this.isRunning) return + this.wanted = true + const generation = this.generation + this.pending = this.run(generation).finally(() => { + this.pending = null + }) + return this.pending + } + + private assertWanted(generation: number): void { + if (!this.wanted || generation !== this.generation) + throw new Error(`${this.options.label} stopped.`) + } + + private async run(generation: number): Promise { + for (;;) { + this.assertWanted(generation) + if (this.attempted) { + this.restarts = this.restarts.filter((at) => Date.now() - at < WINDOW_MS) + if (this.restarts.length >= 3) { + this.failureCode = 'startup' + this.failure = LIMIT_REASON + // A failed lease is terminal. Publish the budget failure on a fresh lease. + this.record = beginRuntimeBackend(this.options.id, this.model || this.options.label) + this.record.fail(LIMIT_REASON) + throw new Error(LIMIT_REASON) + } + this.restarts.push(Date.now()) + await delay(1_000 * this.restarts.length) + this.assertWanted(generation) + } + this.attempted = true + try { + await this.launch(generation) + return + } catch (error) { + this.assertWanted(generation) + if (!this.healthyOnce) throw error + // Recovery attempts that fail to load use the same budget as crashes. + } + } + } + + private async launch(generation: number): Promise { + let paths: Awaited> + try { + paths = await this.options.modelPaths() + } catch (error) { + this.assertWanted(generation) + this.record = beginRuntimeBackend(this.options.id, this.model || this.options.label) + this.record.fail(error) + throw error + } + this.model = paths.model + this.assertWanted(generation) + this.record = beginRuntimeBackend(this.options.id, paths.model) + const failures: string[] = [] + this.failureCode = undefined + try { + reapOrphanProcessesOnPort( + this.options.port, + (command) => /llama-server/i.test(command), + `${this.options.label} llama-server` + ) + const port = await pickFreePort(this.options.port, isPortFree, 20) + if (port === null) + throw new Error(`No private port is available for the ${this.options.label} runtime.`) + this.port = port + const steps = await selectLocalEngines(this.options.modality) + for (const step of steps) { + this.assertWanted(generation) + let stderr = '' + let output = '' + let loading = true + let faultedAt: number | undefined + let spawnError: Error | undefined + const record = this.record + const child = spawn( + step.path, + buildLaunchArgs({ + ...this.options.extraArgs(step), + modelPath: paths.modelPath, + mmProjPath: paths.mmProjPath, + port, + cpuOnly: step.cpuOnly + }), + { + env: { + ...process.env, + ...engineSpawnEnv({ + platform: process.platform, + binDir: path.dirname(step.path), + currentEnv: process.env + }) + }, + stdio: ['ignore', 'ignore', 'pipe'] + } + ) + this.process = child + child.on('error', (error) => { + spawnError = error + }) + child.stderr!.on('data', (chunk) => { + if (loading) { + const text = String(chunk) + stderr = (stderr + text).slice(-16_384) + output = (output + text).slice(-65_536) + record.observe(text) + console.log(`[${this.options.label} runtime] ${text}`) + } else if (GPU_FAULT.test(String(chunk))) { + faultedAt = Date.now() // Do not retain or log request text. + } + }) + child.once('close', (code, signal) => { + if (this.process !== child) return + this.process = null + this.ready = false + if (loading || !this.wanted || generation !== this.generation) return + const now = Date.now() + const gpuFault = faultedAt !== undefined && now - faultedAt < WINDOW_MS + this.failure = gpuFault + ? `${this.options.label} GPU compute failed.` + : `${this.options.label} runtime stopped unexpectedly (${signal ?? code ?? 'unknown'}).` + this.faults = this.faults.filter((fault) => now - fault.at < WINDOW_MS) + if (gpuFault && step.backend !== 'cpu') { + this.faults.push({ backend: step.backend, at: now }) + if (this.faults.filter((fault) => fault.backend === step.backend).length >= 2) + demoteBackend(this.options.modality, step.backend) + } + record.fail(this.failure) + // Wait for a start that has just completed before requesting recovery. + void (async () => { + await this.pending?.catch(() => {}) + if (this.wanted && generation === this.generation) await this.start() + })().catch(() => {}) + }) + try { + const startedAt = Date.now() + for (;;) { + this.assertWanted(generation) + if (spawnError) throw spawnError + if (!this.processAlive) + throw new Error(`${this.options.label} runtime stopped during startup.`) + if (Date.now() - startedAt > (this.options.readyTimeoutMs ?? Infinity)) + throw new Error(`${this.options.label} runtime timed out during startup.`) + if (await this.health()) break + await delay(100) + } + this.assertWanted(generation) + loading = false + this.ready = true + const placement = parseNativeBackend(output) + this.backend = step.cpuOnly ? 'CPU' : (placement?.backend ?? step.backend) + record.ready( + this.backend, + placement?.device, + failures.join(' ') || this.failure || undefined + ) + stderr = '' + output = '' + this.failure = '' + this.healthyOnce = true + return + } catch (error) { + const classified = + classifyLlamaError(stderr) ?? + (/out of memory|cannot allocate|metal.*alloc|cuda.*memory/i.test(stderr) + ? { code: 'out_of_memory', reason: 'The device does not have enough free memory.' } + : null) + const code: string | undefined = classified?.code + if (this.failureCode !== 'out_of_memory') this.failureCode = code ?? this.failureCode + const reason = + classified?.reason ?? (error instanceof Error ? error.message : String(error)) + failures.push(reason) + // Also accepts gpu_fault when the shared classifier adds that code. + if (code === 'gpu_fault' && this.failureCode !== 'out_of_memory') + this.failureCode = 'gpu_fault' + await this.closeProcess() + this.assertWanted(generation) + } + } + throw new Error(failures.join(' ') || `The bundled ${this.options.label} engine is missing.`) + } catch (error) { + this.assertWanted(generation) + this.failure = error instanceof Error ? error.message : String(error) + this.record.fail(this.failure) + throw new Error(this.failure) + } + } + + private async closeProcess(): Promise { + const child = this.process + this.process = null + this.ready = false + if (!child) return + const closed = new Promise((resolve) => child.once('close', () => resolve())) + if (child.exitCode != null || child.signalCode != null) { + // Already exited: its streams close shortly (or closed before this listener was added). + await Promise.race([closed, delay(1_000)]) + return + } + child.kill('SIGTERM') + let timer: ReturnType | undefined + const exited = await Promise.race([ + closed.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), 1_000) + }) + ]) + clearTimeout(timer) + if (!exited) { + child.kill('SIGKILL') + await closed + } + } + + async stop(): Promise { + this.wanted = false + this.generation += 1 + if (this.stopping) return this.stopping + this.record?.stop() + this.stopping = (async () => { + await this.closeProcess() + await this.pending?.catch(() => {}) + this.attempted = false + this.healthyOnce = false + })().finally(() => { + this.stopping = null + }) + return this.stopping + } + + async restart(): Promise { + if (!this.wanted) return + await this.stop() + this.attempted = true + await this.start() + } +} diff --git a/src/main/llm/engine-priority.ts b/src/main/llm/engine-priority.ts deleted file mode 100644 index 0bb95f333..000000000 --- a/src/main/llm/engine-priority.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** Search every binary root for GPU engines before accepting a CPU build. */ -export function enginePriority(serverPath: string): number { - const directory = serverPath.replaceAll('\\', '/').split('/').at(-2) ?? '' - if (directory.endsWith('-cuda')) return 0 - if (directory.endsWith('-cpu')) return 2 - return 1 -} diff --git a/src/main/llm/http-post.ts b/src/main/llm/http-post.ts index 1ae2acb21..841b7a6b2 100644 --- a/src/main/llm/http-post.ts +++ b/src/main/llm/http-post.ts @@ -10,6 +10,7 @@ // integration test drives postCompletionOnce against a real socket-closing server (behaviour). import * as http from 'http' +import { GPU_FAULT_PATTERN } from '../llama-error' /** * WHY a failure happened, decided by the only layer that can see it. @@ -20,12 +21,14 @@ import * as http from 'http' * identical request can never succeed against this model. * - `rejected` — the server is up and refused this request (malformed body, uncompilable grammar). * Also deterministic for this input. + * - `device_fault` — the GPU failed while answering; recover without changing the model. * - `aborted` — the caller cancelled. Not a failure of the request at all. * * A caller must NOT re-derive this from the message text. Capture did exactly that, called every * refusal `capture-model-unavailable`, and retried a permanently broken request 32 times. */ -export type ModelServerFailureKind = 'unavailable' | 'overflow' | 'rejected' | 'aborted' +export type ModelServerFailureKind = + 'unavailable' | 'overflow' | 'rejected' | 'aborted' | 'device_fault' export class ModelServerError extends Error { readonly kind: ModelServerFailureKind @@ -56,7 +59,7 @@ function serverDetail(body: string): string { * llama-server returns a JSON body like {"error":{"message":"request (22825 tokens) exceeds * the available context size (16384 tokens) …"}}; the bare status code alone is useless both * to the user and to a retry policy. */ -export function classifyServerError( +function classifyServerError( statusCode: number | undefined, body: string ): { kind: ModelServerFailureKind; message: string } { @@ -87,6 +90,9 @@ export function classifyServerError( message: `LLM Server Error: ${statusCode ?? '?'}${detail ? ` ${detail}` : ''}` } } + if (GPU_FAULT_PATTERN.test(detail)) { + return { kind: 'device_fault', message: 'The GPU stopped responding while answering.' } + } // A 4xx means the server understood the request and refused it. Anything else — 502, 503, 504, a // bare 500 — is the server failing to service it, which is the definition of an outage. // @@ -98,11 +104,6 @@ export function classifyServerError( return { kind, message: `LLM Server Error: ${statusCode ?? '?'}${detail ? ` ${detail}` : ''}` } } -/** Turn a non-200 model-server response into an ACTIONABLE message. */ -export function describeServerError(statusCode: number | undefined, body: string): string { - return classifyServerError(statusCode, body).message -} - /** The typed rejection for a non-200 response, carrying the reason with the message. */ export function serverResponseError( statusCode: number | undefined, @@ -181,6 +182,18 @@ export function postCompletionOnce( const req = http.request( modelRequestOptions(port, Buffer.byteLength(body), requestPath), (res) => { + res.on('error', (error) => + finish(() => + reject( + new ModelServerError( + 'unavailable', + 'The model server connection closed.', + undefined, + error + ) + ) + ) + ) let data = '' res.on('data', (chunk) => { data += chunk diff --git a/src/main/llm/remote-chat.ts b/src/main/llm/remote-chat.ts index 3ded3949c..46f2ad33f 100644 --- a/src/main/llm/remote-chat.ts +++ b/src/main/llm/remote-chat.ts @@ -199,7 +199,7 @@ export function remoteModelUnreachable(code: string): string | null { /** Keep remote transport errors useful without exposing the endpoint, headers, * API key, or request body. */ -export function remoteTextModelTransportError(error: unknown): Error { +function remoteTextModelTransportError(error: unknown): Error { if (!(error instanceof Error)) return new Error('Remote text model request failed.') const cause = error.cause if (!cause || typeof cause !== 'object') { diff --git a/src/main/llm/select-local-engine.ts b/src/main/llm/select-local-engine.ts index a23d01aab..09f5e1256 100644 --- a/src/main/llm/select-local-engine.ts +++ b/src/main/llm/select-local-engine.ts @@ -1,31 +1,8 @@ -import fs from 'node:fs' -import path from 'node:path' -import { binRoots, exe } from '../runtime-env' -import { gpuDeviceAvailable } from './gpu-device-probe' -import { prioritizeBackend, type BackendPreference } from '../../shared/backend-preferences' +import { nativeLadder, type NativeStep } from '../backend-facts' -/** Dedicated decision and grounding models use the same GPU priority as chat. */ -export async function selectLocalEngine(preference: BackendPreference = 'auto'): Promise { - const platform = process.platform - const base = ['llama-cuda', 'llama', 'llama-cpu', ''] - const directories = preference === 'cpu' - ? ['llama-cpu', ...base.filter((directory) => directory !== 'llama-cpu')] - : prioritizeBackend(base, preference, (directory) => - directory === 'llama-cpu' ? 'cpu' : directory === 'llama-cuda' ? 'cuda' : platform === 'darwin' ? 'metal' : 'vulkan' - ) - for (const directory of directories) { - if (platform !== 'win32' && platform !== 'linux' && directory === 'llama-cuda') continue - for (const root of binRoots()) { - const candidate = path.join(root, directory, exe('llama-server')) - if (!fs.existsSync(candidate)) continue - if (platform === 'win32' || platform === 'linux') { - const backend = - directory === 'llama-cuda' ? 'CUDA' : directory === 'llama' ? 'Vulkan' : null - if (backend && !(await gpuDeviceAvailable(candidate, backend, platform))) continue - } - console.log(`[Local model] selected engine=${directory || 'legacy'}`) - return candidate - } - } - return undefined +/** Grounding, decision and chat share the native backend policy. */ +export function selectLocalEngines( + modality: 'grounding' | 'decision' | 'llm' +): Promise { + return nativeLadder(modality) } diff --git a/src/main/llm/settings-math.ts b/src/main/llm/settings-math.ts index f56df76b8..c8b901fe2 100644 --- a/src/main/llm/settings-math.ts +++ b/src/main/llm/settings-math.ts @@ -97,7 +97,7 @@ export interface LaunchState { ctxSize: number kvCacheType: KvCacheType flashAttn: boolean - gpuLayers: number + gpuLayers?: number | null threads: number | undefined batchSize: number | undefined speculativeDecoding?: SpeculativeDecodingMode @@ -112,11 +112,12 @@ export interface LaunchState { * file sizes) and stays in LLMService; everything here is a pure function of these * inputs so the terminal artifact (the args array handed to llama-server) is testable. */ export interface LaunchArgsInput { + cpuOnly?: boolean modelPath: string mmProjPath: string // empty string = text-only (no --mmproj) port: number effectiveCtxSize: number // already RAM-clamped - gpuLayers: number + gpuLayers?: number | null flashAttn: boolean kvCacheType: KvCacheType threads: number | undefined @@ -177,7 +178,7 @@ export function slotInput(i: LaunchArgsInput): LlmSlotInput { * 16 GB machine 8 GB of cache next to the model invites swapping, so it is capped at a quarter of * RAM; machines with 32 GB or more keep the engine default. */ -export function promptCacheMiB(totalRamBytes: number | undefined): number | undefined { +function promptCacheMiB(totalRamBytes: number | undefined): number | undefined { if (!totalRamBytes || !Number.isFinite(totalRamBytes)) return undefined const quarter = Math.floor(totalRamBytes / 1024 / 1024 / 4) return quarter >= 8192 ? undefined : Math.max(1024, quarter) @@ -201,10 +202,10 @@ export function buildLaunchArgs(i: LaunchArgsInput): string[] { '-c', // Team mode may grow the shared pool past the person's context so every slot gets its share; // it is never smaller. Single mode: exactly the person's context. - String(llmSlotPlan(slotInput(i)).poolCtx), - '-ngl', - String(i.gpuLayers) + String(llmSlotPlan(slotInput(i)).poolCtx) ) + if (i.cpuOnly) args.push('--device', 'none', '-ngl', '0') + else if (typeof i.gpuLayers === 'number') args.push('-ngl', String(i.gpuLayers)) // Hybrid models (Qwen3.5, Bonsai 2) cannot roll recurrent state back past a saved // context checkpoint. At the 8192-token default spacing, any edit earlier in a shorter // conversation (a dropped screenshot, a compacted turn) re-processes the whole prompt; @@ -230,7 +231,7 @@ export function buildLaunchArgs(i: LaunchArgsInput): string[] { if (kv.k !== 'f16' || kv.v !== 'f16') { args.push('--cache-type-k', kv.k, '--cache-type-v', kv.v) } - if (typeof i.threads === 'number') { + if (typeof i.threads === 'number' && i.threads > 0) { args.push('-t', String(i.threads)) } if (typeof i.batchSize === 'number') { @@ -247,7 +248,7 @@ export function buildLaunchArgs(i: LaunchArgsInput): string[] { const draftMax = i.speculativeDraftMax ? String(i.speculativeDraftMax) : '2' args.push('--spec-type', 'draft-mtp', '--spec-draft-n-max', draftMax) if (i.draftModelPath) { - args.push('--spec-draft-model', i.draftModelPath, '--spec-draft-ngl', String(i.gpuLayers)) + args.push('--spec-draft-model', i.draftModelPath) } } else if ( i.draftModelPath && @@ -261,6 +262,10 @@ export function buildLaunchArgs(i: LaunchArgsInput): string[] { ) if (i.speculativeDraftMax) args.push('--spec-draft-n-max', String(i.speculativeDraftMax)) } + if (i.draftModelPath && ['mtp', 'draft', 'dflash'].includes(i.speculativeDecoding ?? '')) { + if (i.cpuOnly) args.push('--spec-draft-ngl', '0') + else if (typeof i.gpuLayers === 'number') args.push('--spec-draft-ngl', String(i.gpuLayers)) + } // Grounding models (UI-TARS / Qwen-VL) need a minimum image-token budget or // clicks land in the wrong place; only set when a projector is present. if (i.mmProjPath && typeof i.imageMinTokens === 'number') { @@ -278,7 +283,7 @@ export function launchArgsChanged( ctxSize?: number kvCacheType?: KvCacheType flashAttn?: boolean - gpuLayers?: number + gpuLayers?: number | null threads?: number batchSize?: number speculativeDecoding?: SpeculativeDecodingMode @@ -295,7 +300,7 @@ export function launchArgsChanged( (typeof patch.ctxSize === 'number' && patch.ctxSize !== current.ctxSize) || (patch.kvCacheType !== undefined && patch.kvCacheType !== current.kvCacheType) || (typeof patch.flashAttn === 'boolean' && patch.flashAttn !== current.flashAttn) || - (typeof patch.gpuLayers === 'number' && patch.gpuLayers !== current.gpuLayers) || + (patch.gpuLayers !== undefined && patch.gpuLayers !== current.gpuLayers) || (typeof patch.threads === 'number' && patch.threads !== current.threads) || (typeof patch.batchSize === 'number' && patch.batchSize !== current.batchSize) || (patch.speculativeDecoding !== undefined && diff --git a/src/main/llm/spawn-env.ts b/src/main/llm/spawn-env.ts index 0c250411d..7892f6e5f 100644 --- a/src/main/llm/spawn-env.ts +++ b/src/main/llm/spawn-env.ts @@ -55,7 +55,13 @@ export interface EngineSpawnEnvInput { */ export function engineSpawnEnv(i: EngineSpawnEnvInput): Record { const pathApi = i.platform === 'win32' ? path.win32 : path.posix - const cudaRuntimeDir = i.binDir.endsWith('-cuda') + const folder = pathApi.basename(i.binDir) + const cudaEnv: Record = + (folder.endsWith('-cuda') || (i.platform === 'win32' && folder === 'whisper')) && + i.currentEnv?.CUDA_CACHE_MAXSIZE === undefined + ? { CUDA_CACHE_MAXSIZE: '4294967296' } + : {} + const cudaRuntimeDir = folder.endsWith('-cuda') ? pathApi.join(pathApi.dirname(i.binDir), 'cuda-runtime') : null if (i.platform === 'linux') { @@ -63,10 +69,12 @@ export function engineSpawnEnv(i: EngineSpawnEnvInput): Record { const libraryDirs = cudaRuntimeDir ? [i.binDir, cudaRuntimeDir] : [i.binDir] if (inherited.LD_LIBRARY_PATH) libraryDirs.push(inherited.LD_LIBRARY_PATH) return { + ...cudaEnv, LD_LIBRARY_PATH: libraryDirs.join(path.posix.delimiter) } } const env: Record = { + ...cudaEnv, // macOS: rpath for the co-located dylibs. Harmless elsewhere. DYLD_LIBRARY_PATH: i.binDir } diff --git a/src/main/llm/stream.ts b/src/main/llm/stream.ts index 7c2f2ab98..4a239353b 100644 --- a/src/main/llm/stream.ts +++ b/src/main/llm/stream.ts @@ -18,7 +18,7 @@ import { type SseTimings, type SseUsage } from './sse-stream' -import { modelRequestOptions, serverResponseError } from './http-post' +import { ModelServerError, modelRequestOptions, serverResponseError } from './http-post' import { generationMetrics, type GenerationMetrics } from '../../shared/generation-metrics' /** Injectable clock so the accumulator's timing is testable without waiting in real time. */ @@ -193,6 +193,19 @@ export function streamCompletion( armIdleTimer() const req = http.request(modelRequestOptions(port, Buffer.byteLength(body)), (res) => { + res.on('error', (error) => { + cleanup() + if (!timedOut && !aborted) { + reject( + new ModelServerError( + 'unavailable', + 'The model server connection closed.', + undefined, + error + ) + ) + } + }) if (res.statusCode !== 200) { // Read the server's error body (small, capped) so we surface an ACTIONABLE // message (e.g. context overflow from too many connectors, or a tool schema diff --git a/src/main/mcp-auth.ts b/src/main/mcp-auth.ts index 97081f90c..65e837373 100644 --- a/src/main/mcp-auth.ts +++ b/src/main/mcp-auth.ts @@ -9,32 +9,19 @@ import path from 'path' import type http from 'http' import { randomBytes } from 'crypto' import { app } from 'electron' -import { authorizeBearer, authorizeBearerAny } from './mcp-auth-logic' +import { authorizeBearer } from './mcp-auth-logic' let cached: string | null = null -/** Supplies the tokens that authorize action tools RIGHT NOW - one per paired device that may - * run this Mac's tools. Registered by the pro sync layer, which owns the pairing set; core - * stays free of any pairing/device knowledge. Un-pairing a device drops its token from this - * list, so its next call is rejected - that is what makes tool access revoke on un-pair. */ -export type ActiveActionTokens = () => readonly string[] - -export interface ActiveActionCredential { +interface ActiveActionCredential { deviceId: string token: string } export type ActiveActionCredentials = () => readonly ActiveActionCredential[] -let activeActionTokens: ActiveActionTokens | null = null let activeActionCredentials: ActiveActionCredentials | null = null -/** Install (or clear, with null) the live per-device token provider. When set, ONLY those - * tokens authorize; the legacy single global token is ignored. */ -export function registerActiveActionTokens(provider: ActiveActionTokens | null): void { - activeActionTokens = provider -} - /** Install the device-bound credentials owned by the paired-device layer. */ export function registerActiveActionCredentials(provider: ActiveActionCredentials | null): void { activeActionCredentials = provider @@ -60,9 +47,6 @@ export function authorizeActionRequest(req: http.IncomingMessage): ActionRequest } return { allowed: deviceId !== null, deviceId } } - if (activeActionTokens) { - return { allowed: authorizeBearerAny(provided, activeActionTokens()), deviceId: null } - } return { allowed: authorizeBearer(provided, getActionToken()), deviceId: null } } @@ -72,7 +56,7 @@ function tokenPath(): string { /** The action-tool token, generating + persisting one on first use. 32 random * bytes (64 hex chars). Owner-only file perms. */ -export function getActionToken(): string { +function getActionToken(): string { if (cached) { return cached } @@ -95,17 +79,6 @@ export function getActionToken(): string { return token } -/** True when the request carries a valid action-tool token. Unauthenticated - * requests still get the open model tools - just not the action tools. - * - * When the pro sync layer has registered a live-token provider, ONLY the tokens - * of currently paired + tools-allowed devices authorize (per-device, revoked on - * un-pair). With no provider (free build / no device sync), it falls back to the - * legacy single global token, which is never distributed in that build anyway. */ -export function isActionAuthorized(req: http.IncomingMessage): boolean { - return authorizeActionRequest(req).allowed -} - /** Dev-only: print the action token so a device can be paired for testing. In a * packaged build the token is NEVER logged (that would defeat the gate); the * shipped path surfaces it in a Settings copy-field instead. */ diff --git a/src/main/mcp-server.ts b/src/main/mcp-server.ts index c70f4ef84..9335b7ab0 100644 --- a/src/main/mcp-server.ts +++ b/src/main/mcp-server.ts @@ -20,6 +20,7 @@ import { llm } from './llm' import { generateImage, imageGenStatus } from './imagegen' import * as tts from './tts' import { embeddings } from './embeddings' +import { DEFAULT_EMBEDDER } from './embedding-models' import { mediaText } from './media-text' import { parseDataUrl } from './mcp-parse-data-url' import { runTool, getToolExtensions } from './tools' @@ -285,7 +286,11 @@ export function buildMcpServer( : await embeddings.generateEmbedding(input) return { content: [{ type: 'text', text: JSON.stringify(embedding) }], - structuredContent: { model: 'all-MiniLM-L6-v2', dimensions: embedding.length, embedding } + structuredContent: { + model: DEFAULT_EMBEDDER.model.replace(/^Xenova\//, ''), + dimensions: embedding.length, + embedding + } } } ) diff --git a/src/main/modality-queue/__tests__/jobs.test.ts b/src/main/modality-queue/__tests__/jobs.test.ts index 9faac8ad9..1667fd49a 100644 --- a/src/main/modality-queue/__tests__/jobs.test.ts +++ b/src/main/modality-queue/__tests__/jobs.test.ts @@ -12,8 +12,13 @@ describe('standard modality jobs', () => { expect(CHAT_JOB).toEqual({ tier: 2, label: 'chat', uses: { id: 'llm' }, evicts: ['image'] }) }) - it('IMAGE_JOB evicts the resident LLM (tier 2)', () => { - expect(IMAGE_JOB).toEqual({ tier: 2, label: 'image', uses: { id: 'image' }, evicts: ['llm'] }) + it('IMAGE_JOB evicts the resident LLM and the task engines (tier 2)', () => { + expect(IMAGE_JOB).toEqual({ + tier: 2, + label: 'image', + uses: { id: 'image' }, + evicts: ['llm', 'grounding', 'decision'] + }) }) it('they evict each other (mutually exclusive on unified memory)', () => { diff --git a/src/main/modality-queue/queue.ts b/src/main/modality-queue/queue.ts index 98f9ea346..8bff59acb 100644 --- a/src/main/modality-queue/queue.ts +++ b/src/main/modality-queue/queue.ts @@ -56,8 +56,8 @@ export interface QueueRequest { // The two standard heavy jobs, defined once instead of spelled out at every // call site (chat was inlined 4× in ipc.ts). A chat/tool turn evicts a resident -// image server; image generation evicts the resident LLM — they can't co-reside -// on unified memory. Frozen so a caller can't mutate the shared descriptor. +// image server; image generation evicts the resident LLM and the task engines (grounding, +// decision) — they can't co-reside on unified memory or on one GPU's VRAM. Frozen so a caller can't mutate the shared descriptor. export const CHAT_JOB: QueueRequest = Object.freeze({ tier: 2, label: 'chat', @@ -68,7 +68,7 @@ export const IMAGE_JOB: QueueRequest = Object.freeze({ tier: 2, label: 'image', uses: { id: 'image' }, - evicts: ['llm'] + evicts: ['llm', 'grounding', 'decision'] }) // Background model work (capture distill, replay-vision, secretary/agent passes). @@ -104,7 +104,7 @@ export function backgroundJob(label: string): QueueRequest { } /** Why a preempted job's model request was aborted. */ -export class PreemptedError extends Error { +class PreemptedError extends Error { constructor(readonly by: string) { super(`Stopped to make room for ${by}; it runs again later.`) this.name = 'PreemptedError' diff --git a/src/main/model-server.ts b/src/main/model-server.ts index d5e22a230..eca88c95e 100644 --- a/src/main/model-server.ts +++ b/src/main/model-server.ts @@ -51,6 +51,7 @@ import { whisperModel } from './media-text' import { getActiveModal } from './active-models' import { modalityForKind } from './active-models-logic' import { embeddings } from './embeddings' +import { DEFAULT_EMBEDDER } from './embedding-models' import { docsText, docsHtml, openApiSpec } from './api-docs' import { handleMcpRequest } from './mcp-server' import { logActionTokenForDev } from './mcp-auth' @@ -78,7 +79,16 @@ import { } from './model-server/chat-messages' import { applyThinkingPayload, requestedThinking } from './llm/chat-payload' import { parseMultipart } from './model-server/multipart' -import { tagLlmEntries, modelEntry, ollamaMirror, specialistModelEntries, installedTextModelEntries, installedImageModelEntries, leadWithActiveChatModel, liveChatCapabilities } from './model-server/models-list' +import { + tagLlmEntries, + modelEntry, + ollamaMirror, + specialistModelEntries, + installedTextModelEntries, + installedImageModelEntries, + leadWithActiveChatModel, + liveChatCapabilities +} from './model-server/models-list' import { buildGatewayModalities, type GatewayModalities } from './model-server/health' import { safeProxyResponse } from './model-server/proxy-response' import { writeDiagnosticLog } from './diagnostics-log' @@ -1527,9 +1537,9 @@ async function handleTeamCompletion( } // ─── Embeddings (local MiniLM) ─────────────────────────────────────────────── -// Served by the in-app embedder (all-MiniLM-L6-v2) rather than proxied to -// llama-server, which isn't started with --embeddings. Same model the RAG/memory -// layer uses, so vectors are consistent across the app. +// Served by the in-app embedder (DEFAULT_EMBEDDER, all-MiniLM-L6-v2) rather than +// proxied to llama-server, which isn't started with --embeddings. Same path every +// in-app embedding takes (embeddings.embed). async function handleEmbeddings( req: http.IncomingMessage, res: http.ServerResponse, @@ -1549,19 +1559,22 @@ async function handleEmbeddings( return } const run = async (): Promise => { - const data: { object: string; index: number; embedding: number[] }[] = [] + const vectors: number[][] = [] if (teamMode()) { // Team mode: the whole input in one forward pass, ahead of background indexing. - const vectors = await embedCoalesced(inputs) - vectors.forEach((embedding, index) => data.push({ object: 'embedding', index, embedding })) - } else for (let i = 0; i < inputs.length; i++) { - const embedding = await embeddings.generateEmbedding(inputs[i]!) - data.push({ object: 'embedding', index: i, embedding }) + vectors.push(...(await embedCoalesced(inputs))) + } else { + // One forward pass per batch: padding to the longest input stays bounded. + for (let i = 0; i < inputs.length; i += 32) { + vectors.push( + ...(await embeddings.embed(inputs.slice(i, i + 32), { model: DEFAULT_EMBEDDER.model })) + ) + } } return { object: 'list', - data, - model: 'all-MiniLM-L6-v2', + data: vectors.map((embedding, index) => ({ object: 'embedding', index, embedding })), + model: DEFAULT_EMBEDDER.model.replace(/^Xenova\//, ''), usage: { prompt_tokens: 0, total_tokens: 0 } } } @@ -1692,9 +1705,15 @@ async function handleModelsList( (whisperModel() ? path.basename(whisperModel() as string) : null) const transcription = sttId ? [tag(sttId, 'transcription')] : [] - const { desktopCatalog, listInstalled, getCatalog, getActiveModel } = await import('./models-manager') + const { desktopCatalog, listInstalled, getCatalog, getActiveModel } = + await import('./models-manager') const installed = await listInstalled() - const catalog = (await getCatalog()).models as { id: string; name: string; kind: string; remoteServerId?: string }[] + const catalog = (await getCatalog()).models as { + id: string + name: string + kind: string + remoteServerId?: string + }[] if (!activeRemote) { const downloaded = installedTextModelEntries(catalog, installed, now) if (downloaded.length) text = downloaded @@ -2319,7 +2338,10 @@ export async function startModelServer(port = GATEWAY_PORT): Promise { json( res, 503, - errBody(locked ? 'API key storage is locked.' : 'API key storage is unavailable.', 'server_error') + errBody( + locked ? 'API key storage is locked.' : 'API key storage is unavailable.', + 'server_error' + ) ) return } diff --git a/src/main/model-sizing.ts b/src/main/model-sizing.ts index 769a10c24..e99bc9d03 100644 --- a/src/main/model-sizing.ts +++ b/src/main/model-sizing.ts @@ -12,15 +12,7 @@ // The RAM-fit verdict is defined once in shared/ and re-exported here so existing // main-process importers (setup.ts, tests) are unchanged while the renderer badge // imports the same source directly — one rule, no drift. -export { - fitLevel, - fitTier, - isLoadableOnDevice, - FIT_OK_FRAC, - FIT_TIGHT_FRAC, - type FitLevel, - type FitTier -} from '../shared/model-fit' +export { fitLevel, fitTier, isLoadableOnDevice } from '../shared/model-fit' import { FIT_SOFT_FRAC, FIT_CEIL_FRAC } from '../shared/model-fit' export type KvCacheType = 'f16' | 'q8_0' | 'q4_0' @@ -136,8 +128,8 @@ export function preferredModelIds(ramGb: number, mode: PerformanceMode): string[ // ceiling, offers "Load anyway" instead of a dead end. Only the physics floor // below can truly refuse a load — the point where the OS would kill the app. -// fitTier / isLoadableOnDevice / FitTier now live in ../shared/model-fit (re-exported -// above) so the renderer browse chip and this loader share ONE definition + fractions. +// fitTier / isLoadableOnDevice / FitTier now live in ../shared/model-fit (the functions are +// re-exported above) so the renderer browse chip and this loader share ONE definition + fractions. /** The hard headroom (GB) a "Load anyway" must still leave free after the model's * weights are resident, or the OS starts killing the app. Below this we refuse @@ -158,14 +150,14 @@ export function checkOverrideSurvival(opts: { availGb: number; incomingWeightsGb export interface LoadAttempt { ctxSize: number - gpuLayers: number + gpuLayers: number | null reason: string } /** Try GPU, then CPU at the selected context. Never reduce the selected value. */ -export function loadAttempts(requestedCtx: number, gpuLayers: number): LoadAttempt[] { +export function loadAttempts(requestedCtx: number, gpuLayers: number | null): LoadAttempt[] { const out: LoadAttempt[] = [{ ctxSize: requestedCtx, gpuLayers, reason: 'requested' }] - if (gpuLayers > 0) { + if (gpuLayers === null || gpuLayers > 0) { out.push({ ctxSize: requestedCtx, gpuLayers: 0, reason: 'CPU-only' }) } return out diff --git a/src/main/models-manager.ts b/src/main/models-manager.ts index 9b328d11a..87c848048 100644 --- a/src/main/models-manager.ts +++ b/src/main/models-manager.ts @@ -649,6 +649,20 @@ export async function getCatalog(): Promise<{ kinds: readonly string[]; models: }) const servers = getRemoteVisionServerSettings().servers const remoteModels = remoteVisionInventoryModels(servers) + // A model whose runtime is not on this machine is not offered for download: Kev without its + // runtime, Parakeet without sherpa-onnx. One already downloaded stays listed, so it can be + // deleted. + const { kevRuntimeInstalled, parakeetRuntimeInstalled } = await import('./runtime-availability') + const kev = kevRuntimeInstalled() + const parakeet = parakeetRuntimeInstalled() + const offered = models.filter((entry) => { + const model = entry as { id?: string; engine?: string; files?: { name: string }[] } + const runtimeMissing = + (!kev && isKevModel(model.id)) || (!parakeet && model.engine === 'parakeet') + if (!runtimeMissing) return true + const files = model.files ?? [] + return files.length > 0 && files.every((file) => present(file.name)) + }) // Every row is named by the one shared resolver, so no screen ever shows a file or repo id. const sources: ModelNameSources = { catalog: CATALOG, @@ -661,7 +675,7 @@ export async function getCatalog(): Promise<{ kinds: readonly string[]; models: // list). One file at a time, so a first read holds one template window in memory, not one per // model; cached afterwards. Unknown (not installed, no template) stays undefined: no icon. const traitsById = new Map() - for (const model of models as CatalogEntry[]) { + for (const model of offered as CatalogEntry[]) { if (model.kind !== 'text' && model.kind !== 'vision') continue const name = primaryFileName(model) if (!name || fileSizeOf(dir, name) <= 0) continue @@ -671,7 +685,7 @@ export async function getCatalog(): Promise<{ kinds: readonly string[]; models: return { kinds: MODEL_KINDS, models: [ - ...models.map((model) => ({ ...model, ...traitsById.get((model as CatalogEntry).id) })), + ...offered.map((model) => ({ ...model, ...traitsById.get((model as CatalogEntry).id) })), ...remoteModels ].map((model) => ({ ...model, @@ -1395,6 +1409,8 @@ export async function deleteModel(modelId: string): Promise { } async function deleteModelFiles(modelId: string): Promise { + if (modelId.startsWith('external:')) + return { success: false, error: 'External models are read only.' } const dir = llm.getModelsDir() if (modelId.startsWith('external:')) return trashExternalModel(modelId, dir) // Imported local model: remove its files + registry entry, clear if active. diff --git a/src/main/native-helper-health.ts b/src/main/native-helper-health.ts index 009561e9a..5c06c4ad9 100644 --- a/src/main/native-helper-health.ts +++ b/src/main/native-helper-health.ts @@ -1,5 +1,6 @@ import type { SystemHealthComponentContract } from '../shared/ipc-contracts' -import { ffmpegBin, whisperBin } from './transcription/whisper-cli' +import { ffmpegBin } from './transcription/whisper-cli' +import { nativeBinaryInstalled } from './backend-facts' interface NativeHelperOwner { id: string @@ -12,7 +13,11 @@ interface NativeHelperOwner { // runtime facts a stable System Health identity and label. const NATIVE_HELPERS: readonly NativeHelperOwner[] = [ { id: 'helper-ffmpeg', label: 'Audio decoder (ffmpeg)', resolve: ffmpegBin }, - { id: 'helper-whisper', label: 'Speech to text helper', resolve: whisperBin } + { + id: 'helper-whisper', + label: 'Speech to text helper', + resolve: () => (nativeBinaryInstalled('stt', 'whisper-cli') ? 'whisper-cli' : null) + } ] export function getNativeHelperHealth(): SystemHealthComponentContract[] { diff --git a/src/main/native-library-env.ts b/src/main/native-library-env.ts index 48acde208..25415b246 100644 --- a/src/main/native-library-env.ts +++ b/src/main/native-library-env.ts @@ -6,14 +6,26 @@ export function nativeLibraryEnv( binDir: string, inherited: Record = {} ): Record { - if (platform === 'darwin') return { DYLD_LIBRARY_PATH: binDir } + const pathApi = platform === 'win32' ? path.win32 : path.posix + const folder = pathApi.basename(binDir) + const cudaEnv: Record = + (folder.endsWith('-cuda') || (platform === 'win32' && folder === 'whisper')) && + inherited.CUDA_CACHE_MAXSIZE === undefined + ? { CUDA_CACHE_MAXSIZE: '4294967296' } + : {} + if (platform === 'darwin') return { ...cudaEnv, DYLD_LIBRARY_PATH: binDir } if (platform === 'linux') { return { + ...cudaEnv, LD_LIBRARY_PATH: inherited.LD_LIBRARY_PATH ? `${binDir}:${inherited.LD_LIBRARY_PATH}` : binDir } } if (platform === 'win32') { - return { PATH: `${binDir}${path.win32.delimiter}${inherited.PATH ?? ''}` } + return { + ...cudaEnv, + GGML_VK_DISABLE_BFLOAT16: '1', + PATH: `${binDir}${path.win32.delimiter}${inherited.PATH ?? ''}` + } } - return {} + return cudaEnv } diff --git a/src/main/nvidia-driver.ts b/src/main/nvidia-driver.ts new file mode 100644 index 000000000..34028eae3 --- /dev/null +++ b/src/main/nvidia-driver.ts @@ -0,0 +1,77 @@ +// The installed NVIDIA driver version. CUDA builds need a minimum driver (shared policy +// NVIDIA_DRIVER_FLOORS); offering them on an older driver ends in an engine that cannot start. + +import fs from 'node:fs' +import path from 'node:path' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) + +/** "Kernel Module 580.82.07" (also "Open Kernel Module for x86_64 580.82.07") → "580.82.07". */ +export function parseLinuxDriverVersion(text: string): string | null { + return /Kernel Module[^\n]*?\s(\d+\.\d+(?:\.\d+)?)\s/i.exec(text)?.[1] ?? null +} + +/** The driver version, 'unknown' when an NVIDIA driver is present but its version cannot be read, + * or null when there is no NVIDIA driver. */ +export async function readNvidiaDriverVersion( + platform: NodeJS.Platform = process.platform +): Promise { + if (platform === 'linux') { + try { + return ( + parseLinuxDriverVersion(fs.readFileSync('/proc/driver/nvidia/version', 'utf8')) ?? 'unknown' + ) + } catch { + return null + } + } + if (platform === 'win32') { + const system32 = path.join( + process.env.SystemRoot ?? process.env.WINDIR ?? 'C:\\Windows', + 'System32' + ) + if (!fs.existsSync(path.join(system32, 'nvcuda.dll'))) return null + try { + const { stdout } = await execFileAsync( + path.join(system32, 'nvidia-smi.exe'), + ['--query-gpu=driver_version', '--format=csv,noheader'], + { timeout: 5_000, windowsHide: true } + ) + return /^\s*(\d+\.\d+)/.exec(stdout)?.[1] ?? 'unknown' + } catch { + return 'unknown' + } + } + return null +} + +let cached: string | null | undefined + +/** The last read driver version (null until the first read finishes, or with no NVIDIA driver). */ +export function cachedNvidiaDriverVersion(): string | null { + return cached ?? null +} + +/** Whether Windows has an NVIDIA driver installed (its CUDA driver library is present). */ +export function hasWindowsNvidiaDriver( + windowsDir = process.env.WINDIR ?? process.env.SystemRoot ?? 'C:\\Windows' +): boolean { + return fs.existsSync(path.join(windowsDir, 'System32', 'nvcuda.dll')) +} + +/** Whether Linux has an NVIDIA driver loaded (it exposes /dev/nvidiaN). */ +export function hasLinuxNvidiaDriver(): boolean { + try { + return fs.readdirSync('/dev').some((name) => /^nvidia\d+$/.test(name)) + } catch { + return false + } +} + +/** Read the driver again: at startup, on wake and when the GPU settings open. */ +export async function refreshNvidiaDriverVersion(): Promise { + cached = await readNvidiaDriverVersion().catch(() => null) + return cached +} diff --git a/src/main/ocr.ts b/src/main/ocr.ts index 984e646a3..32cb12cfe 100644 --- a/src/main/ocr.ts +++ b/src/main/ocr.ts @@ -1,5 +1,5 @@ // OCR a screenshot via the bundled macOS Vision binary (electron/accessibility/ocr). -// The legacy text API remains stable. The boxed API owns structured OCR. +// The boxed API owns structured OCR; runPdfOCR reads scanned PDF pages. import { execFile } from 'child_process' import { promisify } from 'util' @@ -84,18 +84,6 @@ export async function runBoxedOCR(imagePath: string): Promise { } } -export async function runOCR(imagePath: string): Promise { - const bin = ocrBin() - if (!bin) return '' - try { - const { stdout } = await execFileAsync(bin, [imagePath], { maxBuffer: 32 * 1024 * 1024 }) - return stdout.trim() - } catch (e) { - console.error('[OCR] failed:', e) - return '' - } -} - /** The Vision helper is present (macOS builds). */ export function visionOcrAvailable(): boolean { return process.platform === 'darwin' && ocrBin() !== null diff --git a/src/main/onnx-device.ts b/src/main/onnx-device.ts index ec926ebd9..4263821ff 100644 --- a/src/main/onnx-device.ts +++ b/src/main/onnx-device.ts @@ -1,28 +1,4 @@ import type { DeviceType } from '@huggingface/transformers' -import { prioritizeBackend, type BackendPreference } from '../shared/backend-preferences' - -/** - * Try one accelerator at a time. ONNX Runtime does not always continue to the - * next provider when a GPU provider is installed but its driver/runtime cannot - * start, so the caller must create a fresh session for each candidate. - */ -export function onnxDeviceCandidates( - platform = process.platform, - preference: BackendPreference = 'auto' -): DeviceType[] { - const accelerated: DeviceType[] = - platform === 'darwin' - ? ['coreml', 'webgpu'] - : platform === 'win32' - ? ['cuda', 'dml', 'webgpu'] - : platform === 'linux' - ? ['cuda', 'webgpu'] - : ['webgpu'] - - return prioritizeBackend([...accelerated, 'cpu'], preference, (device) => - device === 'coreml' ? 'metal' : device === 'dml' ? 'directml' : device as BackendPreference - ) -} export interface LoadedOnnxRuntime { runtime: T @@ -32,14 +8,18 @@ export interface LoadedOnnxRuntime { export async function loadWithOnnxFallback( load: (device: DeviceType) => Promise, - candidates = onnxDeviceCandidates() + candidates: DeviceType[] ): Promise> { const failures: string[] = [] for (const device of candidates) { try { const runtime = await load(device) console.log(`[ONNX] selected provider=${device}`) - return { runtime, device, ...(failures.length ? { fallbackReason: failures.join('; ') } : {}) } + return { + runtime, + device, + ...(failures.length ? { fallbackReason: failures.join('; ') } : {}) + } } catch (error) { const message = error instanceof Error ? error.message : String(error) failures.push(`${device}: ${message}`) diff --git a/src/main/onnx-host/__tests__/onnx-runtime-env.test.ts b/src/main/onnx-host/__tests__/onnx-runtime-env.test.ts new file mode 100644 index 000000000..3339206fb --- /dev/null +++ b/src/main/onnx-host/__tests__/onnx-runtime-env.test.ts @@ -0,0 +1,161 @@ +// The environment a voice or search helper starts with, over a performance pack installed on disk +// the way the installer leaves it. The operating system is the controlled boundary: the platform +// and architecture are set per test, everything else (manifest, pack folder, check records) is real. +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { configureRuntime } from '../../runtime-env' +import { installedOnnxCudaDir, onnxHelperEnv } from '../onnx-runtime-env' +import { onnxCudaVerified } from '../onnx-cuda-verified' +import { probeOnnxCuda } from '../onnx-cuda-probe' + +const ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-onnx-env-')) +const SHA = 'ab'.repeat(32) +const VERSION = 'test-v1' +const original = { + platform: Object.getOwnPropertyDescriptor(process, 'platform')!, + arch: Object.getOwnPropertyDescriptor(process, 'arch')!, + ldPath: process.env.LD_LIBRARY_PATH, + localAppData: process.env.LOCALAPPDATA +} + +function runOn(platform: 'linux' | 'win32' | 'darwin', arch = 'x64'): void { + Object.defineProperty(process, 'platform', { value: platform, configurable: true }) + Object.defineProperty(process, 'arch', { value: arch, configurable: true }) +} + +/** The bundled manifest naming this platform's pack. */ +function manifest(platform: 'linux' | 'win32'): void { + fs.writeFileSync( + path.join(ROOT, 'performance-packs.json'), + JSON.stringify({ + schemaVersion: 1, + cuda: { + [platform]: { + url: `https://runtime.getoffgridai.co/desktop/cuda/${platform}/${VERSION}.tar.gz`, + sha256: SHA, + bytes: 1024, + version: VERSION + } + } + }) + ) +} + +/** A verified pack with its ONNX CUDA libraries, as the installer leaves it. */ +function installPack(platform: 'linux' | 'win32'): string { + const pack = path.join(ROOT, 'performance-packs', `cuda-${VERSION}-${SHA.slice(0, 12)}`) + const libraries = path.join(pack, 'bin', 'onnx-cuda') + fs.mkdirSync(libraries, { recursive: true }) + fs.writeFileSync(path.join(pack, 'verified.sha256'), `${SHA}\n`) + fs.writeFileSync( + path.join(libraries, platform === 'win32' ? 'onnxruntime.dll' : 'libcublasLt.so.13'), + 'test' + ) + return pack +} + +beforeAll(() => { + delete process.env.LOCALAPPDATA + configureRuntime({ dataDir: ROOT, resourceDirs: [ROOT] }) +}) + +afterEach(() => { + Object.defineProperty(process, 'platform', original.platform) + Object.defineProperty(process, 'arch', original.arch) + fs.rmSync(path.join(ROOT, 'performance-packs'), { recursive: true, force: true }) + fs.rmSync(path.join(ROOT, 'performance-packs.json'), { force: true }) +}) + +afterAll(() => { + if (original.ldPath === undefined) delete process.env.LD_LIBRARY_PATH + else process.env.LD_LIBRARY_PATH = original.ldPath + if (original.localAppData !== undefined) process.env.LOCALAPPDATA = original.localAppData + configureRuntime({ dataDir: undefined, resourceDirs: undefined }) + fs.rmSync(ROOT, { recursive: true, force: true }) +}) + +describe('helper environment', () => { + it('starts a helper with the plain environment when no pack is installed', () => { + runOn('linux') + manifest('linux') + expect(installedOnnxCudaDir()).toBeNull() + const env = onnxHelperEnv() + expect(env.LD_LIBRARY_PATH).toBe(process.env.LD_LIBRARY_PATH) + expect(env.OFFGRID_ONNX_CUDA_VERIFIED).toBeUndefined() + }) + + it('puts the pack’s CUDA libraries first on Linux, once', () => { + runOn('linux') + manifest('linux') + const pack = installPack('linux') + const libraries = path.join(pack, 'bin', 'onnx-cuda') + expect(installedOnnxCudaDir()).toBe(libraries) + + process.env.LD_LIBRARY_PATH = '/usr/lib/existing' + expect(onnxHelperEnv().LD_LIBRARY_PATH).toBe(`${libraries}:/usr/lib/existing`) + // Already on the path: not added again. + process.env.LD_LIBRARY_PATH = `${libraries}:/usr/lib/existing` + expect(onnxHelperEnv().LD_LIBRARY_PATH).toBe(`${libraries}:/usr/lib/existing`) + }) + + it('does not offer the pack on an architecture it has no engines for', () => { + runOn('linux', 'arm64') + manifest('linux') + installPack('linux') + expect(installedOnnxCudaDir()).toBeNull() + }) + + it('loads the Windows CUDA addon only after the CUDA check passed (or for the check itself)', () => { + runOn('win32') + manifest('win32') + const pack = installPack('win32') + const libraries = path.join(pack, 'bin', 'onnx-cuda') + + expect(onnxCudaVerified()).toEqual({ usable: false, reason: { code: 'not-verified' } }) + expect(onnxHelperEnv().OFFGRID_ONNX_CUDA_VERIFIED).toBeUndefined() + + // The check itself runs with the addon. + const forced = onnxHelperEnv('force') + expect(forced.OFFGRID_ONNX_CUDA_VERIFIED).toBe('1') + expect(forced.OFFGRID_PERFORMANCE_PACK_BIN).toBe(path.join(pack, 'bin')) + expect(forced.PATH!.startsWith(`${libraries}${path.win32.delimiter}`)).toBe(true) + + // The check passed for this pack and driver: every helper gets the addon. + fs.writeFileSync( + path.join(pack, 'onnx-cuda-verified.json'), + JSON.stringify({ + packRoot: pack, + driver: null, + at: new Date().toISOString() + }) + ) + expect(onnxCudaVerified()).toEqual({ usable: true }) + expect(onnxHelperEnv().OFFGRID_ONNX_CUDA_VERIFIED).toBe('1') + + // A record for another pack does not count. + fs.writeFileSync( + path.join(pack, 'onnx-cuda-verified.json'), + JSON.stringify({ packRoot: '/elsewhere', driver: null, at: new Date().toISOString() }) + ) + expect(onnxCudaVerified().usable).toBe(false) + }) + + it('says why CUDA voice and search are unavailable', async () => { + runOn('darwin', 'arm64') + expect(onnxCudaVerified()).toEqual({ usable: false, reason: { code: 'no-device' } }) + expect(await probeOnnxCuda()).toBe(false) + + runOn('win32') + manifest('win32') + expect(onnxCudaVerified()).toEqual({ usable: false, reason: { code: 'needs-pack' } }) + // No pack, nothing to check. + expect(await probeOnnxCuda()).toBe(false) + + // A pack but no search model on disk yet: the check waits rather than marking CUDA failed. + const pack = installPack('win32') + expect(await probeOnnxCuda()).toBe(false) + expect(fs.existsSync(path.join(pack, 'onnx-cuda-failed.json'))).toBe(false) + }) +}) diff --git a/src/main/onnx-host/__tests__/utility-host.test.ts b/src/main/onnx-host/__tests__/utility-host.test.ts new file mode 100644 index 000000000..7ba6f02ab --- /dev/null +++ b/src/main/onnx-host/__tests__/utility-host.test.ts @@ -0,0 +1,101 @@ +// Voice and search helpers as separate processes: a helper gets its init message first, answers +// requests, and a crash ends only the helper. After repeated crashes it stays down, saying why, +// until it is cleared. Electron's utilityProcess is the controlled boundary: it is backed by a +// real Node child process, which has the same message, exit and kill behaviour. +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterAll, afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', async () => { + const { fork } = await import('node:child_process') + return { + utilityProcess: { + fork: (entry: string, args: string[], options: { env?: Record }) => { + const child = fork(entry, args, { env: options.env, stdio: 'inherit' }) + return { + postMessage: (message: unknown) => child.send(message as object), + on: (event: string, listener: (...values: unknown[]) => void) => + child.on(event, listener), + once: (event: string, listener: (...values: unknown[]) => void) => + child.once(event, listener), + kill: () => child.kill() + } + } + } + } +}) + +import { clearHelperCrashes, forkHelper, helperStoppedReason } from '../utility-host' + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-utility-host-')) +const helper = path.join(dir, 'helper.js') +// Echoes what it was initialised with, answers 'ping', and dies with code 3 on 'crash'. +fs.writeFileSync( + helper, + `let init = null; +process.on('message', (message) => { + if (message.type === 'init') { init = message.data; return; } + if (message === 'crash') process.exit(3); + process.send({ reply: message, init }); +}); +` +) + +const open: Array> = [] +const start = (serviceName: string): ReturnType => { + const handle = forkHelper(helper, { model: 'search' }, { serviceName, label: 'Search', env: {} }) + open.push(handle) + return handle +} +const exitOf = (handle: ReturnType): Promise => + new Promise((resolve) => handle.on('exit', resolve)) + +afterEach(async () => { + await Promise.all(open.splice(0).map((handle) => handle.terminate())) +}) + +afterAll(() => { + fs.rmSync(dir, { recursive: true, force: true }) +}) + +describe('helper processes', () => { + it('sends the helper its init first, then carries requests both ways', async () => { + const search = start('offgrid-test-echo') + const reply = new Promise((resolve) => search.on('message', resolve)) + search.postMessage('ping') + expect(await reply).toEqual({ reply: 'ping', init: { model: 'search' } }) + }) + + it('counts crashes, stays down after three in a minute, and starts again once cleared', async () => { + const name = 'offgrid-test-crash' + for (let i = 0; i < 3; i++) { + const crashing = start(name) + const exited = exitOf(crashing) + crashing.postMessage('crash') + expect(await exited).toBe(3) + // A request to a helper that has exited goes nowhere instead of throwing. + crashing.postMessage('ping') + } + expect(helperStoppedReason(name)).toBe('the helper exited with code 3') + expect(() => start(name)).toThrow('Search stopped: the helper exited with code 3') + + clearHelperCrashes(name) + expect(helperStoppedReason(name)).toBeNull() + const again = start(name) + const reply = new Promise((resolve) => again.on('message', resolve)) + again.postMessage('ping') + expect(await reply).toMatchObject({ reply: 'ping' }) + }) + + it('does not count a helper it stopped itself as a crash', async () => { + const name = 'offgrid-test-terminate' + for (let i = 0; i < 4; i++) { + const helperProcess = start(name) + await helperProcess.terminate() + // Terminating twice is harmless. + await helperProcess.terminate() + } + expect(helperStoppedReason(name)).toBeNull() + }) +}) diff --git a/src/main/onnx-host/helper-port.ts b/src/main/onnx-host/helper-port.ts new file mode 100644 index 000000000..9ece72b5f --- /dev/null +++ b/src/main/onnx-host/helper-port.ts @@ -0,0 +1,42 @@ +// The child side of a helper process (utility-host.ts): the first message carries the init data, +// every later message is a request. Replies go back through the same port. + +interface ParentPort { + on(event: 'message', listener: (event: { data: unknown }) => void): void + postMessage(message: unknown): void +} + +export interface HelperPort { + post(message: unknown): void + onRequest(listener: (request: Request) => void): void +} + +/** Resolves once the host has sent the init data. Throws outside a utility process. */ +export function connectHelper(): Promise<{ init: Init; port: HelperPort }> { + const parent = (process as unknown as { parentPort?: ParentPort }).parentPort + if (!parent) throw new Error('This helper must be started as an Electron utility process.') + return new Promise((resolve) => { + let init: Init | undefined + const queued: Request[] = [] + let listener: ((request: Request) => void) | null = null + parent.on('message', ({ data }) => { + const message = data as { type?: string; data?: Init } + if (init === undefined && message.type === 'init') { + init = message.data as Init + resolve({ + init, + port: { + post: (reply) => parent.postMessage(reply), + onRequest: (next) => { + listener = next + for (const request of queued.splice(0)) next(request) + } + } + }) + return + } + if (listener) listener(data as Request) + else queued.push(data as Request) + }) + }) +} diff --git a/src/main/onnx-host/onnx-cuda-probe.ts b/src/main/onnx-host/onnx-cuda-probe.ts new file mode 100644 index 000000000..55dc4823f --- /dev/null +++ b/src/main/onnx-host/onnx-cuda-probe.ts @@ -0,0 +1,116 @@ +// Windows: ONNX Runtime's CUDA addon (from the performance pack) replaces the stock addon, which +// carries DirectML and WebGPU. Loading it on a machine where CUDA cannot start would leave voice +// and search on the CPU, so it is loaded only after one embedding has run on CUDA in a helper. +// The result is kept per pack and driver version, so a driver update tests again. + +import fs from 'node:fs' +import path from 'node:path' +import { cachedNvidiaDriverVersion } from '../nvidia-driver' +import { + matches, + onnxCudaVerified, + packRoot, + readRecord, + type ProbeRecord +} from './onnx-cuda-verified' +import { resolveTransformersPackage, modelReadDirectories } from '../model-paths' +import { chosenModelsDir, modelsDir } from '../runtime-env' +import { DEFAULT_EMBEDDER } from '../embedding-models' + +const PROBE_TIMEOUT_MS = 120_000 + +function needsProbe(root: string): boolean { + return ( + !matches(readRecord(path.join(root, 'onnx-cuda-verified.json')), root) && + !matches(readRecord(path.join(root, 'onnx-cuda-failed.json')), root) + ) +} + +let running: Promise | null = null + +/** Run one embedding on CUDA in a helper and record the result. Windows only; true when CUDA ran. + * Waits until the default embedder is on disk, so an offline machine is not marked failed. */ +export function probeOnnxCuda(): Promise { + if (process.platform !== 'win32' || process.arch !== 'x64') return Promise.resolve(false) + const root = packRoot() + if (!root || !needsProbe(root)) return Promise.resolve(onnxCudaVerified().usable) + if ( + !resolveTransformersPackage(DEFAULT_EMBEDDER.model, 'model.onnx', modelReadDirectories()) && + !resolveTransformersPackage( + DEFAULT_EMBEDDER.model, + 'model_quantized.onnx', + modelReadDirectories() + ) + ) { + return Promise.resolve(false) + } + running ??= runProbe(root).finally(() => { + running = null + }) + return running +} + +async function runProbe(root: string): Promise { + const [{ forkHelper }, { onnxHelperEnv }] = await Promise.all([ + import('./utility-host'), + import('./onnx-runtime-env') + ]) + const entry = path.join(__dirname, 'embeddings-worker.js') + const record: ProbeRecord = { + packRoot: root, + driver: cachedNvidiaDriverVersion(), + at: new Date().toISOString() + } + let helper: Awaited> | null = null + try { + helper = forkHelper( + entry, + { + modelsDir: modelsDir(), + chosenModelsDir: chosenModelsDir(), + modelReadDirectories: modelReadDirectories(), + devices: ['cuda'] + }, + { serviceName: 'offgrid-onnx-cuda-probe', label: 'GPU check', env: onnxHelperEnv('force') } + ) + const started = helper + const device = await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error('The CUDA check timed out.')), + PROBE_TIMEOUT_MS + ) + started.on( + 'message', + (response: { id: number; vectors?: unknown; device?: string; error?: string }) => { + if (response.id !== 1 || (!response.vectors && !response.error)) return + clearTimeout(timer) + if (response.error) reject(new Error(response.error)) + else resolve(response.device ?? 'cpu') + } + ) + started.on('exit', (code) => { + clearTimeout(timer) + reject(new Error(`The CUDA check stopped (exit code ${code}).`)) + }) + started.postMessage({ id: 1, texts: ['probe'], model: DEFAULT_EMBEDDER.model }) + }) + if (device !== 'cuda') throw new Error(`ONNX Runtime ran on ${device}, not CUDA.`) + fs.writeFileSync(path.join(root, 'onnx-cuda-verified.json'), JSON.stringify(record)) + fs.rmSync(path.join(root, 'onnx-cuda-failed.json'), { force: true }) + console.log('[ONNX] CUDA verified for voice and search') + return true + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + fs.writeFileSync( + path.join(root, 'onnx-cuda-failed.json'), + JSON.stringify({ ...record, error: message }) + ) + console.warn( + '[ONNX] CUDA check failed; voice and search keep DirectML, WebGPU or CPU:', + message + ) + return false + } finally { + await helper?.terminate() + } +} diff --git a/src/main/onnx-host/onnx-cuda-verified.ts b/src/main/onnx-host/onnx-cuda-verified.ts new file mode 100644 index 000000000..68e06c338 --- /dev/null +++ b/src/main/onnx-host/onnx-cuda-verified.ts @@ -0,0 +1,41 @@ +// Whether helpers may load the Windows ONNX CUDA addon: the record the CUDA check +// (onnx-cuda-probe.ts) leaves beside the performance pack, per pack and driver version. + +import fs from 'node:fs' +import path from 'node:path' +import type { BackendReason } from '@offgrid/models' +import { installedPerformancePackBin } from '../performance-pack' +import { cachedNvidiaDriverVersion } from '../nvidia-driver' + +export interface ProbeRecord { + packRoot: string + driver: string | null + at: string + error?: string +} + +export function packRoot(): string | null { + const bin = installedPerformancePackBin() + return bin ? path.dirname(bin) : null +} + +export function readRecord(file: string): ProbeRecord | null { + try { + return JSON.parse(fs.readFileSync(file, 'utf8')) as ProbeRecord + } catch { + return null + } +} + +export function matches(record: ProbeRecord | null, root: string): boolean { + return !!record && record.packRoot === root && record.driver === cachedNvidiaDriverVersion() +} + +/** Whether helpers may load the CUDA addon on this machine (Windows only). */ +export function onnxCudaVerified(): { usable: boolean; reason?: BackendReason } { + if (process.platform !== 'win32') return { usable: false, reason: { code: 'no-device' } } + const root = packRoot() + if (!root) return { usable: false, reason: { code: 'needs-pack' } } + if (matches(readRecord(path.join(root, 'onnx-cuda-verified.json')), root)) return { usable: true } + return { usable: false, reason: { code: 'not-verified' } } +} diff --git a/src/main/onnx-host/onnx-runtime-env.ts b/src/main/onnx-host/onnx-runtime-env.ts new file mode 100644 index 000000000..e3f8fe2e4 --- /dev/null +++ b/src/main/onnx-host/onnx-runtime-env.ts @@ -0,0 +1,43 @@ +// The environment a voice or search helper process starts with. CUDA libraries for ONNX Runtime +// reach the helper here, at spawn, so the app never restarts itself to change its own library path, +// and the CUDA 13 libraries never leak into the chat, image or speech engines. + +import fs from 'node:fs' +import path from 'node:path' +import { installedPerformancePackBin } from '../performance-pack' +import { onnxCudaVerified } from './onnx-cuda-verified' + +/** The pack's ONNX CUDA library folder, when the pack is installed and has one. */ +export function installedOnnxCudaDir(): string | null { + const bin = installedPerformancePackBin() + if (!bin) return null + const libraries = path.join(bin, 'onnx-cuda') + const marker = process.platform === 'win32' ? 'onnxruntime.dll' : 'libcublasLt.so.13' + return fs.existsSync(path.join(libraries, marker)) ? libraries : null +} + +/** `cuda: 'auto'` adds the CUDA libraries only where they are allowed to load; `'force'` is the + * Windows probe that decides whether they are. */ +export function onnxHelperEnv(cuda: 'auto' | 'force' = 'auto'): Record { + const env: Record = {} + for (const [key, value] of Object.entries(process.env)) if (value !== undefined) env[key] = value + const libraries = installedOnnxCudaDir() + if (!libraries) return env + if (process.platform === 'linux') { + const current = (env.LD_LIBRARY_PATH ?? '').split(':').filter(Boolean) + if (!current.includes(libraries)) env.LD_LIBRARY_PATH = [libraries, ...current].join(':') + return env + } + if ( + process.platform === 'win32' && + process.arch === 'x64' && + (cuda === 'force' || onnxCudaVerified().usable) + ) { + // The patched onnxruntime-node binding loads the pack's CUDA addon only with this flag set + // (scripts/patch-windows-onnx-cuda.mjs); without it the stock addon keeps DirectML and WebGPU. + env.OFFGRID_ONNX_CUDA_VERIFIED = '1' + env.OFFGRID_PERFORMANCE_PACK_BIN = path.dirname(libraries) + env.PATH = `${libraries}${path.win32.delimiter}${env.PATH ?? ''}` + } + return env +} diff --git a/src/main/onnx-host/utility-host.ts b/src/main/onnx-host/utility-host.ts new file mode 100644 index 000000000..dc8794e50 --- /dev/null +++ b/src/main/onnx-host/utility-host.ts @@ -0,0 +1,86 @@ +// ONNX Runtime work (voice, search) runs in Electron utility processes, not worker threads. A GPU +// driver crash inside ONNX Runtime then ends the helper, which restarts on the next request; in a +// worker thread it ended the whole app. The helper also gets its own environment, so CUDA library +// paths reach ONNX Runtime without restarting the app or leaking into other child processes. +// +// The handle mirrors the small part of the Worker API the hosts use (postMessage, message and exit +// events, terminate), so the hosts keep their request queues and protocols unchanged. + +import { utilityProcess, type UtilityProcess } from 'electron' +import { onnxHelperEnv } from './onnx-runtime-env' + +/** Crashes allowed per helper in CRASH_WINDOW_MS before it stays down until restarted. */ +const CRASH_LIMIT = 3 +const CRASH_WINDOW_MS = 60_000 + +export interface HelperProcess { + postMessage(message: unknown): void + on(event: 'message', listener: (message: never) => void): void + on(event: 'exit', listener: (code: number) => void): void + terminate(): Promise +} + +const crashes = new Map() +const lastErrors = new Map() + +/** Why a helper is stopped after repeated crashes, or null while it may run. */ +export function helperStoppedReason(serviceName: string): string | null { + const now = Date.now() + const recent = (crashes.get(serviceName) ?? []).filter((at) => now - at < CRASH_WINDOW_MS) + crashes.set(serviceName, recent) + return recent.length >= CRASH_LIMIT + ? (lastErrors.get(serviceName) ?? 'It stopped repeatedly.') + : null +} + +/** Allow a stopped helper to start again (a retry, or a backend or pack change). */ +export function clearHelperCrashes(serviceName: string): void { + crashes.delete(serviceName) + lastErrors.delete(serviceName) +} + +/** Start `entry` as a utility process and send it `init` as its first message. */ +export function forkHelper( + entry: string, + init: unknown, + options: { serviceName: string; label: string; env?: Record } +): HelperProcess { + const stopped = helperStoppedReason(options.serviceName) + if (stopped) throw new Error(`${options.label} stopped: ${stopped}`) + const child: UtilityProcess = utilityProcess.fork(entry, [], { + serviceName: options.serviceName, + env: options.env ?? onnxHelperEnv(), + stdio: 'inherit', + // ONNX Runtime and its execution providers are native libraries loaded at run time. + ...(process.platform === 'darwin' ? { allowLoadingUnsignedLibraries: true } : {}) + }) + let exited = false + let terminating = false + const exit = new Promise((resolve) => { + child.once('exit', (code) => { + exited = true + if (!terminating && code !== 0) { + const now = Date.now() + crashes.set(options.serviceName, [...(crashes.get(options.serviceName) ?? []), now]) + lastErrors.set(options.serviceName, `the helper exited with code ${code}`) + } + resolve() + }) + }) + child.postMessage({ type: 'init', data: init }) + return { + postMessage: (message) => { + if (!exited) child.postMessage(message) + }, + on: (event: 'message' | 'exit', listener: (value: never) => void) => { + if (event === 'message') child.on('message', listener as (message: unknown) => void) + else child.on('exit', listener as (code: number) => void) + }, + terminate: async () => { + if (exited) return + terminating = true + child.kill() + await exit + } + } +} diff --git a/src/main/parsers/__tests__/claude-desktop-parser.test.ts b/src/main/parsers/__tests__/claude-desktop-parser.test.ts deleted file mode 100644 index aec89c79c..000000000 --- a/src/main/parsers/__tests__/claude-desktop-parser.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Branch coverage for the Claude Desktop parser. parser.test.ts covers the happy path - * (titles, user/assistant grouping, empty input); this file targets the branches that - * test does not: BROWSER_URL, the [TITLE] skip, METADATA timestamp match vs non-match, - * continuation-line bracket stripping, and the "commit only non-empty content" guard. - * One case per branch/condition. - */ -import { describe, it, expect } from 'vitest' -import { parseClaudeDesktopOutput } from '../claude-desktop' - -describe('parseClaudeDesktopOutput — metadata branches', () => { - it('extracts a browser URL', () => { - const r = parseClaudeDesktopOutput('[BROWSER_URL] claude.ai/chat/xyz\n[USER] hi') - expect(r.browserUrl).toBe('claude.ai/chat/xyz') - }) - - it('skips a generic [TITLE] line (never appended to a message)', () => { - const r = parseClaudeDesktopOutput('[USER] question\n[TITLE] Some Header\n[ASSISTANT] answer') - const user = r.messages.find((m) => m.role === 'user') - expect(user?.content).toBe('question') // header not folded into the user turn - expect(r.messages.some((m) => m.content.includes('Some Header'))).toBe(false) - }) - - it('captures a METADATA timestamp that looks like a time and attaches it to the next message', () => { - const r = parseClaudeDesktopOutput('[METADATA] 10:42\n[USER] hello') - expect(r.messages[0]!.timestamp).toBe('10:42') - }) - - it('ignores a METADATA value that is not a time (no timestamp set)', () => { - const r = parseClaudeDesktopOutput('[METADATA] not-a-time\n[USER] hello') - expect(r.messages[0]!.timestamp).toBe('') // lastTimestamp never set - }) - - it('carries the most recent timestamp forward to later messages', () => { - const r = parseClaudeDesktopOutput('[METADATA] 09:15\n[USER] first\n[ASSISTANT] reply') - expect(r.messages[0]!.timestamp).toBe('09:15') - expect(r.messages[1]!.timestamp).toBe('09:15') - }) -}) - -describe('parseClaudeDesktopOutput — role + continuation branches', () => { - it('appends a plain continuation line to the current message', () => { - const r = parseClaudeDesktopOutput('[USER] line one\ncontinued line') - expect(r.messages[0]!.content).toBe('line one\ncontinued line') - }) - - it('strips a leading [BRACKET] tag off a continuation line', () => { - // A line starting with an unrecognized [TAG] under an open role has its tag stripped. - const r = parseClaudeDesktopOutput('[ASSISTANT] start\n[NOTE] tail text') - expect(r.messages[0]!.content).toBe('start\ntail text') - }) - - it('drops a continuation line that is empty after stripping its tag', () => { - const r = parseClaudeDesktopOutput('[USER] only line\n[EMPTY]') - expect(r.messages[0]!.content).toBe('only line') - }) - - it('ignores content before any role is opened (no currentRole)', () => { - const r = parseClaudeDesktopOutput('stray text before any role\n[USER] real') - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('real') - }) - - it('starts a new user turn with empty content when the [USER] line has no text', () => { - // [USER] with no inline text, then a continuation supplies the content. - const r = parseClaudeDesktopOutput('[USER]\nthe actual question') - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('the actual question') - }) - - it('does not commit a role that never received any content', () => { - // A bare [ASSISTANT] with nothing after it must not produce an empty message. - const r = parseClaudeDesktopOutput('[USER] hi\n[ASSISTANT]') - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.role).toBe('user') - }) - - it('merges consecutive same-role turns into one message', () => { - const r = parseClaudeDesktopOutput('[USER] part a\n[USER] part b') - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('part a\npart b') - }) - - it('returns undefined titles/url when none are present', () => { - const r = parseClaudeDesktopOutput('[USER] hi') - expect(r.chatTitle).toBeUndefined() - expect(r.windowTitle).toBeUndefined() - expect(r.browserUrl).toBeUndefined() - }) -}) diff --git a/src/main/parsers/__tests__/parser-branches.test.ts b/src/main/parsers/__tests__/parser-branches.test.ts deleted file mode 100644 index cff307c84..000000000 --- a/src/main/parsers/__tests__/parser-branches.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -// Branch-coverage tests for the four chat-capture parsers. parser.test.ts covers -// the happy path; this file drives the branches it leaves untouched - METADATA -// timestamps, [TITLE] skips, continuation lines, empty tags, consecutive same-role -// tags, the inline role-label detector (exact match vs "label + remainder"), and -// each parser's noise-literal / URL-prefix filtering. One case per branch/path. -import { describe, it, expect } from 'vitest' -import { - parseClaudeDesktopOutput, - parseClaudeWebOutput, - parseChatGPTOutput, - parseGeminiOutput -} from '../index' - -// The four parsers share the same tag-driven skeleton (titles / METADATA / [USER] / -// [ASSISTANT] / continuation). These cases must hold for every one of them, so we -// run them across all four to exercise each parser's copy of that skeleton. -const tagParsers: Array<[string, (t: string) => ReturnType]> = [ - ['claude-desktop', parseClaudeDesktopOutput], - ['claude-web', parseClaudeWebOutput], - ['chatgpt', parseChatGPTOutput], - ['gemini', parseGeminiOutput] -] - -describe.each(tagParsers)('shared tag skeleton: %s', (_name, parse) => { - it('captures a [METADATA] timestamp that matches H:MM and attaches it to the message', () => { - const r = parse(`[METADATA] 10:42\n[USER] hi there`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.timestamp).toBe('10:42') - }) - - it('ignores a [METADATA] line with no time pattern (timestamp stays empty)', () => { - const r = parse(`[METADATA] no time here\n[USER] hi there`) - expect(r.messages[0]!.timestamp).toBe('') - }) - - it('skips a generic [TITLE] line without emitting a message', () => { - const r = parse(`[TITLE] Some Section Header\n[USER] real question`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('real question') - }) - - it('appends a continuation line (no tag) to the current role', () => { - const r = parse(`[ASSISTANT] first line\nsecond line`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('first line\nsecond line') - }) - - it('drops a leading orphan continuation line when no role is active yet', () => { - const r = parse(`orphan text before any role\n[USER] question`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('question') - }) - - it('strips a bracket tag prefix off a continuation line', () => { - const r = parse(`[USER] question\n[IGNORED] continued body`) - expect(r.messages[0]!.content).toBe('question\ncontinued body') - }) - - it('starts a fresh message with an empty body when a [USER] tag has no inline content', () => { - const r = parse(`[ASSISTANT] answer\n[USER]\nfollow up`) - const users = r.messages.filter((m) => m.role === 'user') - expect(users).toHaveLength(1) - expect(users[0]!.content).toBe('follow up') - }) - - it('appends inline content to the same role across repeated [USER] tags', () => { - const r = parse(`[USER] one\n[USER] two`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('one\ntwo') - }) - - it('appends inline content to the same role across repeated [ASSISTANT] tags', () => { - const r = parse(`[ASSISTANT] a\n[ASSISTANT] b`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('a\nb') - }) - - it('does not push an empty continuation of the same role', () => { - // Second [USER] tag with no content and role already user -> nothing appended. - const r = parse(`[USER] only\n[USER]`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('only') - }) - - it('extracts window title, chat title, and browser url tags', () => { - const r = parse(`[WINDOW_TITLE] W\n[BROWSER_URL] example.com/x\n[CHAT_TITLE] C\n[USER] q`) - expect(r.windowTitle).toBe('W') - expect(r.browserUrl).toBe('example.com/x') - expect(r.chatTitle).toBe('C') - }) -}) - -describe('Claude capture parity', () => { - it('keeps desktop and web parsing identical across metadata, roles, and continuations', () => { - const input = [ - '[WINDOW_TITLE] Claude', - '[BROWSER_URL] claude.ai/chat/example', - '[CHAT_TITLE] Parser contract', - '[METADATA] 12:34', - '[USER] first line', - 'continued question', - '[ASSISTANT] answer', - '[NOTE] continued answer' - ].join('\n') - - expect(parseClaudeWebOutput(input)).toEqual(parseClaudeDesktopOutput(input)) - }) -}) - -describe('ChatGPT inline role-label detection + noise', () => { - it('detects a bare "ChatGPT said:" label (exact match, no remainder)', () => { - const r = parseChatGPTOutput(`You said: question\nChatGPT said:\nthe answer body`) - expect(r.messages.map((m) => m.role)).toEqual(['user', 'assistant']) - expect(r.messages[1]!.content).toBe('the answer body') - }) - - it('captures the remainder after a "You said: " label on the same line', () => { - const r = parseChatGPTOutput(`You said: inline question`) - expect(r.messages[0]).toMatchObject({ role: 'user', content: 'inline question' }) - }) - - it('drops a role-label remainder that is itself pure noise', () => { - // "you: new chat" -> role user, remainder "new chat" is a noise literal -> dropped. - const r = parseChatGPTOutput(`you: new chat`) - expect(r.messages).toHaveLength(0) - }) - - it('filters a "chatgpt " model-header line as noise', () => { - const r = parseChatGPTOutput(`[USER] chatgpt 4o\n[USER] real`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('real') - }) - - it('filters lines that start with the chatgpt.com / chat.openai.com hosts', () => { - const r = parseChatGPTOutput( - `[USER] chatgpt.com/c/abc\n[USER] chat.openai.com/c/def\n[USER] kept` - ) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('kept') - }) - - it('recognises the "assistant:" label with a remainder', () => { - const r = parseChatGPTOutput(`assistant: hello from assistant`) - expect(r.messages[0]).toMatchObject({ role: 'assistant', content: 'hello from assistant' }) - }) -}) - -describe('Gemini inline role-label detection + noise', () => { - it('detects a bare "Gemini" label as assistant (exact match)', () => { - const r = parseGeminiOutput(`You question here\nGemini\nassistant reply`) - expect(r.messages.map((m) => m.role)).toEqual(['user', 'assistant']) - expect(r.messages[1]!.content).toBe('assistant reply') - }) - - it('captures the remainder after a "you " label', () => { - const r = parseGeminiOutput(`you asked something`) - expect(r.messages[0]).toMatchObject({ role: 'user', content: 'asked something' }) - }) - - it('filters lines starting with the gemini / bard google hosts', () => { - const r = parseGeminiOutput( - `[ASSISTANT] gemini.google.com/app/1\n[ASSISTANT] bard.google.com/x\n[ASSISTANT] kept` - ) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('kept') - }) - - it('drops a role-label remainder that is pure Gemini noise', () => { - const r = parseGeminiOutput(`you show thinking`) - expect(r.messages).toHaveLength(0) - }) - - it('filters a Gemini UI-chrome literal on a continuation line', () => { - // "extensions"/"feedback" are UI-chrome noise literals that are NOT role-label - // prefixes, so they reach the continuation noise filter and are stripped. - const r = parseGeminiOutput(`[ASSISTANT] real answer\nextensions\nfeedback`) - expect(r.messages).toHaveLength(1) - expect(r.messages[0]!.content).toBe('real answer') - }) -}) - -describe('empty / whitespace input', () => { - it.each(tagParsers)('%s returns no messages for empty input', (_n, parse) => { - expect(parse('').messages).toHaveLength(0) - }) -}) diff --git a/src/main/parsers/chatgpt.ts b/src/main/parsers/chatgpt.ts deleted file mode 100644 index dc4024913..000000000 --- a/src/main/parsers/chatgpt.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { ParseResult } from './types' -import { createNoiseFilter, createRoleLabelDetector, parseTaggedChatOutput } from './tagged-chat' - -const isNoiseLine = createNoiseFilter({ - literals: [ - 'chat history', - 'search chats', - 'images', - 'apps', - 'projects', - 'gpts', - 'explore gpts', - 'your chats', - 'new chat', - 'share', - 'skip to content', - 'settings', - 'help', - 'account', - 'upgrade', - 'plans', - 'billing', - 'log out', - 'logout', - 'chatgpt can make mistakes. check important info.', - 'cookie preferences', - 'improve', - 'reports', - 'critique', - 'write', - 'focus' - ], - prefixes: ['chatgpt.com/', 'chat.openai.com/'], - patterns: [/^chatgpt\s*\d/] -}) - -const detectRoleLabel = createRoleLabelDetector([ - { label: 'you said:', role: 'user' }, - { label: 'you:', role: 'user' }, - { label: 'chatgpt said:', role: 'assistant' }, - { label: 'chatgpt:', role: 'assistant' }, - { label: 'assistant:', role: 'assistant' } -]) - -export function parseChatGPTOutput(text: string): ParseResult { - return parseTaggedChatOutput(text, { detectRoleLabel, isNoiseLine }) -} diff --git a/src/main/parsers/claude-desktop.ts b/src/main/parsers/claude-desktop.ts deleted file mode 100644 index 831b1de51..000000000 --- a/src/main/parsers/claude-desktop.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** Parse tagged OCR output captured from the Claude desktop application. */ -import type { ParseResult } from './types' -import { parseTaggedChatOutput } from './tagged-chat' - -export function parseClaudeDesktopOutput(text: string): ParseResult { - return parseTaggedChatOutput(text) -} diff --git a/src/main/parsers/claude-web.ts b/src/main/parsers/claude-web.ts deleted file mode 100644 index cccfe7cff..000000000 --- a/src/main/parsers/claude-web.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** Parse tagged OCR output captured from claude.ai in a browser. */ -import type { ParseResult } from './types' -import { parseTaggedChatOutput } from './tagged-chat' - -export function parseClaudeWebOutput(text: string): ParseResult { - return parseTaggedChatOutput(text) -} diff --git a/src/main/parsers/gemini.ts b/src/main/parsers/gemini.ts deleted file mode 100644 index 53f496cd6..000000000 --- a/src/main/parsers/gemini.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { ParseResult } from './types' -import { createNoiseFilter, createRoleLabelDetector, parseTaggedChatOutput } from './tagged-chat' - -const isNoiseLine = createNoiseFilter({ - literals: [ - 'gemini', - 'chats', - 'new chat', - 'chat history', - 'history', - 'recent', - 'settings', - 'help', - 'feedback', - 'privacy', - 'terms', - 'apps', - 'extensions', - 'upload', - 'send', - 'stop', - 'regenerate', - 'share', - 'export', - 'sign in', - 'signed out', - 'gemini can make mistakes, so double-check it', - 'show thinking', - 'hide thinking', - 'improve', - 'reports', - 'critique', - 'write', - 'focus' - ], - prefixes: ['gemini.google.com/', 'bard.google.com/'] -}) - -const detectRoleLabel = createRoleLabelDetector([ - { label: 'you', role: 'user' }, - { label: 'you:', role: 'user' }, - { label: 'user:', role: 'user' }, - { label: 'gemini', role: 'assistant' }, - { label: 'gemini:', role: 'assistant' }, - { label: 'assistant:', role: 'assistant' } -]) - -export function parseGeminiOutput(text: string): ParseResult { - return parseTaggedChatOutput(text, { detectRoleLabel, isNoiseLine }) -} diff --git a/src/main/parsers/index.ts b/src/main/parsers/index.ts deleted file mode 100644 index 5bf62f603..000000000 --- a/src/main/parsers/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Parser module index - * - * Re-exports all platform-specific parsers for clean imports. - * Each parser is completely independent - no shared logic between them. - */ - -export { parseClaudeDesktopOutput } from './claude-desktop' -export { parseClaudeWebOutput } from './claude-web' -export { parseChatGPTOutput } from './chatgpt' -export { parseGeminiOutput } from './gemini' diff --git a/src/main/parsers/tagged-chat.ts b/src/main/parsers/tagged-chat.ts deleted file mode 100644 index 2dea724ef..000000000 --- a/src/main/parsers/tagged-chat.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { ParseResult } from './types' - -type MessageRole = 'user' | 'assistant' - -interface RoleLabel { - role: MessageRole - remainder?: string -} - -interface RoleLabelDefinition { - label: string - role: MessageRole -} - -interface NoiseFilterOptions { - literals: readonly string[] - prefixes?: readonly string[] - patterns?: readonly RegExp[] -} - -interface TaggedChatParserConfig { - detectRoleLabel?: (value: string) => RoleLabel | null - isNoiseLine?: (value: string) => boolean -} - -interface TaggedChatMetadata { - browserUrl?: string - chatTitle?: string - lastTimestamp: string - windowTitle?: string -} - -const keepLine = (): boolean => false - -function consumeMetadataLine(trimmed: string, metadata: TaggedChatMetadata): boolean { - if (trimmed.startsWith('[WINDOW_TITLE]')) { - metadata.windowTitle = trimmed.replace('[WINDOW_TITLE]', '').trim() - return true - } - if (trimmed.startsWith('[BROWSER_URL]')) { - metadata.browserUrl = trimmed.replace('[BROWSER_URL]', '').trim() - return true - } - if (trimmed.startsWith('[CHAT_TITLE]')) { - metadata.chatTitle = trimmed.replace('[CHAT_TITLE]', '').trim() - return true - } - if (trimmed.startsWith('[TITLE]')) { - return true - } - if (trimmed.startsWith('[METADATA]')) { - const time = trimmed.replace('[METADATA]', '').trim() - if (/\d{1,2}:\d{2}/.test(time)) { - metadata.lastTimestamp = time - } - return true - } - return false -} - -export function createNoiseFilter({ - literals, - prefixes = [], - patterns = [] -}: NoiseFilterOptions): (value: string) => boolean { - const literalSet = new Set(literals) - - return (value): boolean => { - const lower = value.toLowerCase() - return ( - !lower || - literalSet.has(lower) || - prefixes.some((prefix) => lower.startsWith(prefix)) || - patterns.some((pattern) => pattern.test(lower)) - ) - } -} - -export function createRoleLabelDetector( - definitions: readonly RoleLabelDefinition[] -): (value: string) => RoleLabel | null { - return (value) => { - const lower = value.toLowerCase() - - for (const definition of definitions) { - if (lower === definition.label) { - return { role: definition.role } - } - if (lower.startsWith(`${definition.label} `)) { - return { - role: definition.role, - remainder: value.slice(definition.label.length).trim() - } - } - } - - return null - } -} - -export function parseTaggedChatOutput( - text: string, - { detectRoleLabel, isNoiseLine = keepLine }: TaggedChatParserConfig = {} -): ParseResult { - const messages: ParseResult['messages'] = [] - let currentRole: MessageRole | undefined - let currentContent: string[] = [] - const metadata: TaggedChatMetadata = { lastTimestamp: '' } - - const commitCurrent = (): void => { - if (!currentRole || currentContent.length === 0) { - return - } - - const content = currentContent.join('\n').trim() - if (!isNoiseLine(content)) { - messages.push({ role: currentRole, content, timestamp: metadata.lastTimestamp }) - } - } - - const acceptRoleContent = (role: MessageRole, content: string): void => { - if (currentRole !== role) { - commitCurrent() - currentRole = role - currentContent = [] - } - if (content && !isNoiseLine(content)) { - currentContent.push(content) - } - } - - for (const line of text.split('\n')) { - const trimmed = line.trim() - const lineWithoutTag = trimmed.replace(/^\[.*?\]\s*/, '').trim() - const roleLabel = detectRoleLabel?.(lineWithoutTag) - - if (roleLabel) { - commitCurrent() - currentRole = roleLabel.role - currentContent = [] - if (roleLabel.remainder && !isNoiseLine(roleLabel.remainder)) { - currentContent.push(roleLabel.remainder) - } - continue - } - - if (consumeMetadataLine(trimmed, metadata)) { - continue - } - if (trimmed.startsWith('[USER]')) { - acceptRoleContent('user', trimmed.replace('[USER]', '').trim()) - } else if (trimmed.startsWith('[ASSISTANT]')) { - acceptRoleContent('assistant', trimmed.replace('[ASSISTANT]', '').trim()) - } else if (currentRole) { - const content = trimmed.replace(/^\[.*?\]/, '').trim() - if (content && !isNoiseLine(content)) { - currentContent.push(content) - } - } - } - - commitCurrent() - return { - messages, - chatTitle: metadata.chatTitle, - windowTitle: metadata.windowTitle, - browserUrl: metadata.browserUrl - } -} diff --git a/src/main/parsers/types.ts b/src/main/parsers/types.ts deleted file mode 100644 index 9b1f5a52b..000000000 --- a/src/main/parsers/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Shared types for chat message parsers. - * This is the ONLY shared code between parsers - just the interface definition. - */ - -export interface ParseResult { - messages: { role: string; content: string; timestamp?: string }[] - chatTitle?: string - windowTitle?: string - browserUrl?: string -} diff --git a/src/main/performance-pack.ts b/src/main/performance-pack.ts index dc9031a84..89f51d4ce 100644 --- a/src/main/performance-pack.ts +++ b/src/main/performance-pack.ts @@ -3,30 +3,52 @@ import path from 'node:path' import { createHash } from 'node:crypto' import { execFile } from 'node:child_process' import { promisify } from 'node:util' +import { NVIDIA_DRIVER_FLOORS, nvidiaDriverMeets } from '@offgrid/models' import type { PerformancePackAsset, PerformancePackManifest, PerformancePackStatus } from '../shared/performance-pack' import { dataDir, resourceFile } from './runtime-env' -import { hasWindowsNvidiaDriver, hasLinuxNvidiaDriver } from './transcription/whisper-runtime' +import { + cachedNvidiaDriverVersion, + hasLinuxNvidiaDriver, + hasWindowsNvidiaDriver +} from './nvidia-driver' +import { hasGpuDevice } from './llm/gpu-device-probe' +import { engineSpawnEnv } from './llm/spawn-env' const execFileAsync = promisify(execFile) const SHA256 = /^[a-f0-9]{64}$/i +/** A download that receives nothing for this long pauses, so the user can resume it. */ +const STALL_MS = 60_000 +/** Room left on the drive after the install, so the pack never fills it. */ +const SPACE_MARGIN_BYTES = 1024 ** 3 let controller: AbortController | null = null let task: Promise | null = null let phase: PerformancePackStatus['phase'] | null = null let downloadedBytes = 0 let errorMessage: string | undefined -let installedThisSession = false +let installedDevice: string | undefined let notify: ((status: PerformancePackStatus) => void) | undefined +class StalledDownload extends Error { + constructor() { + super('The download stalled. Resume it to continue.') + } +} + function manifestAsset(): PerformancePackAsset | null { const manifestPath = resourceFile('performance-packs.json') if (!manifestPath) return null try { - const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as Omit & { schemaVersion?: number } + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as Omit< + PerformancePackManifest, + 'schemaVersion' + > & { schemaVersion?: number } if (manifest.schemaVersion !== 1 || !['win32', 'linux'].includes(process.platform)) return null + // The pack carries x64 engines only. + if (process.arch !== 'x64') return null const asset = manifest.cuda[process.platform as 'win32' | 'linux'] if ( !asset || @@ -36,7 +58,8 @@ function manifestAsset(): PerformancePackAsset | null { !Number.isSafeInteger(asset.bytes) || asset.bytes <= 0 || !/^[a-zA-Z0-9._-]+$/.test(asset.version) - ) return null + ) + return null return asset } catch { return null @@ -49,21 +72,57 @@ function hasNvidiaDriver(): boolean { : process.platform === 'linux' && hasLinuxNvidiaDriver() } -function packRoot(asset: PerformancePackAsset): string { - return path.join(dataDir(), 'performance-packs', `cuda-${asset.version}-${asset.sha256.slice(0, 12)}`) +function minDriver(asset: PerformancePackAsset): string { + return asset.minDriver ?? NVIDIA_DRIVER_FLOORS.cuda12[process.platform as 'win32' | 'linux'] +} + +/** A known driver below the pack's floor. An unreadable version is not held against the user. */ +function driverTooOld(asset: PerformancePackAsset): string | null { + const driver = cachedNvidiaDriverVersion() + if (!driver || driver === 'unknown') return null + return nvidiaDriverMeets(driver, minDriver(asset)) ? null : driver +} + +/** + * Where packs live, newest location first. Windows installs go under Local AppData: Roaming + * AppData syncs with roaming profiles, and a 7 GB pack would sync at every sign-in. A pack already + * installed in the older location is still found. + */ +function packParents(): string[] { + const legacy = path.join(dataDir(), 'performance-packs') + const local = + process.platform === 'win32' && process.env.LOCALAPPDATA + ? path.join(process.env.LOCALAPPDATA, 'Off Grid AI Desktop', 'performance-packs') + : null + return local && local !== legacy ? [local, legacy] : [legacy] +} + +function packName(asset: PerformancePackAsset): string { + return `cuda-${asset.version}-${asset.sha256.slice(0, 12)}` } function installedBin(asset: PerformancePackAsset): string | null { - const root = packRoot(asset) - const marker = path.join(root, 'verified.sha256') - try { - if (fs.readFileSync(marker, 'utf8').trim() !== asset.sha256.toLowerCase()) return null - const bin = path.join(root, 'bin') - if (!fs.statSync(bin).isDirectory()) return null - return bin - } catch { - return null + for (const parent of packParents()) { + const root = path.join(parent, packName(asset)) + try { + if ( + fs.readFileSync(path.join(root, 'verified.sha256'), 'utf8').trim() !== + asset.sha256.toLowerCase() + ) + continue + const bin = path.join(root, 'bin') + if (fs.statSync(bin).isDirectory()) return bin + } catch { + // Not installed here. + } } + return null +} + +/** The verified pack's binary root, including before environment activation. */ +export function installedPerformancePackBin(): string | null { + const asset = manifestAsset() + return asset ? installedBin(asset) : null } /** Run once during main-process setup and after installation. */ @@ -74,37 +133,75 @@ export function activateInstalledPerformancePack(): void { else delete process.env.OFFGRID_PERFORMANCE_PACK_BIN } -/** ONNX loads CUDA with dlopen, so Linux must see these libraries at process start. */ -export function prepareInstalledOnnxCudaLibraries(): boolean { - if (process.platform !== 'linux') return false - const asset = manifestAsset() - const bin = asset && installedBin(asset) - if (!bin) return false - const libraries = path.join(bin, 'onnx-cuda') - if (!fs.existsSync(path.join(libraries, 'libcublasLt.so.13')) || - !fs.existsSync(path.join(libraries, 'libcudnn.so.9'))) return false - const current = (process.env.LD_LIBRARY_PATH ?? '').split(':').filter(Boolean) - if (current.includes(libraries)) return false - process.env.LD_LIBRARY_PATH = [libraries, ...current].join(':') - return true +function freeBytes(dir: string): number | null { + try { + const stat = fs.statfsSync(dir) + return Number(stat.bavail) * Number(stat.bsize) + } catch { + return null + } +} + +/** Space the rest of the download, the extracted files and a margin need. */ +function neededBytes(asset: PerformancePackAsset, archive: string): number { + const partial = fs.existsSync(archive) ? Math.min(fs.statSync(archive).size, asset.bytes) : 0 + return asset.bytes - partial + (asset.extractedBytes ?? asset.bytes * 2) + SPACE_MARGIN_BYTES +} + +function newInstallTarget(asset: PerformancePackAsset): { + parent: string + root: string + archive: string +} { + const parent = packParents()[0]! + const root = path.join(parent, packName(asset)) + return { parent, root, archive: `${root}.tar.gz.part` } +} + +/** The nearest existing folder, for a free-space reading before the pack folder exists. */ +function existingAncestor(dir: string): string { + let current = dir + while (!fs.existsSync(current) && path.dirname(current) !== current) + current = path.dirname(current) + return current } export function performancePackStatus(): PerformancePackStatus { const asset = manifestAsset() if (!asset) return { phase: 'unavailable', bytes: 0, downloadedBytes: 0 } if (!hasNvidiaDriver()) return { phase: 'not-needed', bytes: 0, downloadedBytes: 0 } + const driver = cachedNvidiaDriverVersion() ?? undefined if (installedBin(asset)) { return { - phase: 'installed', bytes: asset.bytes, downloadedBytes: asset.bytes, - version: asset.version, restartRequired: installedThisSession + phase: 'installed', + bytes: asset.bytes, + downloadedBytes: asset.bytes, + version: asset.version, + ...(installedDevice ? { device: installedDevice } : {}), + ...(driver ? { driver } : {}) } } - const current = phase ?? 'available' + const tooOld = driverTooOld(asset) + if (tooOld) { + return { + phase: 'driver-too-old', + bytes: asset.bytes, + downloadedBytes: 0, + version: asset.version, + driver: tooOld, + minDriver: minDriver(asset) + } + } + const target = newInstallTarget(asset) + const free = freeBytes(existingAncestor(target.parent)) return { - phase: current, + phase: phase ?? 'available', bytes: asset.bytes, downloadedBytes, version: asset.version, + needBytes: neededBytes(asset, target.archive), + ...(free === null ? {} : { freeBytes: free }), + ...(driver ? { driver } : {}), ...(errorMessage ? { error: errorMessage } : {}) } } @@ -113,93 +210,215 @@ function publish(): void { notify?.(performancePackStatus()) } +function setPhase(next: PerformancePackStatus['phase']): void { + phase = next + publish() +} + export function onPerformancePackChanged(callback: (status: PerformancePackStatus) => void): void { notify = callback } +const installedListeners = new Set<() => void>() + +/** Called once a pack is installed and active. The owners of the engines that use it (voice, + * search, best settings) refresh from here; this module does not know them. */ +export function onPerformancePackInstalled(listener: () => void): () => void { + installedListeners.add(listener) + return () => installedListeners.delete(listener) +} + async function digest(file: string): Promise { const hash = createHash('sha256') for await (const chunk of fs.createReadStream(file)) hash.update(chunk) return hash.digest('hex') } -async function fetchPack(asset: PerformancePackAsset, signal: AbortSignal): Promise { - const root = packRoot(asset) - const parent = path.dirname(root) - fs.mkdirSync(parent, { recursive: true }) - const archive = `${root}.tar.gz.part` - const existing = fs.existsSync(archive) ? fs.statSync(archive).size : 0 - const response = await fetch(asset.url, { - signal, - headers: existing > 0 ? { Range: `bytes=${existing}-` } : undefined - }) - if (!response.ok || !response.body) throw new Error(`Download failed: HTTP ${response.status}`) - const append = existing > 0 && response.status === 206 - if (append && !response.headers.get('content-range')?.startsWith(`bytes ${existing}-`)) { - throw new Error('The server returned an invalid resume range.') - } - const stream = fs.createWriteStream(archive, { flags: append ? 'a' : 'w' }) - downloadedBytes = append ? existing : 0 - publish() - let lastProgressPublishedAt = Date.now() +/** One read of the response body, or a stall error when nothing arrives for STALL_MS. */ +async function readWithStall(read: Promise): Promise { + let timer: ReturnType | undefined try { - const reader = response.body.getReader() - for (;;) { - const { done, value } = await reader.read() - if (done) break - if (signal.aborted) throw signal.reason - if (!stream.write(value)) { - await new Promise((resolve, reject) => { - stream.once('drain', resolve) - stream.once('error', reject) - }) - } - downloadedBytes += value.byteLength - if (downloadedBytes > asset.bytes) throw new Error('The download is larger than expected.') - const now = Date.now() - if (now - lastProgressPublishedAt >= 250 || downloadedBytes === asset.bytes) { - publish() - lastProgressPublishedAt = now + return await Promise.race([ + read, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new StalledDownload()), STALL_MS) + }) + ]) + } finally { + clearTimeout(timer) + } +} + +/** Download the rest of the archive. A complete archive is left as it is. */ +async function download( + asset: PerformancePackAsset, + archive: string, + signal: AbortSignal +): Promise { + for (let attempt = 0; attempt < 2; attempt++) { + let existing = fs.existsSync(archive) ? fs.statSync(archive).size : 0 + if (existing > asset.bytes) { + fs.rmSync(archive, { force: true }) + existing = 0 + } + downloadedBytes = existing + if (existing === asset.bytes) return + const response = await fetch(asset.url, { + signal, + headers: existing > 0 ? { Range: `bytes=${existing}-` } : undefined + }) + // 416: the server has nothing past what is on disk. Start over once. + if (response.status === 416) { + fs.rmSync(archive, { force: true }) + continue + } + if (!response.ok || !response.body) + throw new Error(`The download failed (HTTP ${response.status}). Try again.`) + const append = existing > 0 && response.status === 206 + if (append && !response.headers.get('content-range')?.startsWith(`bytes ${existing}-`)) { + throw new Error('The server returned an invalid resume range.') + } + const stream = fs.createWriteStream(archive, { flags: append ? 'a' : 'w' }) + downloadedBytes = append ? existing : 0 + publish() + let lastProgressPublishedAt = Date.now() + try { + const reader = response.body.getReader() + for (;;) { + const { done, value } = await readWithStall(reader.read()) + if (done) break + if (signal.aborted) throw signal.reason + if (!stream.write(value)) { + await new Promise((resolve, reject) => { + stream.once('drain', resolve) + stream.once('error', reject) + }) + } + downloadedBytes += value.byteLength + if (downloadedBytes > asset.bytes) throw new Error('The download is larger than expected.') + const now = Date.now() + if (now - lastProgressPublishedAt >= 250 || downloadedBytes === asset.bytes) { + publish() + lastProgressPublishedAt = now + } } + await new Promise((resolve, reject) => { + stream.once('error', reject) + stream.end(resolve) + }) + } catch (error) { + stream.destroy() + throw error } - await new Promise((resolve, reject) => { - stream.once('error', reject) - stream.end(resolve) + return + } + throw new Error('The download could not be resumed. Try again.') +} + +/** Windows ships bsdtar in System32; a GNU tar earlier on PATH (Git for Windows) reads "C:\…" as + * a remote host. */ +function tarCommand(): string { + if (process.platform !== 'win32') return 'tar' + const system = path.join( + process.env.SystemRoot ?? process.env.WINDIR ?? 'C:\\Windows', + 'System32', + 'tar.exe' + ) + return fs.existsSync(system) ? system : 'tar' +} + +/** Proof that the pack's CUDA engine starts on this GPU: the device it reports, or null. */ +async function smokeTest(bin: string): Promise { + const server = path.join( + bin, + 'llama-cuda', + process.platform === 'win32' ? 'llama-server.exe' : 'llama-server' + ) + const binDir = path.dirname(server) + try { + const { stdout, stderr } = await execFileAsync(server, ['--list-devices'], { + timeout: 30_000, + maxBuffer: 512 * 1024, + windowsHide: true, + env: { + ...process.env, + ...engineSpawnEnv({ platform: process.platform, binDir, currentEnv: process.env }) + } }) - } catch (error) { - stream.destroy() - throw error + const output = `${stdout}\n${stderr}` + if (!hasGpuDevice(output, 'CUDA')) return null + const devices = output.split(/Available devices:/i)[1] ?? '' + return /^\s*CUDA0\s*:\s*([^(\r\n]+)/m.exec(devices)?.[1]?.trim() || 'NVIDIA GPU' + } catch { + return null + } +} + +async function fetchPack(asset: PerformancePackAsset, signal: AbortSignal): Promise { + const { parent, root, archive } = newInstallTarget(asset) + fs.mkdirSync(parent, { recursive: true }) + const free = freeBytes(parent) + const need = neededBytes(asset, archive) + if (free !== null && free < need) { + const gb = (bytes: number): string => (bytes / 1024 ** 3).toFixed(1) + throw new Error(`Needs ${gb(need)} GB free on this drive; it has ${gb(free)} GB.`) } - if (downloadedBytes !== asset.bytes || (await digest(archive)) !== asset.sha256.toLowerCase()) { + await download(asset, archive, signal) + + setPhase('verifying') + if ( + fs.statSync(archive).size !== asset.bytes || + (await digest(archive)) !== asset.sha256.toLowerCase() + ) { fs.rmSync(archive, { force: true }) - throw new Error('The downloaded performance files failed the size or hash check.') + downloadedBytes = 0 + throw new Error('The download was damaged. Download it again.') } + + setPhase('installing') const staging = `${root}.staging` fs.rmSync(staging, { recursive: true, force: true }) fs.mkdirSync(staging, { recursive: true }) try { - await execFileAsync('tar', ['-xzf', archive, '-C', staging], { timeout: 300_000 }) + // No timeout: 7 GB extracts slowly on some disks. The verified archive stays until the install + // succeeds, so a retry installs again without downloading again. + try { + await execFileAsync(tarCommand(), ['-xzf', archive, '-C', staging], { windowsHide: true }) + } catch { + throw new Error('Could not install the files. Try again.') + } const bin = path.join(staging, 'bin') - const server = path.join(bin, 'llama-cuda', process.platform === 'win32' ? 'llama-server.exe' : 'llama-server') + const server = path.join( + bin, + 'llama-cuda', + process.platform === 'win32' ? 'llama-server.exe' : 'llama-server' + ) if (!fs.statSync(bin).isDirectory() || !fs.statSync(server).isFile()) { throw new Error('The performance pack is missing its CUDA chat engine.') } - if (process.platform === 'linux' && - (!fs.existsSync(path.join(bin, 'onnx-cuda', 'libcublasLt.so.13')) || - !fs.existsSync(path.join(bin, 'onnx-cuda', 'libcudnn.so.9')))) { + if ( + process.platform === 'linux' && + (!fs.existsSync(path.join(bin, 'onnx-cuda', 'libcublasLt.so.13')) || + !fs.existsSync(path.join(bin, 'onnx-cuda', 'libcudnn.so.9'))) + ) { throw new Error('The performance pack is missing its CUDA speech libraries.') } + + setPhase('testing') + const device = await smokeTest(bin) + if (!device) + throw new Error( + 'The GPU did not start with these files. Update your NVIDIA driver, then try again.' + ) + fs.writeFileSync(path.join(staging, 'verified.sha256'), `${asset.sha256.toLowerCase()}\n`) fs.rmSync(root, { recursive: true, force: true }) fs.renameSync(staging, root) fs.rmSync(archive, { force: true }) - installedThisSession = true + installedDevice = device activateInstalledPerformancePack() - // The CUDA engine is now available: auto-managed installs move onto it. - void import('./best-settings').then((m) => { - m.forgetDetectedMachine() - return m.ensureBestSettings() - }).catch((error: unknown) => console.warn('[best-settings] CUDA refresh failed', error)) + // Engines read the pack at their next spawn; their owners refresh onto it. No app restart. + for (const listener of installedListeners) listener() } catch (error) { fs.rmSync(staging, { recursive: true, force: true }) throw error @@ -208,7 +427,8 @@ async function fetchPack(asset: PerformancePackAsset, signal: AbortSignal): Prom export function startPerformancePack(): PerformancePackStatus { const asset = manifestAsset() - if (!asset || !hasNvidiaDriver() || installedBin(asset)) return performancePackStatus() + if (!asset || !hasNvidiaDriver() || installedBin(asset) || driverTooOld(asset)) + return performancePackStatus() if (task !== null) return performancePackStatus() phase = 'downloading' errorMessage = undefined @@ -216,10 +436,15 @@ export function startPerformancePack(): PerformancePackStatus { const signal = controller.signal publish() task = fetchPack(asset, signal) - .then(() => { phase = 'installed' }) + .then(() => { + phase = 'installed' + }) .catch((error: unknown) => { if (signal.aborted) phase = 'paused' - else { + else if (error instanceof StalledDownload) { + phase = 'paused' + errorMessage = error.message + } else { phase = 'failed' errorMessage = error instanceof Error ? error.message : String(error) } diff --git a/src/main/prompts.ts b/src/main/prompts.ts index 0791091ca..587161cf9 100644 --- a/src/main/prompts.ts +++ b/src/main/prompts.ts @@ -16,245 +16,6 @@ export interface PromptDef { // Default templates — extracted from the original hardcoded prompts in ipc.ts // --------------------------------------------------------------------------- -const MASTER_MEMORY_INITIAL = `Create a MASTER MEMORY about this user from the conversation summary below. Write in third person. - -Include sections for: About the User, Projects & Work, Technical Environment, Preferences, Key Decisions, Important Relationships. Only include sections with actual content. - -Be specific — include names, versions, concrete details. Skip generic info. - -Conversation Summary: -{{SUMMARY}} - -Master Memory:` - -const MASTER_MEMORY_INCREMENTAL = `Update this master memory by integrating the new conversation summary. Write in third person. - -Rules: PRESERVE all existing info. ADD new facts. MERGE related info into sections. Only UPDATE if directly contradicted. - -CURRENT MASTER MEMORY: -{{CURRENT_MASTER}} - -NEW SUMMARY: -{{NEW_SUMMARY}} - -Updated Master Memory:` - -const MASTER_MEMORY_BATCH_FIRST = `Create a MASTER MEMORY about this user from the conversation summaries below. Write in third person. - -Include sections for: About the User, Projects & Work, Technical Environment, Preferences, Key Decisions, Important Relationships. Only include sections with actual content. Be specific. - -Conversation Summaries: -{{BATCH_TEXT}} - -Master Memory:` - -const MASTER_MEMORY_BATCH_EXPAND = `Update this master memory by integrating new conversation summaries. PRESERVE existing info, ADD new facts. Write in third person. - -CURRENT MASTER MEMORY: -{{CURRENT_MASTER}} - -ADDITIONAL SUMMARIES (batch {{BATCH_NUM}}): -{{BATCH_TEXT}} - -Updated Master Memory:` - -const MASTER_MEMORY_MERGE = `Merge these partial summaries into one unified MASTER MEMORY about the user. Write in third person. - -Combine all information, remove duplicates, organize into sections: About the User, Projects & Work, Technical Environment, Preferences, Key Decisions, Important Relationships. Be specific — keep all names, versions, concrete details. - -Partial Summaries: -{{PARTIAL_SUMMARIES}} - -Master Memory:` - -const MEMORY_FILTER_LENIENT = `You are a inclusive memory filter. Decide if the following single message should be saved as LONG-TERM memory that a user would want in future chats. - -Store information that could be useful in future conversations, including: -- Personal preferences, facts, or profile information -- Projects, specs, requirements, or technical choices -- Decisions, commitments, or agreements -- Instructions, constraints, or guidelines -- Plans, schedules, goals, or deadlines -- Facts about entities, tools, technologies mentioned -- General knowledge shared that might be referenced again - -Do NOT store: -- Simple greetings or pleasantries -- Generic filler phrases - -If the role is 'assistant', store ONLY if it repeats a verified user fact or a decision the user made. -If unsure, set store to true. - -Return JSON ONLY with this shape: -{ - "store": boolean, - "name": "short 3-5 word title for this memory", - "memory": "short standalone statement suitable for future retrieval" -} - -Role: {{ROLE}} -Message: {{MESSAGE}} -JSON:` - -const MEMORY_FILTER_BALANCED = `You are a balanced memory filter. Decide if the following single message should be saved as LONG-TERM memory that a user would want in future chats. - -Store ONLY explicitly stated, durable, high-signal information such as: -- Personal preferences the user directly and clearly stated ("I prefer X", "I always use Y") -- Concrete project names, product names, or specific technical architecture decisions -- Firm decisions or commitments the user explicitly made ("We decided to go with X", "I chose Y") -- Specific constraints, requirements, or deadlines stated as facts -- Long-term goals or plans the user described in their own words - -Do NOT store: -- Greetings, small talk, pleasantries, or filler -- Code snippets, error messages, stack traces, or log output -- Questions the user asked (unless the question itself reveals a strong preference) -- Assistant explanations, suggestions, or recommendations (unless the user confirmed them as a decision) -- One-off troubleshooting steps, debugging attempts, or ephemeral details -- Generic or common-knowledge statements ("React is a library", "APIs use HTTP") -- Vague or implied preferences ("seems like they like X") -- Redundant facts already implied by the message itself -- Speculation, hypotheticals, or unverified claims -- Anything that won't clearly matter in future conversations - -If the role is 'assistant', store ONLY if it repeats a verified user fact or a decision the user made. -If unsure, set store to false. - -Return JSON ONLY with this shape: -{ - "store": boolean, - "name": "short 3-5 word title for this memory", - "memory": "short standalone statement suitable for future retrieval" -} - -Role: {{ROLE}} -Message: {{MESSAGE}} -JSON:` - -const MEMORY_FILTER_STRICT = `You are a very strict memory filter. Decide if the following single message should be saved as LONG-TERM memory that a user would want in future chats. - -Store ONLY high-value, explicitly stated information: -- Direct personal preferences the user explicitly stated -- Major decisions or commitments clearly made -- Critical project requirements explicitly mentioned -- Core facts about important entities central to the user's work - -Do NOT store: -- Greetings, small talk, or pleasantries -- Troubleshooting steps or temporary solutions -- Generic statements or common knowledge -- Technical details that are easily searchable -- Speculation, suggestions, or hypotheticals -- Anything not explicitly and directly stated by the user -- Inferred or implied information - -If the role is 'assistant', store ONLY if it repeats a verified user fact or a decision the user made. -If unsure, set store to false. - -Return JSON ONLY with this shape: -{ - "store": boolean, - "name": "short 3-5 word title for this memory", - "memory": "short standalone statement suitable for future retrieval" -} - -Role: {{ROLE}} -Message: {{MESSAGE}} -JSON:` - -const ENTITY_EXTRACTION_LENIENT = `You are extracting entities from long-term memory statements. Be inclusive - capture entities that might be useful later. - -Return JSON only with this shape: -{ - "entities": [ - { - "name": "string", - "type": "Person | Organization | Product | Place | Object | Project | Concept | Event | Other", - "facts": ["short factual statements about the entity"] - } - ] -} - -Rules: -- Include all entities mentioned in the memory statements. -- Include people, organizations, products, places, projects, technologies, tools, and concepts. -- Be inclusive - if an entity is mentioned, it's probably worth tracking. -- Facts should be concise and verifiable. -- Return {"entities": []} only if there are no discernible entities at all. - -Memory statements: -{{MEMORY_TEXT}} - -JSON:` - -const ENTITY_EXTRACTION_BALANCED = `You are extracting entities from long-term memory statements. Only keep entities worth remembering for future chats. - -Return JSON only with this shape: -{ - "entities": [ - { - "name": "string", - "type": "Person | Organization | Product | Place | Object | Project | Concept | Event | Other", - "facts": ["short factual statements about the entity"] - } - ] -} - -Rules: -- Only include entities explicitly and repeatedly mentioned or clearly central to the user's work. -- Include ONLY entities the user has a specific, ongoing relationship with (their projects, their tools, people they work with, organizations they belong to). -- Exclude passing mentions, one-off references, generic examples, or entities mentioned only in assistant advice. -- Exclude generic or common single-word entities like "API", "database", "server", "frontend", "backend", "app", "website", "code". -- Exclude entities used only as illustrative examples or in generic advice. -- Entity names must be at least 3 characters long. -- Facts must be concise, specific, durable, and directly from the memory statements — no inferred facts. -- If no entities clearly meet the bar, return {"entities": []}. - -Memory statements: -{{MEMORY_TEXT}} - -JSON:` - -const ENTITY_EXTRACTION_STRICT = `You are extracting entities from long-term memory statements. Be very selective - only keep high-value entities. - -Return JSON only with this shape: -{ - "entities": [ - { - "name": "string", - "type": "Person | Organization | Product | Place | Object | Project | Concept | Event | Other", - "facts": ["short factual statements about the entity"] - } - ] -} - -Rules: -- Only include entities that are CENTRAL to the user's work or life. -- Exclude generic tools, common technologies, libraries, or frameworks unless the user has a specific relationship to them. -- Exclude one-time mentions or incidental references. -- Only include entities the user has explicitly discussed in detail or shown clear importance. -- Facts must be specific, verified, and directly from the memory statements. -- When in doubt, exclude the entity. -- Return {"entities": []} if no entities are clearly significant. - -Memory statements: -{{MEMORY_TEXT}} - -JSON:` - -const ENTITY_SUMMARY = `You are updating an entity profile summary. - -Entity: {{NAME}} -Type: {{TYPE}} - -Existing summary: -{{EXISTING_SUMMARY}} - -New facts: -{{NEW_FACTS}} - -Write a concise, well-structured summary. Include only verified facts. Do not add speculation. Output plain text only.` - const SESSION_SUMMARY = `Extract a user profile summary from this conversation. Focus on the USER, not the assistant. Write in third person. Include: what they're working on, their preferences, decisions made, tools/tech they use, people mentioned. @@ -265,157 +26,11 @@ Conversation: User Profile Summary:` -const RAG_CHAT = `You are a helpful assistant. Answer general questions using your knowledge. Use the provided context when the user asks about their memories, conversations, summaries, or entities. -For personal facts, prefer specific evidence from messages, summaries, entities, and memories over the master memory. Use the master memory only as supplemental context. If the provided context does not contain a personal fact, say you do not have that information and ask a brief follow-up question. -Do not fabricate details. Cite claims drawn from the provided SOURCES list using its tag in square brackets, e.g. [S2] or [S5]. Cite only sources you actually used; general knowledge does not need a memory citation. - -ARTIFACTS: You have a LIVE, sandboxed code canvas built into this app. When the user asks you to CREATE or BUILD something runnable — a React app or component, a web page, an interactive UI, a chart/graph, a diagram, or a vector graphic — emit ONE fenced code block and it runs IMMEDIATELY in a canvas beside the chat. You do NOT have or need a terminal, npm, Vite, build tools, or the user's filesystem. NEVER give installation/setup steps (no "npm create vite", "npm install", "run npm run dev"), NEVER tell them to create files, and NEVER say you "can't execute" or "can't build" it — you can, right here. Just write the finished, self-contained code: -- a React app or component -> \`\`\`jsx — define ONE top-level component named \`App\` containing everything; plain JSX, NO import/export statements; React, ReactDOM, useState, useEffect and the other hooks are already global; it is auto-rendered into #root. Put ALL the code in this one block — no separate App.jsx/index.css/main.jsx files. -- a plain web page or interactive UI (no React) -> \`\`\`html (one complete document; inline all CSS and JS; no external/CDN/network resources) -- a flowchart, sequence, or relationship diagram -> \`\`\`mermaid -- a static graphic, icon, or illustration -> \`\`\`svg -When the user says React/Next/component/hooks, ALWAYS use \`\`\`jsx (never \`\`\`html, never setup instructions). The code is shown only in the canvas — do NOT also paste it in prose. Produce a complete, runnable artifact. At most one short sentence before the block. For ordinary questions, answer normally (no code block). - -SKILLS: The user can install skills (reusable instruction packs) in their .skills folder and invoke one with /skill-name. Installed skills: -{{SKILLS_BLOCK}} -When a message invokes a skill, its full instructions are prepended to the question — follow them for that request. - -CLARIFY: When you genuinely need more information to answer well, you may ask ONE concise multiple-choice question instead of guessing. Emit it as a fenced \`\`\`ask block containing only JSON: {"question": "…", "options": ["…", "…"], "multiSelect": false}. The app turns the options into clickable buttons. Use this sparingly — only when the choice materially changes your answer — otherwise answer directly. - -IMAGE: When the user asks you to generate, create, draw, or make a picture/image/photo/logo/illustration, respond with ONLY a fenced \`\`\`image block containing a single vivid, detailed image-generation prompt (subject, style, lighting, composition) — no other text. The app generates it on-device. Do this only for genuine image requests; for everything else, never emit an image block. -{{HISTORY_BLOCK}} -User question: -{{QUERY}} - -Context: -{{CONTEXT_BLOCK}} - -Answer:` - // --------------------------------------------------------------------------- // Prompt registry // --------------------------------------------------------------------------- export const PROMPT_REGISTRY: PromptDef[] = [ - // Master Memory - { - key: 'masterMemory.initial', - name: 'Initial Creation', - description: 'Creates the first master memory document from a single conversation summary.', - category: 'master-memory', - variables: [{ name: 'SUMMARY', description: 'The conversation summary to build from' }], - defaultTemplate: MASTER_MEMORY_INITIAL - }, - { - key: 'masterMemory.incremental', - name: 'Incremental Update', - description: 'Merges a new conversation summary into the existing master memory.', - category: 'master-memory', - variables: [ - { name: 'CURRENT_MASTER', description: 'The current master memory content' }, - { name: 'NEW_SUMMARY', description: 'The new conversation summary to integrate' } - ], - defaultTemplate: MASTER_MEMORY_INCREMENTAL - }, - { - key: 'masterMemory.batchFirst', - name: 'Batch First', - description: - 'Creates master memory from the first batch of summaries during full regeneration.', - category: 'master-memory', - variables: [{ name: 'BATCH_TEXT', description: 'Formatted batch of conversation summaries' }], - defaultTemplate: MASTER_MEMORY_BATCH_FIRST - }, - { - key: 'masterMemory.batchExpand', - name: 'Batch Expand', - description: 'Expands master memory with additional batches during full regeneration.', - category: 'master-memory', - variables: [ - { name: 'CURRENT_MASTER', description: 'The current master memory content' }, - { name: 'BATCH_NUM', description: 'The current batch number' }, - { name: 'BATCH_TEXT', description: 'Formatted batch of conversation summaries' } - ], - defaultTemplate: MASTER_MEMORY_BATCH_EXPAND - }, - - // Memory Filter - { - key: 'memoryFilter.lenient', - name: 'Lenient Filter', - description: 'Inclusive memory filter that stores most potentially useful information.', - category: 'memory-filter', - variables: [ - { name: 'ROLE', description: 'Message role (user or assistant)' }, - { name: 'MESSAGE', description: 'The message content to evaluate' } - ], - defaultTemplate: MEMORY_FILTER_LENIENT - }, - { - key: 'memoryFilter.balanced', - name: 'Balanced Filter', - description: 'Balanced memory filter that stores explicitly stated, high-signal information.', - category: 'memory-filter', - variables: [ - { name: 'ROLE', description: 'Message role (user or assistant)' }, - { name: 'MESSAGE', description: 'The message content to evaluate' } - ], - defaultTemplate: MEMORY_FILTER_BALANCED - }, - { - key: 'memoryFilter.strict', - name: 'Strict Filter', - description: - 'Very selective memory filter that only stores high-value, directly stated information.', - category: 'memory-filter', - variables: [ - { name: 'ROLE', description: 'Message role (user or assistant)' }, - { name: 'MESSAGE', description: 'The message content to evaluate' } - ], - defaultTemplate: MEMORY_FILTER_STRICT - }, - - // Entity Extraction - { - key: 'entityExtraction.lenient', - name: 'Lenient Extraction', - description: 'Inclusive entity extraction that captures all mentioned entities.', - category: 'entity', - variables: [{ name: 'MEMORY_TEXT', description: 'Formatted list of memory statements' }], - defaultTemplate: ENTITY_EXTRACTION_LENIENT - }, - { - key: 'entityExtraction.balanced', - name: 'Balanced Extraction', - description: 'Balanced entity extraction for entities worth remembering.', - category: 'entity', - variables: [{ name: 'MEMORY_TEXT', description: 'Formatted list of memory statements' }], - defaultTemplate: ENTITY_EXTRACTION_BALANCED - }, - { - key: 'entityExtraction.strict', - name: 'Strict Extraction', - description: 'Very selective entity extraction for only high-value entities.', - category: 'entity', - variables: [{ name: 'MEMORY_TEXT', description: 'Formatted list of memory statements' }], - defaultTemplate: ENTITY_EXTRACTION_STRICT - }, - { - key: 'entitySummary', - name: 'Entity Summary', - description: 'Updates an entity profile summary with new facts.', - category: 'entity', - variables: [ - { name: 'NAME', description: 'Entity name' }, - { name: 'TYPE', description: 'Entity type (Person, Project, etc.)' }, - { name: 'EXISTING_SUMMARY', description: 'Current entity summary or "(none)"' }, - { - name: 'NEW_FACTS', - description: 'New facts to integrate (one per line, prefixed with "- ")' - } - ], - defaultTemplate: ENTITY_SUMMARY - }, // Session { @@ -427,34 +42,6 @@ export const PROMPT_REGISTRY: PromptDef[] = [ { name: 'CONVERSATION_TEXT', description: 'Full conversation text with role labels' } ], defaultTemplate: SESSION_SUMMARY - }, - - // Chat - { - key: 'ragChat', - name: 'Memory Chat', - description: 'System prompt for the RAG-powered memory chat assistant.', - category: 'chat', - variables: [ - { name: 'HISTORY_BLOCK', description: 'Previous conversation history (may be empty)' }, - { name: 'QUERY', description: "The user's current question" }, - { - name: 'CONTEXT_BLOCK', - description: 'Retrieved context from memories, messages, summaries, and entities' - } - ], - defaultTemplate: RAG_CHAT - }, - - // Master Memory - Merge (map-reduce final step) - { - key: 'masterMemory.merge', - name: 'Merge Partials', - description: - 'Merges multiple partial summaries into the final master memory (used in map-reduce regeneration).', - category: 'master-memory', - variables: [{ name: 'PARTIAL_SUMMARIES', description: 'All partial summaries concatenated' }], - defaultTemplate: MASTER_MEMORY_MERGE } ] diff --git a/src/main/records/entities.ts b/src/main/records/entities.ts new file mode 100644 index 000000000..b3f074ab8 --- /dev/null +++ b/src/main/records/entities.ts @@ -0,0 +1,98 @@ +// The people and organisations a question names, as one more list for the records engine: the +// question's words are matched against entity names (entity_fts, visible entities only), and the +// best few entities bring their open to-dos and their most recent screen moments. "What did I +// promise Kunal" finds Kunal's open to-dos even when they never say "promise". Names in scripts +// written without spaces (王小明, 田中) are found through entity_fts's trigram twin, and a one- or +// two-character name by its substring among the entities that twin holds. + +import { + ftsQuery, + ftsTrigramQuery, + needsTrigram, + type EntityRecallPort, + type RecordHit +} from '@offgrid/rag/records' +import { getDB, tableExists } from '../database' +import { actionWhenSql, hydrateRecords, visibleEntity, windowSql } from './sources' + +/** Entities a question can bring in. */ +const MAX_ENTITIES = 3 + +type Db = ReturnType + +/** Visible entities whose name matches `match` in the FTS5 table `fts`, best first. */ +function entitiesByName(db: Db, fts: string, match: string): number[] { + if (!tableExists(db, fts)) return [] + return ( + db + .prepare( + `SELECT e.id FROM ${fts} JOIN entities e ON e.id = ${fts}.rowid + WHERE ${fts} MATCH ? AND ${visibleEntity(db, 'e')} + ORDER BY bm25(${fts}) LIMIT ?` + ) + .all(`name : (${match})`, MAX_ENTITIES) as { id: number }[] + ).map((row) => row.id) +} + +/** Visible entities in the trigram twin whose name holds one of `runs` (one or two characters), + * most recently updated first. */ +function entitiesByShortName(db: Db, runs: string[]): number[] { + if (!runs.length || !tableExists(db, 'entity_fts_tri')) return [] + return ( + db + .prepare( + `SELECT e.id FROM entity_fts_tri_docsize d JOIN entities e ON e.id = d.id + WHERE (${runs.map(() => 'instr(offgrid_fts_text(e.name), ?) > 0').join(' OR ')}) + AND ${visibleEntity(db, 'e')} + ORDER BY e.updated_at DESC LIMIT ?` + ) + .all(...runs, MAX_ENTITIES) as { id: number }[] + ).map((row) => row.id) +} + +export const entityRecall: EntityRecallPort = { + recall(question, window, limit) { + const match = ftsQuery(question) + const trigram = needsTrigram(question) ? ftsTrigramQuery(question) : null + const db = getDB() + const found = [ + ...(match ? entitiesByName(db, 'entity_fts', match) : []), + ...(trigram?.match ? entitiesByName(db, 'entity_fts_tri', trigram.match) : []), + ...(trigram ? entitiesByShortName(db, trigram.short) : []) + ] + const entityIds = [...new Set(found)].slice(0, MAX_ENTITIES) + if (!entityIds.length) return [] + const ids = JSON.stringify(entityIds) + + // Open to-dos first: what was promised or asked of the person is the usual question. + const aw = windowSql(actionWhenSql('a'), window, 'ms') + const actions = tableExists(db, 'action_items') + ? ( + db + .prepare( + `SELECT a.id FROM action_items a + WHERE a.entity_id IN (SELECT value FROM json_each(?)) AND a.status = 'open'${aw.sql} + ORDER BY ${actionWhenSql('a')} DESC LIMIT ?` + ) + .all(ids, ...aw.args, Math.ceil(limit / 3)) as { id: number }[] + ).map((row) => `act:${row.id}`) + : [] + + const when = "CAST(strftime('%s', o.ts) AS INTEGER)" + const w = windowSql('o.ts', window, 'datetime') + const moments = tableExists(db, 'observation_entities') + ? ( + db + .prepare( + `SELECT DISTINCT o.id FROM observation_entities oe + JOIN observations o ON o.id = oe.observation_id + WHERE oe.entity_id IN (SELECT value FROM json_each(?))${w.sql} + ORDER BY ${when} DESC LIMIT ?` + ) + .all(ids, ...w.args, Math.max(1, limit - actions.length)) as { id: number }[] + ).map((row) => `obs:${row.id}`) + : [] + + return hydrateRecords([...actions, ...moments]) as RecordHit[] + } +} diff --git a/src/main/records/index.ts b/src/main/records/index.ts new file mode 100644 index 000000000..43e7612e6 --- /dev/null +++ b/src/main/records/index.ts @@ -0,0 +1,201 @@ +// Memory search on desktop: the shared records engine (@offgrid/rag/records RecordSearchService) +// over this app's stores - keyword sources in SQLite FTS5, record vectors in the encrypted +// database, the people a question names, the ms-marco cross-encoder and the record embedder. +// Universal search (search.ts) and the chat tools both search through recordSearch. + +import { + ftsTypeahead, + parseTimeWindow, + RecordSearchService, + withoutTimePhrase, + type QueryEmbedPort, + type RecordHit, + type RecordRerankPort, + type RecordSearchOptions, + type RecordSearchResult, + type RecordSource +} from '@offgrid/rag/records' +import type { SearchKind, SearchResult } from '../../shared/search-contract' +import { embeddings } from '../embeddings' +import { RECORD_EMBEDDER } from '../embedding-models' +import { entityRecall } from './entities' +import { ensureRecordSchema } from './schema' +import { RECORD_SOURCES, type DesktopRecordHit } from './sources' +import { embedQueuedRecords, hasRecordVectors, recordVectors, recordVectorStatus } from './store' + +export { ensureRecordSchema } from './schema' +export { releaseRecordIndex } from './store' +export { + actionWhenSql, + screenMomentKey, + SYNCED_BULK_MAIL, + visibleEntity, + windowSql +} from './sources' +export type { DesktopRecordHit } from './sources' + +/** The record kinds a search can be narrowed to. Search kinds map onto them ('message' is a + * screen moment from a connected account). */ +const RECORD_KINDS = new Set(RECORD_SOURCES.map((source) => source.kind)) + +function recordKinds(kinds: readonly string[]): string[] { + const out = new Set() + for (const kind of kinds) { + const mapped = kind === 'message' ? 'screen' : kind + if (RECORD_KINDS.has(mapped)) out.add(mapped) + } + return [...out] +} + +const queryEmbedder: QueryEmbedPort = { + async embedQuery(text) { + // Someone is waiting on this one: it goes ahead of queued background indexing. + const [vector] = await embeddings.embed([RECORD_EMBEDDER.queryPrefix + text], { + model: RECORD_EMBEDDER.model, + interactive: true, + source: 'Memory search' + }) + return vector ?? [] + } +} + +/** The ms-marco cross-encoder the document engine reranks with. */ +const recordReranker: RecordRerankPort = { + rerank: (query, passages) => embeddings.rerank(query, passages) +} + +let service: RecordSearchService | null = null + +const deps = { + vectors: recordVectors, + embed: queryEmbedder, + rerank: recordReranker, + entities: entityRecall, + onWarning: (message: string, error: unknown) => console.warn(`[records] ${message}`, error) +} + +function recordService(): RecordSearchService { + return (service ??= new RecordSearchService({ ...deps, sources: RECORD_SOURCES })) +} + +/** A service whose keyword sources all search `match` (a search box's AND-of-word-starts) whatever + * the engine builds from the question; meaning search still reads the question. */ +function keywordTypeaheadService(match: string): RecordSearchService { + return new RecordSearchService({ + ...deps, + sources: RECORD_SOURCES.map((source): RecordSource => ({ + ...source, + keyword: (_match, window, limit) => source.keyword(match, window, limit) + })) + }) +} + +export interface DesktopRecordSearchOptions extends RecordSearchOptions { + /** + * The Search screen: keywords match the way a search box types (every word, each as a word + * start: ftsTypeahead), while meaning search still runs. `typeahead` (the command palette) is + * the same without meaning search. + */ + keywordTypeahead?: boolean +} + +/** + * Search everything the user did or saw. `kinds` may use search kinds ('message' searches screen + * moments); kinds with no records (attachments) match nothing. Meaning search is skipped while no + * record has a vector yet (nothing to compare the question with). + */ +export async function recordSearch( + question: string, + options: DesktopRecordSearchOptions = {} +): Promise { + ensureRecordSchema() + const { keywordTypeahead, kinds, ...rest } = options + const narrowed = kinds?.length ? recordKinds(kinds) : null + if (narrowed && !narrowed.length) return { hits: [], window: null } + const semantic = rest.semantic !== false && !rest.typeahead && hasRecordVectors() + const search = { ...rest, semantic, ...(narrowed ? { kinds: narrowed } : {}) } + if (keywordTypeahead && !rest.typeahead) { + // The words the engine searches: the whole question when a time phrase only boosts. + const words = + rest.timeMode === 'boost' || rest.window === null + ? question + : withoutTimePhrase(question, rest.window ?? parseTimeWindow(question, rest.now)) + const match = ftsTypeahead(words) + if (match) return keywordTypeaheadService(match).search(question, search) + } + return recordService().search(question, search) +} + +/** A snippet reads as prose: chat and notes are stored as markdown, and a result list that + * shows `**bold**`, fences or `` tags reads as source code. Capped at 280 chars. */ +export function plainSnippet(text: string): string { + return (text || '') + .replace(/```[\w-]*\n?/g, ' ') + .replace(/<\/?[a-z!][^<>\n]{0,200}>/gi, ' ') + .replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^\s{0,3}(?:#{1,6}|>|[-*+]|\d+\.)\s+/gm, '') + .replace(/(? void +): Promise { + if (draining) return + ensureRecordSchema() + if (!recordVectorStatus().pending) return + draining = true + const release = embeddings.hold() + try { + let done = 0 + for (;;) { + const batch = await embedQueuedRecords((texts) => + embeddings.embed(texts, { model: RECORD_EMBEDDER.model, source: 'Memory indexing' }) + ) + done += batch.embedded + onProgress?.({ done, remaining: batch.remaining }) + if (batch.remaining === 0 || batch.processed === 0) break + // Searches come first: wait until nobody has searched for a moment before the next batch. + await embeddings.whenQuiet(SEARCH_QUIET_MS) + } + } finally { + release() + draining = false + } +} diff --git a/src/main/records/schema.ts b/src/main/records/schema.ts new file mode 100644 index 000000000..cb083feda --- /dev/null +++ b/src/main/records/schema.ts @@ -0,0 +1,285 @@ +// The tables behind memory search: record_vectors (one float vector per embeddable record, in the +// encrypted database), record_queue (records waiting to be embedded), the action and meeting +// full-text indexes (each with its trigram twin for no-space scripts, database.ts), and the triggers +// that keep all of them in step with their source tables. +// +// Deletion is one transaction: every source table's AFTER DELETE trigger removes the record's +// vector and queue entry inside the statement that deleted the row (Open WebUI deletes the row +// and the vector in two stores, and a failed second delete leaves an orphan). +// +// Pro creates observations, action items and meetings lazily, so each table's triggers and index +// are installed the first time it is seen. Its existing rows are queued in the background after +// start-up, a slice at a time (the cursor survives a restart); rows written after the triggers +// exist are queued by them, so nothing is missed. Nothing here reads a whole table on the request +// path: a first search only installs triggers and creates missing indexes empty. + +import fs from 'fs' +import path from 'path' +import { app } from 'electron' +import { + clearRebuildCursor, + ensureFtsIndex, + getDB, + OBSERVATION_FTS, + queueIndexRebuild, + rebuildCursor, + setRebuildCursor, + tableExists, + type FtsIndexSpec +} from '../database' +import { RECORD_EMBEDDER } from '../embedding-models' +import { actionWhenSql } from './sources' + +type Db = ReturnType + +/** A table whose rows are embedded as records. */ +interface EmbeddedTable { + /** The record kind (RecordSource.kind). */ + kind: string + /** Record keys are `${prefix}:${id}`. */ + prefix: string + table: string + /** The column that identifies a row ('rowid' for tables keyed by text). */ + id: string + /** Columns whose change makes the record embed again. */ + watch: string[] +} + +const EMBEDDED_TABLES: EmbeddedTable[] = [ + { kind: 'screen', prefix: 'obs', table: 'observations', id: 'id', watch: ['summary'] }, + { kind: 'action', prefix: 'act', table: 'action_items', id: 'id', watch: ['text', 'due'] }, + { kind: 'meeting', prefix: 'mtg', table: 'meetings', id: 'id', watch: ['title', 'summary'] }, + { + kind: 'entity', + prefix: 'ent', + table: 'entities', + id: 'id', + watch: ['name', 'type', 'summary'] + }, + { kind: 'summary', prefix: 'sum', table: 'chat_summaries', id: 'rowid', watch: ['summary'] } +] + +/** The retired legacy stores (memories, entity facts): their rows stay, but nothing embeds or + * searches them, so their queue triggers, queued keys, seeding cursors and vectors go. */ +const RETIRED_TABLES = [ + { table: 'memories', prefix: 'mem', kind: 'memory' }, + { table: 'entity_facts', prefix: 'fact', kind: 'fact' } +] + +function retireLegacyRecords(db: Db): void { + db.transaction(() => { + for (const { table, prefix, kind } of RETIRED_TABLES) { + db.exec(` + DROP TRIGGER IF EXISTS ${table}_record_ai; + DROP TRIGGER IF EXISTS ${table}_record_au; + DROP TRIGGER IF EXISTS ${table}_record_ad; + `) + // Range scans on the keys' primary key and the kind index. + db.prepare('DELETE FROM record_queue WHERE key >= ? AND key < ?').run( + `${prefix}:`, + `${prefix};` + ) + db.prepare('DELETE FROM record_vectors WHERE kind = ?').run(kind) + clearRebuildCursor(db, `record_queue:${table}`) + } + })() +} + +/** To-dos by what they say (and the words as first heard). */ +const ACTION_FTS: FtsIndexSpec = { + name: 'action_fts', + source: 'action_items', + rowid: 'id', + columns: ['text', 'raw_text'], + triggers: 'action_items_fts', + trigram: true +} + +/** Meetings by title and summary. Transcripts are documents, searched by the document engine. */ +const MEETING_FTS: FtsIndexSpec = { + name: 'meeting_fts', + source: 'meetings', + rowid: 'id', + columns: ['title', 'summary'], + triggers: 'meetings_fts', + trigram: true +} + +/** Read paths the record sources use, per table; created once the table exists. */ +const READ_INDEXES: Record = { + meetings: ['CREATE INDEX IF NOT EXISTS idx_meetings_started ON meetings(started_at)'], + action_items: [ + `CREATE INDEX IF NOT EXISTS idx_action_items_when ON action_items(${actionWhenSql()})` + ] +} + +function triggerExists(db: Db, name: string): boolean { + return !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name = ?").get(name) +} + +/** Queue, re-queue and forget one table's records as its rows change. Its existing rows are queued + * in the background (queueExistingRecords). */ +function installRecordTriggers(db: Db, spec: EmbeddedTable): void { + const { table, prefix, id, watch } = spec + const key = (row: 'new' | 'old'): string => `'${prefix}:' || ${row}.${id}` + db.transaction(() => { + db.exec(` + CREATE TRIGGER IF NOT EXISTS ${table}_record_ai AFTER INSERT ON ${table} BEGIN + INSERT OR IGNORE INTO record_queue (key) VALUES (${key('new')}); + END; + CREATE TRIGGER IF NOT EXISTS ${table}_record_au AFTER UPDATE OF ${watch.join(', ')} ON ${table} BEGIN + INSERT OR IGNORE INTO record_queue (key) VALUES (${key('new')}); + END; + CREATE TRIGGER IF NOT EXISTS ${table}_record_ad AFTER DELETE ON ${table} BEGIN + DELETE FROM record_vectors WHERE key = ${key('old')}; + DELETE FROM record_queue WHERE key = ${key('old')}; + END; + `) + setRebuildCursor(db, seedName(spec), 0) + })() +} + +const seedName = (spec: EmbeddedTable): string => `record_queue:${spec.table}` + +/** One background slice queueing a table's rows that have no vector, oldest first (so the newest + * get the highest queue positions and embed first). Returns whether more remain. */ +function queueExistingRecords(spec: EmbeddedTable): () => boolean { + const { table, prefix, id } = spec + const name = seedName(spec) + return () => { + const db = getDB() + const cursor = rebuildCursor(db, name) + if (cursor === null) return false + if (!tableExists(db, table)) { + clearRebuildCursor(db, name) + return false + } + const upper = ( + db + .prepare( + `SELECT MAX(r) AS r FROM (SELECT ${id} AS r FROM ${table} WHERE ${id} > ? ORDER BY ${id} LIMIT 2000)` + ) + .get(cursor) as { r: number | null } + ).r + if (upper === null) { + clearRebuildCursor(db, name) + return false + } + db.transaction(() => { + db.prepare( + `INSERT OR IGNORE INTO record_queue (key) + SELECT '${prefix}:' || t.${id} FROM ${table} t + WHERE t.${id} > ? AND t.${id} <= ? + AND NOT EXISTS (SELECT 1 FROM record_vectors v WHERE v.key = '${prefix}:' || t.${id}) + ORDER BY t.${id}` + ).run(cursor, upper) + setRebuildCursor(db, name, upper) + })() + return true + } +} + +/** The LanceDB index this replaces held a plaintext copy of captured text beside the encrypted + * database. It and its receipts table go once the vectors live in SQLCipher. */ +function removeLegacyVectorStore(db: Db): void { + db.exec('DROP TABLE IF EXISTS vec_indexed') + try { + const dir = path.join(app.getPath('userData'), 'lancedb') + void fs.promises.rm(dir, { recursive: true, force: true }).catch((error) => { + console.warn('[records] could not remove the legacy vector store', error) + }) + } catch { + /* no Electron app paths (tests) */ + } +} + +const ready = new WeakMap>() + +/** + * Create the record tables, and the triggers and indexes of every source table that exists. + * Cheap after the first call: a table is checked again only until it has been seen. + */ +export function ensureRecordSchema(): void { + const db = getDB() + let done = ready.get(db) + if (!done) { + db.exec(` + CREATE TABLE IF NOT EXISTS record_vectors ( + key TEXT PRIMARY KEY, + kind TEXT NOT NULL, + ref_id TEXT NOT NULL, + ts INTEGER NOT NULL DEFAULT 0, + hash TEXT NOT NULL, + model TEXT NOT NULL, + vector BLOB NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_record_vectors_kind_ts ON record_vectors(kind, ts); + CREATE INDEX IF NOT EXISTS idx_record_vectors_model ON record_vectors(model); + -- Records to embed: new, edited, or embedded by another model. Newest rowid embeds first. + CREATE TABLE IF NOT EXISTS record_queue (key TEXT PRIMARY KEY); + `) + // A different embedder makes every stored vector stale. They stay (search ignores them, by + // model) until each record is embedded again. + db.prepare( + 'INSERT OR IGNORE INTO record_queue (key) SELECT key FROM record_vectors WHERE model != ?' + ).run(RECORD_EMBEDDER.model) + removeLegacyVectorStore(db) + retireLegacyRecords(db) + done = new Set() + ready.set(db, done) + } + for (const spec of EMBEDDED_TABLES) { + if (done.has(spec.table) || !tableExists(db, spec.table)) continue + if (!triggerExists(db, `${spec.table}_record_ad`)) installRecordTriggers(db, spec) + // Queueing the rows that predate the triggers (resumed after a restart) runs in the background. + if (rebuildCursor(db, seedName(spec)) !== null) + queueIndexRebuild(seedName(spec), queueExistingRecords(spec)) + for (const sql of READ_INDEXES[spec.table] ?? []) db.exec(sql) + done.add(spec.table) + } + // An observation's search text includes its frames' words, so linking a frame embeds it again, + // and unlinking or deleting one (a privacy delete) drops the vector made from its words and + // embeds the moment again from what is left. + if ( + !done.has('observation_frames') && + tableExists(db, 'observation_frames') && + tableExists(db, 'observations') + ) { + db.exec(` + CREATE TRIGGER IF NOT EXISTS observation_frames_record_ai AFTER INSERT ON observation_frames BEGIN + INSERT OR IGNORE INTO record_queue (key) VALUES ('obs:' || new.observation_id); + END; + CREATE TRIGGER IF NOT EXISTS observation_frames_record_ad AFTER DELETE ON observation_frames + WHEN EXISTS (SELECT 1 FROM observations WHERE id = old.observation_id) BEGIN + DELETE FROM record_vectors WHERE key = 'obs:' || old.observation_id; + INSERT OR IGNORE INTO record_queue (key) VALUES ('obs:' || old.observation_id); + END; + `) + done.add('observation_frames') + } + if ( + !done.has('frames') && + tableExists(db, 'frames') && + tableExists(db, 'observation_frames') && + tableExists(db, 'observations') + ) { + db.exec(` + CREATE TRIGGER IF NOT EXISTS frames_record_ad AFTER DELETE ON frames BEGIN + DELETE FROM record_vectors WHERE key IN ( + SELECT 'obs:' || observation_id FROM observation_frames WHERE frame_id = old.id); + INSERT OR IGNORE INTO record_queue (key) + SELECT 'obs:' || of.observation_id FROM observation_frames of + WHERE of.frame_id = old.id + AND EXISTS (SELECT 1 FROM observations o WHERE o.id = of.observation_id); + END; + `) + done.add('frames') + } + // Created empty when missing; one built by an older schema keeps serving and is rebuilt in the + // background after start-up. + for (const spec of [OBSERVATION_FTS, ACTION_FTS, MEETING_FTS]) { + if (done.has(spec.name) || !tableExists(db, spec.source)) continue + ensureFtsIndex(db, spec) + done.add(spec.name) + } +} diff --git a/src/main/records/sources.ts b/src/main/records/sources.ts new file mode 100644 index 000000000..23a9682bd --- /dev/null +++ b/src/main/records/sources.ts @@ -0,0 +1,1032 @@ +// Where each kind of record lives, for the shared records engine: keyword search (FTS5, within a +// time window when the question names one), the newest records in a window, hydration of vector +// hits, and the text each record is embedded with. SQLite is the source of truth; a key that no +// longer has a row simply hydrates to nothing. +// +// Keys: obs: and frame: (screen), act:, mtg:, +// ent:, sum:, chat:, doc:. +// The legacy memories and entity facts are retired: no source searches them and nothing embeds them. +// +// Text in scripts written without spaces (Chinese, Japanese, Thai…) is also searched through each +// index's trigram twin (`${fts}_tri`, database.ts): `trigram` on a source. + +import { + formatDay, + fuseLists, + type RecordHit, + type RecordSource, + type TimeWindow, + type TrigramQuery +} from '@offgrid/rag/records' +import { getDB, tableExists } from '../database' + +/** A desktop record: `synced` marks a screen moment made from a connected account (mail, chat). */ +export interface DesktopRecordHit extends RecordHit { + synced?: boolean +} + +/** One moment, once: capture records the same screen again and again while it stays put, each with + * the same summary, and a result list showed dozens of identical cards. Screen moments with the same + * app and summary share this key (search results, the empty-query feed and the chat tools collapse + * on it); a synced message and every other record keep their own. */ +export function screenMomentKey(hit: RecordHit): string { + if (hit.kind !== 'screen' || (hit as DesktopRecordHit).synced) return hit.key + const summary = (hit.text.split('\n', 1)[0] ?? '').trim() + if (!summary) return hit.key + return `screen\u0000${(hit.surface ?? '').toLowerCase()}\u0000${summary.toLowerCase()}` +} + +type Db = ReturnType +type Row = Record + +/** How much of a moment's on-screen text joins its summary, in search and in its embedding. */ +const EXCERPT_CHARS = 600 +/** Longest text a record is embedded with (MiniLM reads about 256 tokens). */ +const EMBED_CHARS = 2_000 + +/** Bulk mail synced from a connected account (newsletters, promotions) on observation `o`: the one + * predicate for leaving it out of counts and the feed, and for demoting it in results. */ +export const SYNCED_BULK_MAIL = `(COALESCE(o.category, '') = 'consumption' AND o.source_key IS NOT NULL)` +/** Share of a bulk-mail result's score taken away: searchable, never favoured. */ +const BULK_MAIL_PENALTY = 0.5 + +/** A SQLite datetime column as epoch ms. */ +export const epochMs = (column: string): string => + `CAST(strftime('%s', ${column}) AS INTEGER) * 1000` +/** Same text as pro's idx_observations_ts_sec expression, so window reads use that index. */ +const epochSec = (column: string): string => `CAST(strftime('%s', ${column}) AS INTEGER)` + +/** `AND ` and its arguments: the one time-window rule for every store. `unit` is + * how the column stores time (SQLite datetime text, or epoch ms). */ +export function windowSql( + column: string, + window: TimeWindow | null, + unit: 'datetime' | 'ms' +): { sql: string; args: number[] } { + if (!window) return { sql: '', args: [] } + if (unit === 'ms') + return { sql: ` AND ${column} >= ? AND ${column} < ?`, args: [window.start, window.end] } + const expr = epochSec(column) + return { + sql: ` AND ${expr} >= ? AND ${expr} < ?`, + args: [Math.floor(window.start / 1000), Math.ceil(window.end / 1000)] + } +} + +const columnsSeen = new WeakMap>() +/** Whether `table` has `column`. Pro adds some columns later, so only a yes is remembered. */ +function hasColumn(db: Db, table: string, column: string): boolean { + let seen = columnsSeen.get(db) + if (!seen) columnsSeen.set(db, (seen = new Map())) + const id = `${table}.${column}` + if (seen.get(id)) return true + const has = (db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[]).some( + (row) => row.name === column + ) + if (has) seen.set(id, true) + return has +} + +/** The one hidden-entity rule: a SQL condition true for entities the user has not hidden. + * `entities.hidden` is a Pro column; without it nothing is hidden. */ +export function visibleEntity(db: Db, alias: string): string { + return hasColumn(db, 'entities', 'hidden') ? `${alias}.hidden = 0` : '1' +} + +/** The absolute due instant (`due_at`, ms) when this profile has it. */ +function dueAtSql(db: Db, alias: string): string { + return hasColumn(db, 'action_items', 'due_at') ? `${alias}.due_at` : 'NULL' +} + +const json = (values: unknown[]): string => JSON.stringify(values) +const text = (value: unknown): string => (typeof value === 'string' ? value : '') +const num = (value: unknown): number => (typeof value === 'number' ? value : Number(value) || 0) + +function keyParts(key: string): { prefix: string; id: string } { + const at = key.indexOf(':') + return { prefix: key.slice(0, at), id: key.slice(at + 1) } +} + +/** The quoted terms of an FTS5 expression built by ftsQuery or ftsTypeahead, lower-case. */ +function matchTerms(match: string): string[] { + return [...match.matchAll(/"((?:[^"]|"")+)"/g)].map((m) => + m[1]!.replace(/""/g, '"').toLowerCase() + ) +} + +/** The newest rows of a trigram twin a one- or two-character run is looked for in. */ +const SHORT_RUN_SCAN = 5_000 + +/** + * Rowids of `table` (as `s`) for a no-space-script question, from the trigram twin of `fts`: the + * trigram MATCH (runs of three characters or more), best first, then the newest of the twin's rows + * whose text holds a one- or two-character run, which no trigram can match. The twin holds only rows + * in those scripts (its %_docsize shadow table lists them), so the scan never reads the rest. + * `where` and `window` are extra conditions on `s`; `columns` the indexed columns. + */ +function trigramRowids( + db: Db, + options: { + fts: string + table: string + columns: string[] + where?: string + window: { sql: string; args: number[] } + query: TrigramQuery + limit: number + } +): number[][] { + const { table, columns, window, query, limit } = options + const tri = `${options.fts}_tri` + if (!tableExists(db, tri)) return [] + const where = options.where ? ` AND ${options.where}` : '' + const lists: number[][] = [] + if (query.match) { + lists.push( + ( + db + .prepare( + `SELECT s.rowid AS id FROM ${tri} JOIN ${table} s ON s.rowid = ${tri}.rowid + WHERE ${tri} MATCH ?${where}${window.sql} ORDER BY bm25(${tri}) LIMIT ?` + ) + .all(query.match, ...window.args, limit) as { id: number }[] + ).map((row) => row.id) + ) + } + if (query.short.length) { + const contains = query.short + .flatMap(() => columns.map((column) => `instr(offgrid_fts_text(s.${column}), ?) > 0`)) + .join(' OR ') + lists.push( + ( + db + .prepare( + `SELECT s.rowid AS id + FROM (SELECT id FROM ${tri}_docsize ORDER BY id DESC LIMIT ${SHORT_RUN_SCAN}) d + JOIN ${table} s ON s.rowid = d.id + WHERE (${contains})${where}${window.sql} + ORDER BY d.id DESC LIMIT ?` + ) + .all( + ...query.short.flatMap((run) => columns.map(() => run)), + ...window.args, + limit + ) as { id: number }[] + ).map((row) => row.id) + ) + } + return lists +} + +/** Lists of keys fused into one, best first. */ +function fuseKeys(lists: string[][], limit: number): string[] { + return fuseLists( + lists.map((items) => ({ items })), + (key) => key + ) + .slice(0, limit) + .map((entry) => entry.item) +} + +// --------------------------------------------------------------------------------------------- +// Screen moments +// --------------------------------------------------------------------------------------------- + +/** Frames read per moment for its excerpt; a moment that stayed on screen can have dozens. */ +const EXCERPT_FRAMES = 4 + +/** The distinct lines of a moment's on-screen text (its first frames), up to `max` chars. */ +function frameExcerpts(db: Db, observationIds: number[], max = EXCERPT_CHARS): Map { + const out = new Map() + if (!observationIds.length || !tableExists(db, 'observation_frames')) return out + const rows = db + .prepare( + `SELECT id, text FROM ( + SELECT of.observation_id AS id, substr(f.text, 1, 4000) AS text, + ROW_NUMBER() OVER (PARTITION BY of.observation_id ORDER BY f.id) AS n + FROM observation_frames of JOIN frames f ON f.id = of.frame_id + WHERE of.observation_id IN (SELECT value FROM json_each(?)) + AND f.text IS NOT NULL AND COALESCE(f.source, '') != 'connector') + WHERE n <= ${EXCERPT_FRAMES} ORDER BY id, n` + ) + .all(json(observationIds)) as { id: number; text: string }[] + const seen = new Map>() + for (const row of rows) { + let lines = seen.get(row.id) + if (!lines) seen.set(row.id, (lines = new Set())) + let excerpt = out.get(row.id) ?? '' + for (const raw of row.text.split(/\n+/)) { + if (excerpt.length >= max) break + const line = raw.replace(/\s+/g, ' ').trim() + const id = line.toLowerCase() + if (line.length < 3 || lines.has(id)) continue + lines.add(id) + excerpt = excerpt ? `${excerpt} · ${line}` : line + } + out.set(row.id, excerpt.slice(0, max)) + } + return out +} + +function hydrateObservations( + db: Db, + ids: number[], + matched: Map +): DesktopRecordHit[] { + if (!ids.length || !tableExists(db, 'observations')) return [] + const rows = db + .prepare( + `SELECT o.id, o.summary, COALESCE(o.surface, '') AS surface, o.url, ${epochMs('o.ts')} AS ts, + CASE WHEN o.salience > 0 THEN o.salience ELSE 0.5 END AS evidence, + ${SYNCED_BULK_MAIL} AS bulk, + (SELECT f.image_path FROM observation_frames of JOIN frames f ON f.id = of.frame_id + WHERE of.observation_id = o.id AND f.image_path IS NOT NULL + ORDER BY f.id LIMIT 1) AS imagePath, + EXISTS (SELECT 1 FROM observation_frames of JOIN frames f ON f.id = of.frame_id + WHERE of.observation_id = o.id AND f.source = 'connector') AS synced + FROM observations o WHERE o.id IN (SELECT value FROM json_each(?))` + ) + .all(json(ids)) as Row[] + const excerpts = frameExcerpts(db, ids) + return rows.map((row) => { + const id = num(row.id) + const synced = !!row.synced + const body = [matched.get(id), excerpts.get(id)].filter(Boolean).join(' · ') + return { + key: `obs:${id}`, + kind: 'screen', + refId: id, + title: text(row.surface) || 'Screen', + text: body ? `${text(row.summary)}\n${body.slice(0, EXCERPT_CHARS)}` : text(row.summary), + ts: num(row.ts), + surface: text(row.surface), + url: text(row.url) || null, + // A synced message has no screenshot. + imagePath: synced ? null : text(row.imagePath) || null, + evidence: num(row.evidence), + ...(row.bulk ? { penalty: BULK_MAIL_PENALTY } : {}), + synced + } + }) +} + +function hydrateFrames(db: Db, ids: number[]): DesktopRecordHit[] { + if (!ids.length || !tableExists(db, 'frames')) return [] + return ( + db + .prepare( + `SELECT id, COALESCE(surface, '') AS surface, url, image_path AS imagePath, + substr(text, 1, ${EXCERPT_CHARS}) AS text, ${epochMs('ts')} AS ts + FROM frames WHERE id IN (SELECT value FROM json_each(?))` + ) + .all(json(ids)) as Row[] + ).map((row) => ({ + key: `frame:${num(row.id)}`, + kind: 'screen', + refId: num(row.id), + title: text(row.surface) || 'Screen', + text: text(row.text), + ts: num(row.ts), + surface: text(row.surface), + url: text(row.url) || null, + imagePath: text(row.imagePath) || null + })) +} + +/** Screen moments: their summaries (observation_fts) and every word on screen (frame_fts). A frame + * that belongs to a moment counts for that moment; a frame of its own is a hit with its picture. */ +const screenSource: RecordSource = { + kind: 'screen', + decays: true, + keyword(match, window, limit) { + const db = getDB() + const lists: string[][] = [] + const matched = new Map() + if (tableExists(db, 'observation_fts')) { + const w = windowSql('o.ts', window, 'datetime') + lists.push( + ( + db + .prepare( + `SELECT o.id FROM observation_fts f JOIN observations o ON o.id = f.rowid + WHERE observation_fts MATCH ?${w.sql} ORDER BY bm25(observation_fts) LIMIT ?` + ) + .all(match, ...w.args, limit) as { id: number }[] + ).map((row) => `obs:${row.id}`) + ) + } + // Raw screen text is the largest index by far (tens of millions of frames). Scoring every frame + // a common word or short prefix matches took seconds, so frame hits are the NEWEST matches — an + // index walk, no scoring (screenpipe orders screen text by time the same way); relevance comes + // from the moment summaries above and from meaning search. A one- or two-letter prefix is too + // broad to read raw screen text at all. + const tooBroad = /^("[^"]{1,2}"\*(\s+AND\s+|$))+$/u.test(match) + if (!tooBroad && tableExists(db, 'frame_fts')) { + const w = windowSql('fr.ts', window, 'datetime') + const frames = db + .prepare( + `SELECT fr.id, substr(fr.text, 1, 300) AS text, + (SELECT of.observation_id FROM observation_frames of WHERE of.frame_id = fr.id + LIMIT 1) AS observationId + FROM frame_fts x JOIN frames fr ON fr.id = x.rowid + WHERE frame_fts MATCH ?${w.sql} + ORDER BY x.rowid DESC LIMIT ?` + ) + .all(match, ...w.args, limit) as { + id: number + text: string + observationId: number | null + }[] + lists.push( + frames.map((frame) => { + if (frame.observationId == null) return `frame:${frame.id}` + if (!matched.has(frame.observationId)) + matched.set(frame.observationId, frame.text.replace(/\s+/g, ' ').trim()) + return `obs:${frame.observationId}` + }) + ) + } + return hydrateInOrder(db, fuseKeys(lists, limit), matched) + }, + // Moment summaries only: frame_fts has no trigram twin (see database.ts). + trigram(query, window, limit) { + const db = getDB() + const lists = trigramRowids(db, { + fts: 'observation_fts', + table: 'observations', + columns: ['summary'], + window: windowSql('s.ts', window, 'datetime'), + query, + limit + }) + return hydrateInOrder( + db, + fuseKeys( + lists.map((ids) => ids.map((id) => `obs:${id}`)), + limit + ) + ) + }, + recent(window, limit) { + const db = getDB() + if (!tableExists(db, 'observations')) return [] + const w = windowSql('o.ts', window, 'datetime') + const ids = ( + db + .prepare( + `SELECT o.id FROM observations o + WHERE o.summary IS NOT NULL AND o.summary != '' AND NOT ${SYNCED_BULK_MAIL}${w.sql} + ORDER BY ${epochSec('o.ts')} DESC LIMIT ?` + ) + .all(...w.args, limit) as { id: number }[] + ).map((row) => `obs:${row.id}`) + return hydrateInOrder(db, ids) + } +} + +// --------------------------------------------------------------------------------------------- +// Actions, meetings, people, summaries +// --------------------------------------------------------------------------------------------- + +/** When a to-do came from, in ms: `source_ts` is epoch seconds for captured sources and ms for + * manual jots (pro action-schema actionSourceMs); `created_at` (ms) stands in when there is none. + * `alias` is the action_items alias (none in an index expression). */ +export function actionWhenSql(alias = ''): string { + const column = (name: string): string => (alias ? `${alias}.${name}` : name) + const source = column('source_ts') + return `(CASE WHEN ${source} IS NULL OR ${source} <= 0 THEN ${column('created_at')} WHEN ${source} > 1000000000000 THEN ${source} ELSE ${source} * 1000 END)` +} + +const ACTION_WHEN = actionWhenSql('a') + +function hydrateActions(db: Db, ids: number[]): DesktopRecordHit[] { + if (!ids.length || !tableExists(db, 'action_items')) return [] + return ( + db + .prepare( + `SELECT a.id, a.text, a.due, ${dueAtSql(db, 'a')} AS dueAt, a.status, + a.entity_name AS entityName, a.confidence, ${ACTION_WHEN} AS ts + FROM action_items a + WHERE a.id IN (SELECT value FROM json_each(?)) AND a.status != 'dismissed'` + ) + .all(json(ids)) as Row[] + ).map((row) => { + const status = text(row.status) + const label = status === 'done' ? 'Done' : status === 'suggested' ? 'Suggested to-do' : 'To-do' + return { + key: `act:${num(row.id)}`, + kind: 'action', + refId: num(row.id), + title: row.entityName ? `${label} · ${text(row.entityName)}` : label, + text: actionText(row), + ts: num(row.ts), + surface: 'To-do', + url: null, + evidence: row.confidence == null ? 0.5 : num(row.confidence) + } + }) +} + +/** A to-do with its due date: the absolute day when it was resolved, else the words as said. */ +function actionText(row: Row): string { + const due = row.dueAt ? formatDay(num(row.dueAt)) : text(row.due) + return `${text(row.text)}${due ? ` (due ${due})` : ''}` +} + +function hydrateMeetings(db: Db, ids: number[]): DesktopRecordHit[] { + if (!ids.length || !tableExists(db, 'meetings')) return [] + return ( + db + .prepare( + `SELECT id, COALESCE(title, 'Meeting') AS title, + COALESCE(summary, substr(transcript, 1, ${EXCERPT_CHARS}), '') AS text, + COALESCE(started_at, created_at, 0) AS ts + FROM meetings WHERE id IN (SELECT value FROM json_each(?))` + ) + .all(json(ids)) as Row[] + ).map((row) => ({ + key: `mtg:${num(row.id)}`, + kind: 'meeting', + refId: num(row.id), + title: text(row.title), + text: text(row.text), + ts: num(row.ts), + surface: 'Meeting', + url: null + })) +} + +function hydrateEntities(db: Db, ids: number[]): DesktopRecordHit[] { + if (!ids.length) return [] + return ( + db + .prepare( + `SELECT e.id, e.name, e.type, e.summary, ${epochMs('e.updated_at')} AS ts + FROM entities e + WHERE e.id IN (SELECT value FROM json_each(?)) AND ${visibleEntity(db, 'e')}` + ) + .all(json(ids)) as Row[] + ).map((row) => ({ + key: `ent:${num(row.id)}`, + kind: 'entity', + refId: num(row.id), + title: text(row.name), + text: entityText(row), + ts: num(row.ts), + surface: 'Entity', + url: null + })) +} + +function entityText(row: Row): string { + return `${text(row.name)} (${text(row.type) || 'Unknown'})${row.summary ? `: ${text(row.summary)}` : ''}` +} + +function hydrateSummaries(db: Db, ids: number[]): DesktopRecordHit[] { + if (!ids.length) return [] + return ( + db + .prepare( + `SELECT rowid AS id, session_id AS sessionId, summary, ${epochMs('updated_at')} AS ts + FROM chat_summaries WHERE rowid IN (SELECT value FROM json_each(?))` + ) + .all(json(ids)) as Row[] + ).map((row) => ({ + key: `sum:${num(row.id)}`, + kind: 'summary', + refId: num(row.id), + title: 'Chat summary', + text: text(row.summary), + ts: num(row.ts), + surface: 'Chat summary', + // The chat it summarises. + url: text(row.sessionId) || null + })) +} + +const HYDRATORS: Record< + string, + (db: Db, ids: number[], matched: Map) => DesktopRecordHit[] +> = { + obs: hydrateObservations, + frame: hydrateFrames, + act: hydrateActions, + mtg: hydrateMeetings, + ent: hydrateEntities, + sum: hydrateSummaries +} + +/** + * Records for keys, in the keys' order. A key whose row is gone (or hidden, or dismissed) is left + * out. `matched` adds the on-screen words a frame search found to that moment's text. + */ +export function hydrateRecords( + keys: string[], + matched = new Map() +): DesktopRecordHit[] { + return hydrateInOrder(getDB(), keys, matched) +} + +function hydrateInOrder( + db: Db, + keys: string[], + matched = new Map() +): DesktopRecordHit[] { + const byPrefix = new Map() + for (const key of keys) { + const { prefix, id } = keyParts(key) + if (!HYDRATORS[prefix]) continue + const ids = byPrefix.get(prefix) ?? [] + ids.push(Number(id)) + byPrefix.set(prefix, ids) + } + const found = new Map() + for (const [prefix, ids] of byPrefix) { + for (const hit of HYDRATORS[prefix]!(db, ids, matched)) found.set(hit.key, hit) + } + return keys.map((key) => found.get(key)).filter((hit): hit is DesktopRecordHit => !!hit) +} + +/** A keyword source over one FTS5 table whose rowid is the record's id, and its trigram twin. */ +function ftsSource(options: { + kind: string + decays: boolean + prefix: string + fts: string + /** Joined as `s`; `when` reads its time. */ + table: string + /** The indexed columns of `table`. */ + columns: string[] + when: { column: string; unit: 'datetime' | 'ms' } + /** Extra WHERE conditions on `s`. */ + where?: (db: Db) => string +}): RecordSource { + const { kind, decays, prefix, fts, table, columns, when } = options + return { + kind, + decays, + trigram(query, window, limit) { + const db = getDB() + const lists = trigramRowids(db, { + fts, + table, + columns, + ...(options.where ? { where: options.where(db) } : {}), + window: windowSql(when.column, window, when.unit), + query, + limit + }) + return hydrateInOrder( + db, + fuseKeys( + lists.map((ids) => ids.map((id) => `${prefix}:${id}`)), + limit + ) + ) + }, + keyword(match, window, limit) { + const db = getDB() + if (!tableExists(db, fts)) return [] + const w = windowSql(when.column, window, when.unit) + const ids = db + .prepare( + `SELECT s.rowid AS id FROM ${fts} JOIN ${table} s ON s.rowid = ${fts}.rowid + WHERE ${fts} MATCH ?${options.where ? ` AND ${options.where(db)}` : ''}${w.sql} + ORDER BY bm25(${fts}) LIMIT ?` + ) + .all(match, ...w.args, limit) as { id: number }[] + return hydrateInOrder( + db, + ids.map((row) => `${prefix}:${row.id}`) + ) + } + } +} + +const actionSource: RecordSource = { + ...ftsSource({ + kind: 'action', + decays: false, + prefix: 'act', + fts: 'action_fts', + table: 'action_items', + columns: ['text', 'raw_text'], + when: { column: actionWhenSql('s'), unit: 'ms' }, + where: () => "s.status != 'dismissed'" + }), + recent(window, limit) { + const db = getDB() + if (!tableExists(db, 'action_items')) return [] + const ids = db + .prepare( + `SELECT a.id FROM action_items a + WHERE a.status != 'dismissed' AND ${ACTION_WHEN} >= ? AND ${ACTION_WHEN} < ? + ORDER BY ${ACTION_WHEN} DESC LIMIT ?` + ) + .all(window.start, window.end, limit) as { id: number }[] + return hydrateInOrder( + db, + ids.map((row) => `act:${row.id}`) + ) + } +} + +const meetingSource: RecordSource = { + ...ftsSource({ + kind: 'meeting', + decays: false, + prefix: 'mtg', + fts: 'meeting_fts', + table: 'meetings', + columns: ['title', 'summary'], + when: { column: 'COALESCE(s.started_at, s.created_at)', unit: 'ms' } + }), + recent(window, limit) { + const db = getDB() + if (!tableExists(db, 'meetings')) return [] + const ids = db + .prepare( + `SELECT id FROM meetings WHERE started_at >= ? AND started_at < ? + ORDER BY started_at DESC LIMIT ?` + ) + .all(window.start, window.end, limit) as { id: number }[] + return hydrateInOrder( + db, + ids.map((row) => `mtg:${row.id}`) + ) + } +} + +const entitySource: RecordSource = ftsSource({ + kind: 'entity', + decays: false, + prefix: 'ent', + fts: 'entity_fts', + table: 'entities', + columns: ['name', 'summary', 'type'], + when: { column: 's.updated_at', unit: 'datetime' }, + where: (db) => visibleEntity(db, 's') +}) + +const summarySource: RecordSource = ftsSource({ + kind: 'summary', + decays: true, + prefix: 'sum', + fts: 'summary_fts', + table: 'chat_summaries', + columns: ['summary'], + when: { column: 's.updated_at', unit: 'datetime' } +}) + +// --------------------------------------------------------------------------------------------- +// Chats and knowledge bases (keyword only) +// --------------------------------------------------------------------------------------------- + +function chatHit(row: Row): DesktopRecordHit { + const id = text(row.id) + return { + key: `chat:${id}`, + kind: 'chat', + refId: 0, + title: text(row.title) || 'Chat', + text: text(row.text), + ts: num(row.ts), + surface: 'Chat', + // The conversation to open. + url: id + } +} + +/** Chat hits for messages, best first: one per conversation, its first message in `ids`. */ +function chatHitsForMessages(db: Db, ids: number[], limit: number): DesktopRecordHit[] { + if (!ids.length) return [] + const rows = new Map( + ( + db + .prepare( + `SELECT m.id AS mid, m.conversation_id AS id, rc.title, + substr(m.content, 1, ${EXCERPT_CHARS}) AS text, ${epochMs('m.created_at')} AS ts + FROM rag_messages m JOIN rag_conversations rc ON rc.id = m.conversation_id + WHERE m.id IN (SELECT value FROM json_each(?))` + ) + .all(json(ids)) as Row[] + ).map((row) => [num(row.mid), row]) + ) + const seen = new Set() + const hits: DesktopRecordHit[] = [] + for (const id of ids) { + const row = rows.get(id) + if (!row || seen.has(text(row.id))) continue + seen.add(text(row.id)) + hits.push(chatHit(row)) + if (hits.length === limit) break + } + return hits +} + +/** Your chats: one hit per conversation, its best-matching message (rag_message_fts), plus chats + * whose title has the words. The conversation id is the hit's url. */ +const chatSource: RecordSource = { + kind: 'chat', + decays: true, + trigram(query, window, limit) { + const db = getDB() + // Titles are matched by the keyword list (a substring test on each quoted term). + const lists = trigramRowids(db, { + fts: 'rag_message_fts', + table: 'rag_messages', + columns: ['content'], + window: windowSql('s.created_at', window, 'datetime'), + query, + limit: limit * 10 + }) + return fuseLists( + lists.map((ids) => ({ items: chatHitsForMessages(db, ids, limit) })), + (hit) => hit.key + ) + .slice(0, limit) + .map((entry) => entry.item) + }, + keyword(match, window, limit) { + const db = getDB() + const w = windowSql('m.created_at', window, 'datetime') + const byContent = db + .prepare( + `SELECT best.conversation_id AS id, rc.title, substr(m.content, 1, ${EXCERPT_CHARS}) AS text, + ${epochMs('m.created_at')} AS ts + FROM (SELECT conversation_id, mid, MIN(score) AS score + FROM (SELECT rag_message_fts.rowid AS mid, rag_message_fts.conversation_id, + rag_message_fts.rank AS score + FROM rag_message_fts JOIN rag_messages m ON m.id = rag_message_fts.rowid + WHERE rag_message_fts MATCH ?${w.sql} + ORDER BY rag_message_fts.rank LIMIT ?) + GROUP BY conversation_id) best + JOIN rag_conversations rc ON rc.id = best.conversation_id + JOIN rag_messages m ON m.id = best.mid + ORDER BY best.score LIMIT ?` + ) + .all(match, ...w.args, limit * 10, limit) as Row[] + const terms = matchTerms(match) + // A search box's expression needs every word (AND); a question's, any of them. + const join = / AND /.test(match) ? ' AND ' : ' OR ' + const tw = windowSql('rc.updated_at', window, 'datetime') + const byTitle = terms.length + ? (db + .prepare( + `SELECT rc.id, rc.title, + COALESCE((SELECT substr(content, 1, ${EXCERPT_CHARS}) FROM rag_messages + WHERE conversation_id = rc.id ORDER BY id DESC LIMIT 1), '') AS text, + ${epochMs('rc.updated_at')} AS ts + FROM rag_conversations rc + WHERE (${terms.map(() => "instr(lower(COALESCE(rc.title, '')), ?) > 0").join(join)})${tw.sql} + ORDER BY (${terms.map(() => "(instr(lower(COALESCE(rc.title, '')), ?) > 0)").join(' + ')}) DESC, + rc.updated_at DESC + LIMIT ?` + ) + .all(...terms, ...tw.args, ...terms, limit) as Row[]) + : [] + return fuseLists( + [{ items: byContent.map(chatHit) }, { items: byTitle.map(chatHit) }], + (hit) => hit.key + ) + .slice(0, limit) + .map((entry) => entry.item) + }, + recent(window, limit) { + const w = windowSql('rc.updated_at', window, 'datetime') + return ( + getDB() + .prepare( + `SELECT rc.id, rc.title, + COALESCE((SELECT substr(content, 1, ${EXCERPT_CHARS}) FROM rag_messages + WHERE conversation_id = rc.id ORDER BY id DESC LIMIT 1), '') AS text, + ${epochMs('rc.updated_at')} AS ts + FROM rag_conversations rc WHERE 1 = 1${w.sql} + ORDER BY rc.updated_at DESC LIMIT ?` + ) + .all(...w.args, limit) as Row[] + ).map(chatHit) + } +} + +/** Project knowledge-base documents: the document index's keyword search (bm25), best passage per + * file, and the newest files in a window. The owning project id is the hit's url. */ +function docHit(row: Row): DesktopRecordHit { + const id = num(row.id) + return { + key: `doc:${id}`, + kind: 'doc', + refId: id, + title: text(row.name), + text: text(row.text), + ts: num(row.ts), + surface: 'Knowledge base', + url: text(row.projectId) || null + } +} + +/** + * Knowledge-base documents for passages, best first: one hit per file, its best passage. `passages` + * is a subquery of (document_id, position, rank), lowest rank best; `args` are its arguments. + */ +function docHitsForPassages( + db: Db, + passages: string, + args: unknown[], + window: TimeWindow | null, + limit: number +): DesktopRecordHit[] { + const w = windowSql('d.created_at', window, 'datetime') + const rows = db + .prepare( + `SELECT d.id, d.name, d.project_id AS projectId, substr(c.text, 1, ${EXCERPT_CHARS}) AS text, + ${epochMs('d.created_at')} AS ts + FROM (${passages}) f + JOIN rag_documents d ON d.document_id = f.document_id + JOIN attached_document_chunks c ON c.document_id = f.document_id AND c.position = f.position + WHERE 1 = 1${w.sql} + ORDER BY f.rank` + ) + .all(...args, ...w.args) as Row[] + const seen = new Set() + const hits: DesktopRecordHit[] = [] + for (const row of rows) { + const id = num(row.id) + if (seen.has(id)) continue + seen.add(id) + hits.push(docHit(row)) + if (hits.length === limit) break + } + return hits +} + +const DOC_TRIGRAM = 'attached_document_fts_tri' + +const docSource: RecordSource = { + kind: 'doc', + decays: false, + keyword(match, window, limit) { + const db = getDB() + if (!tableExists(db, 'attached_document_fts') || !tableExists(db, 'rag_documents')) return [] + return docHitsForPassages( + db, + `SELECT document_id, position, bm25(attached_document_fts) AS rank + FROM attached_document_fts WHERE attached_document_fts MATCH ? + ORDER BY rank LIMIT ?`, + [match, limit * 20], + window, + limit + ) + }, + // The passage index's trigram twin holds its own copy of each passage in a no-space script. + trigram(query, window, limit) { + const db = getDB() + if (!tableExists(db, DOC_TRIGRAM) || !tableExists(db, 'rag_documents')) return [] + const lists: DesktopRecordHit[][] = [] + if (query.match) + lists.push( + docHitsForPassages( + db, + `SELECT document_id, position, bm25(${DOC_TRIGRAM}) AS rank + FROM ${DOC_TRIGRAM} WHERE ${DOC_TRIGRAM} MATCH ? + ORDER BY rank LIMIT ?`, + [query.match, limit * 20], + window, + limit + ) + ) + if (query.short.length) + lists.push( + docHitsForPassages( + db, + `SELECT document_id, position, -rowid AS rank + FROM (SELECT rowid, document_id, position, body FROM ${DOC_TRIGRAM} + ORDER BY rowid DESC LIMIT ${SHORT_RUN_SCAN}) + WHERE ${query.short.map(() => 'instr(body, ?) > 0').join(' OR ')} + LIMIT ?`, + [...query.short, limit * 20], + window, + limit + ) + ) + return fuseLists( + lists.map((items) => ({ items })), + (hit) => hit.key + ) + .slice(0, limit) + .map((entry) => entry.item) + }, + recent(window, limit) { + const db = getDB() + if (!tableExists(db, 'rag_documents')) return [] + const w = windowSql('d.created_at', window, 'datetime') + return ( + db + .prepare( + `SELECT d.id, d.name, d.project_id AS projectId, ${epochMs('d.created_at')} AS ts, + COALESCE((SELECT substr(c.text, 1, ${EXCERPT_CHARS}) FROM attached_document_chunks c + WHERE c.document_id = d.document_id ORDER BY c.position LIMIT 1), '') AS text + FROM rag_documents d WHERE 1 = 1${w.sql} + ORDER BY d.created_at DESC LIMIT ?` + ) + .all(...w.args, limit) as Row[] + ).map(docHit) + } +} + +/** Every keyword source, in no particular order (the engine fuses them with equal votes). */ +export const RECORD_SOURCES: RecordSource[] = [ + screenSource, + actionSource, + meetingSource, + entitySource, + summarySource, + chatSource, + docSource +] + +// --------------------------------------------------------------------------------------------- +// What each record is embedded with +// --------------------------------------------------------------------------------------------- + +export interface EmbeddableRecord { + kind: string + refId: string + ts: number + text: string +} + +function rowsFor(db: Db, table: string, sql: string, ids: number[]): Row[] { + if (!ids.length || !tableExists(db, table)) return [] + return db.prepare(sql).all(json(ids)) as Row[] +} + +/** + * The text each key is embedded with, and its kind and time. Keys whose row is gone are absent. + * A moment is its summary plus a de-duplicated excerpt of its frames' words (no raw frame is + * embedded on its own); a person is their name, type and summary; a to-do is its text and due date; + * a meeting is its title and summary. + */ +export function embeddableRecords(keys: string[]): Map { + const db = getDB() + const byPrefix = new Map() + for (const key of keys) { + const { prefix, id } = keyParts(key) + const ids = byPrefix.get(prefix) ?? [] + ids.push(Number(id)) + byPrefix.set(prefix, ids) + } + const out = new Map() + const put = (prefix: string, kind: string, row: Row, body: string): void => { + out.set(`${prefix}:${num(row.id)}`, { + kind, + refId: String(num(row.id)), + ts: num(row.ts), + text: body.replace(/\s+/g, ' ').trim().slice(0, EMBED_CHARS) + }) + } + const obsIds = byPrefix.get('obs') ?? [] + const excerpts = frameExcerpts(db, obsIds) + for (const row of rowsFor( + db, + 'observations', + `SELECT id, summary, ${epochMs('ts')} AS ts FROM observations + WHERE id IN (SELECT value FROM json_each(?))`, + obsIds + )) + put('obs', 'screen', row, `${text(row.summary)}\n${excerpts.get(num(row.id)) ?? ''}`) + for (const row of rowsFor( + db, + 'action_items', + `SELECT a.id, a.text, a.due, ${dueAtSql(db, 'a')} AS dueAt, ${ACTION_WHEN} AS ts + FROM action_items a + WHERE a.id IN (SELECT value FROM json_each(?))`, + byPrefix.get('act') ?? [] + )) + put('act', 'action', row, actionText(row)) + for (const row of rowsFor( + db, + 'meetings', + `SELECT id, title, summary, COALESCE(started_at, created_at, 0) AS ts FROM meetings + WHERE id IN (SELECT value FROM json_each(?))`, + byPrefix.get('mtg') ?? [] + )) + put('mtg', 'meeting', row, [text(row.title), text(row.summary)].filter(Boolean).join('. ')) + for (const row of rowsFor( + db, + 'entities', + `SELECT id, name, type, summary, ${epochMs('updated_at')} AS ts FROM entities + WHERE id IN (SELECT value FROM json_each(?))`, + byPrefix.get('ent') ?? [] + )) + put('ent', 'entity', row, entityText(row)) + for (const row of rowsFor( + db, + 'chat_summaries', + `SELECT rowid AS id, summary, ${epochMs('updated_at')} AS ts FROM chat_summaries + WHERE rowid IN (SELECT value FROM json_each(?))`, + byPrefix.get('sum') ?? [] + )) + put('sum', 'summary', row, text(row.summary)) + return out +} diff --git a/src/main/records/store.ts b/src/main/records/store.ts new file mode 100644 index 000000000..661eeef76 --- /dev/null +++ b/src/main/records/store.ts @@ -0,0 +1,338 @@ +// Record vectors: float32 BLOBs in record_vectors (the encrypted database), searched through an +// in-memory int8 index (@offgrid/rag Int8Index: per-dimension calibration, ×4 oversample, exact +// float rescore of the shortlist). The index is built on the first meaning search, a chunk at a +// time so the main thread keeps breathing, and released after a few idle minutes; the floats +// never leave SQLite except for a shortlist being rescored. +// +// The embed queue (record_queue, filled by triggers) is drained in the background: each batch +// reads the records' text, skips those whose text and model are unchanged (hash), embeds the rest +// in one call and writes them in one transaction. + +import crypto from 'crypto' +import { + calibrate, + Int8Index, + type RecordHit, + type RecordVectorPort, + type TimeWindow +} from '@offgrid/rag/records' +import { getDB } from '../database' +import { RECORD_EMBEDDER } from '../embedding-models' +import { embeddableRecords, hydrateRecords } from './sources' + +/** The index is dropped after this long without a meaning search. */ +const IDLE_RELEASE_MS = 5 * 60_000 +/** Vectors the int8 ranges are calibrated on (the newest stored). */ +const CALIBRATION_SAMPLE = 2_000 +/** Rows read per slice while the index loads. */ +const LOAD_CHUNK = 2_000 +/** Records embedded per call; one forward pass each. Small, because a search that arrives while a + * batch runs waits for it. On the CPU a long record costs about 0.37 s whatever the batch size + * (measured 2026-10-15, 32 and 8 the same), so a small batch costs no throughput. */ +const EMBED_BATCH = 4 + +interface Meta { + kind: string + ts: number +} + +interface LoadedIndex { + vectors: Int8Index + meta: Map +} + +let index: LoadedIndex | null = null +let loading: Promise | null = null +/** Bumped on release, so a load that finishes after it is thrown away. */ +let generation = 0 +/** Writes made while the index loads, applied once it has (null: removed). */ +const pendingWrites = new Map() +let idleTimer: ReturnType | null = null + +function toBlob(vector: ArrayLike): Buffer { + return Buffer.from(new Float32Array(vector).buffer) +} + +function fromBlob(blob: Buffer): Float32Array { + // Copy: a Buffer may share a pooled ArrayBuffer at an offset that is not 4-byte aligned. + const copy = new Uint8Array(blob.byteLength) + copy.set(blob) + return new Float32Array(copy.buffer) +} + +function textHash(text: string): string { + return crypto.createHash('sha256').update(text).digest('hex') +} + +/** Drop the in-memory index (idle, or the data behind it was deleted). The next search reloads it. */ +export function releaseRecordIndex(): void { + generation++ + index = null + loading = null + pendingWrites.clear() + if (idleTimer) clearTimeout(idleTimer) + idleTimer = null +} + +function touch(): void { + if (idleTimer) clearTimeout(idleTimer) + idleTimer = setTimeout(releaseRecordIndex, IDLE_RELEASE_MS) + idleTimer.unref() +} + +async function loadIndex(started: number): Promise { + const db = getDB() + const model = RECORD_EMBEDDER.model + const read = db.prepare( + `SELECT rowid, key, kind, ts, vector FROM record_vectors + WHERE model = ? AND rowid < ? ORDER BY rowid DESC LIMIT ?` + ) + const count = ( + db.prepare('SELECT COUNT(*) AS c FROM record_vectors WHERE model = ?').get(model) as { + c: number + } + ).c + let built: LoadedIndex | null = null + let cursor = Number.MAX_SAFE_INTEGER + for (;;) { + if (generation !== started) return null + const rows = read.all(model, cursor, LOAD_CHUNK) as { + rowid: number + key: string + kind: string + ts: number + vector: Buffer + }[] + if (!rows.length) break + cursor = rows[rows.length - 1]!.rowid + const vectors = rows.map((row) => fromBlob(row.vector)) + if (!built) { + const dimensions = vectors[0]!.length + built = { + vectors: new Int8Index( + dimensions, + calibrate(vectors.slice(0, CALIBRATION_SAMPLE), dimensions), + Math.max(1024, count) + ), + meta: new Map() + } + } + rows.forEach((row, i) => { + built!.vectors.add(row.key, vectors[i]!) + built!.meta.set(row.key, { kind: row.kind, ts: row.ts }) + }) + // Let the window and IPC run between slices. + await new Promise((resolve) => setImmediate(resolve)) + } + if (generation !== started || !built) return null + for (const [key, write] of pendingWrites) { + if (write) { + built.vectors.add(key, write.vector) + built.meta.set(key, write.meta) + } else { + built.vectors.remove(key) + built.meta.delete(key) + } + } + pendingWrites.clear() + return built +} + +async function loadedIndex(): Promise { + touch() + if (index) return index + if (!loading) { + const started = generation + loading = loadIndex(started) + .then((loaded) => { + if (generation === started) index = loaded + return loaded + }) + .finally(() => { + if (generation === started) loading = null + }) + } + return loading +} + +function noteWrite(key: string, meta: Meta, vector: Float32Array): void { + if (index) { + index.vectors.add(key, vector) + index.meta.set(key, meta) + } else if (loading) pendingWrites.set(key, { meta, vector }) +} + +function noteRemoval(key: string): void { + if (index) { + index.vectors.remove(key) + index.meta.delete(key) + } else if (loading) pendingWrites.set(key, null) +} + +/** Exact float vectors for a shortlist of keys. */ +function readVectors(keys: string[]): Map { + const rows = getDB() + .prepare( + `SELECT key, vector FROM record_vectors + WHERE key IN (SELECT value FROM json_each(?)) AND model = ?` + ) + .all(JSON.stringify(keys), RECORD_EMBEDDER.model) as { key: string; vector: Buffer }[] + return new Map(rows.map((row) => [row.key, fromBlob(row.vector)])) +} + +function inWindow(ts: number, window: TimeWindow | null | undefined): boolean { + return !window || (ts >= window.start && ts < window.end) +} + +/** The engine's meaning search over record_vectors. */ +export const recordVectors: RecordVectorPort = { + async search(query, limit, filter) { + const loaded = await loadedIndex() + if (!loaded) return [] + return loaded.vectors.search(query, limit, { + accept: (key) => { + const meta = loaded.meta.get(key) + return ( + !!meta && + (!filter.kinds || filter.kinds.has(meta.kind)) && + inWindow(meta.ts, filter.window) + ) + }, + rescore: readVectors, + oversample: 4 + }) + }, + hydrate(keys) { + const hits: RecordHit[] = hydrateRecords(keys) + // A key with no row was deleted since the index loaded; it leaves the index too. + if (index && hits.length < keys.length) { + const found = new Set(hits.map((hit) => hit.key)) + const stored = new Set( + ( + getDB() + .prepare('SELECT key FROM record_vectors WHERE key IN (SELECT value FROM json_each(?))') + .all(JSON.stringify(keys.filter((key) => !found.has(key)))) as { key: string }[] + ).map((row) => row.key) + ) + for (const key of keys) if (!found.has(key) && !stored.has(key)) noteRemoval(key) + } + return hits + } +} + +/** Whether any record has a vector from the current embedder (one index probe). Until one does, + * a meaning search has nothing to compare the question with, so the question is not embedded. */ +export function hasRecordVectors(): boolean { + if (index) return true + return !!getDB() + .prepare('SELECT 1 FROM record_vectors WHERE model = ? LIMIT 1') + .get(RECORD_EMBEDDER.model) +} + +/** Stored vectors (current model) and records waiting to be embedded. */ +export function recordVectorStatus(): { vectors: number; pending: number } { + const db = getDB() + const vectors = ( + db + .prepare('SELECT COUNT(*) AS c FROM record_vectors WHERE model = ?') + .get(RECORD_EMBEDDER.model) as { + c: number + } + ).c + const pending = (db.prepare('SELECT COUNT(*) AS c FROM record_queue').get() as { c: number }).c + return { vectors, pending } +} + +/** + * Embed one batch from the queue, newest first. Returns how many were embedded, how many left the + * queue (embedded, unchanged or empty) and how many wait. + * A record that was deleted, or edited, while it was being embedded is not written: the delete + * trigger already removed it from the queue, and an edit is embedded again from the queue. + */ +export async function embedQueuedRecords( + embed: (texts: string[]) => Promise, + limit = EMBED_BATCH +): Promise<{ embedded: number; processed: number; remaining: number }> { + const db = getDB() + const model = RECORD_EMBEDDER.model + const keys = db + .prepare('SELECT key FROM record_queue ORDER BY rowid DESC LIMIT ?') + .pluck() + .all(limit) as string[] + if (!keys.length) return { embedded: 0, processed: 0, remaining: 0 } + + const records = embeddableRecords(keys) + const stored = new Map( + ( + db + .prepare( + 'SELECT key, hash, model FROM record_vectors WHERE key IN (SELECT value FROM json_each(?))' + ) + .all(JSON.stringify(keys)) as { key: string; hash: string; model: string }[] + ).map((row) => [row.key, row]) + ) + const unchanged: string[] = [] + const empty: string[] = [] + const work: { key: string; hash: string }[] = [] + for (const key of keys) { + const record = records.get(key) + if (!record?.text) { + empty.push(key) + continue + } + const hash = textHash(record.text) + const previous = stored.get(key) + if (previous?.hash === hash && previous.model === model) unchanged.push(key) + else work.push({ key, hash }) + } + + const vectors = work.length + ? await embed(work.map(({ key }) => RECORD_EMBEDDER.passagePrefix + records.get(key)!.text)) + : [] + + const dequeue = db.prepare('DELETE FROM record_queue WHERE key = ?') + const forget = db.prepare('DELETE FROM record_vectors WHERE key = ?') + const write = db.prepare( + `INSERT INTO record_vectors (key, kind, ref_id, ts, hash, model, vector) + VALUES (@key, @kind, @refId, @ts, @hash, @model, @vector) + ON CONFLICT(key) DO UPDATE SET kind = excluded.kind, ref_id = excluded.ref_id, + ts = excluded.ts, hash = excluded.hash, model = excluded.model, vector = excluded.vector` + ) + const written: { key: string; meta: Meta; vector: Float32Array }[] = [] + const removed: string[] = [] + db.transaction(() => { + for (const key of unchanged) dequeue.run(key) + for (const key of empty) { + dequeue.run(key) + if (forget.run(key).changes) removed.push(key) + } + // Read the text again inside the transaction: only what is still there, unchanged, is written. + const current = embeddableRecords(work.map(({ key }) => key)) + work.forEach(({ key, hash }, i) => { + const vector = vectors[i] + const record = current.get(key) + if (!vector?.length || !record || textHash(record.text) !== hash) return + if (!dequeue.run(key).changes) return + const floats = new Float32Array(vector) + write.run({ + key, + kind: record.kind, + refId: record.refId, + ts: record.ts, + hash, + model, + vector: toBlob(floats) + }) + written.push({ key, meta: { kind: record.kind, ts: record.ts }, vector: floats }) + }) + })() + for (const { key, meta, vector } of written) noteWrite(key, meta, vector) + for (const key of removed) noteRemoval(key) + + const remaining = (db.prepare('SELECT COUNT(*) AS c FROM record_queue').get() as { c: number }).c + return { + embedded: written.length, + processed: unchanged.length + empty.length + written.length, + remaining + } +} diff --git a/src/main/runtime-availability.ts b/src/main/runtime-availability.ts new file mode 100644 index 000000000..18bd533d7 --- /dev/null +++ b/src/main/runtime-availability.ts @@ -0,0 +1,74 @@ +// Whether an optional runtime is on this machine. A model or voice whose runtime is missing is not +// offered: downloading 9 GB of Kev weights, or a voice's assets, only to fail at first use is the +// worst order of events. + +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { binRoots, exe, modelsDir, resourceDirs } from './runtime-env' + +/** macOS major version, or null elsewhere (Darwin 22 = macOS 13). */ +function macosMajor(): number | null { + if (process.platform !== 'darwin') return null + const darwin = Number(os.release().split('.')[0]) + return Number.isFinite(darwin) ? darwin - 9 : null +} + +/** Kev's runtime (bundled on macOS, in the CUDA pack on Windows and Linux). Its MLX build on macOS + * needs macOS 14. */ +export function kevRuntimeInstalled(): boolean { + const supported = + (process.platform === 'darwin' && process.arch === 'arm64' && (macosMajor() ?? 0) >= 14) || + process.platform === 'win32' || + (process.platform === 'linux' && process.arch === 'x64') + if (!supported) return false + const python = + process.platform === 'win32' + ? path.join('kev-runtime', 'python', 'python.exe') + : path.join('kev-runtime', 'python', 'bin', 'python3') + // A source checkout runs Kev from its development virtualenv (models-manager resolves the same). + const developmentPython = path.join( + path.dirname(modelsDir()), + 'decision-models', + 'kev-source', + '.venv', + 'bin', + 'python' + ) + return ( + (binRoots().some((root) => fs.existsSync(path.join(root, python))) || + fs.existsSync(developmentPython)) && + binRoots().some((root) => fs.existsSync(path.join(root, 'kev-local-server.py'))) + ) +} + +/** The sherpa-onnx offline CLI that runs Parakeet. */ +export function parakeetBinary(): string | null { + const name = exe('sherpa-onnx-offline') + for (const root of binRoots()) { + for (const candidate of [ + path.join(root, 'parakeet', 'bin', name), + path.join(root, 'parakeet', name) + ]) { + if (fs.existsSync(candidate)) return candidate + } + } + return null +} + +export function parakeetRuntimeInstalled(): boolean { + return parakeetBinary() !== null +} + +/** The ExecuTorch speech runtime that speaks every non-English voice. */ +export function executorchSpeechBinary(): string | null { + const candidates = [ + ...resourceDirs().map((root) => path.join(root, 'bin', 'executorch-speech')), + path.resolve(process.cwd(), '../executorch-speech/native/bin/executorch-speech') + ] + return candidates.find((candidate) => fs.existsSync(candidate)) ?? null +} + +export function executorchSpeechInstalled(): boolean { + return executorchSpeechBinary() !== null +} diff --git a/src/main/runtime-backends.ts b/src/main/runtime-backends.ts index 021a83ed6..d765ffc3b 100644 --- a/src/main/runtime-backends.ts +++ b/src/main/runtime-backends.ts @@ -8,8 +8,27 @@ export function runtimeBackendSnapshot(): RuntimeBackend[] { } /** A process-scoped lease prevents late output/exit from replacing a newer model. */ -export function beginRuntimeBackend(id: RuntimeId, model: string): { recordRequest: (request?: ReturnType) => void; ready: (backend?: string, device?: string, detail?: string) => void; observe: (chunk: string, loaded?: boolean) => void; stop: () => void; fail: (error: unknown) => void; } { - const record: RuntimeBackend = { id, model, state: 'loading' } +export function beginRuntimeBackend( + id: RuntimeId, + model: string +): { + recordRequest: (request?: ReturnType) => void + ready: (backend?: string, device?: string, detail?: string) => void + observe: (chunk: string, loaded?: boolean) => void + stop: () => void + fail: (error: unknown) => void +} { + // A crash is kept until the next successful load, and an engine that loads per job keeps + // what it last ran on, so the next attempt does not wipe either from Settings. + const previous = live.get(id) + const record: RuntimeBackend = { + id, + model, + state: 'loading', + loadingSince: Date.now(), + ...(previous?.lastError ? { lastError: previous.lastError } : {}), + ...(previous?.lastRun ? { lastRun: previous.lastRun } : {}) + } const startingRequest = currentAIRequest() live.set(id, record) const update = (value: Partial): void => { @@ -24,26 +43,42 @@ export function beginRuntimeBackend(id: RuntimeId, model: string): { recordReque recordRequest: (request = currentAIRequest()): void => { request?.useRuntime(record) }, - ready: (backend?: string, device?: string, detail?: string): void => + ready: (backend?: string, device?: string, detail?: string): void => { update({ state: 'loaded', + loadingSince: undefined, + lastError: undefined, ...(backend ? { backend } : {}), ...(device ? { device } : {}), - ...(detail ? { detail } : {}) - }), + ...(detail ? { detail, fallbackReason: detail } : { fallbackReason: undefined }) + }) + }, observe: (chunk: string, loaded = false): void => { output = (output + chunk).slice(-65536) const placement = parseNativeBackend(output) if (placement) update({ ...placement, ...(loaded ? { state: 'loaded' as const } : {}) }) }, - stop: (): void => update({ state: 'stopped', backend: undefined, device: undefined }), - fail: (error: unknown): void => + stop: (): void => + update({ + state: 'stopped', + loadingSince: undefined, + ...(record.backend + ? { lastRun: { backend: record.backend, device: record.device, at: Date.now() } } + : {}), + backend: undefined, + device: undefined + }), + fail: (error: unknown): void => { + const message = String(error instanceof Error ? error.message : error).slice(0, 1000) update({ state: 'error', + loadingSince: undefined, backend: undefined, device: undefined, - detail: String(error instanceof Error ? error.message : error).slice(0, 1000) + detail: message, + lastError: message }) + } } } @@ -61,7 +96,7 @@ export function parseNativeBackend( ? 'CUDA' : name.startsWith('vulkan') ? 'Vulkan' - : name === 'metal' || name.startsWith('mtl') + : name === 'metal' || name.startsWith('mtl') ? 'Metal' : 'CPU' } diff --git a/src/main/runtime-manager.ts b/src/main/runtime-manager.ts index 8904fa63b..36ba05412 100644 --- a/src/main/runtime-manager.ts +++ b/src/main/runtime-manager.ts @@ -17,11 +17,7 @@ import { readFileSync } from 'node:fs' import os from 'node:os' import type { ResidentType } from '@offgrid/models' import { modalityQueue, type ModalityQueue } from './modality-queue/queue' -import { getResidencyMode, type Modality, type ResidencyMode } from './runtime-residency' -import type { RuntimeBackend } from '../shared/runtime-backends' -import { sdServer } from './sd-server' -import { findSdBinaries, imageBackendForRuntime } from './imagegen/sd-runtime' -import { getBackendPreference } from './backend-preferences' +import type { Modality, ResidencyMode } from './runtime-residency' /** A model runtime the ModalityQueue can free + bring back. Every engine implements * this identically — the queue never knows which concrete engine it holds. */ @@ -98,7 +94,6 @@ let shuttingDown = false export interface RegisterDeps { queue?: ModalityQueue readMode?: (m: Modality) => ResidencyMode - readChatBackend?: () => RuntimeBackend } /** Register a runtime into the queue through the ONE residency seam. The queue calls evict() @@ -110,7 +105,6 @@ export function registerRuntime(rt: ManagedRuntime, deps: RegisterDeps = {}): vo return } const queue = deps.queue ?? modalityQueue - const readMode = deps.readMode ?? getResidencyMode registry.set(rt.modality, rt) queue.registerEvictable(rt.modality, { type: RESIDENT_TYPE[rt.modality], @@ -120,29 +114,9 @@ export function registerRuntime(rt: ManagedRuntime, deps: RegisterDeps = {}): vo // loaded on-demand engine frees it, not the model it would otherwise squeeze out. footprintMB: () => rt.footprintMB?.() ?? 0, ...(rt.expectedMB ? { expectedMB: () => rt.expectedMB!() } : {}), - evict: () => { - // Dedicated CUDA runtimes have their own allocation/auto-fit policy. Do not - // apply the unified-memory Mac's unconditional chat/image teardown there. - // Explicit Unload and shutdown still call rt.evict() directly. - if ( - process.platform !== 'darwin' && - readMode(rt.modality) === 'resident' && - (rt.modality === 'llm' || rt.modality === 'image') - ) { - const imageBinary = - rt.modality === 'image' - ? sdServer.getBinaryPath() - : findSdBinaries('sd-server', getBackendPreference('image'))[0] - const chat = rt.modality === 'llm' ? deps.readChatBackend?.() : undefined - if ( - imageBinary && - imageBackendForRuntime(process.platform, imageBinary) === 'CUDA' && - (rt.modality === 'image' || (chat?.state === 'loaded' && chat.backend?.includes('CUDA'))) - ) - return - } - return rt.evict() - }, + // Chat and image evict each other on every platform when the plan needs the room: a resident + // chat model and a resident image model on one GPU run out of VRAM mid-job. + evict: () => rt.evict(), recover: () => rt.release() }) } diff --git a/src/main/sd-server.ts b/src/main/sd-server.ts index 786bcb26f..82c9e4e59 100644 --- a/src/main/sd-server.ts +++ b/src/main/sd-server.ts @@ -25,7 +25,6 @@ import os from 'os' import { isPackaged } from './runtime-env' import { killOrphansOnPort as reapOrphansOnPort } from './kill-orphan-port' import { findSdBinaries, sdRuntimeLibraryEnv } from './imagegen/sd-runtime' -import { getBackendPreference } from './backend-preferences' import { getResidencyMode } from './runtime-residency' import { filesMB } from './model-footprint' @@ -312,24 +311,37 @@ class SdServerService { } private async startWithFallback(ctx: SdServerContext, key: string, epoch: number): Promise { - const binaries = findSdBinaries('sd-server', getBackendPreference('image')) - if (!binaries.length) - throw new Error('Image server binary (sd-server) not found in resources/bin/sd.') - for (const [index, binary] of binaries.entries()) { + const binaries = await findSdBinaries('sd-server') + if (!binaries.length) throw new Error('No image engine is installed for this choice.') + const failures: string[] = [] + for (const [index, step] of binaries.entries()) { if (epoch !== this.startEpoch) throw new Error('Image server startup canceled.') try { - await this.spawn(ctx, key, binary) + await this.spawn( + ctx, + key, + step.path, + step.cpuOnly && step.backend !== 'cpu', + failures.length ? `Earlier engines did not start: ${failures.join('; ')}` : undefined + ) if (epoch !== this.startEpoch) throw new Error('Image server startup canceled.') return } catch (error) { this.stopProcess() + failures.push(error instanceof Error ? error.message : String(error)) if (epoch !== this.startEpoch || index === binaries.length - 1) throw error console.warn(`[sd-server] Runtime failed; retrying ${binaries[index + 1]}`, error) } } } - private async spawn(ctx: SdServerContext, key: string, bin: string): Promise { + private async spawn( + ctx: SdServerContext, + key: string, + bin: string, + cpuOnly: boolean, + fallbackReason?: string + ): Promise { if (!fs.existsSync(ctx.modelPath)) throw new Error(`Image model not found: ${ctx.modelPath}`) const binDir = path.dirname(bin) @@ -345,9 +357,10 @@ class SdServerService { // Kill an orphaned sd-server from a prior app process still holding the port, // but never a foreign process that merely happened to bind it. - this.killOrphanOnPort() + await this.killOrphanOnPort() const args = buildSdServerContextArgs({ ...ctx, port: this.port }) + if (cpuOnly) args.push('--backend', 'cpu') // cwd at the binary dir so @executable_path rpath resolves libstable-diffusion.dylib. // Windows: prepend binDir to PATH so the co-located stable-diffusion DLLs resolve. const proc = spawn(bin, args, { @@ -386,7 +399,7 @@ class SdServerService { try { await this.waitForReady() - backendState.ready() + backendState.ready(cpuOnly ? 'CPU' : undefined, undefined, fallbackReason) } catch (error) { backendState.fail(error) throw error @@ -518,19 +531,19 @@ class SdServerService { } } - private killOrphanOnPort(): void { + private async killOrphanOnPort(): Promise { // Shared cross-platform reaper (kill-orphan-port.ts) — adds the Windows branch this // used to lack (a crashed sd-server would keep holding the port after a restart). // posix matches our OWN bundled binary by FULL PATH (a user's separately-run sd-server // is left untouched); win32's tasklist only exposes the image name, so match that. - const ownBins = findSdBinaries('sd-server', getBackendPreference('image')) + const ownBins = await findSdBinaries('sd-server') if (!ownBins.length) return reapOrphansOnPort( this.port, (info) => process.platform === 'win32' ? /sd-server/i.test(info) - : ownBins.some((bin) => info.includes(bin)), + : ownBins.some((bin) => info.includes(bin.path)), 'sd-server' ) } diff --git a/src/main/search-backfill-queries.ts b/src/main/search-backfill-queries.ts deleted file mode 100644 index 673e80580..000000000 --- a/src/main/search-backfill-queries.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { epochMsSql } from './search-ranking' - -// Keep the vec_indexed anti-join first in every source predicate. SQLite short-circuits the -// predicate for rows already handled by the backfill, so it does not read their large OCR/text -// values merely to prove that there is no work left. -export const PENDING_SOURCE_SQL = [ - `SELECT 'frame:'||f.id AS key, 'screen' AS kind, f.id AS refId, f.text AS text, - COALESCE(f.surface,'') AS surface, COALESCE(f.url,'') AS url, - ${epochMsSql('f.ts')} AS ts - FROM frames f - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'frame:'||f.id) - AND f.text IS NOT NULL AND COALESCE(f.source, '') != 'connector' - LIMIT ?`, - `SELECT 'obs:'||o.id AS key, 'screen' AS kind, o.id AS refId, o.summary AS text, - COALESCE(o.surface,'') AS surface, COALESCE(o.url,'') AS url, - ${epochMsSql('o.ts')} AS ts - FROM observations o - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'obs:'||o.id) - AND o.summary IS NOT NULL AND length(o.summary) > 0 - LIMIT ?`, - `SELECT 'sum:'||s.rowid AS key, 'meeting' AS kind, s.rowid AS refId, - s.summary AS text, 'Meeting' AS surface, '' AS url, 0 AS ts - FROM chat_summaries s - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'sum:'||s.rowid) - AND s.summary IS NOT NULL - LIMIT ?`, - `SELECT 'mtg:'||m.id AS key, 'meeting' AS kind, m.id AS refId, - COALESCE(m.title,'Meeting')||'. '||COALESCE(m.summary, substr(m.transcript,1,2000)) AS text, - 'Meeting' AS surface, '' AS url, COALESCE(m.started_at,0) AS ts - FROM meetings m - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'mtg:'||m.id) - AND COALESCE(m.summary, m.transcript) IS NOT NULL - LIMIT ?`, - `SELECT 'mem:'||m.id AS key, 'memory' AS kind, m.id AS refId, m.content AS text, - COALESCE(m.source_app,'') AS surface, '' AS url, 0 AS ts - FROM memories m - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'mem:'||m.id) - AND m.content IS NOT NULL - LIMIT ?`, - `SELECT 'ent:'||e.id AS key, 'entity' AS kind, e.id AS refId, - e.name||' '||COALESCE(e.summary,'') AS text, 'Entity' AS surface, '' AS url, 0 AS ts - FROM entities e - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'ent:'||e.id) - AND e.hidden = 0 - LIMIT ?`, - `SELECT 'fact:'||f.id AS key, 'fact' AS kind, f.entity_id AS refId, - f.fact AS text, 'Fact' AS surface, '' AS url, 0 AS ts - FROM entity_facts f - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'fact:'||f.id) - LIMIT ?` -] as const - -export const PENDING_COUNT_SQL = `SELECT - (SELECT COUNT(*) FROM frames f - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'frame:'||f.id) - AND f.text IS NOT NULL) + - (SELECT COUNT(*) FROM observations o - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'obs:'||o.id) - AND o.summary IS NOT NULL AND length(o.summary) > 0) + - (SELECT COUNT(*) FROM chat_summaries s - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'sum:'||s.rowid) - AND s.summary IS NOT NULL) + - (SELECT COUNT(*) FROM meetings m - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'mtg:'||m.id) - AND COALESCE(m.summary, m.transcript) IS NOT NULL) + - (SELECT COUNT(*) FROM memories m - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'mem:'||m.id) - AND m.content IS NOT NULL) + - (SELECT COUNT(*) FROM entities e - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'ent:'||e.id) - AND e.hidden = 0) + - (SELECT COUNT(*) FROM entity_facts f - WHERE NOT EXISTS (SELECT 1 FROM vec_indexed v WHERE v.key = 'fact:'||f.id)) AS c` diff --git a/src/main/search-ranking.ts b/src/main/search-ranking.ts deleted file mode 100644 index b4fa9021c..000000000 --- a/src/main/search-ranking.ts +++ /dev/null @@ -1,201 +0,0 @@ -// Pure search-ranking math, extracted so the ranking behaviour is unit-testable -// without the DB / Electron (mirrors model-sizing.ts). No imports, no side effects. -// -// These encode the search behaviours added this session: -// 1. Recency bias — recent hits float up (this week > last month > last quarter). -// 2. Own-content nudge — your deliberate chats / KB docs aren't buried under -// thousands of ambient screen captures. -// 3. "Match" sort — rank by literal term overlap in title/snippet. - -import { escapeFtsMatch } from './fts-match' -import type { SearchKind, SearchResult, SearchSort } from '../shared/search-contract' - -export type { SearchKind, SearchResult, SearchSort } from '../shared/search-contract' - -/** Recency boost added to a fused score. Tiered + sized to ~one rank of RRF, so a - * strong older match can still beat a weak recent one. No timestamp → 0. */ -export function recencyBoost(ts: number, now: number): number { - if (!ts) return 0 - const ageDays = (now - ts) / 86_400_000 - if (ageDays < 7) return 0.016 - if (ageDays < 30) return 0.009 - if (ageDays < 90) return 0.004 - return 0 -} - -/** Small visibility nudge for the user's own deliberate content (chats / KB docs). */ -export function kindBoost(kind: string): number { - return kind === 'chat' || kind === 'doc' ? 0.012 : 0 -} - -/** Literal term-overlap count in a haystack (for the "Match" sort). Case-insensitive. */ -export function matchScore(haystack: string, terms: string[]): number { - const hay = (haystack || '').toLowerCase() - return terms.reduce((n, t) => n + (t ? hay.split(t).length - 1 : 0), 0) -} - -// --------------------------------------------------------------------------- -// Query tokenisation + FTS expression (pure — no DB). One source of truth for -// how a free-text query splits into terms, shared by every LIKE/facet path. -// --------------------------------------------------------------------------- - -/** Extract lowercased alphanumeric terms from a free-text query, capped at `max`. - * This is the single tokeniser used by every LIKE / facet source. */ -export function queryTerms(query: string, max: number): string[] { - return (query.toLowerCase().match(/[\p{L}\p{N}]+/gu) || []).slice(0, max) -} - -/** Sanitize a free-text query into an FTS5 MATCH expression: each token becomes a - * prefix-match term; punctuation is dropped so user input can't be a syntax error. */ -export function ftsExpr(query: string): string { - return escapeFtsMatch(queryTerms(query, 12), { join: ' ', prefix: true }) -} - -/** - * SQL fragment converting a SQLite datetime column (stored as text) to epoch - * milliseconds — `CAST(strftime('%s', ) AS INTEGER)*1000`. The idiom was - * hand-written 6× across search.ts's SELECT lists; defined once here so a change - * to the conversion is one edit. `col` is a caller-controlled column identifier - * (never user input), safe to interpolate. - */ -export function epochMsSql(col: string): string { - return `CAST(strftime('%s', ${col}) AS INTEGER)*1000` -} - -/** - * Column sets for the no-FTS LIKE search of each source. THE SINGLE SOURCE OF - * TRUTH shared by the facet COUNT queries and the hit builders in search.ts — a - * source's facet count used to be able to diverge from its results because the - * WHERE clause was hand-written in both places. Add a column here and both update. - */ -export const LIKE_COLUMNS = { - chat: ['rm.content', 'rc.title'], - doc: ['c.content', 'd.name'], - meeting: ['title', 'summary', 'transcript'], - frame: ['text'] -} as const - -/** - * Build the term-AND-ed LIKE WHERE fragment AND its bound params for a column set: - * each term contributes one OR-group over the columns (`lower(col) LIKE ?`), the - * groups AND-ed together; params are lowercased `%term%` wildcards, columns.length - * per term. Returning both together guarantees the clause and the params always - * agree in count — the facet counter and the hit query call this identically. - */ -export function likeMatch( - columns: readonly string[], - terms: string[] -): { where: string; args: string[] } { - const perTerm = '(' + columns.map((c) => `lower(${c}) LIKE ?`).join(' OR ') + ')' - return { - where: terms.map(() => perTerm).join(' AND '), - args: terms.flatMap((t) => columns.map(() => `%${t}%`)) - } -} - -// --------------------------------------------------------------------------- -// Reciprocal-rank fusion + result assembly (pure — takes already-fetched hits) -// --------------------------------------------------------------------------- - -/** A raw hit fetched from one source (FTS / LIKE / semantic), before fusion. */ -export interface RawHit { - key: string - kind: SearchKind - refId: number - title: string - snippet: string - surface: string - url: string | null - ts: number -} - -/** A snippet reads as prose: chat and notes are stored as markdown, and a result list that - * shows `**bold**`, fences or `` tags reads as source code. Capped at 280 chars. */ -export function plainSnippet(text: string): string { - return (text || '') - .replace(/```[\w-]*\n?/g, ' ') - .replace(/<\/?[a-z!][^<>\n]{0,200}>/gi, ' ') - .replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1') - .replace(/^\s{0,3}(?:#{1,6}|>|[-*+]|\d+\.)\s+/gm, '') - .replace(/(? { - const fused = new Map() - for (const list of lists) { - list.forEach((hit, rank) => { - const existing = fused.get(hit.key) - if (existing) { - existing.score += rrf(rank) - return - } - fused.set(hit.key, { - key: hit.key, - kind: hit.kind, - refId: hit.refId, - title: hit.title || hit.kind, - snippet: plainSnippet(hit.snippet), - surface: hit.surface || '', - url: hit.url, - ts: hit.ts || 0, - imagePath: null, - score: rrf(rank) - }) - }) - } - return fused -} - -/** Add the recency bias + own-content nudge to every fused result, in place. */ -export function applyBoosts(results: Iterable, now: number): void { - for (const r of results) { - r.score += recencyBoost(r.ts, now) - r.score += kindBoost(r.kind) - } -} - -/** Filter + sort fused results into final ranked order. Pure: takes the fused - * results, applies the source / current-chat filters, then sorts by the chosen - * mode (relevance = blended score, recency = newest first, match = literal term - * overlap in title/snippet with score as tie-break). */ -export function rankResults( - results: SearchResult[], - opts: { - query: string - sources?: string[] - kinds?: SearchKind[] - excludeChatId?: string - sort?: SearchSort - } -): SearchResult[] { - const sourceSet = opts.sources?.length ? new Set(opts.sources.map((s) => s.toLowerCase())) : null - const kindSet = opts.kinds?.length ? new Set(opts.kinds) : null - let ordered = results - if (sourceSet) ordered = ordered.filter((r) => sourceSet.has((r.surface || '').toLowerCase())) - if (kindSet) ordered = ordered.filter((r) => kindSet.has(r.kind)) - // Don't let an answer cite the very conversation it's being asked in. - if (opts.excludeChatId) ordered = ordered.filter((r) => r.key !== `chat:${opts.excludeChatId}`) - const sort = opts.sort ?? 'relevance' - if (sort === 'recency') { - ordered.sort((a, b) => (b.ts || 0) - (a.ts || 0) || b.score - a.score) - } else if (sort === 'match') { - const terms = queryTerms(opts.query, Infinity) - const m = (r: SearchResult): number => matchScore(`${r.title} ${r.snippet}`, terms) - ordered.sort((a, b) => m(b) - m(a) || b.score - a.score) - } else { - ordered.sort((a, b) => b.score - a.score) - } - return ordered -} diff --git a/src/main/search.ts b/src/main/search.ts index 1a28d4717..050bd8704 100644 --- a/src/main/search.ts +++ b/src/main/search.ts @@ -1,606 +1,390 @@ -// Universal search — the single front door over everything Off Grid AI has seen. -// Hybrid: FTS5 keyword (exact words you saw) + LanceDB semantic (NLP recall), -// fused with reciprocal-rank fusion. Plus a background backfill that embeds the -// observation/frame/transcript backlog (using the in-app MiniLM model) so the -// semantic half actually covers your captured life. All local, all offline. -import { getDB } from './database' -import { frameTextIndexBackfilling } from './frame-text-index' -import { embeddings } from './embeddings' +// Universal search — the single front door over everything Off Grid AI has seen. It is the +// records engine (records/, over @offgrid/rag/records): keyword (FTS5) and meaning (record +// vectors) per source, fused with reciprocal-rank fusion, the people a question names, a time +// window read from the question, and an optional cross-encoder. This file adds what the Search +// screen and the command palette need on top: sources (surfaces) to filter by, facet counts, the +// sort modes, the newest-first feed for an empty query, and the embedding backlog. All local, all +// offline. +import { getDB, tableExists } from './database' import { ensureRagStoreSchema } from './rag/store' -import { ftsQuery } from './documents/sqlite-store' -import { addChunks, searchVectors, vectorCount, type VecChunk } from './vectors' import { - applyBoosts, - ftsExpr, - fuseHits, - queryTerms, - rankResults, - likeMatch, - LIKE_COLUMNS, - epochMsSql, - type RawHit, - type SearchKind, - type SearchResult, - type SearchSort -} from './search-ranking' -import { PENDING_COUNT_SQL, PENDING_SOURCE_SQL } from './search-backfill-queries' + ftsTrigramQuery, + ftsTypeahead, + needsTrigram, + parseTimeWindow, + withoutTimePhrase, + type RecordHit, + type TimeWindow +} from '@offgrid/rag/records' +import { + embedRecordBacklog, + ensureRecordSchema, + recordIndexStatus, + recordSearch, + recordToSearchResult, + screenMomentKey +} from './records' +import { + actionWhenSql, + epochMs, + hydrateRecords, + SYNCED_BULK_MAIL, + windowSql +} from './records/sources' +import type { SearchKind, SearchResult, SearchSort } from '../shared/search-contract' + +export type { SearchResult, SearchSort } from '../shared/search-contract' -export type { SearchResult, SearchSort } from './search-ranking' +/** Most results one search returns (the Search screen pages 50 at a time). */ +const MAX_RESULTS = 200 +/** Results asked of the engine beyond the page, for the moments Replay folds together. */ +const RESULT_MARGIN = 10 // --------------------------------------------------------------------------- -// Backfill: embed the backlog into LanceDB (keys tracked in SQLite to skip work) +// Backfill: embed the record backlog (record_queue) into record_vectors // --------------------------------------------------------------------------- -interface PendingRow { - key: string - kind: SearchKind - refId: number - text: string - surface: string - url: string - ts: number -} - -function ensureIndexTable(): void { - getDB().exec('CREATE TABLE IF NOT EXISTS vec_indexed (key TEXT PRIMARY KEY)') -} - -// The full seven-source count scans every capture table, so the backfill loop does not run it -// after every batch. It keeps an estimate that each batch decrements by the keys it newly -// marked, and replaces the estimate with an exact count when it reaches zero (to confirm the -// backlog is really empty) or when it is older than PENDING_RECOUNT_MS (to notice new items). -// An empty pending scan across all seven sources is an exact zero on its own. -const PENDING_RECOUNT_MS = 30_000 -let pendingEstimate: { count: number; countedAt: number } | undefined - -function pendingCount(): number { - ensureIndexTable() - const row = getDB().prepare(PENDING_COUNT_SQL).get() as { c: number } - pendingEstimate = { count: row.c, countedAt: Date.now() } - return row.c -} - -function remainingAfterBatch(marked: number): number { - if (!pendingEstimate) return pendingCount() - const count = Math.max(0, pendingEstimate.count - marked) - if (count === 0 || Date.now() - pendingEstimate.countedAt >= PENDING_RECOUNT_MS) { - return pendingCount() - } - pendingEstimate = { count, countedAt: pendingEstimate.countedAt } - return count -} - -function pendingRows(limit: number): PendingRow[] { - const db = getDB() - for (const sql of PENDING_SOURCE_SQL) { - const rows = db.prepare(sql).all(limit) as PendingRow[] - if (rows.length) return rows - } - return [] -} - -/** Embed one batch of un-indexed items into LanceDB. Returns progress. */ -async function indexBatch(limit = 48): Promise<{ indexed: number; remaining: number }> { - ensureIndexTable() - const db = getDB() - const rows = pendingRows(limit) - if (!rows.length) { - pendingEstimate = { count: 0, countedAt: Date.now() } - return { indexed: 0, remaining: 0 } - } - - const chunks: VecChunk[] = [] - for (const r of rows) { - const text = (r.text || '').trim().slice(0, 1000) - // Keep the frame noise threshold without asking SQLite to read every large OCR value just to - // calculate length(text). On a multi-gigabyte capture database that synchronous scan blocked - // Electron's main thread for seconds. Rows skipped here are still marked below, once only. - if (!text || (r.key.startsWith('frame:') && text.length <= 20)) continue - const vector = await embeddings.generateEmbedding(text) - chunks.push({ - key: r.key, - kind: r.kind, - refId: r.refId, - vector, - text: text.slice(0, 300), - surface: r.surface, - url: r.url, - ts: r.ts || 0 - }) - } - await addChunks(chunks) - - const mark = db.prepare('INSERT OR IGNORE INTO vec_indexed (key) VALUES (?)') - // Only keys inserted here leave the backlog; a key another writer marked meanwhile is ignored. - const marked = db.transaction(() => rows.reduce((sum, r) => sum + mark.run(r.key).changes, 0))() - return { indexed: chunks.length, remaining: remainingAfterBatch(marked) } -} - -let backfilling = false -/** Drain the backlog in the background, one throttled batch at a time. */ +/** Drain the embedding backlog in the background, one throttled batch at a time. */ export async function runBackfill( onProgress?: (p: { done: number; remaining: number }) => void ): Promise { - if (backfilling) return - backfilling = true - // Each run starts from an exact count, so items added since the last run are included. - pendingEstimate = undefined - try { - let done = 0 - for (;;) { - const { indexed, remaining } = await indexBatch() - done += indexed - onProgress?.({ done, remaining }) - if (remaining === 0) break - await new Promise((r) => setTimeout(r, 50)) // breathe — don't starve the LLM/UI - } - } finally { - backfilling = false - } + await embedRecordBacklog(onProgress) } export async function searchStatus(): Promise<{ vectors: number; pending: number }> { - return { vectors: await vectorCount(), pending: pendingCount() } + return recordIndexStatus() } -/** Data sources available to filter by (surfaces seen, busiest first, + meetings). */ -/** Bulk mail synced from a connected account (newsletters, promotions): searchable, but never counted - * or featured, as the ambient rules have it. Screens of videos or feeds you watched stay. */ -const SYNCED_BULK_MAIL = `(COALESCE(o.category, '') = 'consumption' AND o.source_key IS NOT NULL)` +// --------------------------------------------------------------------------- +// Sources (surfaces) and facet counts +// --------------------------------------------------------------------------- /** Distinct moments, as the result list shows them: one per app and summary, not one per capture. */ const DISTINCT_MOMENTS = 'COUNT(DISTINCT lower(trim(o.summary)))' +/** Surfaces that are not apps a moment was seen in, and the record kind each one is. Any other + * surface is a screen moment's app. */ +const SURFACE_KINDS: Record = { + meeting: ['meeting'], + chat: ['chat'], + 'knowledge base': ['doc'], + 'to-do': ['action'], + entity: ['entity'], + 'chat summary': ['summary'] +} + /** Totals are a full pass over every observation; the rail asks for them on every visit. */ const SOURCE_TOTALS_TTL_MS = 60_000 let sourceTotals: { at: number; rows: { source: string; count: number }[] } | null = null +/** Data sources available to filter by (surfaces seen, busiest first, + meetings, to-dos, chats + * and knowledge bases). Bulk mail synced from a connected account is searchable but not counted. */ export function searchSources(): { source: string; count: number }[] { const now = Date.now() if (sourceTotals && now - sourceTotals.at < SOURCE_TOTALS_TTL_MS) return sourceTotals.rows - const rows = countSources() + const rows = countSources(null) sourceTotals = { at: now, rows } return rows } -function countSources(): { source: string; count: number }[] { +/** Items per source, within `window` when given: what the empty-query feed and a time-only query + * ("yesterday") list. Only sources with at least one item. */ +function countSources(window: TimeWindow | null): { source: string; count: number }[] { ensureRagStoreSchema() const db = getDB() - const rows = db - .prepare( - `SELECT o.surface AS source, ${DISTINCT_MOMENTS} AS count FROM observations o - WHERE o.surface IS NOT NULL AND o.surface != '' AND NOT ${SYNCED_BULK_MAIL} - GROUP BY o.surface ORDER BY count DESC LIMIT 20` + const out: { source: string; count: number }[] = [] + if (tableExists(db, 'observations')) { + const w = windowSql('o.ts', window, 'datetime') + out.push( + ...(db + .prepare( + `SELECT o.surface AS source, ${DISTINCT_MOMENTS} AS count FROM observations o + WHERE o.surface IS NOT NULL AND o.surface != '' AND NOT ${SYNCED_BULK_MAIL} + AND o.summary IS NOT NULL AND o.summary != ''${w.sql} + GROUP BY o.surface ORDER BY count DESC LIMIT 20` + ) + .all(...w.args) as { source: string; count: number }[]) ) - .all() as { source: string; count: number }[] - const mtg = db.prepare('SELECT COUNT(*) AS c FROM meetings').get() as { c: number } - if (mtg.c) rows.push({ source: 'Meeting', count: mtg.c }) + } + const counted = (source: string, table: string, sql: string, args: unknown[]): void => { + if (!tableExists(db, table)) return + const c = (db.prepare(sql).get(...args) as { c: number }).c + if (c) out.push({ source, count: c }) + } + const meetings = windowSql('started_at', window, 'ms') + counted('Meeting', 'meetings', `SELECT COUNT(*) AS c FROM meetings WHERE 1 = 1${meetings.sql}`, [ + ...meetings.args + ]) + const actions = windowSql(actionWhenSql('a'), window, 'ms') + counted( + 'To-do', + 'action_items', + `SELECT COUNT(*) AS c FROM action_items a WHERE a.status != 'dismissed'${actions.sql}`, + actions.args + ) // Your own data, not just captured surfaces: chats and project knowledge bases. - const chat = db.prepare('SELECT COUNT(*) AS c FROM rag_conversations').get() as { c: number } - if (chat.c) rows.push({ source: 'Chat', count: chat.c }) - const kb = db.prepare('SELECT COUNT(*) AS c FROM rag_documents').get() as { c: number } - if (kb.c) rows.push({ source: 'Knowledge base', count: kb.c }) - return rows + const chats = windowSql('updated_at', window, 'datetime') + counted( + 'Chat', + 'rag_conversations', + `SELECT COUNT(*) AS c FROM rag_conversations WHERE 1 = 1${chats.sql}`, + chats.args + ) + const documents = windowSql('created_at', window, 'datetime') + counted( + 'Knowledge base', + 'rag_documents', + `SELECT COUNT(*) AS c FROM rag_documents WHERE 1 = 1${documents.sql}`, + documents.args + ) + return out } -/** Per-source MATCH counts for a query — drives the Sources rail facet counts so - * the numbers reflect the current search (Chat: 1, Knowledge base: 0, …). Empty +/** + * How a search box reads a query, the same for results and facet counts. A time phrase alone + * ("yesterday") lists that window. With other words, the time only ranks items near it higher + * ("monday standup", "Black Friday" may be words), and every word must match as a word start. + */ +function readSearchBox(query: string): { window: TimeWindow | null; timeOnly: boolean } { + const window = parseTimeWindow(query) + return { window, timeOnly: !!window && !ftsTypeahead(withoutTimePhrase(query, window)) } +} + +/** Per-source counts for a query — drives the Sources rail so the numbers reflect the current + * search (Chat: 1, Knowledge base: 0, …), read the way universalSearch reads it: a time-only query + * counts what each source has in that window, any other query counts its keyword matches. Empty * query → total counts (searchSources). Only sources with ≥1 match are returned. */ export function searchFacets(query: string): { source: string; count: number }[] { ensureRagStoreSchema() + ensureRecordSchema() const q = query.trim() if (!q) return searchSources() + const { window, timeOnly } = readSearchBox(q) + if (timeOnly) return countSources(window) + const match = ftsTypeahead(q) + if (!match) return [] const db = getDB() + // A query in a script written without spaces also counts what its trigram twins match (runs of + // three characters or more; shorter runs are only found by the search itself). + const trigram = needsTrigram(q) ? ftsTrigramQuery(q)?.match : null + /** `column` among the rows `fts` matches, or its trigram twin matches. */ + const matched = (fts: string, column: string, key = 'rowid'): { sql: string; args: string[] } => { + const twin = `${fts}_tri` + const own = `${column} IN (SELECT ${key} FROM ${fts} WHERE ${fts} MATCH ?)` + if (!trigram || !tableExists(db, twin)) return { sql: own, args: [match] } + return { + sql: `(${own} OR ${column} IN (SELECT ${key} FROM ${twin} WHERE ${twin} MATCH ?))`, + args: [match, trigram] + } + } const out: { source: string; count: number }[] = [] - const m = ftsExpr(q) - if (m) { + const counted = (source: string, index: string, sql: string, args: unknown[]): void => { + if (!tableExists(db, index)) return + const c = (db.prepare(sql).get(...args) as { c: number }).c + if (c) out.push({ source, count: c }) + } + if (tableExists(db, 'observation_fts')) { + const m = matched('observation_fts', 'o.id') out.push( ...(db .prepare( `SELECT COALESCE(o.surface,'') AS source, ${DISTINCT_MOMENTS} AS count - FROM observation_fts f JOIN observations o ON o.id = f.rowid - WHERE observation_fts MATCH ? AND o.surface IS NOT NULL AND o.surface != '' + FROM observations o + WHERE ${m.sql} AND o.surface IS NOT NULL AND o.surface != '' AND NOT ${SYNCED_BULK_MAIL} GROUP BY o.surface ORDER BY count DESC` ) - .all(m) as { source: string; count: number }[]) + .all(...m.args) as { source: string; count: number }[]) ) } - const terms = queryTerms(q, 6) - if (terms.length) { - const chatM = likeMatch(LIKE_COLUMNS.chat, terms) - const chat = db - .prepare( - `SELECT COUNT(*) AS c FROM (SELECT rc.id FROM rag_messages rm JOIN rag_conversations rc ON rc.id=rm.conversation_id WHERE ${chatM.where} GROUP BY rc.id)` - ) - .get(...chatM.args) as { c: number } - if (chat.c) out.push({ source: 'Chat', count: chat.c }) - const kb = knowledgeBaseHits(q, 1000).length - if (kb) out.push({ source: 'Knowledge base', count: kb }) - const mtgM = likeMatch(LIKE_COLUMNS.meeting, terms) - const mtg = db - .prepare(`SELECT COUNT(*) AS c FROM meetings WHERE ${mtgM.where}`) - .get(...mtgM.args) as { c: number } - if (mtg.c) out.push({ source: 'Meeting', count: mtg.c }) + if (tableExists(db, 'rag_message_fts')) { + const m = matched('rag_message_fts', 'm.id') + counted( + 'Chat', + 'rag_messages', + `SELECT COUNT(DISTINCT m.conversation_id) AS c FROM rag_messages m WHERE ${m.sql}`, + m.args + ) } - return out -} - -// --------------------------------------------------------------------------- -// Query: keyword (FTS5) + semantic (LanceDB), fused with RRF -// --------------------------------------------------------------------------- - -// One FTS source → ranked raw hits (best first). `sql` must SELECT the RawHit columns. -function ftsHits(sql: string, match: string, limit: number): RawHit[] { - if (!match) return [] - return getDB().prepare(sql).all(match, limit) as RawHit[] -} - -function keywordHits(query: string, perSource: number): RawHit[][] { - const m = ftsExpr(query) - const epochMs = epochMsSql('o.ts') - return [ - // Screen captures (distilled observation summaries) - ftsHits( - `SELECT 'obs:'||o.id AS key, 'screen' AS kind, o.id AS refId, COALESCE(o.surface,'Screen') AS title, - o.summary AS snippet, COALESCE(o.surface,'') AS surface, o.url AS url, ${epochMs} AS ts - FROM observation_fts f JOIN observations o ON o.id = f.rowid - WHERE observation_fts MATCH ? ORDER BY bm25(observation_fts) LIMIT ?`, - m, - perSource - ), - // Meeting / session transcripts - ftsHits( - `SELECT 'sum:'||o.rowid AS key, 'meeting' AS kind, o.rowid AS refId, 'Meeting' AS title, - o.summary AS snippet, 'Meeting' AS surface, NULL AS url, 0 AS ts - FROM summary_fts f JOIN chat_summaries o ON o.rowid = f.rowid - WHERE summary_fts MATCH ? ORDER BY bm25(summary_fts) LIMIT ?`, - m, - perSource - ), - // Entities - ftsHits( - `SELECT 'ent:'||o.id AS key, 'entity' AS kind, o.id AS refId, o.name AS title, - COALESCE(o.summary,o.type) AS snippet, 'Entity' AS surface, NULL AS url, 0 AS ts - FROM entity_fts f JOIN entities o ON o.id = f.rowid - WHERE entity_fts MATCH ? AND o.hidden = 0 ORDER BY bm25(entity_fts) LIMIT ?`, - m, - perSource - ), - // Entity facts - ftsHits( - `SELECT 'fact:'||o.id AS key, 'fact' AS kind, o.entity_id AS refId, - e.name AS title, - o.fact AS snippet, 'Fact' AS surface, NULL AS url, 0 AS ts - FROM entity_fact_fts f JOIN entity_facts o ON o.id = f.rowid - JOIN entities e ON e.id = o.entity_id - WHERE entity_fact_fts MATCH ? AND e.hidden = 0 - ORDER BY bm25(entity_fact_fts) LIMIT ?`, - m, - perSource - ), - // Memories - ftsHits( - `SELECT 'mem:'||o.id AS key, 'memory' AS kind, o.id AS refId, 'Memory' AS title, - o.content AS snippet, COALESCE(o.source_app,'') AS surface, NULL AS url, 0 AS ts - FROM memory_fts f JOIN memories o ON o.id = f.rowid - WHERE memory_fts MATCH ? ORDER BY bm25(memory_fts) LIMIT ?`, - m, - perSource - ), - // Recorded meeting transcripts (no FTS table) — LIKE over title/summary/transcript. - likeMeetingHits(query, perSource), - // Every word on screen, from the frame text index (raw OCR the summaries dropped). - frameHits(query, perSource), - // Your own chat conversations (title + message content), one hit per chat. - likeChatHits(query, perSource), - // Project knowledge-base documents, from the document index. - knowledgeBaseHits(query, perSource) - ] -} - -// Chat conversations have no FTS index — LIKE over message content OR the chat -// title, one hit per conversation (newest first). The conversation id (TEXT) is -// carried in `url` so the renderer can open that exact chat. -function likeChatHits(query: string, limit: number): RawHit[] { - const terms = queryTerms(query, 6) - if (!terms.length) return [] - const { where, args } = likeMatch(LIKE_COLUMNS.chat, terms) - return getDB() - .prepare( - `SELECT 'chat:'||rc.id AS key, 'chat' AS kind, 0 AS refId, COALESCE(rc.title,'Chat') AS title, - substr(rm.content,1,300) AS snippet, 'Chat' AS surface, rc.id AS url, - ${epochMsSql('rc.updated_at')} AS ts - FROM rag_messages rm JOIN rag_conversations rc ON rc.id = rm.conversation_id - WHERE ${where} GROUP BY rc.id ORDER BY rc.updated_at DESC LIMIT ?` + if (tableExists(db, 'rag_documents') && tableExists(db, 'attached_document_fts')) { + const m = matched('attached_document_fts', 'd.document_id', 'document_id') + counted( + 'Knowledge base', + 'rag_documents', + `SELECT COUNT(DISTINCT d.id) AS c FROM rag_documents d WHERE ${m.sql}`, + m.args ) - .all(...args, limit) as RawHit[] -} - -// Knowledge-base documents (per project): the document index's keyword search (bm25), best -// passage per file. The owning project_id is carried in `url` so the renderer can open it. -function knowledgeBaseHits(query: string, limit: number): RawHit[] { - const match = ftsQuery(query) - if (!match) return [] - let rows: RawHit[] - try { - rows = getDB() - .prepare( - `SELECT 'doc:'||d.id AS key, 'doc' AS kind, d.id AS refId, d.name AS title, - substr(c.text,1,300) AS snippet, 'Knowledge base' AS surface, d.project_id AS url, - ${epochMsSql('d.created_at')} AS ts - FROM (SELECT document_id, position, bm25(attached_document_fts) AS rank - FROM attached_document_fts WHERE attached_document_fts MATCH ? - ORDER BY rank LIMIT ?) f - JOIN rag_documents d ON d.document_id = f.document_id - JOIN attached_document_chunks c ON c.document_id = f.document_id AND c.position = f.position - ORDER BY f.rank` - ) - .all(match, limit * 20) as RawHit[] - } catch { - return [] } - const seen = new Set() - return rows.filter((row) => !seen.has(row.key) && !!seen.add(row.key)).slice(0, limit) -} - -function likeMeetingHits(query: string, limit: number): RawHit[] { - const terms = queryTerms(query, 6) - if (!terms.length) return [] - const { where, args } = likeMatch(LIKE_COLUMNS.meeting, terms) - return getDB() - .prepare( - `SELECT 'mtg:'||id AS key, 'meeting' AS kind, id AS refId, COALESCE(title,'Meeting') AS title, - substr(COALESCE(summary, transcript),1,300) AS snippet, 'Meeting' AS surface, NULL AS url, - COALESCE(started_at,0) AS ts - FROM meetings WHERE ${where} ORDER BY started_at DESC LIMIT ?` + if (tableExists(db, 'meeting_fts')) { + const m = matched('meeting_fts', 's.id') + counted('Meeting', 'meetings', `SELECT COUNT(*) AS c FROM meetings s WHERE ${m.sql}`, m.args) + } + if (tableExists(db, 'action_fts')) { + const m = matched('action_fts', 's.id') + counted( + 'To-do', + 'action_items', + `SELECT COUNT(*) AS c FROM action_items s WHERE ${m.sql} AND s.status != 'dismissed'`, + m.args ) - .all(...args, limit) as RawHit[] -} - -/** One moment, once. Capture records the same screen again and again while it stays put, each - * with the same summary; a result list showed dozens of identical cards. Keeps the first (best - * ranked or newest) of each app and summary. */ -function collapseRepeats(results: SearchResult[]): SearchResult[] { - const seen = new Set() - return results.filter((result) => { - if (result.kind !== 'screen') return true - const identity = `${result.surface.toLowerCase()}\u0000${result.snippet.trim().toLowerCase()}` - if (seen.has(identity)) return false - seen.add(identity) - return true - }) -} - -/** Observations made from a connected account (synced mail and chat) are messages, not screens: - * they have no screenshot, and calling them SCREEN with an empty picture misled. */ -function markSyncedMessages(results: SearchResult[]): void { - const ids = results.filter((r) => r.key.startsWith('obs:')).map((r) => r.refId) - if (!ids.length) return - const synced = new Set( - ( - getDB() - .prepare( - `SELECT DISTINCT of.observation_id AS id FROM observation_frames of - JOIN frames f ON f.id = of.frame_id - WHERE f.source = 'connector' AND of.observation_id IN (${ids.map(() => '?').join(',')})` - ) - .all(...ids) as { id: number }[] - ).map((row) => row.id) - ) - for (const result of results) { - if (result.key.startsWith('obs:') && synced.has(result.refId)) { - result.kind = 'message' - result.imagePath = null - } } + return out } -/** The newest things in the index, for an empty query: captured moments, recorded meetings and - * your chats, newest first, within the chosen sources. Search opens on this feed. */ -function recentHits(limit: number, sources?: string[]): RawHit[] { +// --------------------------------------------------------------------------- +// The empty-query feed +// --------------------------------------------------------------------------- + +/** The newest things in the index, for an empty query: captured moments (a screen that stayed put + * is one moment, not one per capture), recorded meetings, your chats and, when that source is + * chosen, your to-dos — newest first, within the chosen sources. Search opens on this feed. */ +function recentResults(limit: number, sources?: string[]): SearchResult[] { const db = getDB() const wanted = sources?.length ? sources.map((source) => source.toLowerCase()) : null const want = (source: string): boolean => !wanted || wanted.includes(source.toLowerCase()) - const observations = db - .prepare( - `SELECT 'obs:'||o.id AS key, 'screen' AS kind, o.id AS refId, COALESCE(o.surface,'Screen') AS title, - o.summary AS snippet, COALESCE(o.surface,'') AS surface, o.url AS url, ${epochMsSql('o.ts')} AS ts - FROM observations o - WHERE o.summary IS NOT NULL AND o.summary != '' - AND NOT ${SYNCED_BULK_MAIL} - ${wanted ? `AND lower(o.surface) IN (${wanted.map(() => '?').join(',')})` : ''} - ORDER BY o.ts DESC LIMIT ?` - ) - .all(...(wanted ?? []), limit) as RawHit[] - const meetings = want('Meeting') + const result = (row: Omit): SearchResult => ({ + ...row, + imagePath: null, + score: 0 + }) + // Newest moments, read light (id, app, summary) and folded on the same key search results use, + // so only the moments shown are hydrated. + const momentRows = tableExists(db, 'observations') ? (db .prepare( - `SELECT 'mtg:'||id AS key, 'meeting' AS kind, id AS refId, COALESCE(title,'Meeting') AS title, - substr(COALESCE(summary, transcript, ''),1,300) AS snippet, 'Meeting' AS surface, - NULL AS url, COALESCE(started_at,0) AS ts - FROM meetings ORDER BY started_at DESC LIMIT ?` + `SELECT o.id, COALESCE(o.surface, '') AS surface, o.summary FROM observations o + WHERE o.summary IS NOT NULL AND o.summary != '' + AND NOT ${SYNCED_BULK_MAIL} + ${wanted ? `AND lower(o.surface) IN (${wanted.map(() => '?').join(',')})` : ''} + ORDER BY o.ts DESC LIMIT ?` ) - .all(limit) as RawHit[]) + .all(...(wanted ?? []), Math.min(400, limit * 4)) as { + id: number + surface: string + summary: string + }[]) : [] + const seen = new Set() + const momentKeys: string[] = [] + for (const row of momentRows) { + const hit: RecordHit = { + key: `obs:${row.id}`, + kind: 'screen', + refId: row.id, + title: row.surface, + text: row.summary, + ts: 0, + surface: row.surface + } + const id = screenMomentKey(hit) + if (seen.has(id)) continue + seen.add(id) + momentKeys.push(hit.key) + if (momentKeys.length === limit) break + } + // To-dos only when chosen: an open feed is what was seen and said, not the task list. + const actionKeys = + wanted && want('To-do') && tableExists(db, 'action_items') + ? ( + db + .prepare( + `SELECT a.id FROM action_items a WHERE a.status != 'dismissed' + ORDER BY ${actionWhenSql('a')} DESC LIMIT ?` + ) + .all(limit) as { id: number }[] + ).map((row) => `act:${row.id}`) + : [] + // Moments and to-dos come through the records' hydration, so they carry their picture, synced + // state and due date. + const records = hydrateRecords([...momentKeys, ...actionKeys]).map(recordToSearchResult) + const meetings = + want('Meeting') && tableExists(db, 'meetings') + ? ( + db + .prepare( + `SELECT 'mtg:'||id AS key, id AS refId, COALESCE(title,'Meeting') AS title, + substr(COALESCE(summary, transcript, ''),1,300) AS snippet, + COALESCE(started_at,0) AS ts + FROM meetings ORDER BY started_at DESC LIMIT ?` + ) + .all(limit) as { + key: string + refId: number + title: string + snippet: string + ts: number + }[] + ).map((row) => result({ ...row, kind: 'meeting', surface: 'Meeting', url: null })) + : [] const chats = want('Chat') - ? (db - .prepare( - `SELECT 'chat:'||rc.id AS key, 'chat' AS kind, 0 AS refId, COALESCE(rc.title,'Chat') AS title, - COALESCE((SELECT substr(rm.content,1,300) FROM rag_messages rm - WHERE rm.conversation_id = rc.id ORDER BY rm.rowid DESC LIMIT 1),'') AS snippet, - 'Chat' AS surface, rc.id AS url, ${epochMsSql('rc.updated_at')} AS ts - FROM rag_conversations rc ORDER BY rc.updated_at DESC LIMIT ?` - ) - .all(limit) as RawHit[]) + ? ( + db + .prepare( + `SELECT 'chat:'||rc.id AS key, COALESCE(rc.title,'Chat') AS title, + COALESCE((SELECT substr(rm.content,1,300) FROM rag_messages rm + WHERE rm.conversation_id = rc.id ORDER BY rm.rowid DESC LIMIT 1),'') AS snippet, + rc.id AS url, ${epochMs('rc.updated_at')} AS ts + FROM rag_conversations rc ORDER BY rc.updated_at DESC LIMIT ?` + ) + .all(limit) as { key: string; title: string; snippet: string; url: string; ts: number }[] + ).map((row) => result({ ...row, kind: 'chat', refId: 0, surface: 'Chat' })) : [] - return [...observations, ...meetings, ...chats] + return [...records, ...meetings, ...chats] .sort((a, b) => (b.ts || 0) - (a.ts || 0)) .slice(0, limit) } -/** Frames whose on-screen text matches, from the frame text index. While older frames are still - * being indexed (or before the index exists), recent frames are also matched word for word. */ -function frameHits(query: string, limit: number): RawHit[] { - const db = getDB() - const indexed = !!db - .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'frame_fts'") - .get() - const match = ftsExpr(query) - const hits: RawHit[] = - indexed && match - ? (db - .prepare( - `SELECT 'frame:'||f.id AS key, 'screen' AS kind, f.id AS refId, COALESCE(f.surface,'Screen') AS title, - substr(f.text,1,300) AS snippet, COALESCE(f.surface,'') AS surface, f.url AS url, - ${epochMsSql('f.ts')} AS ts - FROM frame_fts x JOIN frames f ON f.id = x.rowid - WHERE frame_fts MATCH ? ORDER BY bm25(frame_fts) LIMIT ?` - ) - .all(match, limit) as RawHit[]) - : [] - if (indexed && !frameTextIndexBackfilling()) return hits - const seen = new Set(hits.map((hit) => hit.key)) - return [...hits, ...likeFrameHits(query, limit).filter((hit) => !seen.has(hit.key))] -} - -// Raw OCR text on all tokens (AND), newest first. -function likeFrameHits(query: string, limit: number): RawHit[] { - const terms = queryTerms(query, 6) - if (!terms.length) return [] - const { where, args } = likeMatch(LIKE_COLUMNS.frame, terms) - // Only while the frame text index is incomplete. Scanning all raw OCR (hundreds of MB) took - // seconds on the main thread, so this reads recent frames only, through the ts index. - const since = new Date(Date.now() - RAW_FRAME_TEXT_DAYS * 86_400_000).toISOString().slice(0, 10) - return getDB() - .prepare( - `SELECT 'frame:'||id AS key, 'screen' AS kind, id AS refId, COALESCE(surface,'Screen') AS title, - substr(text,1,300) AS snippet, COALESCE(surface,'') AS surface, url AS url, - ${epochMsSql('ts')} AS ts - FROM frames - WHERE ts >= ? AND text IS NOT NULL AND COALESCE(source, '') != 'connector' AND ${where} - ORDER BY ts DESC LIMIT ?` - ) - .all(since, ...args, limit) as RawHit[] -} - -/** How far back search reads raw on-screen text word for word. */ -const RAW_FRAME_TEXT_DAYS = 30 - -/** Ranking penalty for bulk mail (newsletters, promos): still searchable, never on top of the - * user's real conversations just because it is recent. */ -const BULK_MAIL_PENALTY = 0.02 +// --------------------------------------------------------------------------- +// Query +// --------------------------------------------------------------------------- -/** Push bulk-mail observations down the ranking. Reads their category in one query. */ -function demoteBulkMail(results: Iterable): void { - const byId = new Map() - for (const r of results) { - if (r.key.startsWith('obs:')) byId.set(Number(r.key.slice(4)), r) - } - if (!byId.size) return - try { - const ids = JSON.stringify([...byId.keys()]) - const rows = getDB() - .prepare( - "SELECT id FROM observations WHERE id IN (SELECT value FROM json_each(?)) AND category = 'consumption'" - ) - .all(ids) as { id: number }[] - for (const { id } of rows) byId.get(id)!.score -= BULK_MAIL_PENALTY - } catch { - // Observations are a Pro table; without it there is nothing to demote. - } +/** Lower-cased words of a query, for the "Match" sort. */ +function queryTerms(query: string): string[] { + return query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [] } -function sourceRowExists(sql: string, id: number): boolean { - return getDB().prepare(sql).get(id) !== undefined +/** Literal term-overlap count in a haystack (for the "Match" sort). Case-insensitive. */ +function matchScore(haystack: string, terms: string[]): number { + const hay = (haystack || '').toLowerCase() + return terms.reduce((n, t) => n + (t ? hay.split(t).length - 1 : 0), 0) } -/** Confirm a cached semantic row still has a visible source-of-truth record. */ -function semanticSourceExists(hit: VecChunk): boolean { - const id = Number(hit.key.slice(hit.key.indexOf(':') + 1)) - if (!Number.isSafeInteger(id) || id <= 0) return false - if (hit.key.startsWith('frame:')) { - return sourceRowExists('SELECT 1 FROM frames WHERE id = ?', id) - } - if (hit.key.startsWith('obs:')) { - return sourceRowExists('SELECT 1 FROM observations WHERE id = ?', id) - } - if (hit.key.startsWith('sum:')) { - return sourceRowExists('SELECT 1 FROM chat_summaries WHERE rowid = ?', id) - } - if (hit.key.startsWith('mtg:')) { - return sourceRowExists('SELECT 1 FROM meetings WHERE id = ?', id) - } - if (hit.key.startsWith('mem:')) { - return sourceRowExists('SELECT 1 FROM memories WHERE id = ?', id) - } - if (hit.key.startsWith('ent:')) { - return sourceRowExists('SELECT 1 FROM entities WHERE id = ? AND hidden = 0', id) - } - if (hit.key.startsWith('fact:')) { - return sourceRowExists( - `SELECT 1 FROM entity_facts f JOIN entities e ON e.id = f.entity_id - WHERE f.id = ? AND e.hidden = 0`, - id - ) +function sortResults(results: SearchResult[], query: string, sort: SearchSort): SearchResult[] { + if (sort === 'recency') + return results.sort((a, b) => (b.ts || 0) - (a.ts || 0) || b.score - a.score) + if (sort === 'match') { + const terms = queryTerms(query) + const m = (r: SearchResult): number => matchScore(`${r.title} ${r.snippet}`, terms) + return results.sort((a, b) => m(b) - m(a) || b.score - a.score) } - return false -} - -/** Query the semantic index while treating SQLite as the source of truth. - * Stale vectors are harmless cache entries and never become user-visible hits. - */ -export async function searchSemanticSources(vector: number[], limit: number): Promise { - const hits = await searchVectors(vector, limit) - return hits.filter(semanticSourceExists).map((h) => ({ - key: h.key, - kind: h.kind as SearchKind, - refId: h.refId, - title: h.surface || h.kind, - snippet: h.text, - surface: h.surface, - url: h.url || null, - ts: h.ts - })) + return results } -async function semanticHits(query: string, limit: number): Promise { - // Someone is waiting on this one: it goes ahead of queued background indexing. - const vector = await embeddings.generateEmbedding(query, { interactive: true }) - return searchSemanticSources(vector, limit) -} - -// Best thumbnail for a hit: a frame's own image, or an observation's linked frame. -function thumbFor(hit: RawHit): string | null { - const db = getDB() - if (hit.key.startsWith('frame:')) { - const r = db.prepare('SELECT image_path FROM frames WHERE id = ?').get(hit.refId) as - | { image_path?: string } - | undefined - return r?.image_path ?? null - } - if (hit.kind === 'screen') { - const r = db - .prepare( - 'SELECT f.image_path FROM observation_frames of JOIN frames f ON f.id = of.frame_id WHERE of.observation_id = ? LIMIT 1' - ) - .get(hit.refId) as { image_path?: string } | undefined - return r?.image_path ?? null +/** The record kinds a source (surface) filter needs searched. */ +function kindsForSources(sources: string[]): string[] { + const kinds = new Set() + for (const source of sources) { + for (const kind of SURFACE_KINDS[source.toLowerCase()] ?? ['screen']) kinds.add(kind) } - return null + return [...kinds] } -/** Hybrid universal search. `semantic` adds the LanceDB pass (slower first call). */ // NOTE: artifacts are intentionally NOT in universal search yet — a hit can't be // deep-linked (it carries no conversation context and there's no standalone // artifact viewer), so clicking one used to jump to a meaningless Replay moment. -// Re-add an `artifactHits` source here once artifacts have an openable target. +// Re-add an artifact source once artifacts have an openable target. +/** Universal search, for search boxes: every word must match as a word start, and a time phrase + * ranks items near it higher (a time phrase alone lists that window). `semantic` adds meaning + * search (slower first call: the index loads); without it (the command palette's type-ahead) + * only words match. `rerank` runs the cross-encoder over the best results. */ export async function universalSearch( query: string, opts: { limit?: number semantic?: boolean + /** Cross-encoder over the head of the results; only where someone waits on the answer. */ + rerank?: boolean sources?: string[] kinds?: SearchKind[] collapseScreenMoments?: boolean @@ -611,70 +395,61 @@ export async function universalSearch( } = {} ): Promise { ensureRagStoreSchema() + ensureRecordSchema() const q = query.trim() - const limit = opts.limit ?? 30 + const limit = Math.min(MAX_RESULTS, Math.max(1, Math.floor(opts.limit ?? 30))) + const sourceSet = opts.sources?.length + ? new Set(opts.sources.map((source) => source.toLowerCase())) + : null + const kindSet = opts.kinds?.length ? new Set(opts.kinds) : null + const excluded = opts.excludeChatId ? `chat:${opts.excludeChatId}` : null + const keep = (result: SearchResult): boolean => + (!sourceSet || sourceSet.has((result.surface || '').toLowerCase())) && + // A synced message is a screen moment that came from a connected account. + (!kindSet || + kindSet.has(result.kind) || + (result.kind === 'message' && kindSet.has('screen'))) && + result.key !== excluded + if (!q) { // No words yet: the newest of everything, so Search never opens on an empty page. - const recent = rankResults(Array.from(fuseHits([recentHits(limit * 4, opts.sources)]).values()), { - query: '', - sources: opts.sources, - kinds: opts.kinds, - excludeChatId: opts.excludeChatId, - sort: 'recency' - }) - const page = collapseRepeats(recent).slice(0, limit) - markSyncedMessages(page) - for (const r of page) - if (r.kind !== 'message') - r.imagePath = thumbFor({ key: r.key, kind: r.kind, refId: r.refId } as RawHit) - return page + return recentResults(limit, opts.sources).filter(keep).slice(0, limit) } - // When filtering by source, cast a wider net per source so enough survive the filter. - const perSource = opts.sources?.length ? 80 : Math.min(40, limit + 10) - const lists = keywordHits(q, perSource) - opts.signal?.throwIfAborted() - if (opts.semantic !== false) { - try { - lists.push(await semanticHits(q, perSource)) - } catch { - /* embedding model not ready — keyword results still fine */ - } - opts.signal?.throwIfAborted() + // A source filter narrows the kinds searched; the engine leaves out what the filters do not keep + // before it ranks, so the page needs no wider net. + let kinds = opts.kinds?.length ? (opts.kinds as string[]) : undefined + if (opts.sources?.length) { + const fromSources = kindsForSources(opts.sources) + kinds = kinds + ? kinds.filter((kind) => fromSources.includes(kind === 'message' ? 'screen' : kind)) + : fromSources + if (!kinds.length) return [] } - - // Reciprocal-rank fusion across all lists, keyed by the unique chunk key. - const fused = fuseHits(lists) - - // Add a recency bias so recent hits float up (this week first, then progressively - // older), plus a small nudge for your own deliberate content (chats / KB docs) so - // it isn't buried under thousands of ambient screen captures. - applyBoosts(fused.values(), Date.now()) - demoteBulkMail(fused.values()) - - // Filter (source / current chat) then sort (relevance / recency / match). - const ordered = rankResults(Array.from(fused.values()), { - query: q, - sources: opts.sources, - kinds: opts.kinds, - excludeChatId: opts.excludeChatId, - sort: opts.sort + opts.signal?.throwIfAborted() + const { timeOnly } = readSearchBox(q) + const semantic = opts.semantic !== false + const { hits } = await recordSearch(q, { + limit: Math.min(MAX_RESULTS, limit + RESULT_MARGIN), + semantic, + rerank: opts.rerank === true, + // Type-ahead keywords (AND, word starts); the palette also skips meaning search. + ...(semantic ? { keywordTypeahead: true } : { typeahead: true }), + timeMode: timeOnly ? 'filter' : 'boost', + ...(kinds ? { kinds } : {}), + collapseKey: screenMomentKey, + ...(sourceSet || kindSet || excluded + ? { exclude: (hit: RecordHit) => !keep(recordToSearchResult(hit)) } + : {}) }) opts.signal?.throwIfAborted() - const distinct = collapseRepeats(ordered) - markSyncedMessages(distinct.slice(0, limit * 2)) - for (const r of distinct) - if (r.kind !== 'message') - r.imagePath = thumbFor({ key: r.key, kind: r.kind, refId: r.refId } as RawHit) - - if (!opts.collapseScreenMoments) return distinct.slice(0, limit) + const ordered = sortResults(hits.map(recordToSearchResult), q, opts.sort ?? 'relevance') + if (!opts.collapseScreenMoments) return ordered.slice(0, limit) - // One captured moment is indexed twice: its raw OCR frame and its distilled - // observation. Replay shows moments, not index records, so collapse both forms - // after ranking and thumbnail resolution. General search keeps both records. + // Replay shows moments, not records: a moment's own frame and the moment collapse to one picture. const seenMoments = new Set() const moments: SearchResult[] = [] - for (const result of distinct) { + for (const result of ordered) { const identity = result.kind === 'screen' ? result.imagePath || `${String(result.ts)}:${result.surface}` diff --git a/src/main/shutdown.ts b/src/main/shutdown.ts index 68116884b..714ca4209 100644 --- a/src/main/shutdown.ts +++ b/src/main/shutdown.ts @@ -68,7 +68,10 @@ export function commitApplicationRelaunch(source: ApplicationRelaunchSource): vo if (!pendingRelaunch) return const { options } = pendingRelaunch pendingRelaunch = null - source.relaunch(options) + // An AppImage runs from a temporary mount that is gone once this process exits: relaunch the + // AppImage file itself. + const appImage = process.env.APPIMAGE + source.relaunch(appImage ? { ...options, execPath: appImage } : options) } export interface ShutdownFailure { diff --git a/src/main/startup-stages.ts b/src/main/startup-stages.ts index fa017d36d..d4ba6023c 100644 --- a/src/main/startup-stages.ts +++ b/src/main/startup-stages.ts @@ -10,7 +10,7 @@ export type StartupStageResult = readonly durationMs: number } -export interface StartupStageContext { +interface StartupStageContext { readonly signal: AbortSignal isOwner(): boolean commit(label: string, apply: () => R): R | undefined diff --git a/src/main/task-settings-events.ts b/src/main/task-settings-events.ts index 2b43c7f25..6bf69c5f4 100644 --- a/src/main/task-settings-events.ts +++ b/src/main/task-settings-events.ts @@ -4,7 +4,7 @@ import { BrowserWindow } from 'electron' -export const TASK_SETTINGS_CHANGED = 'task-settings:changed' +const TASK_SETTINGS_CHANGED = 'task-settings:changed' export function announceTaskSettings(key: string, value: unknown): void { // The model manager also runs without Electron (no windows to tell), as runtime-env allows. diff --git a/src/main/tasks/__tests__/task-execution-plan-service.test.ts b/src/main/tasks/__tests__/task-execution-plan-service.test.ts index 8490502ce..19eab2fb5 100644 --- a/src/main/tasks/__tests__/task-execution-plan-service.test.ts +++ b/src/main/tasks/__tests__/task-execution-plan-service.test.ts @@ -45,7 +45,7 @@ describe('task execution plan service', () => { 'put each route, date, filter, or other constraint in exactly one setup phase' ) expect(prompt).toContain('Do not repeat those constraints in a later results phase') - expect(prompt).toContain('Return 1 to 4 outcome-based phases') + expect(prompt).toContain('Return 1 to 16 outcome-based phases') expect(prompt).toContain('Do not add a separate verification, reporting, or summary phase') expect(prompt).toContain('Name the visible result that completes the final phase') expect(prompt).toContain('Do not use a generic phase') diff --git a/src/main/tasks/task-history-store.ts b/src/main/tasks/task-history-store.ts index 29b0c2af6..5503af09d 100644 --- a/src/main/tasks/task-history-store.ts +++ b/src/main/tasks/task-history-store.ts @@ -25,7 +25,7 @@ import { import { taskDisplayTitle } from '../../shared/task-display-title' export type TaskRunKind = AutomationTaskKind -export type TaskRunStatus = AutomationTaskReadStatus +type TaskRunStatus = AutomationTaskReadStatus export interface TaskRunSnapshot { taskId: string diff --git a/src/main/tasks/task-result-chat.ts b/src/main/tasks/task-result-chat.ts index f5f3708e1..bbff5d003 100644 --- a/src/main/tasks/task-result-chat.ts +++ b/src/main/tasks/task-result-chat.ts @@ -48,7 +48,7 @@ function taskResultDetail(task: TaskRunSnapshot): string { /** The task state owner supplies the status and detail. This one-way Chat * projection makes the state explicit, so a failed or stopped task can never * look successful because of optimistic summary text. */ -export function taskResultChatContent(task: TaskRunSnapshot): string | undefined { +function taskResultChatContent(task: TaskRunSnapshot): string | undefined { if (!CHAT_RESULT_STATUSES.has(task.status)) return undefined const detail = taskResultDetail(task) if (!detail) return undefined diff --git a/src/main/tasks/task-retry.ts b/src/main/tasks/task-retry.ts index 419da3744..039ac967b 100644 --- a/src/main/tasks/task-retry.ts +++ b/src/main/tasks/task-retry.ts @@ -66,7 +66,7 @@ interface TaskRetryServiceOptions { } export const TASK_RETRY_TRACE = 'RETRY · Resumed from the failed checkpoint.' -export const TASK_CONTINUE_TRACE = 'CONTINUE · Resumed from the saved checkpoint.' +const TASK_CONTINUE_TRACE = 'CONTINUE · Resumed from the saved checkpoint.' function taskGuidanceId(context: string | null): string | undefined { if (!context) return undefined @@ -78,7 +78,7 @@ function taskGuidanceId(context: string | null): string | undefined { } } -export function retryGuidanceFromMessages( +function retryGuidanceFromMessages( taskId: string, messages: readonly Pick[] ): string[] { diff --git a/src/main/tasks/task-step-details.ts b/src/main/tasks/task-step-details.ts index 34811f15c..4af8aeb43 100644 --- a/src/main/tasks/task-step-details.ts +++ b/src/main/tasks/task-step-details.ts @@ -81,7 +81,7 @@ function phase(value: unknown): ComputerUsePhase | undefined { } /** Remove model-only reasoning while retaining the action and tool-call output users can audit. */ -export function visibleComputerUseModelOutput(value: unknown): string | undefined { +function visibleComputerUseModelOutput(value: unknown): string | undefined { if (typeof value !== 'string') return undefined const withoutClosedReasoning = value.replace( /<(?:think|analysis|reasoning)\b[^>]*>[\s\S]*?<\/(?:think|analysis|reasoning)>/gi, diff --git a/src/main/tools.ts b/src/main/tools.ts index 6febc1486..2f03cd765 100644 --- a/src/main/tools.ts +++ b/src/main/tools.ts @@ -9,11 +9,16 @@ import { offgridGuideTool } from './tools/offgrid-guide-tool' import { llm } from './llm' import type { GenerationMetrics } from '../shared/generation-metrics' import type { ResponseCutoffContract } from '../shared/ipc-contracts' -import { SEARCH_KB_TOOL, sanitizePromptExcerpt } from '@offgrid/rag' +import { SEARCH_KB_TOOL } from '@offgrid/rag' import { stripChatControlTokens } from '@offgrid/sync' import { isMemoryToolAllowed } from './tools/memory-scope' import { parseToolCallsFromText, stripQwenToolCallMarkup } from './tools/tool-call-parse' -import { getSetting, saveSetting, searchProjectConversations } from './database' +import { + getSetting, + saveSetting, + searchProjectConversations, + type ProjectConversationMessage +} from './database' import { projectDocumentsMode, type ProjectDocumentsMode @@ -47,6 +52,13 @@ import { } from '@offgrid/models' import { toolPromptChars, toolPromptTokens } from './tools/prompt-budget' import { checkCitations } from '@offgrid/rag/documents' +import { + describeRecordStatus, + recordBlock, + timeAnchor, + type RecordHit, + type RecordSearchStatus +} from '@offgrid/rag/records' import { noteDocumentPrefill } from './documents' import { buildPageBlock, buildPageFence, pageBudgetChars } from './tools/page-fence' import { @@ -107,7 +119,8 @@ export interface ToolContext { /** The active project (if the chat is in one), so search_knowledge_base can query * that project's documents, chats and (when the project includes it) captured memory. */ projectId?: string - /** The `[S#]` number of a document passage in this turn's sources, adding it when new. */ + /** The `[S#]` number of a document passage or memory record in this turn's sources, adding it + * when new. */ citeDocumentPassage?: (source: UnifiedSource) => number /** Per-turn location result. Nearby task tools use this to fail closed when * the device location request did not return usable coordinates. */ @@ -119,7 +132,7 @@ export interface ToolContext { completedPrerequisites?: string[] } -export interface ToolConversationTurn { +interface ToolConversationTurn { role: 'user' | 'assistant' content: string } @@ -163,42 +176,55 @@ type ToolDef = { ) => Promise | string | ToolResult } -/** The one retrieval path used by memory and Replay chat tools. A caller may - * narrow the source kind, but search, ranking, thumbnails, and citation shape - * stay owned by universalSearch. */ +/** The one retrieval path used by memory, Replay and meeting chat tools: the records engine + * (src/main/records over @offgrid/rag/records). A caller may narrow the kinds; ranking, dates and + * the shape stay owned by the engine. With the turn's `ctx`, every hit takes its `[S#]` + * from the turn's citation registry, so it is in the Sources panel under the number the model + * sees; `sources` still lists the cited hits (the loop skips ones already registered). */ export async function searchMemoryToolResult( query: string, options: { limit?: number kinds?: SearchKind[] + /** Repeated screen moments always collapse now; kept so existing callers need no change. */ collapseScreenMoments?: boolean excludeChatId?: string emptyText?: string errorSubject?: string - } = {} + } = {}, + ctx?: ToolContext ): Promise { try { - const { universalSearch } = await import('./search') + const { recordSearch, recordToSearchResult, screenMomentKey } = await import('./records') const limit = Math.min(20, Math.max(1, Number(options.limit) || 8)) - const hits = await universalSearch(query, { + const excluded = options.excludeChatId ? `chat:${options.excludeChatId}` : null + const onActivity = ctx?.onActivity + const { hits } = await recordSearch(query, { limit, semantic: true, - kinds: options.kinds, - collapseScreenMoments: options.collapseScreenMoments, - excludeChatId: options.excludeChatId + rerank: true, + ...(options.kinds?.length ? { kinds: options.kinds } : {}), + // The model writes the query: a time phrase in it is a window, not words to match. + timeMode: 'filter', + // The same screen seen again and again is one moment, in every chat tool. + collapseKey: screenMomentKey, + // The chat being asked in cannot cite itself. + ...(excluded ? { exclude: (hit: RecordHit) => hit.key === excluded } : {}), + ...(onActivity + ? { + onStatus: (status: RecordSearchStatus) => + onActivity({ kind: 'documents', label: describeRecordStatus(status) }) + } + : {}) }) - const sources: UnifiedSource[] = hits.map((hit) => ({ ...hit })) - const text = hits.length - ? hits - .map((hit) => { - const when = hit.ts - ? ` · ${new Date(hit.ts).toISOString().slice(0, 16).replace('T', ' ')}` - : '' - return `(${hit.surface || hit.kind}${when}) ${hit.title ? `${hit.title} — ` : ''}${hit.snippet}` - }) - .join('\n') - : (options.emptyText ?? 'Nothing found in memory for that.') - return { text, sources } + if (!hits.length) return { text: options.emptyText ?? 'Nothing found in memory for that.' } + const sources: UnifiedSource[] = [] + const block = recordBlock(query, hits, (hit) => { + const source = recordToSearchResult(hit) + sources.push(source) + return ctx?.citeDocumentPassage?.(source) ?? sources.length + }) + return { text: block.text, sources } } catch (error) { return { text: `Error searching ${options.errorSubject ?? 'memory'}: ${(error as Error).message}` @@ -206,6 +232,25 @@ export async function searchMemoryToolResult( } } +/** A sibling project chat's matching message, as a record the turn can cite. The key is the + * message itself, so the same passage keeps one number however many searches find it. */ +function projectChatHit(message: ProjectConversationMessage): RecordHit { + const title = message.title || 'Project conversation' + // SQLite's CURRENT_TIMESTAMP text is UTC without a zone. + const ts = Date.parse(`${message.created_at.replace(' ', 'T')}Z`) + return { + key: `project-chat:${message.conversation_id}:${message.role}:${message.content.slice(0, 120)}`, + kind: 'chat', + refId: 0, + title, + text: `${message.role}: ${message.content}`, + ts: Number.isFinite(ts) ? ts : 0, + surface: 'Project chat', + // The conversation to open. + url: message.conversation_id + } +} + // --- HTML helpers for the web tools live in ./tools-parsers (pure, unit-tested). // Fetch a URL and return its readable text (shared by the read_url tool and the // deterministic "read this URL, then build" flow). Works for localhost too. @@ -419,10 +464,11 @@ const TOOLS: ToolDef[] = [ // interactive citations. Excludes the current conversation (ctx) so it can't // cite itself. The loop dedups the returned sources across rounds. run: (args, context) => - searchMemoryToolResult(String(args.query ?? ''), { - limit: Number(args.limit) || 8, - excludeChatId: context.conversationId - }) + searchMemoryToolResult( + String(args.query ?? ''), + { limit: Number(args.limit) || 8, excludeChatId: context.conversationId }, + context + ) }, { // Only OFFERED in a project chat (gated in schemas() by projectId). The project's documents @@ -442,30 +488,43 @@ const TOOLS: ToolDef[] = [ if (!query) return { text: 'Give a query to search for.' } const { searchDocumentsForTool } = await import('./documents/tool-extension') const { projectIncludesMemory } = await import('./rag') + const { recordToSearchResult } = await import('./records') const conversations = searchProjectConversations(ctx.projectId, query, ctx.conversationId) const documents = await searchDocumentsForTool(ctx, query) - const memory = projectIncludesMemory(ctx.projectId) - ? await searchMemoryToolResult(query, { - limit: 5, - // Captured memory only: other projects' files and chats stay out of this project. - kinds: ['screen', 'message', 'meeting', 'memory', 'entity', 'fact'], - excludeChatId: ctx.conversationId, - emptyText: 'No matching captured memory.' - }) - : null + // Sibling chats are sources like the rest: each passage takes the turn's next [S#]. + const chatSources: UnifiedSource[] = [] const chats = conversations.length - ? conversations - .map( - (message, index) => - `[C${index + 1}] ${sanitizePromptExcerpt(message.title || 'Project conversation')} · ${message.role}: ${sanitizePromptExcerpt(message.content.slice(0, 1500))}` - ) - .join('\n\n') + ? recordBlock( + query, + conversations.map(projectChatHit), + (hit) => { + const source = recordToSearchResult(hit) + chatSources.push(source) + return ctx.citeDocumentPassage?.(source) ?? chatSources.length + }, + { labels: { chat: 'Project conversations' }, maxChars: 1500, budgetTokens: 2400 } + ).text : 'No matching project conversations found.' + const memory = projectIncludesMemory(ctx.projectId) + ? await searchMemoryToolResult( + query, + { + limit: 5, + // Captured memory only: other projects' files and chats stay out of this project. + kinds: ['screen', 'message', 'meeting', 'entity'], + excludeChatId: ctx.conversationId, + emptyText: 'No matching captured memory.' + }, + ctx + ) + : null return { text: `Project documents:\n${documents}\n\nProject conversations:\n${chats}${ memory ? `\n\nCaptured memory:\n${memory.text}` : '' }`, - ...(memory?.sources?.length ? { sources: memory.sources } : {}) + ...(chatSources.length || memory?.sources?.length + ? { sources: [...chatSources, ...(memory?.sources ?? [])] } + : {}) } } catch (e) { return { text: 'Error searching the knowledge base: ' + (e as Error).message } @@ -985,6 +1044,10 @@ export async function toolChat( // turn to turn, so the model server reuses its cached prefix; this turn's passages sit beside // the question. const historyTokens = toolPromptTokens(messages.slice(1, -1)) + // The system prompt ends with today's date and time, after everything that stays the same from + // turn to turn (instructions, whole documents), so "today" in history or retrieved items is never + // read as now. God's own context dates itself as well. + const anchor = `\n\n${timeAnchor(Date.now())}` const turnReplyReserve = Math.min(1024, Math.max(256, Math.floor(ctx * 0.1))) let documentSources: UnifiedSource[] = [] const { prepareTurnDocuments } = await import('./documents/chat-turn') @@ -997,7 +1060,7 @@ export async function toolChat( budget: { ctx, replyReserve: turnReplyReserve, - systemTokens: toolPromptTokens([messages[0]]), + systemTokens: toolPromptTokens([{ ...messages[0], content: messages[0].content + anchor }]), requiredToolTokens: opts.assistantOnly ? Math.max(0, toolPromptTokens([], tools)) : 0, historyTokens, questionTokens: toolPromptTokens([{ role: 'user', content: query }]), @@ -1032,6 +1095,7 @@ export async function toolChat( } documentSources = attachedDocuments.sources } + messages[0].content = `${messages[0].content}${anchor}` // One model call folds every earlier turn into a structured summary that stands in for them // from now on. If it fails, the oldest turns fall back to short excerpts as before. const summary = @@ -1077,9 +1141,12 @@ export async function toolChat( // Document passages lead the sources, so `[S1]…[Sn]` in the prompt are the first n sources. const unified: UnifiedSource[] = [...documentSources] const unifiedKeys = new Set(documentSources.map((source) => source.key)) - // A document search tool cites passages by their place in these sources: a passage the turn + // Whether the model was shown any `[S#]` tag this turn (documents, memory, meetings, chats). + let citedSources = documentSources.length > 0 + // A search tool cites passages and records by their place in these sources: one the turn // already gave keeps its number, a new one is added now so its number is fixed. toolContext.citeDocumentPassage = (source) => { + citedSources = true const known = unified.findIndex((existing) => existing.key === source.key) if (known >= 0) return known + 1 unifiedKeys.add(source.key) @@ -1116,7 +1183,7 @@ export async function toolChat( ...result, // A citation that points at no source would lead nowhere in the Sources panel. answer: - documentSources.length || result.unified.some((source) => source.kind === 'attachment') + citedSources || result.unified.some((source) => source.kind === 'attachment') ? checkCitations(answer, result.unified.length) : answer, metrics: { diff --git a/src/main/tools/chat-tool-selection.ts b/src/main/tools/chat-tool-selection.ts index 1002c63e3..1ba787657 100644 --- a/src/main/tools/chat-tool-selection.ts +++ b/src/main/tools/chat-tool-selection.ts @@ -6,9 +6,9 @@ import { mergeStickyTools, previousOfferedTools, rememberOfferedTools } from './ * has to fetch the model (~23MB) first, and offline it may never arrive; the turn must not stall. * The load keeps going in the background, so later turns route semantically. */ -export const SEMANTIC_ROUTING_WAIT_MS = 2_000 +const SEMANTIC_ROUTING_WAIT_MS = 2_000 /** When MiniLM is already on disk the wait is only its load (a few seconds on a cold start). */ -export const LOCAL_ROUTING_MODEL_WAIT_MS = 20_000 +const LOCAL_ROUTING_MODEL_WAIT_MS = 20_000 // The routing model embeddings-core loads (Xenova/all-MiniLM-L6-v2, fp32 or quantized). async function routingModelOnDisk(): Promise { @@ -81,15 +81,23 @@ export async function selectChatToolSchemas( if (!namedTool && !unavailableImageTool) { try { let embed = options.embed + let embedMany: ((texts: string[]) => Promise) | undefined if (!embed) { const { embeddings } = await import('../embeddings') - embed = (t: string): Promise => embeddings.generateEmbedding(t, { interactive: true }) + const many = (texts: string[]): Promise => + embeddings.embed(texts, { interactive: true }) + embedMany = many + embed = async (t: string): Promise => (await many([t]))[0] ?? [] } const { contextualToolRoutingText, selectRelevantToolsSemantic } = await import('./tool-embedding-ranking') const routingText = contextualToolRoutingText(query, history) const ranked = await withinRoutingWait( - selectRelevantToolsSemantic(routingText, rawTools, { embed }), + selectRelevantToolsSemantic( + routingText, + rawTools, + embedMany ? { embed, embedMany } : { embed } + ), options.semanticWaitMs ?? ((await routingModelOnDisk()) ? LOCAL_ROUTING_MODEL_WAIT_MS : SEMANTIC_ROUTING_WAIT_MS) ) diff --git a/src/main/tools/nativeActionToolExtension-logic.ts b/src/main/tools/nativeActionToolExtension-logic.ts index 5d3f4b205..12ec76066 100644 --- a/src/main/tools/nativeActionToolExtension-logic.ts +++ b/src/main/tools/nativeActionToolExtension-logic.ts @@ -7,7 +7,7 @@ import { COMPUTER_USE_ACTION_TYPE, WEB_USE_ACTION_TYPE } from '@offgrid/use' import type { ActionRisk } from '../actions/approval' -export const WEB_USE_TOOL_NAME = WEB_USE_ACTION_TYPE +const WEB_USE_TOOL_NAME = WEB_USE_ACTION_TYPE export interface NativeToolSpec { /** Model-facing tool name. */ @@ -357,7 +357,7 @@ export const WINDOWS_TOOL_NAMES: ReadonlySet = new Set([ // Location and the link opener are also available without task tools. export const LINUX_TOOL_NAMES: ReadonlySet = new Set(['open_url', 'get_current_location']) -export const TASK_USE_TOOL_NAMES: ReadonlySet = new Set([WEB_USE_TOOL_NAME, 'computer_use']) +const TASK_USE_TOOL_NAMES: ReadonlySet = new Set([WEB_USE_TOOL_NAME, 'computer_use']) export function specsForPlatform( platform: NodeJS.Platform, diff --git a/src/main/tools/page-fence.ts b/src/main/tools/page-fence.ts index e0e5c07fc..d12c16c76 100644 --- a/src/main/tools/page-fence.ts +++ b/src/main/tools/page-fence.ts @@ -15,8 +15,8 @@ export function buildPageFence(pageText?: string): string { // Reserve roughly a quarter of context for page text at four characters per token. // The floor keeps small-model excerpts useful; the ceiling stays below the wire cap // and leaves space for instructions, history, tools, and the model's answer. -export const PAGE_CONTEXT_FRACTION = 0.25 -export const PAGE_CHARS_PER_TOKEN = 4 +const PAGE_CONTEXT_FRACTION = 0.25 +const PAGE_CHARS_PER_TOKEN = 4 export const MIN_PAGE_BUDGET_CHARS = 4_000 export const MAX_PAGE_BUDGET_CHARS = 60_000 diff --git a/src/main/tools/planner-logic.ts b/src/main/tools/planner-logic.ts index aa18526e1..bf76a88ba 100644 --- a/src/main/tools/planner-logic.ts +++ b/src/main/tools/planner-logic.ts @@ -12,7 +12,7 @@ * The llm call and the tool dispatch are injected in planner.ts / plan-executor.ts. */ -export interface PlanBinding { +interface PlanBinding { /** The arg on THIS step to fill. */ arg: string /** The earlier step (0-based) whose result supplies the value. */ diff --git a/src/main/tools/planner.ts b/src/main/tools/planner.ts index b87670493..a4283bf5b 100644 --- a/src/main/tools/planner.ts +++ b/src/main/tools/planner.ts @@ -15,7 +15,7 @@ import { type ToolCatalogEntry } from './planner-logic' -export class PlanValidationError extends Error { +class PlanValidationError extends Error { constructor(readonly validationErrors: readonly string[]) { super(`Invalid structured plan after retry: ${validationErrors.join('; ')}`) this.name = 'PlanValidationError' diff --git a/src/main/tools/sticky-tool-set.ts b/src/main/tools/sticky-tool-set.ts index 422889728..e449fb58a 100644 --- a/src/main/tools/sticky-tool-set.ts +++ b/src/main/tools/sticky-tool-set.ts @@ -11,7 +11,7 @@ type ToolSchema = unknown const MAX_CONVERSATIONS = 64 const offeredByConversation = new Map() -export function toolSchemaName(schema: ToolSchema): string | undefined { +function toolSchemaName(schema: ToolSchema): string | undefined { const name = (schema as { function?: { name?: unknown } } | null)?.function?.name return typeof name === 'string' && name ? name : undefined } diff --git a/src/main/tools/tool-embedding-ranking.ts b/src/main/tools/tool-embedding-ranking.ts index 9600989e8..813921ca0 100644 --- a/src/main/tools/tool-embedding-ranking.ts +++ b/src/main/tools/tool-embedding-ranking.ts @@ -9,6 +9,8 @@ export interface EmbedDeps { /** Returns a NORMALIZED embedding (so cosine similarity == dot product). */ embed: (text: string) => Promise + /** The same for several texts in one call: uncached tools are embedded together when given. */ + embedMany?: (texts: string[]) => Promise } const MAX_RELEVANT_TOOLS = 6 @@ -79,6 +81,22 @@ async function embedTool(text: string, embed: EmbedDeps['embed']): Promise { + if (!deps.embedMany) return + const missing = [...new Set(tools.map(toolText))].filter((text) => !toolVecCache.has(hash(text))) + if (!missing.length) return + try { + const vectors = await deps.embedMany(missing) + missing.forEach((text, i) => { + const vector = vectors[i] + if (vector?.length) toolVecCache.set(hash(text), vector) + }) + } catch { + /* embedTool retries each tool on its own */ + } +} + /** Reorder tools so the built-ins (first `keepFirst`) stay put and the connector * tools after them are sorted by DESCENDING cosine similarity to `query`. Returns * the input unchanged when there's nothing to gain (≤1 connector tool, empty @@ -97,6 +115,7 @@ export async function rankConnectorToolsSemantic( const qv = await deps.embed(query) // may throw → caller falls back to lexical const builtins = tools.slice(0, keepFirst) const connectors = tools.slice(keepFirst) + await cacheToolVectors(connectors, deps) const scored = await Promise.all( connectors.map(async (tool, i) => { try { @@ -122,6 +141,7 @@ export async function selectRelevantToolsSemantic( if (!query.trim() || tools.length === 0) return [] const qv = await deps.embed(query) + await cacheToolVectors(tools, deps) const scored = await Promise.all( tools.map(async (tool, index) => { try { diff --git a/src/main/transcription/__tests__/whisper-runtime.test.ts b/src/main/transcription/__tests__/whisper-runtime.test.ts index 5e9d15bb5..4697c9b24 100644 --- a/src/main/transcription/__tests__/whisper-runtime.test.ts +++ b/src/main/transcription/__tests__/whisper-runtime.test.ts @@ -7,7 +7,10 @@ describe('Whisper CUDA libraries', () => { whisperRuntimeLibraryEnv('linux', '/app/bin/whisper-cuda/whisper-cli', { LD_LIBRARY_PATH: '/host/lib' }) - ).toEqual({ LD_LIBRARY_PATH: '/app/bin/whisper-cuda:/app/bin/cuda-runtime:/host/lib' }) + ).toEqual({ + CUDA_CACHE_MAXSIZE: '4294967296', + LD_LIBRARY_PATH: '/app/bin/whisper-cuda:/app/bin/cuda-runtime:/host/lib' + }) }) it('uses bundled Windows CUDA libraries for the accelerated Whisper binary', () => { @@ -15,12 +18,16 @@ describe('Whisper CUDA libraries', () => { whisperRuntimeLibraryEnv('win32', 'C:\\app\\bin\\whisper\\whisper-cli.exe', { PATH: 'C:\\Windows' }) - ).toEqual({ PATH: 'C:\\app\\bin\\whisper;C:\\app\\bin\\cuda-runtime;C:\\Windows' }) + ).toEqual({ + CUDA_CACHE_MAXSIZE: '4294967296', + GGML_VK_DISABLE_BFLOAT16: '1', + PATH: 'C:\\app\\bin\\whisper;C:\\app\\bin\\cuda-runtime;C:\\Windows' + }) }) it('does not add CUDA libraries to the CPU binary', () => { - expect( - whisperRuntimeLibraryEnv('linux', '/app/bin/whisper-cpu/whisper-cli', {}) - ).toEqual({ LD_LIBRARY_PATH: '/app/bin/whisper-cpu' }) + expect(whisperRuntimeLibraryEnv('linux', '/app/bin/whisper-cpu/whisper-cli', {})).toEqual({ + LD_LIBRARY_PATH: '/app/bin/whisper-cpu' + }) }) }) diff --git a/src/main/transcription/parakeet-cli.ts b/src/main/transcription/parakeet-cli.ts index c3af0effb..633bf0e8d 100644 --- a/src/main/transcription/parakeet-cli.ts +++ b/src/main/transcription/parakeet-cli.ts @@ -17,7 +17,7 @@ import { binRoots } from '../runtime-env' import { modelReadDirectories } from '../model-paths' import { getActiveModal } from '../active-models' import { ffmpegBin } from './whisper-cli' -import { existing } from './bin-resolution' +import { parakeetBinary } from '../runtime-availability' import { modelsByEngine } from './classify' import { decodeToWavArgs, DECODE_TIMEOUT_MS } from './ffmpeg-decode' import type { TranscriptionService, Transcript, TranscribeOptions } from './types' @@ -27,10 +27,7 @@ import { runNativeTranscriptionProcess } from './native-process' * stager preserves the prebuilt's bin/ + lib/ structure (so its @rpath finds the * dylibs), hence bin/sherpa-onnx-offline; the flat path is a fallback. */ function parakeetBin(): string | null { - return existing([ - ...binRoots().map((r) => path.join(r, 'parakeet', 'bin', 'sherpa-onnx-offline')), - ...binRoots().map((r) => path.join(r, 'parakeet', 'sherpa-onnx-offline')) - ]) + return parakeetBinary() } /** The four files a sherpa-onnx offline transducer needs (bundled-default names). */ diff --git a/src/main/transcription/whisper-cli.ts b/src/main/transcription/whisper-cli.ts index a29044d49..318f50a6a 100644 --- a/src/main/transcription/whisper-cli.ts +++ b/src/main/transcription/whisper-cli.ts @@ -1,3 +1,4 @@ +import { nativeBinaryInstalled } from '../backend-facts' // WhisperCliTranscription — the only TranscriptionService implementation today. // Wraps the bundled whisper-cli + ffmpeg. The binary/model resolvers live here // (extractors.ts re-exports them for back-compat) so model selection and the @@ -20,15 +21,14 @@ import type { TranscriptionService, Transcript, TranscribeOptions, Seg } from '. import { runNativeTranscriptionProcess } from './native-process' import { HINDI_SCRIPT_RECOVERY_MESSAGE } from '../../shared/transcription-recovery' import { findWhisperBinary, findWhisperBinaries, whisperRuntimeLibraryEnv } from './whisper-runtime' -import { getBackendPreference } from '../backend-preferences' const HINDI_DEVANAGARI_PROMPT = 'यह ऑडियो हिंदी में है। हिंदी को केवल देवनागरी लिपि में लिखें।' /** Resolve the bundled whisper-cli across dev / packaged layouts. System Health * reuses this exact runtime resolver so its Installed claim cannot drift from * the executable the transcription service will actually launch. */ -export function whisperBin(): string | null { - return findWhisperBinary('whisper-cli', getBackendPreference('stt')) +export function whisperBin(): Promise { + return findWhisperBinary('whisper-cli') } /** Resolve ffmpeg: bundled first, then common system locations. */ @@ -81,7 +81,7 @@ function sizeRank(f: string): number { /** A Hindi transcription should contain Devanagari when it contains Hindi text. A small * multilingual Whisper model can occasionally return Latin or Perso-Arabic text even with * `-l hi`; that is a model-quality miss, not a request to translate the recording. */ -export function hindiTranscriptNeedsQualityRetry(text: string, language: string): boolean { +function hindiTranscriptNeedsQualityRetry(text: string, language: string): boolean { if (language !== 'hi' || !text.trim()) return false return !/\p{Script=Devanagari}/u.test(text) } @@ -89,7 +89,7 @@ export function hindiTranscriptNeedsQualityRetry(text: string, language: string) /** Pick a stronger downloaded multilingual Whisper model for one quality retry. English-only * `.en` models cannot correct a Hindi script miss. The active model still runs first; this only * returns a model whose catalog-size rank is higher than the one that produced the bad result. */ -export function strongerMultilingualWhisperModel( +function strongerMultilingualWhisperModel( currentModel: string, files: readonly string[], dir: string @@ -98,13 +98,17 @@ export function strongerMultilingualWhisperModel( const candidate = files .filter((file) => !/\.en\.bin$/i.test(file) && sizeRank(file) > currentRank) .sort((a, b) => sizeRank(b) - sizeRank(a))[0] - return candidate ? (resolveModelAsset(candidate, [dir]) ?? resolveModelAsset(candidate) ?? path.join(dir, candidate)) : null + return candidate + ? (resolveModelAsset(candidate, [dir]) ?? + resolveModelAsset(candidate) ?? + path.join(dir, candidate)) + : null } /** Run the selected model first, then make one quality retry when Hindi text has the wrong * script and a stronger multilingual model is already on the device. `run` is the native-process * boundary, which keeps the decision deterministic and testable without replacing Off Grid code. */ -export async function transcribeWithHindiQualityRetry({ +async function transcribeWithHindiQualityRetry({ language, model, modelFiles, @@ -189,7 +193,7 @@ class WhisperCliTranscription implements TranscriptionService { // pre-converted 16 kHz WAV (alreadyWav16k:true) succeeds without it. Report // available if whisper + a model are present — the transcribe() call validates // ffmpeg at that point when it's actually needed. - return !!whisperBin() && !!whisperModel() + return nativeBinaryInstalled('stt', 'whisper-cli') && !!whisperModel() } async transcribe(input: { path: string }, opts: TranscribeOptions = {}): Promise { @@ -202,8 +206,9 @@ class WhisperCliTranscription implements TranscriptionService { }, async (log) => { await log.inputFile(input.path) - const binaries = findWhisperBinaries('whisper-cli', getBackendPreference('stt')) - if (binaries.length === 0) throw new Error('Transcription runtime (whisper) is not installed.') + const binaries = await findWhisperBinaries('whisper-cli') + if (binaries.length === 0) + throw new Error('Transcription runtime (whisper) is not installed.') const model = resolveModel(opts.model) log.update({ model: model ?? undefined, backend: 'Unknown' }) if (!model) @@ -255,18 +260,21 @@ class WhisperCliTranscription implements TranscriptionService { const runArgs = [...args] runArgs[1] = modelPath let lastError: unknown - for (const [index, binary] of binaries.entries()) { + for (const [index, step] of binaries.entries()) { + const binary = step.path + const attemptArgs = + step.cpuOnly && step.backend !== 'cpu' ? [...runArgs, '-ng'] : runArgs try { const result = await recordAIRequest( { modality: 'stt', source: 'Whisper attempt', model: modelPath, - request: { args: runArgs, binary }, + request: { args: attemptArgs, binary }, signal: opts.signal }, async () => - runNativeTranscriptionProcess(binary, runArgs, { + runNativeTranscriptionProcess(binary, attemptArgs, { runtimeModel: modelPath, maxBuffer: 64 * 1024 * 1024, timeout: 30 * 60_000, @@ -281,7 +289,10 @@ class WhisperCliTranscription implements TranscriptionService { } catch (error) { lastError = error if (opts.signal?.aborted || index === binaries.length - 1) throw error - console.warn(`[transcription] Whisper runtime failed; retrying ${binaries[index + 1]}`, error) + console.warn( + `[transcription] Whisper runtime failed; retrying ${binaries[index + 1]}`, + error + ) } } throw lastError diff --git a/src/main/transcription/whisper-runtime.ts b/src/main/transcription/whisper-runtime.ts index c45aa883a..e46a12d18 100644 --- a/src/main/transcription/whisper-runtime.ts +++ b/src/main/transcription/whisper-runtime.ts @@ -1,69 +1,20 @@ -import fs from 'fs' import path from 'path' -import { binRoots, exe } from '../runtime-env' import { nativeLibraryEnv } from '../native-library-env' -import { prioritizeBackend, type BackendPreference } from '../../shared/backend-preferences' +import { nativeLadder, type NativeStep } from '../backend-facts' +export { hasWindowsNvidiaDriver, hasLinuxNvidiaDriver } from '../nvidia-driver' -/** NVIDIA's Windows display driver installs this API library. */ -export function hasWindowsNvidiaDriver( - windowsDir = process.env.WINDIR ?? process.env.SystemRoot ?? 'C:\\Windows' -): boolean { - return fs.existsSync(path.join(windowsDir, 'System32', 'nvcuda.dll')) +export async function findWhisperBinary( + name: 'whisper-cli' | 'whisper-server' +): Promise { + return (await findWhisperBinaries(name))[0]?.path ?? null } -/** Linux GPU device nodes are created by the active NVIDIA or DRM driver. */ -export function hasLinuxGpuDevice(): boolean { - if (hasLinuxNvidiaDriver()) return true - try { - return fs.readdirSync('/dev/dri').some((name) => name.startsWith('renderD')) - } catch { - return false - } -} - -/** NVIDIA's Linux device node is available only when its driver is active. */ -export function hasLinuxNvidiaDriver(): boolean { - try { - // Containers can expose a GPU with a nonzero host device index. - return fs.readdirSync('/dev').some((name) => /^nvidia\d+$/.test(name)) - } catch { - return false - } -} - -/** Prefer accelerated Whisper, with a separate CPU runtime as the safe fallback. */ -export function findWhisperBinary(name: 'whisper-cli' | 'whisper-server', preference: BackendPreference = 'auto'): string | null { - return findWhisperBinaries(name, preference)[0] ?? null -} - -/** Keep the CPU binary available when a present GPU driver cannot start Whisper. */ -export function findWhisperBinaries(name: 'whisper-cli' | 'whisper-server', preference: BackendPreference = 'auto'): string[] { - const gpuAvailable = - process.platform === 'win32' - ? hasWindowsNvidiaDriver() - : process.platform === 'linux' - ? hasLinuxGpuDevice() - : true - const accelerated = - process.platform === 'linux' && hasLinuxNvidiaDriver() - ? ['whisper-cuda', 'whisper', 'whisper-cpu'] - : gpuAvailable - ? ['whisper', 'whisper-cpu'] - : ['whisper-cpu', 'whisper'] - // macOS and older packages keep the resident binary in its own directory. - const directories = name === 'whisper-server' ? ['whisper-server', ...accelerated] : accelerated - const binaries = directories.flatMap((directory) => - binRoots() - .map((root) => path.join(root, directory, exe(name))) - .filter((candidate) => fs.existsSync(candidate)) - ) - return prioritizeBackend(binaries, preference, (binary) => { - const directory = path.basename(path.dirname(binary)) - if (directory === 'whisper-cpu') return 'cpu' - if (directory === 'whisper-cuda' || (process.platform === 'win32' && directory === 'whisper')) return 'cuda' - if (process.platform === 'darwin' && directory !== 'whisper-cpu') return 'metal' - return 'vulkan' - }) +export async function findWhisperBinaries( + name: 'whisper-cli' | 'whisper-server' +): Promise { + const steps = await nativeLadder('stt', name) + if (!steps.length) throw new Error('No transcription engine is installed for this choice.') + return steps } /** Let Whisper load the bundled CUDA libraries before the system runtime path. */ diff --git a/src/main/transcription/whisper-server-transcription.ts b/src/main/transcription/whisper-server-transcription.ts index 0200979d5..4593c9dc0 100644 --- a/src/main/transcription/whisper-server-transcription.ts +++ b/src/main/transcription/whisper-server-transcription.ts @@ -1,3 +1,4 @@ +import { nativeBinaryInstalled } from '../backend-facts' import { randomUUID } from 'crypto' import fs from 'fs' import { recordAIRequest } from '../ai-request-log' @@ -15,7 +16,7 @@ export class WhisperServerTranscription implements TranscriptionService { constructor(private readonly svc: WhisperServerService = whisperServer) {} isAvailable(): boolean { - return !!this.svc.findBinary() && !!whisperModel() + return nativeBinaryInstalled('stt', 'whisper-server') && !!whisperModel() } async transcribe(input: { path: string }, opts: TranscribeOptions = {}): Promise { diff --git a/src/main/transcription/whisper-server.ts b/src/main/transcription/whisper-server.ts index 0cdd3fc32..0bf01982a 100644 --- a/src/main/transcription/whisper-server.ts +++ b/src/main/transcription/whisper-server.ts @@ -22,8 +22,7 @@ import path from 'path' import fs from 'fs' import os from 'os' import { isPackaged } from '../runtime-env' -import { findWhisperBinary, whisperRuntimeLibraryEnv } from './whisper-runtime' -import { getBackendPreference } from '../backend-preferences' +import { findWhisperBinary, findWhisperBinaries, whisperRuntimeLibraryEnv } from './whisper-runtime' import type { Transcript } from './types' import { killOrphansOnPort as reapOrphansOnPort } from '../kill-orphan-port' import { Mutex } from 'async-mutex' @@ -126,6 +125,7 @@ export class WhisperServerService { private activeKey: string | null = null // whisperContextKey of the loaded model, null when down private activeModelPath: string | null = null private startPromise: Promise | null = null + private startEpoch = 0 private idleTimer: ReturnType | null = null private idleMs = 5 * 60_000 // keep the model hot for 5 min of inactivity, then evict private stderrTail: string[] = [] @@ -146,8 +146,8 @@ export class WhisperServerService { } /** Resolve the bundled whisper-server binary across dev / packaged layouts. */ - findBinary(): string | null { - return findWhisperBinary('whisper-server', getBackendPreference('stt')) + findBinary(): Promise { + return findWhisperBinary('whisper-server') } /** Ensure a server is up with EXACTLY this context; restart on a model/thread @@ -158,18 +158,46 @@ export class WhisperServerService { if (this.server && this.activeKey === key) return // already the right model if (this.server && this.activeKey !== key) this.stopProcess() // swap -> restart if (this.startPromise !== null) return this.startPromise - this.startPromise = this.spawn(ctx, key).finally(() => { + this.startPromise = this.startWithFallback(ctx, key).finally(() => { this.startPromise = null }) return this.startPromise } - private async spawn(ctx: WhisperServerContext, key: string): Promise { - const bin = this.findBinary() - if (!bin) - throw new Error( - 'Resident STT engine (whisper-server) not found in resources/bin/whisper-server.' - ) + private async startWithFallback(ctx: WhisperServerContext, key: string): Promise { + let epoch = this.startEpoch + const steps = await findWhisperBinaries('whisper-server') + const failures: string[] = [] + for (const [index, step] of steps.entries()) { + if (epoch !== this.startEpoch) throw new Error('Transcription server startup canceled.') + try { + await this.spawn( + ctx, + key, + step, + epoch, + failures.length ? `Earlier engines did not start: ${failures.join('; ')}` : undefined + ) + return + } catch (error) { + if (epoch !== this.startEpoch) throw error + failures.push(error instanceof Error ? error.message : String(error)) + this.stopProcess() + epoch = this.startEpoch + if (index === steps.length - 1) throw error + console.warn(`[whisper-server] Runtime failed; retrying ${steps[index + 1]?.path}`, error) + } + } + } + + private async spawn( + ctx: WhisperServerContext, + key: string, + step: Awaited>[number], + epoch: number, + fallbackReason?: string + ): Promise { + const bin = step.path if (!fs.existsSync(ctx.modelPath)) throw new Error(`Transcription model not found: ${ctx.modelPath}`) const binDir = path.dirname(bin) @@ -186,9 +214,11 @@ export class WhisperServerService { // Kill an orphaned whisper-server from a prior app process still holding the // port, but never a foreign process that merely happened to bind it. - this.killOrphanOnPort() + await this.killOrphanOnPort() + if (epoch !== this.startEpoch) throw new Error('Transcription server startup canceled.') const args = buildWhisperServerArgs({ ...ctx, port: this.port }) + if (step.cpuOnly && step.backend !== 'cpu') args.push('-ng') // cwd at the binary dir so @rpath resolves the co-located libwhisper/libggml dylibs. const proc = spawn(bin, args, { cwd: binDir, @@ -219,7 +249,7 @@ export class WhisperServerService { try { await this.waitForReady() - backendState.ready() + backendState.ready(step.cpuOnly ? 'CPU' : undefined, undefined, fallbackReason) } catch (error) { backendState.fail(error) throw error @@ -310,6 +340,7 @@ export class WhisperServerService { } private stopProcess(): void { + this.startEpoch++ this.backendState?.stop() if (this.server) { try { @@ -339,13 +370,13 @@ export class WhisperServerService { } } - private killOrphanOnPort(): void { + private async killOrphanOnPort(): Promise { // Shared cross-platform reaper (kill-orphan-port.ts) — adds the Windows branch this // used to lack (a crashed whisper-server would keep holding the port after a restart). // Match policy: posix matches our OWN bundled binary by FULL PATH (a user's separately // run whisper-server is left untouched); win32's tasklist only exposes the image name, // so match the exe name there. - const ownBin = this.findBinary() + const ownBin = await this.findBinary() if (!ownBin) return reapOrphansOnPort( this.port, diff --git a/src/main/tts-ipc.ts b/src/main/tts-ipc.ts index 16bfbd7ba..8d1c5650d 100644 --- a/src/main/tts-ipc.ts +++ b/src/main/tts-ipc.ts @@ -27,6 +27,10 @@ export function setupTtsIpc(): void { } } + // Parakeet is offered only where its sherpa-onnx runtime ships. + ipcMain.handle('transcription:parakeet-available', async () => + (await import('./runtime-availability')).parakeetRuntimeInstalled() + ) ipcMain.handle('tts:voices', async () => { const { listVoiceCatalog } = await import('./tts') try { diff --git a/src/main/tts-onnx-worker.ts b/src/main/tts-onnx-worker.ts index 6c6a25471..c14c99e1f 100644 --- a/src/main/tts-onnx-worker.ts +++ b/src/main/tts-onnx-worker.ts @@ -1,29 +1,28 @@ -import { parentPort, workerData } from 'node:worker_threads' import { writeFile } from 'node:fs/promises' import type { DeviceType, ProgressInfo } from '@huggingface/transformers' import { KokoroTTS } from 'kokoro-js' import { configureTransformersEnv } from './embeddings-env' import { requireModelStorage } from './runtime-env' import { resolveTransformersPackage } from './model-paths' -import { loadWithOnnxFallback, onnxDeviceCandidates } from './onnx-device' -import type { BackendPreference } from '../shared/backend-preferences' +import { loadWithOnnxFallback } from './onnx-device' +import type { OnnxDevice } from '@offgrid/models' +import { connectHelper, type HelperPort } from './onnx-host/helper-port' const MODEL_ID = 'onnx-community/Kokoro-82M-v1.0-ONNX' -if (!parentPort) throw new Error('tts-onnx-worker must be started as a worker thread') -const port = parentPort -const { - modelsDir, - modelReadDirectories = [modelsDir], - backendPreference = 'auto' -} = workerData as { +/** Sent by the host (tts-onnx.ts) as the helper's first message. */ +export interface TtsWorkerInit { modelsDir: string modelReadDirectories?: string[] - backendPreference?: BackendPreference + devices: OnnxDevice[] } -configureTransformersEnv(modelsDir) -interface TtsWorkerRequest { +let port: HelperPort +let modelsDir = '' +let modelReadDirectories: string[] = [] +let devices: OnnxDevice[] = ['cpu'] + +export interface TtsWorkerRequest { id: number type: 'prepare' | 'synthesize' text?: string @@ -73,11 +72,7 @@ const unusableDevices = new Set() const lastProgress = new Map() function speechDeviceCandidates(): DeviceType[] { - const candidates = onnxDeviceCandidates(process.platform, backendPreference) - // On Linux, CPU Kokoro is substantially faster than WebGPU when CUDA cannot - // produce audio. Keep an explicit WebGPU choice ahead of CPU. - if (process.platform !== 'linux' || backendPreference === 'webgpu') return candidates - return [...candidates.filter((device) => device !== 'webgpu'), ...candidates.filter((device) => device === 'webgpu')] + return devices } function hasAudibleSamples(samples: Float32Array): boolean { @@ -93,7 +88,7 @@ function reportProgress(id: number, info: ProgressInfo): void { const percentage = Math.max(0, Math.min(100, Math.floor(info.progress))) if (lastProgress.get(id) === percentage) return lastProgress.set(id, percentage) - port.postMessage({ + port.post({ id, type: 'progress', downloadedBytes: info.loaded, @@ -149,7 +144,12 @@ async function runtime(id: number): Promise<{ runtime: KokoroRuntime; device: De speechDeviceCandidates().filter((device) => !unusableDevices.has(device)) ) .then((value) => { - port.postMessage({ id, type: 'ready', device: value.device, fallbackReason: value.fallbackReason } satisfies TtsWorkerResponse) + port.post({ + id, + type: 'ready', + device: value.device, + fallbackReason: value.fallbackReason + } satisfies TtsWorkerResponse) loaded = value return value }) @@ -188,23 +188,33 @@ async function synthesizeWithFallback(request: TtsWorkerRequest): Promise { +function handle(request: TtsWorkerRequest): void { void (async () => { try { - const device = request.type === 'synthesize' - ? await synthesizeWithFallback(request) - : (await runtime(request.id)).device - port.postMessage({ + const device = + request.type === 'synthesize' + ? await synthesizeWithFallback(request) + : (await runtime(request.id)).device + port.post({ id: request.id, type: 'complete', device } satisfies TtsWorkerResponse) } catch (error) { - port.postMessage({ + port.post({ id: request.id, type: 'error', error: error instanceof Error ? error.message : String(error) } satisfies TtsWorkerResponse) } })() +} + +void connectHelper().then(({ init, port: connected }) => { + port = connected + modelsDir = init.modelsDir + modelReadDirectories = init.modelReadDirectories ?? [init.modelsDir] + devices = init.devices + configureTransformersEnv(modelsDir) + port.onRequest(handle) }) diff --git a/src/main/tts-onnx.ts b/src/main/tts-onnx.ts index bf9df2d35..03eaebee3 100644 --- a/src/main/tts-onnx.ts +++ b/src/main/tts-onnx.ts @@ -2,12 +2,12 @@ import { beginRuntimeBackend, providerLabel } from './runtime-backends' import { currentAIRequest, type AIRequestHandle } from './ai-request-log' import { existsSync } from 'node:fs' import path from 'node:path' -import { Worker } from 'node:worker_threads' +import { clearHelperCrashes, forkHelper, type HelperProcess } from './onnx-host/utility-host' import type { DownloadProgress } from '@offgrid/executorch-speech' import { modelsDir } from './runtime-env' import { modelReadDirectories } from './model-paths' -import type { TtsWorkerResponse } from './tts-onnx-worker' -import { getBackendPreference } from './backend-preferences' +import type { TtsWorkerInit, TtsWorkerResponse } from './tts-onnx-worker' +import { onnxLadder } from './backend-facts' export interface OnnxSynthesisInput { text: string @@ -33,28 +33,38 @@ interface PendingRequest { onProgress?: (progress: DownloadProgress) => void } +const SERVICE_NAME = 'offgrid-voice' + function workerEntry(): string | null { const built = path.join(__dirname, 'tts-onnx-worker.js') return existsSync(built) ? built : null } export class OnnxSpeechRuntime { - private worker: Worker | null = null + private worker: HelperProcess | null = null private nextId = 1 private readonly pending = new Map() - private spawn(): Worker { + private spawn(): HelperProcess { if (this.worker) return this.worker const entry = workerEntry() if (!entry) throw new Error('The ONNX speech worker is not built.') const backendState = beginRuntimeBackend('speech', 'Kokoro') - const worker = new Worker(entry, { - workerData: { - modelsDir: modelsDir(), - modelReadDirectories: modelReadDirectories(), - backendPreference: getBackendPreference('tts') - } - }) + let worker: HelperProcess + try { + worker = forkHelper( + entry, + { + modelsDir: modelsDir(), + modelReadDirectories: modelReadDirectories(), + devices: onnxLadder('tts') + } satisfies TtsWorkerInit, + { serviceName: SERVICE_NAME, label: 'Voice' } + ) + } catch (error) { + backendState.fail(error) + throw error + } worker.on('message', (response: TtsWorkerResponse) => { if (response.type === 'ready') { backendState.ready(providerLabel(response.device!), undefined, response.fallbackReason) @@ -74,7 +84,8 @@ export class OnnxSpeechRuntime { } this.pending.delete(response.id) backendState.recordRequest(pending.log) - if (response.type === 'error') pending.reject(new Error(response.error || 'ONNX speech failed.')) + if (response.type === 'error') + pending.reject(new Error(response.error || 'ONNX speech failed.')) else pending.resolve(response.device || 'cpu') }) const fail = (error: Error): void => { @@ -84,14 +95,17 @@ export class OnnxSpeechRuntime { for (const request of this.pending.values()) request.reject(error) this.pending.clear() } - worker.on('error', fail) worker.on('exit', (code) => { - backendState.stop() - if (this.worker !== worker) return - if (code !== 0) fail(new Error(`ONNX speech worker exited with code ${code}.`)) - else this.worker = null + if (this.worker !== worker) { + backendState.stop() + return + } + if (code !== 0) fail(new Error(`The voice helper stopped (exit code ${code}).`)) + else { + backendState.stop() + this.worker = null + } }) - worker.unref() this.worker = worker return worker } @@ -103,7 +117,13 @@ export class OnnxSpeechRuntime { const worker = this.spawn() const id = this.nextId++ return new Promise((resolve, reject) => { - this.pending.set(id, { voice: message.voice, resolve, reject, onProgress, log: currentAIRequest() }) + this.pending.set(id, { + voice: message.voice, + resolve, + reject, + onProgress, + log: currentAIRequest() + }) worker.postMessage({ ...message, id }) }) } @@ -125,6 +145,22 @@ export class OnnxSpeechRuntime { ) } + /** True while a voice helper process is running. */ + get isRunning(): boolean { + return this.worker !== null + } + + /** Start the next request on a fresh helper with the current backend choice and environment. + * Waits up to 10 s for requests in flight. */ + async restart(): Promise { + clearHelperCrashes(SERVICE_NAME) + const deadline = Date.now() + 10_000 + while (this.pending.size > 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)) + } + await this.close() + } + /** Syntheses this worker is running now. */ get inFlight(): number { return this.pending.size @@ -139,7 +175,8 @@ export class OnnxSpeechRuntime { const worker = this.worker this.worker = null if (!worker) return - for (const request of this.pending.values()) request.reject(new Error('Speech runtime unloaded.')) + for (const request of this.pending.values()) + request.reject(new Error('Speech runtime unloaded.')) this.pending.clear() await worker.terminate() } diff --git a/src/main/tts.ts b/src/main/tts.ts index 32a3230d1..0bb5c9add 100644 --- a/src/main/tts.ts +++ b/src/main/tts.ts @@ -33,6 +33,7 @@ import type { ManagedRuntime } from './runtime-manager' import { modalityQueue, SPEECH_JOB } from './modality-queue/queue' import { chooseVoice, DEFAULT_VOICE } from './tts-logic' import { OnnxSpeechRuntime } from './tts-onnx' +import { executorchSpeechBinary, executorchSpeechInstalled } from './runtime-availability' import { MEDIA_WORKER_IDLE_MS, mayStartMediaWorker } from '@offgrid/models' import { measuredAvailableMB } from './runtime-manager' @@ -46,7 +47,9 @@ const ONNX_VOICES = new Set( .filter(({ language }) => language === 'en-us' || language === 'en-gb') .map(({ id }) => id) ) -const onnxSpeech = new OnnxSpeechRuntime() +/** The Kokoro voice helper process; index.ts restarts it on wake and after a CUDA check. */ +export const ttsOnnxRuntime = new OnnxSpeechRuntime() +const onnxSpeech = ttsOnnxRuntime /** Extra Kokoro workers for team gateway mode, each its own ONNX session (330 MB each). */ const teamOnnxSpeech: OnnxSpeechRuntime[] = [] @@ -99,11 +102,7 @@ function messageOf(error: unknown): string { } function executablePath(): string { - const candidates = [ - ...resourceDirs().map((root) => path.join(root, 'bin', 'executorch-speech')), - path.resolve(process.cwd(), '../executorch-speech/native/bin/executorch-speech') - ] - const executable = candidates.find((candidate) => fs.existsSync(candidate)) + const executable = executorchSpeechBinary() if (!executable) throw new Error('The local voice runtime is not installed in this build.') return executable } @@ -194,12 +193,17 @@ export async function listVoiceCatalog( const remote = getActiveRemoteVisionServerForModality('voice') // Every remote provider owns its speakers; the remote model, not this machine, decides them. if (remote) return listRemoteVoices(remote) - const voices = speechCapabilities.voices.map(({ id, language }) => ({ - id, - label: kokoroVoiceLabel(id), - language: LANGUAGE_TAGS[language] ?? language, - languageLabel: speechLanguageLabel(LANGUAGE_TAGS[language] ?? language) - })) + // Every voice but the English Kokoro ones speaks through the ExecuTorch runtime; without it, + // offering them only leads to a download that cannot play. + const executorch = executorchSpeechInstalled() + const voices = speechCapabilities.voices + .filter(({ id }) => executorch || ONNX_VOICES.has(id)) + .map(({ id, language }) => ({ + id, + label: kokoroVoiceLabel(id), + language: LANGUAGE_TAGS[language] ?? language, + languageLabel: speechLanguageLabel(LANGUAGE_TAGS[language] ?? language) + })) onProgress?.(100) return voices } @@ -220,6 +224,9 @@ async function prepareVoiceAssetsInner( voice: string, onProgress?: (progress: DownloadProgress) => void ): Promise { + if (!ONNX_VOICES.has(voice) && !executorchSpeechInstalled()) { + throw new Error('This voice needs the local voice runtime, which is not in this build.') + } if (ONNX_VOICES.has(voice)) { try { await onnxSpeech.prepare(voice, onProgress) @@ -232,7 +239,7 @@ async function prepareVoiceAssetsInner( } /** One finished speech pass: the voice can start on pass 0 while the rest is still being made. */ -export interface SpeechPass { +interface SpeechPass { readonly index: number readonly count: number readonly dataUrl: string diff --git a/src/main/vectors-predicates.ts b/src/main/vectors-predicates.ts deleted file mode 100644 index e39165517..000000000 --- a/src/main/vectors-predicates.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Pure SQL-predicate builders extracted from vectors.ts so the string escaping / -// flooring can be unit-tested without opening a LanceDB connection. Behaviour is -// unchanged — vectors.ts imports these back for its delete queries. - -/** Build the `kind IN (...)` predicate for a set of kinds, SQL-escaping single - * quotes (`'` -> `''`) in each value. */ -export function kindsPredicate(kinds: string[]): string { - const list = kinds.map((k) => `'${String(k).replace(/'/g, "''")}'`).join(', ') - return `kind IN (${list})` -} - -/** Append the age filter to a kinds predicate. `cutoffMs` is floored to an - * integer so a fractional epoch-ms never produces a malformed literal. */ -export function olderThanPredicate(kinds: string[], cutoffMs: number): string { - return `${kindsPredicate(kinds)} AND ts < ${Math.floor(cutoffMs)}` -} diff --git a/src/main/vectors.ts b/src/main/vectors.ts deleted file mode 100644 index 708354050..000000000 --- a/src/main/vectors.ts +++ /dev/null @@ -1,100 +0,0 @@ -// LanceDB vector store (Apache-2.0, embedded/offline) — the semantic half of -// universal search. Lives in a Lance dataset under userData/lancedb, keyed by -// `${kind}:${refId}` back to the SQLite source of truth. MiniLM 384-dim vectors. -import * as lancedb from '@lancedb/lancedb' -import path from 'path' -import { app } from 'electron' -import { kindsPredicate, olderThanPredicate } from './vectors-predicates' - -export interface VecChunk { - key: string // unique `${kind}:${refId}` - kind: string // screen | meeting | memory | entity | fact - refId: number - vector: number[] // 384-dim, MiniLM (normalized) - text: string // short snippet for display - surface: string - url: string - ts: number // epoch ms (0 if unknown) -} - -const TABLE = 'chunks' -let connPromise: Promise | null = null -let tablePromise: Promise | null = null - -function conn(): Promise { - if (!connPromise) connPromise = lancedb.connect(path.join(app.getPath('userData'), 'lancedb')) - return connPromise -} - -async function loadTable(): Promise { - const db = await conn() - const names = await db.tableNames() - return names.includes(TABLE) ? db.openTable(TABLE) : null -} - -function table(): Promise { - if (!tablePromise) tablePromise = loadTable() - return tablePromise -} - -/** Append chunks; lazily creates the table (inferring schema) on first batch. */ -export async function addChunks(rows: VecChunk[]): Promise { - if (!rows.length) return - const db = await conn() - const data = rows as unknown as Record[] - const tbl = await table() - if (tbl) { - await tbl.add(data) - return - } - const created = await db.createTable(TABLE, data, { mode: 'create' }) - tablePromise = Promise.resolve(created) -} - -/** k-NN over the store. Returns chunks with `_distance` (smaller = closer). */ -export async function searchVectors( - vector: number[], - limit: number -): Promise<(VecChunk & { _distance: number })[]> { - const tbl = await table() - if (!tbl) return [] - return (await tbl.query().nearestTo(vector).limit(limit).toArray()) as unknown as (VecChunk & { - _distance: number - })[] -} - -export async function vectorCount(): Promise { - const tbl = await table() - return tbl ? tbl.countRows() : 0 -} - -/** Delete only the rows of the given kinds (screen | meeting | memory | entity | - * fact). Used by data-privacy "clear category" so wiping one category doesn't - * nuke the shared dataset — and operates through the live table handle so the - * cached connection stays valid (no stale/dangling handle). */ -export async function deleteByKinds(kinds: string[]): Promise { - if (!kinds.length) return - const tbl = await table() - if (!tbl) return - // Let failures propagate: the privacy path must not report success when the - // semantic index wasn't actually cleared. - await tbl.delete(kindsPredicate(kinds)) -} - -/** Like deleteByKinds, but only rows older than `cutoffMs` (epoch ms) — for - * age-based retention cleanup so pruned captures/meetings don't leave ghost - * vector hits behind. */ -export async function deleteByKindsOlderThan(kinds: string[], cutoffMs: number): Promise { - if (!kinds.length) return - const tbl = await table() - if (!tbl) return - await tbl.delete(olderThanPredicate(kinds, cutoffMs)) -} - -/** Drop the cached connection + table handles. Call after the lancedb dir is - * removed out-of-band (e.g. delete-all), so the next access reopens cleanly - * instead of using a dangling handle. */ -export function resetVectors(): void { - connPromise = null - tablePromise = null -} diff --git a/src/main/vision.ts b/src/main/vision.ts index 828c7ebec..86459df15 100644 --- a/src/main/vision.ts +++ b/src/main/vision.ts @@ -193,42 +193,4 @@ class VisionService { } } -/** - * The focused window's rectangle expressed as a FRACTION of its display — used to - * crop a full-display screenshot down to exactly that window. Window bounds are - * global (screen) coords; we subtract the window's display origin and divide by - * the display size. desktopCapturer thumbnails preserve the display's aspect - * ratio, so these fractions map cleanly onto the screenshot pixels. Returns null - * if bounds are missing/degenerate (caller then OCRs the whole screen). - */ -export function windowRegionOnDisplay(bounds?: { - x: number - y: number - width: number - height: number -}): { x: number; y: number; w: number; h: number } | null { - if (!bounds || !Number.isFinite(bounds.x) || bounds.width < 50 || bounds.height < 50) return null - try { - const center = { - x: Math.round(bounds.x + bounds.width / 2), - y: Math.round(bounds.y + bounds.height / 2) - } - const db = screen.getDisplayNearestPoint(center).bounds - if (!db.width || !db.height) return null - const clamp = (n: number): number => Math.min(1, Math.max(0, n)) - const r = { - x: clamp((bounds.x - db.x) / db.width), - y: clamp((bounds.y - db.y) / db.height), - w: clamp(bounds.width / db.width), - h: clamp(bounds.height / db.height) - } - // Ignore if it's basically the whole display (no useful crop) or degenerate. - if (r.w < 0.15 || r.h < 0.15) return null - if (r.x <= 0.01 && r.y <= 0.01 && r.w >= 0.99 && r.h >= 0.99) return null - return r - } catch { - return null - } -} - export const vision = new VisionService() diff --git a/src/main/vision/__tests__/grounder-runtime.test.ts b/src/main/vision/__tests__/grounder-runtime.test.ts index cd97d84f3..95e0aeb01 100644 --- a/src/main/vision/__tests__/grounder-runtime.test.ts +++ b/src/main/vision/__tests__/grounder-runtime.test.ts @@ -104,7 +104,7 @@ describe('GrounderRuntime', () => { writeEngine(true) configureRuntime({ binRoots: [binaries] }) runtime = new GrounderRuntime() - await expect(runtime.connection(modelId)).rejects.toThrow('does not have enough free memory') + await expect(runtime.connection(modelId)).rejects.toThrow('Out of memory - this model is too large') expect(runtime.running).toBe(false) } ) diff --git a/src/main/vision/focused-input.ts b/src/main/vision/focused-input.ts index 3a48bca50..bd4596a30 100644 --- a/src/main/vision/focused-input.ts +++ b/src/main/vision/focused-input.ts @@ -4,7 +4,7 @@ import { accessibilityHelperPath } from '../accessibility/ax-helper' const execFileAsync = promisify(execFile) -export type FocusedInputState = 'safe' | 'secure' | 'unknown' +type FocusedInputState = 'safe' | 'secure' | 'unknown' export interface FocusedInputTarget { state: FocusedInputState diff --git a/src/main/vision/grounder-loader.ts b/src/main/vision/grounder-loader.ts index f729b58dc..dac0d8d93 100644 --- a/src/main/vision/grounder-loader.ts +++ b/src/main/vision/grounder-loader.ts @@ -27,11 +27,6 @@ export interface GrounderTiming { swapOutMs: number } -/** The wall-clock a swap adds on top of the task run. */ -export function grounderSwapOverheadMs(t: GrounderTiming): number { - return t.swapInMs + t.swapOutMs -} - interface GrounderRemoteSelection { id: string model: string diff --git a/src/main/vision/grounder-runtime.ts b/src/main/vision/grounder-runtime.ts index a288d7a92..d760490ee 100644 --- a/src/main/vision/grounder-runtime.ts +++ b/src/main/vision/grounder-runtime.ts @@ -1,39 +1,61 @@ -import { beginRuntimeBackend, parseNativeBackend } from '../runtime-backends' -import { spawn, type ChildProcess } from 'node:child_process' import fs from 'node:fs' import { totalmem } from 'node:os' import path from 'node:path' import { Mutex } from 'async-mutex' import { prepareModelMemory, registerModelEvictor } from '../model-memory' import type { RemoteTextModelConnection } from '../llm/remote-chat' -import { buildLaunchArgs } from '../llm/settings-math' -import { engineSpawnEnv } from '../llm/spawn-env' -import { selectLocalEngine } from '../llm/select-local-engine' -import { getBackendPreference } from '../backend-preferences' -import { isPortFree, pickFreePort } from '../free-port' -import { reapOrphanProcessesOnPort } from '../kill-orphan-port' import { resolveComputerUseModelArtifact } from '../models-manager' import { llm } from '../llm' import { readGgufMtpSupport } from '../models/gguf-metadata' import { MTP_COMPANIONS } from '../models/catalog-logic' +import { DedicatedLlamaRuntime } from '../llm/dedicated-llama-runtime' const GROUNDER_PORT = 8489 -/** Dedicated local GUI-grounding runtime. It stays resident between delegated - * grounding calls so the reasoning model never unloads or shares this server. */ +/** Dedicated local GUI-grounding runtime, resident between delegated calls. */ export class GrounderRuntime { - private process: ChildProcess | null = null - private port = GROUNDER_PORT private modelId: string | null = null - private stderr = '' - private backendOutput = '' - private backend: string | undefined - private backendState?: ReturnType private readonly mutex = new Mutex() private speculativeSettingsKey = '' + private readonly runtime = new DedicatedLlamaRuntime({ + id: 'grounding', + label: 'Grounding', + modality: 'grounding', + port: GROUNDER_PORT, + modelPaths: async () => { + const artifact = await resolveComputerUseModelArtifact(this.modelId!) + if (!artifact) throw new Error('The selected grounding model is not installed.') + return { + model: this.modelId!, + modelPath: artifact.primaryPath, + mmProjPath: artifact.projectorPath ?? '' + } + }, + extraArgs: () => this.launchSettings + }) + private launchSettings: Omit< + Parameters[0], + 'modelPath' | 'mmProjPath' | 'port' | 'cpuOnly' + > = { + effectiveCtxSize: 16_384, + flashAttn: true, + kvCacheType: 'q8_0', + speculativeDecoding: 'off', + threads: undefined, + batchSize: 1_024 + } get running(): boolean { - return this.process !== null && this.process.exitCode === null + return this.runtime.isRunning + } + health(): Promise { + return this.runtime.health() + } + restart(): Promise { + return this.runtime.restart() + } + shouldBeRunning(): boolean { + return this.runtime.shouldBeRunning() } async connection(modelId: string): Promise { @@ -41,149 +63,70 @@ export class GrounderRuntime { await this.mutex.runExclusive(async () => { const settings = llm.getSettings() const speculativeSettingsKey = `${settings.speculativeDecoding}:${settings.speculativeDraftMax ?? 0}` - if (!this.running || this.modelId !== modelId || this.speculativeSettingsKey !== speculativeSettingsKey) + if ( + !this.running || + this.modelId !== modelId || + this.speculativeSettingsKey !== speculativeSettingsKey + ) await this.start(modelId) - this.backendState?.recordRequest() + this.runtime.recordRequest() }) return { id: `local-grounder:${modelId}`, name: 'Local grounding specialist', provider: 'custom', - endpoint: `http://127.0.0.1:${this.port}/v1`, + endpoint: `http://127.0.0.1:${this.runtime.port}/v1`, model: modelId, - computeBackend: this.backend, + computeBackend: this.runtime.backend, apiKey: '' } } private async start(modelId: string): Promise { + const settingsKey = `${llm.getSettings().speculativeDecoding}:${llm.getSettings().speculativeDraftMax ?? 0}` + if (this.modelId === modelId && this.speculativeSettingsKey === settingsKey) { + await this.runtime.start() + return + } await this.shutdown() const artifact = await resolveComputerUseModelArtifact(modelId) if (!artifact) throw new Error('The selected grounding model is not installed.') - reapOrphanProcessesOnPort( - GROUNDER_PORT, - (command) => /llama-server/i.test(command), - 'Grounding llama-server' - ) - const port = await pickFreePort(GROUNDER_PORT, isPortFree, 20) - if (port === null) throw new Error('No private port is available for the grounding runtime.') - const serverPath = await selectLocalEngine(getBackendPreference('grounding')) - if (!serverPath) throw new Error('The bundled grounding engine is missing.') - const settings = llm.getSettings() const companion = MTP_COMPANIONS['bartowski/tencent_UI-Mate-9B-GGUF+MTP'] const companionPath = path.join(path.dirname(artifact.primaryPath), companion.name) - const matchingCompanion = /^tencent_UI-Mate-9B-(?:Q|IQ)[A-Z0-9_]+(?:\+MTP)?\.gguf$/i.test(path.basename(artifact.primaryPath)) && - readGgufMtpSupport(companionPath, fs, undefined, companion.sizeBytes) - const useMtp = settings.speculativeDecoding === 'mtp' && + const matchingCompanion = + /^tencent_UI-Mate-9B-(?:Q|IQ)[A-Z0-9_]+(?:\+MTP)?\.gguf$/i.test( + path.basename(artifact.primaryPath) + ) && readGgufMtpSupport(companionPath, fs, undefined, companion.sizeBytes) + const useMtp = + settings.speculativeDecoding === 'mtp' && (readGgufMtpSupport(artifact.primaryPath, fs) || matchingCompanion) this.speculativeSettingsKey = `${settings.speculativeDecoding}:${settings.speculativeDraftMax ?? 0}` - this.port = port this.modelId = modelId - this.stderr = '' - this.backendOutput = '' - this.backend = undefined - const backendState = beginRuntimeBackend('grounding', this.modelId!) - this.backendState = backendState - const process = spawn( - serverPath, - buildLaunchArgs({ - modelPath: artifact.primaryPath, - mmProjPath: artifact.projectorPath ?? '', - port, - effectiveCtxSize: 16_384, - gpuLayers: getBackendPreference('grounding') === 'cpu' ? 0 : 99, - flashAttn: true, - kvCacheType: 'q8_0', - speculativeDecoding: useMtp ? 'mtp' : 'off', - speculativeDraftMax: settings.speculativeDraftMax, - draftModelPath: useMtp && matchingCompanion ? companionPath : undefined, - threads: undefined, - batchSize: 1_024, - imageMinTokens: 1_024, - reportModelPlacement: true, - // Team gateway mode opens slots here too, so several clients can ground at once. - // (Single mode passes nothing new, so its argv is unchanged.) - ...(llm.gatewayMode() === 'team' - ? { gatewayMode: 'team' as const, totalRamBytes: totalmem() } - : {}) - }), - { - env: { - ...globalThis.process.env, - ...engineSpawnEnv({ - platform: globalThis.process.platform, - binDir: path.dirname(serverPath), - currentEnv: globalThis.process.env - }) - }, - stdio: ['ignore', 'ignore', 'pipe'] - } - ) - this.process = process - let loading = true - process.stderr!.on('data', (chunk) => { - if (loading) { - const output = String(chunk) - this.stderr = `${this.stderr}${output}`.slice(-16_384) - this.backendOutput = `${this.backendOutput}${output}`.slice(-65_536) - backendState.observe(output) - console.log(`[Grounding runtime] ${output}`) - } - }) - process.once('close', () => { - backendState.stop() - if (this.process === process) this.process = null - }) - - try { - await this.waitUntilReady() - loading = false // Level 4 can contain request data after startup. - this.backend = parseNativeBackend(this.backendOutput)?.backend - backendState.ready(this.backend) - } catch (error) { - await this.shutdown() - const detail = /out of memory|cannot allocate|metal.*alloc/i.test(this.stderr) - ? ' The device does not have enough free memory.' - : '' - throw new Error( - `${error instanceof Error ? error.message : 'The grounding runtime did not start.'}${detail}` - ) - } - } - - private async waitUntilReady(): Promise { - for (;;) { - if (!this.running) throw new Error('The grounding runtime stopped during startup.') - try { - const response = await fetch(`http://127.0.0.1:${this.port}/health`) - if (response.ok) return - } catch { - /* The server is still loading. */ - } - await new Promise((resolve) => setTimeout(resolve, 100)) + this.launchSettings = { + effectiveCtxSize: 16_384, + flashAttn: true, + kvCacheType: 'q8_0', + speculativeDecoding: useMtp ? 'mtp' : 'off', + speculativeDraftMax: settings.speculativeDraftMax, + draftModelPath: useMtp && matchingCompanion ? companionPath : undefined, + threads: undefined, + batchSize: 1_024, + imageMinTokens: 1_024, + reportModelPlacement: true, + // Team gateway mode opens slots here too, so several clients can ground at once. + // (Single mode passes nothing new, so its argv is unchanged.) + ...(llm.gatewayMode() === 'team' + ? { gatewayMode: 'team' as const, totalRamBytes: totalmem() } + : {}) } + await this.runtime.start() } async shutdown(): Promise { - this.backendState?.stop() - const process = this.process - this.process = null + await this.runtime.stop() this.modelId = null - if (!process || process.exitCode !== null) return - const closedPromise = new Promise((resolve) => { - process.once('close', () => resolve()) - }) - process.kill('SIGTERM') - const closed = await Promise.race([ - closedPromise.then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 1_000)) - ]) - if (!closed) { - process.kill('SIGKILL') - await closedPromise - } } } diff --git a/src/main/vision/grounder-selection.ts b/src/main/vision/grounder-selection.ts index 34406eeea..06ce8390f 100644 --- a/src/main/vision/grounder-selection.ts +++ b/src/main/vision/grounder-selection.ts @@ -5,7 +5,7 @@ import { getActiveModal } from '../active-models' import { getComputerUseSettings } from '../computer-use-settings' -export const DEFAULT_GROUNDER_MODEL_ID = 'mradermacher/UI-TARS-1.5-7B-GGUF' +const DEFAULT_GROUNDER_MODEL_ID = 'mradermacher/UI-TARS-1.5-7B-GGUF' export function selectedGrounderModelId(): string { return ( diff --git a/src/main/vision/model-adapters/continuation-capsule.ts b/src/main/vision/model-adapters/continuation-capsule.ts index c9cb4efc0..d05601a01 100644 --- a/src/main/vision/model-adapters/continuation-capsule.ts +++ b/src/main/vision/model-adapters/continuation-capsule.ts @@ -1,6 +1,6 @@ import type { VisionContinuationCapsule } from './types' -export const MAX_CONTINUATION_DONE_ITEMS = 10 +const MAX_CONTINUATION_DONE_ITEMS = 10 const MAX_DONE_CHARS = 96 const MAX_FIELD_CHARS = 160 diff --git a/src/main/vision/model-adapters/general-vision-operator.ts b/src/main/vision/model-adapters/general-vision-operator.ts index 8749e0a77..90546f4ed 100644 --- a/src/main/vision/model-adapters/general-vision-operator.ts +++ b/src/main/vision/model-adapters/general-vision-operator.ts @@ -149,7 +149,7 @@ export function buildCanonicalVisionOperatorRequest( return buildVisionOperatorRequest(input, 'general') } -export function isBonsaiQwenVisionModel(model: { id: string; primaryFile: string }): boolean { +function isBonsaiQwenVisionModel(model: { id: string; primaryFile: string }): boolean { return /(?:^|[/_-])(?:(?:ternary-)?bonsai-2-27b|qwen3[._-]?8)(?:[/_.-]|$)/i.test( `${model.id}/${model.primaryFile}` ) diff --git a/src/main/vision/model-adapters/general-vision-tools.ts b/src/main/vision/model-adapters/general-vision-tools.ts index 2387151b8..db1f29c61 100644 --- a/src/main/vision/model-adapters/general-vision-tools.ts +++ b/src/main/vision/model-adapters/general-vision-tools.ts @@ -7,15 +7,9 @@ import type { VisionPolicyToolCall } from './types' -export const GENERAL_VISION_TOOL_NAMES = [ - 'complete_milestone', - 'perform_action', - 'rethink', - 'call_user' -] as const -export const DIRECTION_VERDICTS = ['aligned', 'off_course'] as const +const DIRECTION_VERDICTS = ['aligned', 'off_course'] as const -type ToolName = (typeof GENERAL_VISION_TOOL_NAMES)[number] +type ToolName = 'complete_milestone' | 'perform_action' | 'rethink' | 'call_user' type Direction = (typeof DIRECTION_VERDICTS)[number] type ObjectValue = Record type DecisionResult = diff --git a/src/main/vision/model-adapters/holo-3-1.ts b/src/main/vision/model-adapters/holo-3-1.ts index c31b0ff6b..91081286c 100644 --- a/src/main/vision/model-adapters/holo-3-1.ts +++ b/src/main/vision/model-adapters/holo-3-1.ts @@ -75,7 +75,7 @@ export const holo31Adapter: VisionModelAdapter = { } } -export function holo31PolicyFailure( +function holo31PolicyFailure( response: VisionPolicyResponse, bounds: Parameters[1] ): string | undefined { diff --git a/src/main/vision/model-adapters/ui-mate/policy.ts b/src/main/vision/model-adapters/ui-mate/policy.ts index 85adcf077..80de3e3ed 100644 --- a/src/main/vision/model-adapters/ui-mate/policy.ts +++ b/src/main/vision/model-adapters/ui-mate/policy.ts @@ -8,7 +8,7 @@ * execute input, or decide whether an action is allowed. */ -export const UI_MATE_ACTIONS = [ +const UI_MATE_ACTIONS = [ 'left_click', 'right_click', 'middle_click', @@ -27,8 +27,8 @@ export const UI_MATE_ACTIONS = [ 'finished' ] as const -export type UIMateActionName = (typeof UI_MATE_ACTIONS)[number] | 'subtask_complete' -export type UIMateControl = 'WAIT' | 'USER' | 'DONE' | 'FAIL' +type UIMateActionName = (typeof UI_MATE_ACTIONS)[number] | 'subtask_complete' +type UIMateControl = 'WAIT' | 'USER' | 'DONE' | 'FAIL' export const UI_MATE_GENERATION_CONFIG = { temperature: 1, @@ -57,7 +57,7 @@ export interface UIMateParsedResponse { error?: string } -export interface UIMateHistoryStep { +interface UIMateHistoryStep { /** The response returned by UI-Mate at this step. */ response: string /** The short action text parsed from the response. */ @@ -66,7 +66,7 @@ export interface UIMateHistoryStep { screenshotDataUrl?: string } -export type UIMateContentBlock = +type UIMateContentBlock = | { type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string } } diff --git a/src/main/vision/model-adapters/ui-tars.ts b/src/main/vision/model-adapters/ui-tars.ts index b72444bcf..89d731d2a 100644 --- a/src/main/vision/model-adapters/ui-tars.ts +++ b/src/main/vision/model-adapters/ui-tars.ts @@ -41,7 +41,7 @@ function uiTarsInstruction(input: Parameters } /** UI-TARS remains on its native single action-text protocol. */ -export function parseUiTarsPolicyResponse( +function parseUiTarsPolicyResponse( response: string, bounds: Parameters[1] ): VisionPolicyDecision { diff --git a/src/main/vision/vision-agent.ts b/src/main/vision/vision-agent.ts index 5524bd8e4..6112409ce 100644 --- a/src/main/vision/vision-agent.ts +++ b/src/main/vision/vision-agent.ts @@ -20,7 +20,7 @@ import type { VisionActionEffect, VisionSemanticElement } from './vision-common- export type { VisionActionEffect, VisionSemanticElement } from './vision-common-types' -export interface VisionCaptureMetadata { +interface VisionCaptureMetadata { path?: string geometry?: ScreenshotGeometry /** Native title of the captured target window, when accessibility provides it. */ @@ -34,7 +34,7 @@ export interface VisionCaptureMetadata { verificationSemanticElements?: readonly VisionSemanticElement[] } -export interface VisionActuationResult { +interface VisionActuationResult { mappedAction?: VisionAction /** Execution-boundary handoff. The private value is intentionally absent. */ handoff?: string @@ -149,7 +149,7 @@ export interface VisionTaskProgress { action: string } -export interface VisionTaskReasoning { +interface VisionTaskReasoning { step: number content: string live: boolean diff --git a/src/main/vision/vision-host.ts b/src/main/vision/vision-host.ts index a1c1da814..36c517e6b 100644 --- a/src/main/vision/vision-host.ts +++ b/src/main/vision/vision-host.ts @@ -40,7 +40,7 @@ import { waitForVisionUser } from './vision-controller' import { hideSupervisorWindow, showSupervisorWindow } from './supervisor-window' -import { loadActuation, actuationAvailable, type ActuationPort } from '../input/actuation' +import { loadActuation, type ActuationPort } from '../input/actuation' import { checkAccessibilityPermission, checkScreenRecordingPermission } from '../permissions' import { mapActionToScreen, type DisplayGeometry } from '../input/coordinate-mapping' import { @@ -125,14 +125,6 @@ async function activateAndConfirmTarget(targetLabel: string): Promise { } } -export type { ActuationPort } - -/** Back-compat alias: the rail-neutral availability check now lives in the - * shared actuation module (both vision and the accessibility rail use it). */ -export function visionActuationAvailable(): boolean { - return actuationAvailable() -} - function makeScreen(input: { actuation: ActuationPort guard: VisionGuard diff --git a/src/main/vision/vision-prompt.ts b/src/main/vision/vision-prompt.ts index e85d771c3..30b75df64 100644 --- a/src/main/vision/vision-prompt.ts +++ b/src/main/vision/vision-prompt.ts @@ -11,7 +11,7 @@ * credential step is a handoff to the user, never typed by the agent. */ -export const VISION_ACTION_SPACE = [ +const VISION_ACTION_SPACE = [ "click(point='x y'), left_double(...), right_single(...),", "drag(start_box='(x,y)', end_box='(x,y)'), type(content='...'), hotkey(key='...'),", "scroll(point='x y', direction='down'), navigate(url='https://...') (Web Use only), wait(), subtask_complete(), finished(content='...'), call_user(content='...')." diff --git a/src/main/vision/vision-task-graph.ts b/src/main/vision/vision-task-graph.ts index 4a21dbb98..73fbe8411 100644 --- a/src/main/vision/vision-task-graph.ts +++ b/src/main/vision/vision-task-graph.ts @@ -1342,7 +1342,7 @@ function clickTarget(action: VisionAction): { x: number; y: number } | null { } /** Treat nearby click variants as one target strategy, even after capture resizing. */ -export function isEquivalentClickTarget( +function isEquivalentClickTarget( action: VisionAction, currentFrame: ReturnType, previous: diff --git a/src/main/voice-transcription-ipc.ts b/src/main/voice-transcription-ipc.ts index 495852832..b0786c0c2 100644 --- a/src/main/voice-transcription-ipc.ts +++ b/src/main/voice-transcription-ipc.ts @@ -17,7 +17,7 @@ export type TranscriptionServiceProvider = () => Promise const active = new Map() /** A transcription that has not finished by this deadline is treated as stalled and ended. */ -export const TRANSCRIBE_TIMEOUT_MS = 5 * 60_000 +const TRANSCRIBE_TIMEOUT_MS = 5 * 60_000 function requestKey(senderId: number, requestId: string): string { return `${senderId}:${requestId}` diff --git a/src/preload/index.ts b/src/preload/index.ts index 0df07459e..06415104c 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -218,13 +218,6 @@ const offGridApi = { getMediaUrl: (absPath: string) => ipcRenderer.invoke('media:url', absPath), // Clipboard manager is a Pro feature: its renderer reaches IPC through the // generic proInvoke / proOn passthrough above (no dedicated namespace here). - getMemories: (limit: number, appName?: string) => - ipcRenderer.invoke('db:get-memories', limit, appName), - addMemory: (content: string, source?: string) => - ipcRenderer.invoke('db:add-memory', content, source), - searchMemories: (query: string) => ipcRenderer.invoke('db:search-memories', query), - getStats: () => ipcRenderer.invoke('db:get-stats'), - getDashboardStats: () => ipcRenderer.invoke('db:get-dashboard-stats'), extractMemory: (text: string) => ipcRenderer.invoke('llm:extract', text), // Chat Summaries @@ -233,15 +226,9 @@ const offGridApi = { ipcRenderer.invoke('db:get-memories-for-session', sessionId), getEntitiesForSession: (sessionId: string) => ipcRenderer.invoke('db:get-entities-for-session', sessionId), - getMemoryRecordsForSession: (sessionId: string) => - ipcRenderer.invoke('db:get-memory-records-for-session', sessionId), summarizeSession: (sessionId: string) => ipcRenderer.invoke('llm:summarize-session', sessionId), deleteSession: (sessionId: string) => ipcRenderer.invoke('db:delete-session', sessionId), - // Master Memory - getMasterMemory: () => ipcRenderer.invoke('db:get-master-memory'), - regenerateMasterMemory: () => ipcRenderer.invoke('db:regenerate-master-memory'), - // RAG Chat - updated to support conversation history + live streaming ragChat: ( query: string, @@ -367,11 +354,7 @@ const offGridApi = { deleteRagConversation: (id: string) => ipcRenderer.invoke('rag:delete-conversation', id), // Entities - getEntities: (appName?: string) => ipcRenderer.invoke('db:get-entities', appName), - getEntityDetails: (entityId: number, appName?: string) => - ipcRenderer.invoke('db:get-entity-details', entityId, appName), deleteEntity: (entityId: number) => ipcRenderer.invoke('db:delete-entity', entityId), - deleteMemory: (memoryId: number) => ipcRenderer.invoke('db:delete-memory', memoryId), // User Profile getUserProfile: () => ipcRenderer.invoke('db:get-user-profile'), @@ -409,31 +392,11 @@ const offGridApi = { consoleStatus: () => ipcRenderer.invoke('console:status'), consoleSyncNow: () => ipcRenderer.invoke('console:sync-now'), consoleDisconnect: () => ipcRenderer.invoke('console:disconnect'), - reprocessAllSessions: (clean?: boolean) => - ipcRenderer.invoke('db:reprocess-all-sessions', clean ?? false), - - // Master Memory Progress - onMasterMemoryProgress: (callback: (data: { current: number; total: number }) => void) => { - const subscription = (_event: unknown, data: { current: number; total: number }): void => - callback(data) - ipcRenderer.on('master-memory:progress', subscription) - return unsubscribe('master-memory:progress', subscription) - }, // Prompts getPrompts: () => ipcRenderer.invoke('prompts:get-all'), savePrompt: (key: string, value: string) => ipcRenderer.invoke('prompts:save', key, value), resetPrompt: (key: string) => ipcRenderer.invoke('prompts:reset', key), - onReprocessProgress: ( - callback: (data: { phase: string; processed: number; total: number }) => void - ) => { - const subscription = ( - _event: unknown, - data: { phase: string; processed: number; total: number } - ): void => callback(data) - ipcRenderer.on('reprocess:progress', subscription) - return unsubscribe('reprocess:progress', subscription) - }, // Auto-update — fired when a new version finished downloading and is staged. // installUpdate() forces the quit+swap (Squirrel only applies on a graceful @@ -450,6 +413,11 @@ const offGridApi = { residencyGet: () => ipcRenderer.invoke('runtime:residency:get'), residencySet: (modality: string, mode: string) => ipcRenderer.invoke('runtime:residency:set', modality, mode), + backendChoices: (): Promise< + Record + > => ipcRenderer.invoke('runtime:backend:choices'), + backendRetry: (modality: string): Promise => + ipcRenderer.invoke('runtime:backend:retry', modality), backendPreferencesGet: () => ipcRenderer.invoke('runtime:backend:get'), backendPreferenceSet: (modality: string, preference: string) => ipcRenderer.invoke('runtime:backend:set', modality, preference), @@ -642,7 +610,6 @@ const offGridApi = { status: (): Promise => ipcRenderer.invoke('performance-pack:status'), start: (): Promise => ipcRenderer.invoke('performance-pack:start'), pause: (): Promise => ipcRenderer.invoke('performance-pack:pause'), - restart: (): Promise => ipcRenderer.invoke('performance-pack:restart'), onChanged: (callback: (status: PerformancePackStatus) => void): (() => void) => { const subscription = (_event: unknown, status: PerformancePackStatus): void => callback(status) @@ -752,7 +719,7 @@ const offGridApi = { systemPrompt?: string kvCacheType?: 'f16' | 'q8_0' | 'q4_0' flashAttn?: boolean - gpuLayers?: number + gpuLayers?: number | null threads?: number batchSize?: number speculativeDecoding?: 'off' | 'ngram' | 'mtp' | 'draft' | 'dflash' @@ -871,6 +838,7 @@ const offGridApi = { // --- Voice output (text-to-speech via Kokoro) --- ttsVoices: () => ipcRenderer.invoke('tts:voices'), + parakeetAvailable: (): Promise => ipcRenderer.invoke('transcription:parakeet-available'), prepareTtsVoice: (voice: string) => ipcRenderer.invoke('tts:prepare-voice', voice), onTtsVoiceProgress: ( callback: (data: { @@ -1025,8 +993,6 @@ const offGridApi = { ipcRenderer.invoke('crm:entity-record', entityId, opts), crmObservationFrames: (observationId: number) => ipcRenderer.invoke('crm:observation-frames', observationId), - crmSearch: (query: string, entityId?: number) => - ipcRenderer.invoke('crm:search', query, entityId), searchFacets: (query: string) => ipcRenderer.invoke('search:facets', query), universalSearch: ( query: string, diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 80fda6e86..2fc94a027 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -39,11 +39,10 @@ import { currentPlatform, primaryModifier } from './lib/device' import { NotificationProvider } from './hooks/NotificationProvider' import { useNotifications } from './hooks/useNotifications' import { ToastProvider } from './hooks/ToastProvider' -import { ReprocessingProvider, useReprocessing } from './hooks/useReprocessing' import { useState, useEffect, useLayoutEffect, useCallback, useRef } from 'react' import { Sidebar, SidebarBody } from './components/ui/sidebar' import { NavThemeToggle } from './components/ThemeToggle' -import { motion, AnimatePresence } from 'motion/react' +import { AnimatePresence } from 'motion/react' import { IconMessageCircle, IconSettings, @@ -100,7 +99,6 @@ type ViewMode = | 'connectors' | 'meetings' | 'chats' - | 'memories' | 'entities' | 'memory-chat' | 'tasks' @@ -129,54 +127,10 @@ interface NavigationState { subroute: string | null settingsSection: string | null selectedSessionId: string | null - selectedMemoryId: number | null selectedEntityId: number | null selectedProjectId: string | null } -function ReprocessingBanner(): React.JSX.Element | null { - const { reprocessing, progress } = useReprocessing() - if (!reprocessing) return null - - const pct = - progress && progress.total > 0 ? Math.round((progress.processed / progress.total) * 100) : 0 - - return ( - - - - {progress?.phase === 'cleared' - ? 'Data cleared. Rebuilding memories and entities...' - : progress - ? `Reprocessing session ${progress.processed} of ${progress.total}...` - : 'Reprocessing sessions...'} - - {progress && progress.total > 0 && ( -
- -
- )} - {progress && progress.total > 0 && ( - {pct}% - )} - - ) -} - // One rule for the look of EVERY sidebar row - nav items, the model-status row, the mobile-app // link. The Tailwind palette is remapped onto the theme-aware --og-* tokens in assets/main.css, // so these classes already flip with data-theme and no `dark:` variant belongs here: `dark:` is @@ -347,7 +301,6 @@ function AppContent(): React.JSX.Element { const [activeModelsOpen, setActiveModelsOpen] = useState(false) const [modelSettingsTab, setModelSettingsTab] = useState('model') const [selectedSessionId, setSelectedSessionId] = useState(null) - const [selectedMemoryId, setSelectedMemoryId] = useState(null) // Version of a downloaded-and-staged update (null = none). Surfaced as a banner // with a "Restart to update" button — Squirrel only applies on a clean quit, so // we drive the install explicitly instead of waiting for one. @@ -458,7 +411,6 @@ function AppContent(): React.JSX.Element { setModelSettingsOpen(false) setModelSettingsTab('model') setSelectedSessionId(null) - setSelectedMemoryId(null) setSelectedEntityId(null) setSelectedProjectId(null) setSearchQuery('') @@ -516,7 +468,6 @@ function AppContent(): React.JSX.Element { '/connectors': 'connectors', '/meetings': 'meetings', '/chat': CHAT_VIEW, - '/memories': 'memories', '/entities': 'entities', '/models': 'models', '/gateway': 'gateway', @@ -632,7 +583,6 @@ function AppContent(): React.JSX.Element { 'memory-chat': '/chat', chats: '/chats', tasks: '/tasks', - memories: '/memories', entities: '/entities', models: '/models', gateway: '/gateway', @@ -675,7 +625,6 @@ function AppContent(): React.JSX.Element { subroute: isInternalTabView(viewMode) ? navigationSubroute : null, settingsSection: viewMode === 'settings' ? settingsSection : null, selectedSessionId, - selectedMemoryId, selectedEntityId, selectedProjectId } @@ -687,7 +636,6 @@ function AppContent(): React.JSX.Element { lastState.subroute === currentState.subroute && lastState.settingsSection === currentState.settingsSection && lastState.selectedSessionId === currentState.selectedSessionId && - lastState.selectedMemoryId === currentState.selectedMemoryId && lastState.selectedEntityId === currentState.selectedEntityId && lastState.selectedProjectId === currentState.selectedProjectId @@ -706,7 +654,6 @@ function AppContent(): React.JSX.Element { navigationSubroute, settingsSection, selectedSessionId, - selectedMemoryId, selectedEntityId, selectedProjectId, syncNavFlags @@ -751,7 +698,6 @@ function AppContent(): React.JSX.Element { setNavigationSubroute(previousState.subroute) setSettingsSection(previousState.settingsSection) setSelectedSessionId(previousState.selectedSessionId) - setSelectedMemoryId(previousState.selectedMemoryId) setSelectedEntityId(previousState.selectedEntityId) setSelectedProjectId(previousState.selectedProjectId) syncNavFlags() @@ -772,7 +718,6 @@ function AppContent(): React.JSX.Element { setNavigationSubroute(nextState.subroute) setSettingsSection(nextState.settingsSection) setSelectedSessionId(nextState.selectedSessionId) - setSelectedMemoryId(nextState.selectedMemoryId) setSelectedEntityId(nextState.selectedEntityId) setSelectedProjectId(nextState.selectedProjectId) syncNavFlags() @@ -785,17 +730,11 @@ function AppContent(): React.JSX.Element { navigateBack() }, [navigateBack]) - // Navigation handlers for Dashboard and MemoryChat - const handleSelectChat = useCallback( - (sessionId: string) => { - navigateTo('chats', () => setSelectedSessionId(sessionId)) - }, - [navigateTo] - ) - - const handleSelectMemory = useCallback( - (memoryId: number) => { - navigateTo('memories', () => setSelectedMemoryId(memoryId)) + // Navigation handlers for MemoryChat + // A chat cited in an answer is one of your own conversations: open it in Chat, as Search does. + const handleOpenConversation = useCallback( + (conversationId: string) => { + navigateTo('memory-chat', () => setChatTarget({ conversationId })) }, [navigateTo] ) @@ -807,13 +746,15 @@ function AppContent(): React.JSX.Element { [navigateTo] ) + // Pro navigation is defined below; search hits (a to-do) reach it through this ref. + const handleProNavigateRef = useRef<((intent: ProNavigationIntent) => void) | null>(null) + // Universal-search result → jump to the exact thing: open its source URL, the // owning entity/memory/meeting, or seek Replay to that captured moment. const handleOpenHit = useCallback( (hit: SearchHit) => { navigateSearchHit(hit, { selectEntity: handleSelectEntity, - selectMemory: handleSelectMemory, openMeeting: (meetingId) => { navigateTo('meetings', () => setMeetingTarget(meetingId)) }, @@ -823,12 +764,13 @@ function AppContent(): React.JSX.Element { openReplay: (timestamp) => { navigateTo('replay', () => setReplayTarget(timestamp)) }, + openAction: (actionId) => handleProNavigateRef.current?.({ view: 'actions', actionId }), openLink: (url) => { window.open(url, '_blank', 'noopener') } }) }, - [handleSelectEntity, handleSelectMemory, navigateTo] + [handleSelectEntity, navigateTo] ) const openSearch = useCallback( @@ -876,6 +818,9 @@ function AppContent(): React.JSX.Element { }, [navigateTo] ) + useEffect(() => { + handleProNavigateRef.current = handleProNavigate + }, [handleProNavigate]) useEffect(() => { if (!proReady || !isPro) return @@ -1057,7 +1002,6 @@ function AppContent(): React.JSX.Element { setNavigationSubroute(isInternalTabView(view) ? subroute : null) setSettingsSection(view === 'settings' ? subroute : null) setSelectedSessionId(null) - setSelectedMemoryId(null) setSelectedEntityId(null) setReplayTarget(null) }) @@ -1346,10 +1290,6 @@ function AppContent(): React.JSX.Element {
- {/* Global reprocessing banner */} - - - {/* Content Area */}
{ - navigateTo('memories', () => { - setSelectedMemoryId(memoryId) - setSelectedSessionId(null) - }) - }} /> ) : ( <> @@ -1392,8 +1326,9 @@ function AppContent(): React.JSX.Element { + handleProNavigate({ view: 'actions', actionId }), + onNavigateToChat: handleOpenConversation, onNavigateToMeeting: (meetingId) => handleProNavigate({ view: 'meetings', meetingId }), onNavigateToEntity: handleSelectEntity, @@ -1410,8 +1345,10 @@ function AppContent(): React.JSX.Element { ) ) : viewMode === 'memory-chat' ? ( + handleProNavigate({ view: 'actions', actionId }) + } + onNavigateToChat={handleOpenConversation} onNavigateToMeeting={(meetingId) => handleProNavigate({ view: 'meetings', meetingId }) } @@ -1488,12 +1425,9 @@ function AppContent(): React.JSX.Element { onSearchSourcesChange: setSearchSources, searchSort, onSearchSortChange: setSearchSort, - selectedMemoryId, - setSelectedMemoryId, selectedEntityId, rec, onSelectEntity: handleSelectEntity, - onSelectMemory: handleSelectMemory, onOpenHit: handleOpenHit, openChatOwner: handleOpenChatOwner } satisfies ProViewContext) ?? ( @@ -1579,9 +1513,7 @@ function App(): React.JSX.Element | null { - - - + diff --git a/src/renderer/src/__tests__/harness/app-boundary.ts b/src/renderer/src/__tests__/harness/app-boundary.ts index a188b7a00..c6b040acf 100644 --- a/src/renderer/src/__tests__/harness/app-boundary.ts +++ b/src/renderer/src/__tests__/harness/app-boundary.ts @@ -54,7 +54,6 @@ export function installAppBoundary(overrides: Record = {}): voi universalSearch: async () => [], onNewAction: eventSubscription, onUpdateDownloaded: eventSubscription, - onReprocessProgress: eventSubscription, onNavigate: eventSubscription, onMeetingState: eventSubscription, onModelProgress: eventSubscription, diff --git a/src/renderer/src/bootstrap/proView.ts b/src/renderer/src/bootstrap/proView.ts index 43b7619f2..76d4efbe8 100644 --- a/src/renderer/src/bootstrap/proView.ts +++ b/src/renderer/src/bootstrap/proView.ts @@ -30,14 +30,11 @@ export interface ProViewContext { onSearchSourcesChange: (s: string[]) => void searchSort: 'relevance' | 'recency' | 'match' onSearchSortChange: (s: 'relevance' | 'recency' | 'match') => void - selectedMemoryId: number | null - setSelectedMemoryId: (id: number | null) => void selectedEntityId: number | null // The meeting-recorder handle (typed loosely so core needn't know its shape). // eslint-disable-next-line @typescript-eslint/no-explicit-any rec: any onSelectEntity: (id: number) => void - onSelectMemory: (id: number) => void onOpenHit: (hit: SearchHit) => void openChatOwner: (target: { conversationId?: string; openGallery?: boolean }) => void } diff --git a/src/renderer/src/components/ChatDetail.tsx b/src/renderer/src/components/ChatDetail.tsx index 11f5fa20b..8b3da3bd6 100644 --- a/src/renderer/src/components/ChatDetail.tsx +++ b/src/renderer/src/components/ChatDetail.tsx @@ -14,32 +14,21 @@ import { ProgressiveBlur } from './ui/progressive-blur' import { openChatLink } from '@renderer/lib/chat-link' import { primaryModifier } from '@renderer/lib/device' -interface Memory { - id: number - content: string - raw_text: string | null - source_app: string | null - session_id: string - created_at: string -} - interface Entity { id: number name: string type: string summary: string | null updated_at: string - fact_count: number } interface ChatDetailProps { sessionId: string onBack: () => void onSelectEntity?: (entityId: number) => void - onSelectMemory?: (memoryId: number) => void } -type ExpandedSection = 'summary' | 'memories' | 'entities' | null +type ExpandedSection = 'summary' | 'entities' | null // Markdown components for summary display const markdownComponents: Components = { @@ -93,55 +82,6 @@ const markdownComponents: Components = { ) } -// Memory card - full version for expanded view -function MemoryCardFull({ memory, onClick }: { memory: Memory; onClick: () => void }): React.JSX.Element { - const formatTime = (dateStr: string): string => parseSqliteUtc(dateStr).toLocaleString() - - return ( - -

{memory.content}

-
- {formatTime(memory.created_at)} - - View in Memories - - - - -
- - ) -} - -// Compact memory card for bento view -function MemoryCard({ memory, onClick }: { memory: Memory; onClick: () => void }): React.JSX.Element { - return ( - -

{memory.content}

-
- - View details - - - - -
- - ) -} - // Entity card - full version for expanded view function EntityCardFull({ entity, onClick }: { entity: Entity; onClick: () => void }): React.JSX.Element { const formatTime = (dateStr: string): string => parseSqliteUtc(dateStr).toLocaleString() @@ -159,9 +99,6 @@ function EntityCardFull({ entity, onClick }: { entity: Entity; onClick: () => vo {entity.type || 'Unknown'} - - {entity.fact_count} facts -
{entity.summary && (

{entity.summary}

@@ -200,7 +137,7 @@ function EntityCard({ entity, onClick }: { entity: Entity; onClick: () => void } )}
- {entity.fact_count} facts · View details + View details @@ -256,8 +193,7 @@ function SectionActions({ } -export function ChatDetail({ sessionId, onBack, onSelectEntity, onSelectMemory }: ChatDetailProps): React.JSX.Element { - const [memories, setMemories] = useState([]) +export function ChatDetail({ sessionId, onBack, onSelectEntity }: ChatDetailProps): React.JSX.Element { const [entities, setEntities] = useState([]) const [summary, setSummary] = useState(null) const [loading, setLoading] = useState(false) @@ -293,12 +229,10 @@ export function ChatDetail({ sessionId, onBack, onSelectEntity, onSelectMemory } const fetchData = async (): Promise => { setLoading(true) try { - const [memoriesData, entitiesData, sessionsData] = await Promise.all([ - window.api.getMemoryRecordsForSession(sessionId), + const [entitiesData, sessionsData] = await Promise.all([ window.api.getEntitiesForSession(sessionId), window.api.getChatSessions() ]) - setMemories(memoriesData) setEntities(entitiesData) const thisSession = sessionsData.find((s) => s.session_id === sessionId) @@ -407,66 +341,6 @@ export function ChatDetail({ sessionId, onBack, onSelectEntity, onSelectMemory } )} - {/* Expanded Memories View */} - {expandedSection === 'memories' && ( - -
- -
handleExpand('memories')} - isExpanded - /> - } - className="relative z-10 flex h-full flex-col" - > -
-
- {memories.length > 0 ? ( -
- {memories.map((memory) => ( - onSelectMemory?.(memory.id)} - /> - ))} -
- ) : ( - - - No memories yet - - - )} -
- {memories.length > 4 && ( - - )} -
-
-
- - )} - {/* Expanded Entities View */} {expandedSection === 'entities' && ( -
handleExpand('memories')} - /> - } - > -
handleExpand('memories')} - onKeyDown={(event) => { - if (event.target !== event.currentTarget) return - if (event.key === 'Enter' || event.key === ' ') { - event.preventDefault() - handleExpand('memories') - } - }} - > - -
- {memories.length > 0 ? ( - memories - .slice(0, 3) - .map((memory) => ( - onSelectMemory?.(memory.id)} - /> - )) - ) : ( - - - No memories yet - - - )} - {memories.length > 3 && ( -

- +{memories.length - 3} more -

- )} -
-
-
-
navigation.onNavigateToEntity?.(entityId), - selectMemory: (memoryId) => navigation.onNavigateToMemory?.(memoryId), openMeeting: (meetingId) => { if (meetingId != null) navigation.onNavigateToMeeting?.(meetingId) }, @@ -66,7 +56,8 @@ export function openUnifiedContext(item: UnifiedContextItem, navigation: Context if (target?.conversationId) navigation.onNavigateToChat?.(target.conversationId) if (target?.projectId) navigation.onOpenProject?.(target.projectId) }, - openReplay: (timestamp) => navigation.onSeekReplay?.(timestamp) + openReplay: (timestamp) => navigation.onSeekReplay?.(timestamp), + openAction: (actionId) => navigation.onNavigateToAction?.(actionId) } ) } @@ -188,53 +179,6 @@ function SourceScoresSectionComponent({ ) } -function MasterMemorySectionComponent({ - content -}: Readonly<{ content?: string | null }>): React.JSX.Element | null { - if (!content) return null - return ( -
-
Master memory
-
- - {preprocessChatMarkdown(content)} - -
-
- ) -} - -function MemoriesContextSectionComponent({ - memories, - onNavigate -}: Readonly<{ - memories?: readonly RagMemory[] - onNavigate?: (memoryId: number) => void -}>): React.JSX.Element | null { - if (!memories?.length) return null - return ( -
-
- Memories ({memories.length}) -
-
- {memories.slice(0, 5).map((memory, index) => ( - - ))} -
-
- ) -} - function SummariesContextSectionComponent({ summaries, onNavigate @@ -298,28 +242,6 @@ function EntitiesContextSectionComponent({ ) } -function EntityFactsContextSectionComponent({ - facts -}: Readonly<{ facts?: readonly RagEntityFact[] }>): React.JSX.Element | null { - if (!facts?.length) return null - return ( -
-
- Entity facts ({facts.length}) -
-
- {facts.slice(0, 5).map((fact, index) => ( -
-

- {typeof fact === 'string' ? fact : fact.fact} -

-
- ))} -
-
- ) -} - function ContextDisclosureComponent({ context, navigation @@ -359,11 +281,6 @@ function ContextDisclosureComponent({ - - - ) } export const UnifiedContextSection = memo(UnifiedContextSectionComponent) -export const SourceScoresSection = memo(SourceScoresSectionComponent) -export const MasterMemorySection = memo(MasterMemorySectionComponent) -export const MemoriesContextSection = memo(MemoriesContextSectionComponent) -export const SummariesContextSection = memo(SummariesContextSectionComponent) -export const EntitiesContextSection = memo(EntitiesContextSectionComponent) -export const EntityFactsContextSection = memo(EntityFactsContextSectionComponent) +const SourceScoresSection = memo(SourceScoresSectionComponent) +const SummariesContextSection = memo(SummariesContextSectionComponent) +const EntitiesContextSection = memo(EntitiesContextSectionComponent) export const ContextDisclosure = memo(ContextDisclosureComponent) diff --git a/src/renderer/src/components/MemoryChat/helper.tsx b/src/renderer/src/components/MemoryChat/helper.tsx index 6561e39b8..1634fc1fd 100644 --- a/src/renderer/src/components/MemoryChat/helper.tsx +++ b/src/renderer/src/components/MemoryChat/helper.tsx @@ -68,7 +68,7 @@ export function attachmentsOf(message: { attachments?: StoredAttachment[] }): At })) } -export function parseRagContext(context: unknown): RagContext | undefined { +function parseRagContext(context: unknown): RagContext | undefined { if (typeof context === 'string') { try { return JSON.parse(context) as RagContext @@ -79,7 +79,7 @@ export function parseRagContext(context: unknown): RagContext | undefined { return context && typeof context === 'object' ? (context as RagContext) : undefined } -export function readRagProvenance(message: RawRagMessage): ChatMessage['provenance'] { +function readRagProvenance(message: RawRagMessage): ChatMessage['provenance'] { if ( typeof message.origin_device_id !== 'string' || typeof message.origin_device_name !== 'string' diff --git a/src/renderer/src/components/MemoryChat/index.tsx b/src/renderer/src/components/MemoryChat/index.tsx index 253ff57af..98149617b 100644 --- a/src/renderer/src/components/MemoryChat/index.tsx +++ b/src/renderer/src/components/MemoryChat/index.tsx @@ -294,7 +294,7 @@ const MESSAGE_ROW_ENTER = { /** God's own voice turn mode (hands-free unless changed in its voice composer). */ export function MemoryChat({ - onNavigateToMemory, + onNavigateToAction, onNavigateToChat, onNavigateToMeeting, onNavigateToEntity, @@ -3448,6 +3448,51 @@ export function MemoryChat({ return } if (data.type === 'done') return + if ( + data.type === 'step' && + data.step && + typeof data.step === 'object' && + 'kind' in data.step && + data.step.kind === 'restarting_model' + ) { + // The same model restarts after a GPU fault; the reply placeholder starts over. + reasoningByStream.current[data.streamId] = '' + answerByStream.current[data.streamId] = '' + timelineByStream.current[data.streamId] = [] + toolCallsByStream.current[data.streamId] = [] + resetStreamViewMessage(data.streamId) + } + if ( + data.type === 'step' && + data.step && + typeof data.step === 'object' && + 'kind' in data.step && + data.step.kind === 'device_fallback' && + 'message' in data.step && + typeof data.step.message === 'string' + ) { + // Chat moved to the CPU: one notice above the reply. The same model answers, so the reply + // (which may already be complete) is left as it is. + const content = data.step.message + const noticeId = crypto.randomUUID() + setConvMessages(cid, (previous) => { + const notice: ChatMessage = { id: noticeId, role: 'assistant', content, notice: true } + const streamIndex = previous.findIndex((message) => message.id === data.streamId) + if (streamIndex < 0) return [...previous, notice] + return [...previous.slice(0, streamIndex), notice, ...previous.slice(streamIndex)] + }) + void window.api + .addRagMessage(cid, 'assistant', content, { notice: true }) + .then((stored) => + setConvMessages(cid, (previous) => + previous.map((message) => + message.id === noticeId ? { ...message, id: stored.uuid } : message + ) + ) + ) + .catch((error) => console.warn('Could not save the CPU notice', error)) + return + } if ( data.type === 'step' && data.step && @@ -3826,7 +3871,7 @@ export function MemoryChat({ const installedSkillNames = useMemo(() => skills.map((skill) => skill.name), [skills]) const messageNavigation = useMemo( () => ({ - onNavigateToMemory, + onNavigateToAction, onNavigateToChat, onNavigateToMeeting, onNavigateToEntity, @@ -3866,7 +3911,7 @@ export function MemoryChat({ onNavigateToChat, onNavigateToEntity, onNavigateToMeeting, - onNavigateToMemory, + onNavigateToAction, onOpenProject, onOpenSkillPreset, onSeekReplay diff --git a/src/renderer/src/components/MemoryChat/interfaces.tsx b/src/renderer/src/components/MemoryChat/interfaces.tsx index f0d6c13a4..85f6a146a 100644 --- a/src/renderer/src/components/MemoryChat/interfaces.tsx +++ b/src/renderer/src/components/MemoryChat/interfaces.tsx @@ -25,7 +25,7 @@ export interface GodChatState { * all memory, and takes what it knows right now with every question. Generation details and the * tools-sent list are not shown. */ -export interface GodChatOptions { +interface GodChatOptions { /** The companion's name, shown as the chat's title. */ readonly name: string /** What God knows right now and how it should answer, added to every turn. */ @@ -55,7 +55,8 @@ export interface GodChatOptions { } export interface MemoryChatProps { - readonly onNavigateToMemory?: (memoryId: number) => void + /** Open a to-do on the Actions screen (Pro). */ + readonly onNavigateToAction?: (actionId: number) => void readonly onNavigateToChat?: (sessionId: string) => void readonly onNavigateToMeeting?: (meetingId: number) => void readonly onNavigateToEntity?: (entityId: number) => void diff --git a/src/renderer/src/components/MemoryChat/types.tsx b/src/renderer/src/components/MemoryChat/types.tsx index b1709dd74..0db1caf96 100644 --- a/src/renderer/src/components/MemoryChat/types.tsx +++ b/src/renderer/src/components/MemoryChat/types.tsx @@ -18,10 +18,8 @@ import type { TaskSession } from '@renderer/lib/task-session-store' import type { Artifact } from '../ArtifactCanvas' import type { DemoPreset } from '../explore/presetCatalog' -export type RagMemory = { id: number; content?: string; text?: string } export type RagSummary = { session_id: string; summary?: string; title?: string; app_name?: string } export type RagEntity = { id: number; name?: string } -export type RagEntityFact = { fact?: string } | string export type RagContext = { branches?: { question: string; answers: ResponseVersion[] }[] @@ -30,12 +28,11 @@ export type RagContext = { notice?: boolean /** On a compaction notice: the summary that stands in for every turn before it. */ compactionSummary?: string - masterMemory?: string | null - memories?: RagMemory[] + // masterMemory, memories and entityFacts in older saved contexts are from the retired legacy + // stores and are no longer shown. messages?: unknown[] summaries?: RagSummary[] entities?: RagEntity[] - entityFacts?: RagEntityFact[] unified?: Array< Omit & { key?: string @@ -213,7 +210,8 @@ export type StoredMessageAttachment = NonNullable[nu export type OpenImage = { url: string; path?: string; title?: string } export type ContextNavigation = Readonly<{ - onNavigateToMemory?: (memoryId: number) => void + /** Open a to-do on the Actions screen (Pro). */ + onNavigateToAction?: (actionId: number) => void onNavigateToChat?: (sessionId: string) => void onNavigateToMeeting?: (meetingId: number) => void onNavigateToEntity?: (entityId: number) => void @@ -247,7 +245,7 @@ export type MessageRowState = Readonly<{ regenerationDisabled: boolean }> -export type AskOptionSelection = Readonly<{ +type AskOptionSelection = Readonly<{ message: ChatMessage ask: AskBlock option: string diff --git a/src/renderer/src/components/MemoryChat/utlis.tsx b/src/renderer/src/components/MemoryChat/utlis.tsx index fcfaf51c4..61d008b37 100644 --- a/src/renderer/src/components/MemoryChat/utlis.tsx +++ b/src/renderer/src/components/MemoryChat/utlis.tsx @@ -5,6 +5,7 @@ import { isSupportingChatContext, preprocessChatMarkdown } from '@offgrid/sync' +import { splitCitations } from '@offgrid/rag/documents' import { toSpeakableText } from '@renderer/lib/speakable' import { runningToolLabel } from '@renderer/lib/tool-display' import type { ImageGenerationJobContract } from '../../../../shared/image-generation-contract' @@ -17,9 +18,8 @@ export const EMPTY_MSGS: ChatMessage[] = [] export const OPEN_CHAT_TABS_KEY = 'offgrid:chat:open-tabs' export const ACTIVE_CHAT_TAB_KEY = 'offgrid:chat:active-tab' -export const ASK_FENCE = /```ask\s*\n[\s\S]*?```/i +const ASK_FENCE = /```ask\s*\n[\s\S]*?```/i const ARTIFACT_FENCE = /```(?:html|svg|mermaid|jsx|tsx|react|image)\s*\n[\s\S]*?```/gi -const CITATION = /\[S(\d+)\]/g export const IMAGE_MESSAGE_COLUMN_WIDTH = 'w-full max-w-2xl' export function imageProgressLabel( @@ -75,9 +75,16 @@ export function parseAsk(content: string): AskBlock | null { return null } +/** Each `[S#]` tag (grouped tags too) replaced, read the way the shared engine reads them. */ +function mapCitations(text: string, citation: (index: number) => string): string { + return splitCitations(text) + .map((segment) => (segment.kind === 'citation' ? citation(segment.index) : segment.text)) + .join('') +} + export function messageToSpeakable(raw: string): string { return toSpeakableText( - (raw || '').replace(ASK_FENCE, '').replace(ARTIFACT_FENCE, '').replace(CITATION, '').trim() + mapCitations((raw || '').replace(ASK_FENCE, '').replace(ARTIFACT_FENCE, ''), () => '').trim() ) } @@ -88,6 +95,8 @@ export function activityLabel(activity?: { label?: string }): string { if (!activity) return '' + if (activity.kind === 'restarting_model') return 'Restarting the model…' + if (activity.kind === 'waiting') return '' if (activity.kind === 'planning') return 'Planning next action…' if (activity.kind === 'preparing_tool_calls') return activity.name === 'generate_image' ? 'Preparing image requests…' : 'Preparing actions…' @@ -98,12 +107,7 @@ export function activityLabel(activity?: { if (activity.kind === 'searching') return 'Searching your memory…' if (activity.kind === 'memory') { const counts = activity.counts || {} - const total = - (counts.memories || 0) + - (counts.summaries || 0) + - (counts.entities || 0) + - (counts.facts || 0) + - (counts.unified || 0) + const total = (counts.summaries || 0) + (counts.entities || 0) + (counts.unified || 0) return `Searched your memory — ${total} result${total === 1 ? '' : 's'}` } if (activity.kind === 'project') { @@ -143,10 +147,7 @@ export function renderedMessageContent(message: ChatMessage): string { const selected = selectedMessageContent(message) if (message.role !== 'assistant') return preprocessChatMarkdown(selected) return preprocessChatMarkdown( - selected - .replace(ASK_FENCE, '') - .replace(/\[S(\d+)\]/g, '[S$1](cite:$1)') - .trim() + mapCitations(selected.replace(ASK_FENCE, ''), (index) => `[S${index}](cite:${index})`).trim() ) } @@ -209,10 +210,8 @@ export function assistantWorkIsSettled(message: ChatMessage): boolean { export function contextResultCount(context: RagContext): number { return ( (context.sources?.length ?? 0) + - (context.memories?.length ?? 0) + (context.summaries?.length ?? 0) + (context.entities?.length ?? 0) + - (context.entityFacts?.length ?? 0) + (context.unified?.length ?? 0) ) } diff --git a/src/renderer/src/components/ProcessingControls.tsx b/src/renderer/src/components/ProcessingControls.tsx index 7a6719a34..3197d44f6 100644 --- a/src/renderer/src/components/ProcessingControls.tsx +++ b/src/renderer/src/components/ProcessingControls.tsx @@ -1,7 +1,9 @@ +import { backendReasonText, type BackendChoice } from '@offgrid/models' import { useEffect, useState } from 'react' import { SettingsSelect } from './SettingsSelect' import { SettingsRow } from './SettingsRow' import { + BACKEND_MODALITIES, backendChoices, normalizeBackendPreferences, type BackendModality, @@ -9,10 +11,13 @@ import { type BackendPreferences } from '../../../shared/backend-preferences' import { - runtimeBackendLabel, + runtimeBackendStatus, type RuntimeBackend, type RuntimeId } from '../../../shared/runtime-backends' +import { ArrowClockwise } from '@phosphor-icons/react' +import { Button } from './ui/button' +import { IconTip } from './IconTip' import type { PerformancePackStatus } from '../../../shared/performance-pack' import { persistToggle } from '@renderer/lib/persist-toggle' import { @@ -36,7 +41,7 @@ interface QueueLive { queued: { label: string; tier: number }[] } -export function RuntimeResidencySection(): React.ReactElement { +function RuntimeResidencySection(): React.ReactElement { // eslint-disable-next-line @typescript-eslint/no-explicit-any const api = (window as any).api const [modes, setModes] = useState>({}) @@ -263,6 +268,30 @@ export function BackendPreferencesSection({ const [running, setRunning] = useState([]) const [packStatus, setPackStatus] = useState(null) const [error, setError] = useState('') + const [choices, setChoices] = useState>>({}) + + useEffect(() => { + // The main process knows which engines and devices this machine has. Without that answer + // (an older preload), every stored choice is offered and Auto still picks what runs. + const offerAll = (): Partial> => + Object.fromEntries( + BACKEND_MODALITIES.map((modality) => [ + modality, + backendChoices(modality, platform).map((value) => ({ value, available: true })) + ]) + ) + if (typeof api.backendChoices !== 'function') { + setChoices(offerAll()) + return + } + void api + .backendChoices() + .then((value: Partial> | undefined) => + setChoices(value ?? offerAll()) + ) + .catch(() => setError('Backend choices could not load.')) + /* eslint-disable-next-line react-hooks/exhaustive-deps */ + }, [packStatus, preferences]) useEffect(() => { if (api.backendPreferencesGet) { @@ -299,6 +328,17 @@ export function BackendPreferencesSection({ /* eslint-disable-next-line react-hooks/exhaustive-deps */ }, []) + const retry = (modality: BackendModality): void => { + setError('') + void api + .backendRetry(modality) + .then(() => api.backendChoices?.()) + .then((value: Partial> | undefined) => { + if (value) setChoices(value) + }) + .catch(() => setError('The engine could not be restarted.')) + } + const change = (modality: BackendModality, preference: BackendPreference): void => { const previous = preferences setPreferences({ ...preferences, [modality]: preference }) @@ -322,37 +362,53 @@ export function BackendPreferencesSection({ {modalities.map((modality) => { const runtime = running.find((entry) => entry.id === RUNTIME_IDS[modality]) const label = `${BACKEND_ROW_LABELS[modality]} backend` - const packOwnsCuda = ['llm', 'image', 'stt', 'grounding', 'decision'].includes(modality) - const cudaNeedsPack = - packOwnsCuda && - packStatus !== null && - ['available', 'downloading', 'paused', 'failed', 'not-needed'].includes(packStatus.phase) + const status = runtimeBackendStatus(runtime) + const demoted = (choices[modality] ?? []).some( + (choice) => choice.reason?.code === 'failed-this-session' + ) + const canRetry = status.tone === 'warn' || status.tone === 'error' || demoted return ( - ({ - value, - label: - value === 'cuda' && cudaNeedsPack - ? 'CUDA (download required)' - : BACKEND_LABELS[value], - disabled: value === 'cuda' && cudaNeedsPack - }))} - onValueChange={(value) => change(modality, value)} - /> +
+ {canRetry ? ( + + + + ) : null} + ({ + value: choice.value, + label: + choice.value === 'metal' && + platform === 'darwin' && + ['tts', 'embeddings'].includes(modality) + ? 'Core ML' + : BACKEND_LABELS[choice.value], + disabled: !choice.available, + hint: + !choice.available && choice.reason + ? backendReasonText(choice.reason) + : undefined + }))} + onValueChange={(value) => change(modality, value)} + /> +
) })} diff --git a/src/renderer/src/components/SettingsPanel.tsx b/src/renderer/src/components/SettingsPanel.tsx index 7aaea9bc7..83131e715 100644 --- a/src/renderer/src/components/SettingsPanel.tsx +++ b/src/renderer/src/components/SettingsPanel.tsx @@ -98,7 +98,7 @@ type LlmSettings = { systemPrompt?: string kvCacheType?: KvCacheType flashAttn?: boolean - gpuLayers?: number + gpuLayers?: number | null threads?: number batchSize?: number speculativeDecoding?: SpeculativeDecodingMode @@ -198,7 +198,7 @@ const DEFAULTS: LlmSettings = { systemPrompt: '', kvCacheType: 'f16', flashAttn: false, - gpuLayers: 99, + gpuLayers: null, threads: 0, batchSize: 512, speculativeDecoding: 'off', @@ -1094,16 +1094,28 @@ export function SettingsPanel({ )} - String(value)} - onDraft={(value) => setS((current) => ({ ...current, gpuLayers: value }))} - onCommit={(value) => set({ gpuLayers: value })} - /> +
+ {s.gpuLayers != null && ( + String(value)} + onDraft={(value) => setS((current) => ({ ...current, gpuLayers: value }))} + onCommit={(value) => set({ gpuLayers: value })} + /> + )} + +
diff --git a/src/renderer/src/components/SettingsSelect.tsx b/src/renderer/src/components/SettingsSelect.tsx index 67226b819..4ca6ff954 100644 --- a/src/renderer/src/components/SettingsSelect.tsx +++ b/src/renderer/src/components/SettingsSelect.tsx @@ -35,6 +35,7 @@ export interface SettingsSelectOption { value: T label: string disabled?: boolean + hint?: string /** Capability meta shown muted on the right of the row, so the list helps you choose. */ meta?: React.ReactNode } @@ -138,7 +139,12 @@ export function SettingsSelect({ {options.map((option) => ( - {option.label} + + {option.label} + {option.hint && ( + {option.hint} + )} + ))} diff --git a/src/renderer/src/components/__tests__/ChatDetail.keyboard.test.tsx b/src/renderer/src/components/__tests__/ChatDetail.keyboard.test.tsx index 362952a60..74bd086c6 100644 --- a/src/renderer/src/components/__tests__/ChatDetail.keyboard.test.tsx +++ b/src/renderer/src/components/__tests__/ChatDetail.keyboard.test.tsx @@ -8,11 +8,8 @@ const originalApi = window.api beforeEach(() => { // The Electron preload is the external boundary. Render the real panel and controls - // against the same session-record contract returned by its three read-only IPC calls. - const sessionApi: Pick< - Window['api'], - 'getChatSessions' | 'getEntitiesForSession' | 'getMemoryRecordsForSession' - > = { + // against the same session-record contract returned by its two read-only IPC calls. + const sessionApi: Pick = { getChatSessions: async () => [ { session_id: 'keyboard-session', @@ -22,8 +19,7 @@ beforeEach(() => { summary: 'Read the [source](https://example.test/source).' } ], - getEntitiesForSession: async () => [], - getMemoryRecordsForSession: async () => [] + getEntitiesForSession: async () => [] } window.api = { ...originalApi, ...sessionApi } }) @@ -34,7 +30,7 @@ afterEach(() => { }) describe('Chat Detail keyboard access', () => { - for (const section of ['summary', 'memories', 'entities']) { + for (const section of ['summary', 'entities']) { for (const key of ['{Enter}', ' ']) { it(`opens ${section} with ${key} and closes with Escape`, async () => { const user = userEvent.setup() diff --git a/src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx b/src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx index 4b21d9263..33ca1db0c 100644 --- a/src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx +++ b/src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx @@ -1915,9 +1915,10 @@ describe(' - chat lifecycle integration (#36-#42, #47-#48)', () => await send('Follow up', user) await waitFor(() => expect(boundary.calls).toHaveLength(2)) expect(vi.mocked(boundary.api.toolChat).mock.calls[1]![1]).toEqual( + // Earlier turns carry their day (a dated header and createdAt); the pair is what matters. expect.arrayContaining([ - { role: 'user', content: 'Original question' }, - { role: 'assistant', content: 'Original answer' } + expect.objectContaining({ role: 'user', content: expect.stringContaining('Original question') }), + expect.objectContaining({ role: 'assistant', content: 'Original answer' }) ]) ) boundary.resolve(1, 'Follow up answer') @@ -1964,8 +1965,8 @@ describe(' - chat lifecycle integration (#36-#42, #47-#48)', () => const history = vi.mocked(boundary.api.toolChat).mock.calls[2]![1] expect(history).toEqual( expect.arrayContaining([ - { role: 'user', content: 'Edited question' }, - { role: 'assistant', content: 'Edited first answer' } + expect.objectContaining({ role: 'user', content: expect.stringContaining('Edited question') }), + expect.objectContaining({ role: 'assistant', content: 'Edited first answer' }) ]) ) expect(JSON.stringify(history)).not.toContain('Old branch answer') diff --git a/src/renderer/src/components/__tests__/MemoryChat.compaction-history.integration.test.tsx b/src/renderer/src/components/__tests__/MemoryChat.compaction-history.integration.test.tsx index 57f96c430..296dbaca3 100644 --- a/src/renderer/src/components/__tests__/MemoryChat.compaction-history.integration.test.tsx +++ b/src/renderer/src/components/__tests__/MemoryChat.compaction-history.integration.test.tsx @@ -7,6 +7,10 @@ import { ChatBoundary, installBoundary, renderChat, send } from './harness/chat- afterEach(cleanup) +/** A turn's text as written: the first turn of each day carries that day's marker line. */ +const said = (turn: { content: string }): string => + turn.content.replace(/^\[(?:Earlier conversation|Today) — [^\]]+\]\n/, '') + it('keeps the active turn and a short earlier excerpt after a saved compaction notice', async () => { ;(Element.prototype as unknown as { scrollIntoView: () => void }).scrollIntoView = () => {} const boundary = new ChatBoundary() @@ -21,8 +25,8 @@ it('keeps the active turn and a short earlier excerpt after a saved compaction n getLlmSettings: async () => ({ ctxSize: 4096 }), toolChat: async (_query: string, history: Array<{ role: string; content: string }>) => ({ answer: - history.some((turn) => turn.content === 'hi') && - history.some((turn) => turn.content.startsWith('Earlier chat excerpts')) && + history.some((turn) => said(turn) === 'hi') && + history.some((turn) => said(turn).startsWith('Earlier chat excerpts')) && history.every((turn) => !turn.content.includes('END-OF-OLD-ANSWER')) ? 'The compacted history carried forward.' : 'The full old history returned.', @@ -63,11 +67,11 @@ it('sends the saved summary in place of the compacted turns after reopening the toolChat: async (_query: string, history: Array<{ role: string; content: string }>) => ({ answer: history[0]?.content.includes('## Goal\nA three-day Lisbon trip in May.') && - history.some((turn) => turn.content === 'Make it three days') && + history.some((turn) => said(turn) === 'Make it three days') && history.every( (turn) => !turn.content.includes('END-OF-OLD-ANSWER') && - !turn.content.startsWith('Earlier chat excerpts') + !said(turn).startsWith('Earlier chat excerpts') ) ? 'The summary carried forward.' : 'The summary was lost.', diff --git a/src/renderer/src/components/__tests__/SettingsPanel.best-settings.integration.test.tsx b/src/renderer/src/components/__tests__/SettingsPanel.best-settings.integration.test.tsx index fc1ea310e..16996f84c 100644 --- a/src/renderer/src/components/__tests__/SettingsPanel.best-settings.integration.test.tsx +++ b/src/renderer/src/components/__tests__/SettingsPanel.best-settings.integration.test.tsx @@ -75,7 +75,7 @@ it('offers the fastest settings in one slim row, applies them, then slides the r expect(await screen.findByText('Fastest settings on for Apple M5, Metal GPU')).toBeTruthy() await waitFor(() => expect(screen.getByRole('combobox', { name: 'Chat and vision backend' }).textContent).toContain( - 'Metal' + 'Auto' ) ) await waitFor( diff --git a/src/renderer/src/components/actions/ApprovalSetup.tsx b/src/renderer/src/components/actions/ApprovalSetup.tsx index 9b0e95b62..294398b7c 100644 --- a/src/renderer/src/components/actions/ApprovalSetup.tsx +++ b/src/renderer/src/components/actions/ApprovalSetup.tsx @@ -34,7 +34,7 @@ function parsedArgs(record: ApprovalSetupRecord): Record { } } -export function buildApprovalChatPrompt( +function buildApprovalChatPrompt( record: ApprovalSetupRecord, instruction: string, context: string, diff --git a/src/renderer/src/components/actions/action-approval-presentation.ts b/src/renderer/src/components/actions/action-approval-presentation.ts index c74ef744e..0f1260e29 100644 --- a/src/renderer/src/components/actions/action-approval-presentation.ts +++ b/src/renderer/src/components/actions/action-approval-presentation.ts @@ -5,7 +5,7 @@ export interface ActionApprovalRequestView { risk: string } -export interface ActionApprovalDetail { +interface ActionApprovalDetail { key: string label: string value: string @@ -72,7 +72,7 @@ function normalizedPhrase(value: string): string { .trim() } -export function isApprovalChatter(value: unknown): boolean { +function isApprovalChatter(value: unknown): boolean { return typeof value === 'string' && APPROVAL_CHATTER.has(normalizedPhrase(value)) } diff --git a/src/renderer/src/components/explore/presetCatalog.ts b/src/renderer/src/components/explore/presetCatalog.ts index 64f9ff4c4..c739fed9d 100644 --- a/src/renderer/src/components/explore/presetCatalog.ts +++ b/src/renderer/src/components/explore/presetCatalog.ts @@ -17,9 +17,9 @@ import { } from '@phosphor-icons/react' export type PresetCapability = 'browser' | 'computer-use' | 'memory' | 'phone' | 'creation' -export type DemoReadiness = 'robust' | 'needs-setup' | 'needs-data' +type DemoReadiness = 'robust' | 'needs-setup' | 'needs-data' export type PresetRequirement = 'pro' | 'phone-paired' | 'capture-history' | 'image-model' -export type PresetFieldKind = +type PresetFieldKind = | 'text' | 'textarea' | 'select' @@ -29,7 +29,7 @@ export type PresetFieldKind = | 'style' | 'image' -export interface PresetFieldOption { +interface PresetFieldOption { value: string label: string } @@ -48,7 +48,7 @@ export interface PresetIntakeField { promptSuffix?: string } -export interface PresetIntake { +interface PresetIntake { title: string description: string recommendedConnector?: string diff --git a/src/renderer/src/components/navigation/SidebarNavigationMenu.tsx b/src/renderer/src/components/navigation/SidebarNavigationMenu.tsx index d8f17109b..20f1e5b00 100644 --- a/src/renderer/src/components/navigation/SidebarNavigationMenu.tsx +++ b/src/renderer/src/components/navigation/SidebarNavigationMenu.tsx @@ -2,7 +2,7 @@ import { useState, type ReactNode } from 'react' import { IconChevronDown } from '@tabler/icons-react' import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '../ui/collapsible' -export interface SidebarNavigationGroup { +interface SidebarNavigationGroup { label: string icon: ReactNode items: readonly Item[] diff --git a/src/renderer/src/components/setup/PerformancePackGate.tsx b/src/renderer/src/components/setup/PerformancePackGate.tsx index 01bce9106..9750a2658 100644 --- a/src/renderer/src/components/setup/PerformancePackGate.tsx +++ b/src/renderer/src/components/setup/PerformancePackGate.tsx @@ -4,20 +4,43 @@ import type { PerformancePackStatus } from '../../../../shared/performance-pack' const SKIPPED_KEY = 'performance_pack_intro_complete' -export function PerformancePackGate({ children }: { children: React.ReactNode }): React.ReactElement { +export function PerformancePackGate({ + children +}: { + children: React.ReactNode +}): React.ReactElement { const [status, setStatus] = useState(null) const [dismissed, setDismissed] = useState(() => localStorage.getItem(SKIPPED_KEY) === 'true') useEffect(() => { - const pack = window.api.performancePack as Partial | undefined + const pack = window.api.performancePack as + Partial | undefined if (typeof pack?.status !== 'function') return - void pack.status().then(setStatus).catch(() => setStatus({ - phase: 'unavailable', bytes: 0, downloadedBytes: 0 - })) + void pack + .status() + .then(setStatus) + .catch(() => + setStatus({ + phase: 'unavailable', + bytes: 0, + downloadedBytes: 0 + }) + ) }, []) if (!status) return <>{children} - if (dismissed || !['available', 'paused', 'downloading', 'failed'].includes(status.phase)) { + if ( + dismissed || + ![ + 'available', + 'paused', + 'downloading', + 'verifying', + 'installing', + 'testing', + 'failed' + ].includes(status.phase) + ) { return <>{children} } @@ -29,7 +52,9 @@ export function PerformancePackGate({ children }: { children: React.ReactNode }) return (
-

Optional setup

+

+ Optional setup +

diff --git a/src/renderer/src/components/setup/PerformancePackPanel.tsx b/src/renderer/src/components/setup/PerformancePackPanel.tsx index d657712e7..c34a493a7 100644 --- a/src/renderer/src/components/setup/PerformancePackPanel.tsx +++ b/src/renderer/src/components/setup/PerformancePackPanel.tsx @@ -1,13 +1,26 @@ import { useEffect, useState } from 'react' -import { DownloadSimple, Pause, Play, ArrowClockwise, CheckCircle } from '@phosphor-icons/react' +import { + DownloadSimple, + Pause, + Play, + ArrowClockwise, + ArrowSquareOut, + CheckCircle, + WarningCircle +} from '@phosphor-icons/react' import { projectProgress } from '@offgrid/ui' import { Button } from '../ui/button' import { GlyphSwap, IconTip } from '../IconTip' import type { PerformancePackStatus } from '../../../../shared/performance-pack' import { currentPlatform } from '@renderer/lib/device' +import { openExternal } from '../../constants/links' import { formatStorageBytes } from './storage-format' import { EngineUpdatesSection } from './EngineUpdatesSection' -import { Progress, ProgressIndicator, ProgressTrack } from '@offgrid/operator-ui/operator/coss/progress' +import { + Progress, + ProgressIndicator, + ProgressTrack +} from '@offgrid/operator-ui/operator/coss/progress' import { TextMorph } from '@offgrid/operator-ui/operator/motion/text-morph' import { AnimatedNumber } from '@offgrid/operator-ui/operator/motion/animated-number' import { TextShimmer } from '@offgrid/operator-ui/operator/text-shimmer' @@ -17,6 +30,15 @@ interface PerformancePackPanelProps { showUnavailable?: boolean } +const NVIDIA_DRIVERS_URL = 'https://www.nvidia.com/Download/index.aspx' + +/** The phases after the download, shown as one moving label. */ +const WORKING_LABELS: Partial> = { + verifying: 'Checking the download…', + installing: 'Installing…', + testing: 'Testing your GPU…' +} + const GPU_COMPONENTS = [ { name: 'Chat and vision', detail: 'NVIDIA chat engine' }, { name: 'Image generation', detail: 'NVIDIA image engine' }, @@ -24,24 +46,37 @@ const GPU_COMPONENTS = [ { name: 'Computer Use', detail: 'NVIDIA grounding and decision engines' } ] as const -export function PerformancePackPanel({ onSkip, showUnavailable = false }: PerformancePackPanelProps): React.ReactElement | null { +export function PerformancePackPanel({ + onSkip, + showUnavailable = false +}: PerformancePackPanelProps): React.ReactElement | null { const [status, setStatus] = useState(null) useEffect(() => { - const pack = window.api.performancePack as Partial | undefined + const pack = window.api.performancePack as + Partial | undefined if (typeof pack?.status !== 'function') return - void pack.status().then(setStatus).catch(() => {}) + void pack + .status() + .then(setStatus) + .catch(() => {}) return pack.onChanged?.(setStatus) }, []) - if (!status) return showUnavailable ?

Checking GPU components...

: null + if (!status) + return showUnavailable ? ( +

+ Checking GPU components... +

+ ) : null if (status.phase === 'not-needed' || status.phase === 'unavailable') { if (!showUnavailable) return null - const message = currentPlatform() === 'darwin' - ? 'Metal support is included on macOS. No extra GPU download is needed.' - : status.phase === 'not-needed' - ? 'No NVIDIA driver was found. The CUDA download is only for devices with an NVIDIA GPU.' - : 'No optional GPU download is available in this build.' + const message = + currentPlatform() === 'darwin' + ? 'Metal support is included on macOS. No extra GPU download is needed.' + : status.phase === 'not-needed' + ? 'No NVIDIA driver was found. The CUDA download is only for devices with an NVIDIA GPU.' + : 'No optional GPU download is available in this build.' return

{message}

} @@ -49,8 +84,40 @@ export function PerformancePackPanel({ onSkip, showUnavailable = false }: Perfor downloadedBytes: status.downloadedBytes, totalBytes: status.bytes }) + if (status.phase === 'driver-too-old') { + return ( +
+ +

+ The NVIDIA speed-up needs driver {status.minDriver} or newer. This computer has{' '} + {status.driver}. +

+ + + + {onSkip ? ( + + ) : null} +
+ ) + } + const installed = status.phase === 'installed' const downloading = status.phase === 'downloading' + const working = WORKING_LABELS[status.phase] + const shortOfSpace = + status.needBytes !== undefined && + status.freeBytes !== undefined && + status.freeBytes < status.needBytes const packActionLabel = downloading ? 'Pause' : status.phase === 'paused' @@ -64,17 +131,35 @@ export function PerformancePackPanel({ onSkip, showUnavailable = false }: Perfor
- {installed ? : } + {installed ? ( + + ) : ( + + )}

Use your NVIDIA GPU

- One download of {formatStorageBytes(status.bytes)} adds GPU support for the model types below. - You can use the app during the download or skip this step. + One download of {formatStorageBytes(status.bytes)} adds GPU support for the model types + below. You can use the app during the download or skip this step.

+ {!installed && status.needBytes !== undefined ? ( +

+ Needs {formatStorageBytes(status.needBytes)} free + {status.freeBytes !== undefined + ? ` (this drive has ${formatStorageBytes(status.freeBytes)})` + : ''} + . +

+ ) : null} {installed ? (

- {status.restartRequired ? 'Installed. Restart the app to use it.' : 'Installed and ready.'} + {status.device ? `CUDA ready on ${status.device}.` : 'Installed and ready.'} +

+ ) : null} + {working ? ( +

+ {working}

) : null} {status.error ?

{status.error}

: null} @@ -87,7 +172,9 @@ export function PerformancePackPanel({ onSkip, showUnavailable = false }: Perfor

{component.name}

{component.detail}

- + {installed ? 'Ready' : 'Included'}
@@ -96,7 +183,7 @@ export function PerformancePackPanel({ onSkip, showUnavailable = false }: Perfor

Voice replies and search use separate GPU settings.

- {status.downloadedBytes > 0 && !installed ? ( + {status.downloadedBytes > 0 && !installed && !working ? (
{status.phase === 'paused' ? 'Paused' : 'Downloading'} @@ -113,15 +200,12 @@ export function PerformancePackPanel({ onSkip, showUnavailable = false }: Perfor
) : null}
- {installed && status.restartRequired ? ( - - ) : !installed ? ( + {!installed && !working ? ( + ) : null}
{installed && showUnavailable ? : null} diff --git a/src/renderer/src/components/use-chat-voice-turns.ts b/src/renderer/src/components/use-chat-voice-turns.ts index 926ea9bb0..917d21eac 100644 --- a/src/renderer/src/components/use-chat-voice-turns.ts +++ b/src/renderer/src/components/use-chat-voice-turns.ts @@ -17,7 +17,7 @@ import { captureUrlForPath } from '../../../shared/ogcapture-url' export type ChatVoicePhase = 'idle' | 'starting' | 'listening' | 'recording' | 'transcribing' -export interface ChatVoiceClip { +interface ChatVoiceClip { url: string duration: number path?: string diff --git a/src/renderer/src/env.d.ts b/src/renderer/src/env.d.ts index fd17ec526..3abd61eda 100644 --- a/src/renderer/src/env.d.ts +++ b/src/renderer/src/env.d.ts @@ -11,66 +11,11 @@ interface ProLicenseInfo { type OffGridPermissionStatus = import('../../shared/ipc-contracts').PermissionStatusContract -interface DashboardStats { - totalChats: number - totalMemories: number - totalEntities: number - totalRelationships: number - totalMessages: number - totalFacts: number - todayChats: number - todayMemories: number - todayEntities: number - recentChats: Array<{ - session_id: string - title: string | null - app_name: string - memory_count: number - entity_count: number - updated_at: string - }> - recentMemories: Array<{ - id: number - content: string - source_app: string - created_at: string - }> - topEntities: Array<{ - id: number - name: string - type: string - fact_count: number - session_count: number - }> - entityTypeCounts: Array<{ - type: string - count: number - }> - appDistribution: Array<{ - app_name: string - chat_count: number - memory_count: number - }> - activityByDay: Array<{ - date: string - chats: number - memories: number - }> -} - type RagConversation = import('../../shared/ipc-contracts').RagConversationContract type RagMessage = import('../../shared/ipc-contracts').RagMessageContract type RagChatResult = import('../../shared/ipc-contracts').RagChatResultContract -// DUPLICATE (ambient decl). Canonical shape: the `reprocess:progress` IPC payload -// emitted in src/main/ipc.ts. Keep in sync; guarded by ipc-type-parity.test.ts. -interface ReprocessProgress { - phase: string - processed: number - total: number -} - interface AppSettings { memoryStrictness?: 'lenient' | 'balanced' | 'strict' entityStrictness?: 'lenient' | 'balanced' | 'strict' @@ -80,35 +25,16 @@ interface AppSettings { interface ChatSessionRecord { session_id: string last_activity: string - memory_count: number entity_count: number summary: string | null } -interface SessionMemoryRecord { - id: number - content: string - raw_text: string | null - source_app: string | null - session_id: string - created_at: string -} - -interface MemoryListRecord { - id: number - name: string | null - content: string - source_app: string - created_at: string -} - interface SessionEntityRecord { id: number name: string type: string summary: string | null updated_at: string - fact_count: number } type ArtifactKind = import('../../shared/ipc-contracts').ArtifactKindContract @@ -301,27 +227,14 @@ interface RendererAPIOverrides { relaunch: () => Promise onChanged: (cb: (info: ProLicenseInfo) => void) => () => void } - onMasterMemoryProgress?: ( - callback: (data: { current: number; total: number }) => void - ) => () => void - getMemories: (limit: number, appName?: string) => Promise - addMemory: (content: string, source?: string) => Promise<{ id: number }> - searchMemories: (query: string) => Promise - getStats: () => Promise> - getDashboardStats: () => Promise extractMemory: (text: string) => Promise<{ summary: string; entities: string[]; topic: string }> getChatSessions: () => Promise getMemoriesForSession: (sessionId: string) => Promise getEntitiesForSession: (sessionId: string) => Promise - getMemoryRecordsForSession: (sessionId: string) => Promise summarizeSession: (sessionId: string) => Promise deleteSession: (sessionId: string) => Promise - // Master Memory - getMasterMemory: () => Promise<{ content: string | null; updated_at: string | null }> - regenerateMasterMemory: () => Promise - // RAG Chat ragChat: ( query: string, @@ -418,12 +331,8 @@ interface RendererAPIOverrides { lastSync: number }> consoleDisconnect: () => Promise - reprocessAllSessions: (clean?: boolean) => Promise<{ processed: number; total: number }> - getEntities: (appName?: string) => Promise - getEntityDetails: (entityId: number, appName?: string) => Promise deleteEntity: (entityId: number) => Promise - deleteMemory: (memoryId: number) => Promise // Artifacts library saveArtifact: (a: { @@ -502,7 +411,6 @@ interface RendererAPIOverrides { sourceApp: string }) => void ) => () => void - onReprocessProgress: (callback: (data: ReprocessProgress) => void) => () => void onUpdateDownloaded: (callback: (data: { version: string }) => void) => () => void getStagedUpdateVersion: () => Promise installUpdate: () => Promise diff --git a/src/renderer/src/hooks/useReprocessing.tsx b/src/renderer/src/hooks/useReprocessing.tsx deleted file mode 100644 index 5d9c1b96a..000000000 --- a/src/renderer/src/hooks/useReprocessing.tsx +++ /dev/null @@ -1,73 +0,0 @@ -import { createContext, useContext, useState, useEffect, useCallback, ReactNode } from 'react' - -interface ReprocessProgress { - phase: string - processed: number - total: number -} - -interface ReprocessResult { - processed: number - total: number -} - -interface ReprocessingContextType { - reprocessing: boolean - progress: ReprocessProgress | null - result: ReprocessResult | null - startReprocess: (clean: boolean) => void - clearResult: () => void -} - -const ReprocessingContext = createContext(undefined) - -export function ReprocessingProvider({ children }: { children: ReactNode }): React.JSX.Element { - const [reprocessing, setReprocessing] = useState(false) - const [progress, setProgress] = useState(null) - const [result, setResult] = useState(null) - - // Listen for progress events from the main process - useEffect(() => { - const unsub = window.api.onReprocessProgress((data) => { - setProgress(data) - }) - return unsub - }, []) - - const clearResult = useCallback(() => setResult(null), []) - - const startReprocess = useCallback( - async (clean: boolean) => { - if (reprocessing) return - setReprocessing(true) - setResult(null) - setProgress(null) - try { - const res = await window.api.reprocessAllSessions(clean) - setResult(res) - } catch (e) { - console.error('Reprocess failed:', e) - } finally { - setReprocessing(false) - setProgress(null) - } - }, - [reprocessing] - ) - - return ( - - {children} - - ) -} - -export function useReprocessing(): ReprocessingContextType { - const context = useContext(ReprocessingContext) - if (!context) { - throw new Error('useReprocessing must be used within a ReprocessingProvider') - } - return context -} diff --git a/src/renderer/src/lib/__tests__/search-navigation.integration.test.ts b/src/renderer/src/lib/__tests__/search-navigation.integration.test.ts index 2bb027b60..830b38061 100644 --- a/src/renderer/src/lib/__tests__/search-navigation.integration.test.ts +++ b/src/renderer/src/lib/__tests__/search-navigation.integration.test.ts @@ -20,15 +20,12 @@ describe('universal-search typed navigation seam', () => { const opened: string[] = [] const ports: SearchNavigationPorts = { selectEntity: (id) => opened.push(`entity:${String(id)}`), - selectMemory: (id) => opened.push(`memory:${String(id)}`), openMeeting: (id) => opened.push(`meeting:${String(id)}`), openChat: (target) => opened.push(JSON.stringify(target)), openReplay: (timestamp) => opened.push(`replay:${String(timestamp)}`) } navigateSearchHit(hit('entity', 11), ports) - navigateSearchHit(hit('fact', 11), ports) - navigateSearchHit(hit('memory', 12), ports) navigateSearchHit(hit('meeting', 13), ports) navigateSearchHit(hit('chat', 0, 'conversation-14'), ports) navigateSearchHit(hit('doc', 15, 'project-15'), ports) @@ -36,8 +33,6 @@ describe('universal-search typed navigation seam', () => { expect(opened).toEqual([ 'entity:11', - 'entity:11', - 'memory:12', 'meeting:13', JSON.stringify({ conversationId: 'conversation-14' }), JSON.stringify({ projectId: 'project-15' }), @@ -49,7 +44,6 @@ describe('universal-search typed navigation seam', () => { const opened: string[] = [] const ports: SearchNavigationPorts = { selectEntity: () => undefined, - selectMemory: () => undefined, openMeeting: (id) => opened.push(`meeting:${String(id)}`), openChat: (target) => opened.push(String(target)), openReplay: (timestamp) => opened.push(`replay:${String(timestamp)}`) diff --git a/src/renderer/src/lib/chat-queue-persistence.ts b/src/renderer/src/lib/chat-queue-persistence.ts index ff105f650..b865a251c 100644 --- a/src/renderer/src/lib/chat-queue-persistence.ts +++ b/src/renderer/src/lib/chat-queue-persistence.ts @@ -9,7 +9,7 @@ export type QueuedAttachmentReference = Pick< > & { text?: string } export type PersistedQueuedTurn = Omit & { atts: QueuedAttachmentReference[] } -export function persistedQueuedTurn(item: QueuedTurn): PersistedQueuedTurn { +function persistedQueuedTurn(item: QueuedTurn): PersistedQueuedTurn { return { ...(item.id ? { id: item.id } : {}), text: item.text, diff --git a/src/renderer/src/lib/chat-turn-replacement.ts b/src/renderer/src/lib/chat-turn-replacement.ts index ffa5eed07..7909074ca 100644 --- a/src/renderer/src/lib/chat-turn-replacement.ts +++ b/src/renderer/src/lib/chat-turn-replacement.ts @@ -3,7 +3,7 @@ import type { ResponseVersion } from '../../../shared/chat-versions' import { attachmentsOf } from '../components/MemoryChat/helper' import { priorAnswerVersions } from './chat-versions' -export interface ReplacementSend { +interface ReplacementSend { regen: true atts: Attachment[] conversationId: string diff --git a/src/renderer/src/lib/current-view.ts b/src/renderer/src/lib/current-view.ts index c0647b686..73a8620f4 100644 --- a/src/renderer/src/lib/current-view.ts +++ b/src/renderer/src/lib/current-view.ts @@ -31,7 +31,7 @@ export function setCurrentView(view: string): void { for (const listener of listeners) listener() } -export function getCurrentView(): string { +function getCurrentView(): string { return currentView } diff --git a/src/renderer/src/lib/internal-tab-route.ts b/src/renderer/src/lib/internal-tab-route.ts index c5fc2ca8e..600fa821b 100644 --- a/src/renderer/src/lib/internal-tab-route.ts +++ b/src/renderer/src/lib/internal-tab-route.ts @@ -1,12 +1,6 @@ import { modelKindLabel } from './model-kind-labels' -export const INTERNAL_TAB_VIEWS = [ - 'devices', - 'entities', - 'meetings', - 'models', - 'notifications' -] as const +const INTERNAL_TAB_VIEWS = ['devices', 'entities', 'meetings', 'models', 'notifications'] as const export type InternalTabView = (typeof INTERNAL_TAB_VIEWS)[number] export interface InternalTabRoute { @@ -78,7 +72,7 @@ export function internalTabSubroute(view: InternalTabView, id: string): string | } /** Reject unknown tabs and screen-owned detail segments under tabs that do not allow them. */ -export function canonicalInternalTabSubroute( +function canonicalInternalTabSubroute( view: InternalTabView, subroute: string | null ): string | null { diff --git a/src/renderer/src/lib/paletteScreens.ts b/src/renderer/src/lib/paletteScreens.ts index bbc69596b..258900b4a 100644 --- a/src/renderer/src/lib/paletteScreens.ts +++ b/src/renderer/src/lib/paletteScreens.ts @@ -30,13 +30,12 @@ export function internalTabPaletteScreens( * Screens whose name is not the word people reach for. Matched in addition to the label, so * "preferences" finds Settings and "sync" finds Devices. */ -export const SCREEN_ALIASES: Record = { +const SCREEN_ALIASES: Record = { settings: ['preferences', 'config'], devices: ['sync', 'mesh', 'phone', 'pairing'], models: ['download', 'gguf', 'llm'], connectors: ['integrations', 'mcp', 'tools'], 'memory-chat': ['chat', 'ask'], - memories: ['memory', 'notes'], replay: ['timeline', 'movie', 'history'], day: ['today', 'timeline'], reflect: ['reflection', 'mind share'], diff --git a/src/renderer/src/lib/reply-sound.ts b/src/renderer/src/lib/reply-sound.ts index ee0f5b9cc..e47d4fc99 100644 --- a/src/renderer/src/lib/reply-sound.ts +++ b/src/renderer/src/lib/reply-sound.ts @@ -7,7 +7,7 @@ export const REPLY_SOUND_SETTING = 'replySound' const CHANGED = 'offgrid:reply-sound-changed' /** On unless turned off. */ -export function readReplySound(settings: Record): boolean { +function readReplySound(settings: Record): boolean { return settings[REPLY_SOUND_SETTING] !== false } diff --git a/src/renderer/src/lib/residency-rows.ts b/src/renderer/src/lib/residency-rows.ts index bca7527ff..9fa4c600d 100644 --- a/src/renderer/src/lib/residency-rows.ts +++ b/src/renderer/src/lib/residency-rows.ts @@ -9,7 +9,7 @@ // stopped matching anything. Deriving both sides from here makes a rename fail a test // instead of orphaning one. -export type ResidencyModality = import('../../../shared/runtime-residency-types').Modality +type ResidencyModality = import('../../../shared/runtime-residency-types').Modality export interface ResidencyRow { modality: ResidencyModality diff --git a/src/renderer/src/lib/search-navigation.ts b/src/renderer/src/lib/search-navigation.ts index 25d7f2068..0db3e761f 100644 --- a/src/renderer/src/lib/search-navigation.ts +++ b/src/renderer/src/lib/search-navigation.ts @@ -2,10 +2,11 @@ import type { SearchHit } from '../types' export interface SearchNavigationPorts { selectEntity(entityId: number): void - selectMemory(memoryId: number): void openMeeting(meetingId: number | null): void openChat(target: { conversationId?: string; projectId?: string } | null): void openReplay(timestamp: number): void + /** A to-do opens on the Actions screen, focused on it (Pro). Without it a to-do opens nothing. */ + openAction?(actionId: number): void /** A synced email or chat opens where it lives (its link), when it has one. */ openLink?(url: string): void } @@ -20,19 +21,24 @@ export function navigateSearchHit( ports: SearchNavigationPorts, now = Date.now() ): void { - if (hit.kind === 'entity' || hit.kind === 'fact') { + // A source saved before the legacy stores were retired may still say 'fact' (its refId is the + // entity it is about) or 'memory' (nothing shows those any more, so it opens nothing). + const kind: string = hit.kind + if (kind === 'entity' || kind === 'fact') { ports.selectEntity(hit.refId) return } - if (hit.kind === 'memory') { - ports.selectMemory(hit.refId) - return - } + if (kind === 'memory') return if (hit.kind === 'meeting') { ports.openMeeting(hit.refId || null) return } - if (hit.kind === 'chat') { + if (hit.kind === 'action') { + if (hit.refId) ports.openAction?.(hit.refId) + return + } + // A chat, or a chat summary: the conversation it belongs to. + if (hit.kind === 'chat' || hit.kind === 'summary') { ports.openChat(hit.url ? { conversationId: hit.url } : null) return } diff --git a/src/shared/__tests__/backend-preferences.test.ts b/src/shared/__tests__/backend-preferences.test.ts index 678e947fc..1a6c07663 100644 --- a/src/shared/__tests__/backend-preferences.test.ts +++ b/src/shared/__tests__/backend-preferences.test.ts @@ -1,25 +1,22 @@ import { describe, expect, it } from 'vitest' -import { backendChoices, normalizeBackendPreferences, prioritizeBackend } from '../backend-preferences' +import { backendChoices, normalizeBackendPreferences } from '../backend-preferences' -describe('model backend choices', () => { - it('offers native backends by operating system and ONNX providers by model type', () => { +describe('stored backend choices', () => { + it('accepts every value that can apply on the operating system', () => { expect(backendChoices('llm', 'darwin')).toEqual(['auto', 'metal', 'cpu']) expect(backendChoices('image', 'linux')).toEqual(['auto', 'cuda', 'vulkan', 'cpu']) - expect(backendChoices('stt', 'win32')).toEqual(['auto', 'cuda', 'cpu']) - expect(backendChoices('tts', 'win32')).toEqual(['auto', 'directml', 'webgpu', 'cpu']) + expect(backendChoices('stt', 'win32')).toEqual(['auto', 'cuda', 'vulkan', 'cpu']) + expect(backendChoices('tts', 'win32')).toEqual(['auto', 'cuda', 'directml', 'webgpu', 'cpu']) expect(backendChoices('embeddings', 'linux')).toEqual(['auto', 'cuda', 'webgpu', 'cpu']) }) it('discards invalid or unsupported saved values', () => { - const result = normalizeBackendPreferences({ llm: 'vulkan', tts: 'webgpu', image: 'unknown' }, 'darwin') + const result = normalizeBackendPreferences( + { llm: 'vulkan', tts: 'webgpu', image: 'unknown' }, + 'darwin' + ) expect(result.llm).toBe('auto') expect(result.tts).toBe('webgpu') expect(result.image).toBe('auto') }) - - it('tries a selected GPU first but keeps fallback engines', () => { - const candidates = ['cuda', 'vulkan', 'cpu'] as const - expect(prioritizeBackend([...candidates], 'vulkan', (value) => value)).toEqual(['vulkan', 'cuda', 'cpu']) - expect(prioritizeBackend([...candidates], 'cpu', (value) => value)).toEqual(['cpu']) - }) }) diff --git a/src/shared/__tests__/best-settings.test.ts b/src/shared/__tests__/best-settings.test.ts index 9133711fd..34dd57223 100644 --- a/src/shared/__tests__/best-settings.test.ts +++ b/src/shared/__tests__/best-settings.test.ts @@ -17,11 +17,11 @@ const appleM5: MachineProfile = { const qwen9b = { weightsGb: 6.4, supportsMtp: true, modelMaxCtx: 262_144 } describe('planBestSettings', () => { - it('runs everything on Metal with MTP, a full-precision cache and warm engines on a 32 GB Mac', () => { + it('lets every engine pick its fastest backend, with MTP, a full-precision cache and warm engines on a 32 GB Mac', () => { const plan = planBestSettings(appleM5, qwen9b) expect(plan.chat).toEqual({ - gpuLayers: 99, + gpuLayers: null, flashAttn: true, kvCacheType: 'f16', ctxSize: 16_384, @@ -30,7 +30,7 @@ describe('planBestSettings', () => { speculativeDraftMax: 2, draftModel: '' }) - expect(Object.values(plan.backends)).toEqual(Array(7).fill('metal')) + expect(Object.values(plan.backends)).toEqual(Array(7).fill('auto')) expect(plan.residency).toEqual({ image: 'resident', stt: 'resident', @@ -39,14 +39,15 @@ describe('planBestSettings', () => { }) expect(plan.screenshotSize).toBe('balanced') expect(plan.summary[0]).toBe( - 'Chat and vision run on the Apple M5 GPU (Metal) with every layer offloaded, MTP with 2 draft tokens, a full-precision (f16) cache and a 16K context.' + 'Chat and vision run on the Apple M5 GPU (Metal) with layers placed on the GPU automatically, MTP with 2 draft tokens, a full-precision (f16) cache and a 16K context.' ) + expect(plan.summary[1]).toBe('Auto picks the fastest processor that works on this computer.') expect(plan.summary[2]).toBe( 'Kept ready in memory: image generation, transcription, voice, search.' ) }) - it('uses CUDA for the native engines and DirectML for ONNX models on a Windows NVIDIA PC', () => { + it('leaves an NVIDIA PC on Auto, which tries CUDA first wherever it can run', () => { const plan = planBestSettings( { platform: 'win32', @@ -58,18 +59,11 @@ describe('planBestSettings', () => { qwen9b ) - expect(plan.backends).toEqual({ - llm: 'cuda', - image: 'cuda', - stt: 'cuda', - tts: 'directml', - grounding: 'cuda', - decision: 'cuda', - embeddings: 'directml' - }) + expect(Object.values(plan.backends)).toEqual(Array(7).fill('auto')) + expect(plan.chat.gpuLayers).toBeNull() }) - it('keeps choices the platform does not offer on Auto for a Linux Vulkan GPU', () => { + it('leaves a Linux Vulkan GPU on Auto, which tries Vulkan before the CPU', () => { const plan = planBestSettings( { platform: 'linux', @@ -81,8 +75,8 @@ describe('planBestSettings', () => { qwen9b ) - expect(plan.backends.llm).toBe('vulkan') - expect(plan.backends.stt).toBe('vulkan') + expect(plan.backends.llm).toBe('auto') + expect(plan.backends.stt).toBe('auto') expect(plan.backends.tts).toBe('auto') expect(plan.backends.embeddings).toBe('auto') expect(plan.residency).toEqual({ stt: 'resident', tts: 'resident', embeddings: 'resident' }) diff --git a/src/shared/__tests__/runtime-backends.test.ts b/src/shared/__tests__/runtime-backends.test.ts index 544f4cba1..c46013583 100644 --- a/src/shared/__tests__/runtime-backends.test.ts +++ b/src/shared/__tests__/runtime-backends.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { modelRuntimeBackend, runtimeBackendLabel, + runtimeBackendStatus, type RuntimeBackend } from '../runtime-backends' @@ -11,9 +12,9 @@ describe('runtime backend display', () => { expect(runtimeBackendLabel()).toBe('Not loaded') expect(runtimeBackendLabel(value)).toBe('Loading model…') expect(runtimeBackendLabel({ ...value, state: 'loaded' })).toBe('Backend not confirmed') - expect(runtimeBackendLabel({ ...value, state: 'loaded', backend: 'Metal', device: 'GPU' })).toBe( - 'Metal · GPU' - ) + expect( + runtimeBackendLabel({ ...value, state: 'loaded', backend: 'Metal', device: 'GPU' }) + ).toBe('Metal · GPU') expect(runtimeBackendLabel({ ...value, state: 'error' })).toBe('Engine stopped') expect(runtimeBackendLabel({ ...value, state: 'stopped' })).toBe('Not loaded') expect(runtimeBackendLabel({ ...value, state: 'unavailable' })).toBe('Status unavailable') @@ -26,8 +27,57 @@ describe('runtime backend display', () => { { id: 'image', model: 'selected.gguf', state: 'loaded', backend: 'CPU' } ] expect(modelRuntimeBackend(values, 'chat', ['selected.gguf'])).toEqual(values[1]) - values.unshift({ id: 'chat', model: '/models/selected.gguf', state: 'loaded', backend: 'Metal' }) + values.unshift({ + id: 'chat', + model: '/models/selected.gguf', + state: 'loaded', + backend: 'Metal' + }) expect(modelRuntimeBackend(values, 'chat', ['selected.gguf'])).toEqual(values[0]) expect(modelRuntimeBackend(values, 'chat', ['/models/selected.gguf'])).toEqual(values[0]) }) }) + +describe('the Settings status line', () => { + const now = Date.parse('2026-10-15T12:00:00Z') + const loaded: RuntimeBackend = { + id: 'chat', + state: 'loaded', + backend: 'CUDA', + device: 'RTX 4070' + } + + it('names the device, the backend and how much of the model is on the GPU', () => { + expect(runtimeBackendStatus({ ...loaded, gpuLayers: 28, totalLayers: 28 }, now)).toEqual({ + text: 'RTX 4070 · CUDA · all layers on GPU', + tone: 'ok' + }) + expect(runtimeBackendStatus({ ...loaded, gpuLayers: 20, totalLayers: 40 }, now).text).toBe( + 'RTX 4070 · CUDA · 20 of 40 layers on GPU' + ) + }) + + it('says why it fell back to the CPU and why an engine stopped', () => { + expect( + runtimeBackendStatus( + { id: 'chat', state: 'loaded', backend: 'CPU', fallbackReason: 'No usable GPU found' }, + now + ) + ).toEqual({ text: 'CPU', tone: 'warn', reason: 'No usable GPU found' }) + expect( + runtimeBackendStatus({ id: 'image', state: 'error', lastError: 'Out of GPU memory' }, now) + ).toEqual({ text: 'Stopped', tone: 'error', reason: 'Out of GPU memory' }) + }) + + it('shows the last run for an engine that loads per job, and names a slow first GPU load', () => { + expect( + runtimeBackendStatus( + { id: 'image', state: 'stopped', lastRun: { backend: 'Metal', at: now - 2 * 60_000 } }, + now + ).text + ).toBe('Last run: Metal · 2 min ago') + expect( + runtimeBackendStatus({ id: 'chat', state: 'loading', loadingSince: now - 30_000 }, now).text + ).toBe('Preparing the GPU (first time only)…') + }) +}) diff --git a/src/shared/__tests__/task-execution-plan.test.ts b/src/shared/__tests__/task-execution-plan.test.ts index 6f4ea2e82..87d94dae3 100644 --- a/src/shared/__tests__/task-execution-plan.test.ts +++ b/src/shared/__tests__/task-execution-plan.test.ts @@ -41,6 +41,31 @@ describe('task execution plan', () => { ).toBe(1) }) + it('preserves a sixteen-phase plan and restores progress through its final phase', () => { + const plan = normalizeTaskExecutionPlan({ + phases: Array.from({ length: 16 }, (_, index) => ({ + title: `Set form field ${index + 1}`, + operation: { kind: 'type', target: `Field ${index + 1}`, value: 'Example' } + })) + })! + expect(plan.phases).toHaveLength(16) + expect(decodeTaskExecutionPlan(encodeTaskExecutionPlan(plan))).toEqual(plan) + expect(taskExecutionPlanProgress(encodeTaskExecutionPlanProgress(plan, 15))).toEqual({ + plan, + activePhaseIndex: 15 + }) + expect( + taskExecutionPlanProgress([ + ...encodeTaskExecutionPlanProgress(plan, 15), + encodeTaskPhaseResume('phase-10') + ]) + ).toEqual({ plan, activePhaseIndex: 9 }) + for (const id of ['phase-0', 'phase-01', 'phase-17']) { + expect(decodeTaskPhase(encodeTaskPhase(id))).toBeNull() + expect(decodeTaskPhaseResume(encodeTaskPhaseResume(id))).toBeNull() + } + }) + it('provides computer-use phases without web-only copy', () => { const plan = fallbackTaskExecutionPlan('Messages', 'computer') expect(plan.phases.map((phase) => phase.title)).toEqual([ diff --git a/src/shared/ai-request-log.ts b/src/shared/ai-request-log.ts index 96cab3e82..b54e4eaba 100644 --- a/src/shared/ai-request-log.ts +++ b/src/shared/ai-request-log.ts @@ -1,6 +1,6 @@ export const AI_MODALITIES = ['text', 'image', 'tts', 'stt', 'embedding'] as const export type AIModality = (typeof AI_MODALITIES)[number] -export type AIRequestStatus = 'running' | 'completed' | 'failed' | 'cancelled' | 'interrupted' +type AIRequestStatus = 'running' | 'completed' | 'failed' | 'cancelled' | 'interrupted' export interface AIRequestSummary { id: string parentId?: string diff --git a/src/shared/backend-preferences.ts b/src/shared/backend-preferences.ts index 3986efaf3..b6e8e497c 100644 --- a/src/shared/backend-preferences.ts +++ b/src/shared/backend-preferences.ts @@ -1,41 +1,38 @@ -export const BACKEND_MODALITIES = [ - 'llm', 'image', 'stt', 'tts', 'grounding', 'decision', 'embeddings' -] as const - -export type BackendModality = (typeof BACKEND_MODALITIES)[number] -export type BackendPreference = 'auto' | 'cuda' | 'vulkan' | 'metal' | 'webgpu' | 'directml' | 'cpu' -export type BackendPreferences = Record +import { + BACKEND_MODALITIES, + normalizeBackendPreference, + type BackendModality, + type BackendPreference, + type BackendPreferences +} from '@offgrid/models' +export { + BACKEND_MODALITIES, + type BackendModality, + type BackendPreference, + type BackendPreferences +} from '@offgrid/models' export function backendChoices(modality: BackendModality, platform: string): BackendPreference[] { const native = platform === 'darwin' ? ['metal'] : ['cuda', 'vulkan'] - const onnx = platform === 'darwin' - ? ['metal', 'webgpu'] - : platform === 'win32' - ? ['directml', 'webgpu'] - : ['cuda', 'webgpu'] - const accelerated = modality === 'tts' || modality === 'embeddings' - ? onnx - : modality === 'stt' && platform === 'win32' - ? ['cuda'] - : native + const onnx = + platform === 'darwin' + ? ['metal', 'webgpu'] + : platform === 'win32' + ? ['cuda', 'directml', 'webgpu'] + : ['cuda', 'webgpu'] + const accelerated = modality === 'tts' || modality === 'embeddings' ? onnx : native return ['auto', ...accelerated, 'cpu'] as BackendPreference[] } export function normalizeBackendPreferences(value: unknown, platform: string): BackendPreferences { - const saved = value && typeof value === 'object' ? value as Record : {} - return Object.fromEntries(BACKEND_MODALITIES.map((modality) => { - const choice = saved[modality] - return [modality, backendChoices(modality, platform).includes(choice as BackendPreference) - ? choice : 'auto'] - })) as BackendPreferences -} - -/** CPU is explicit. Other choices are tried first and can fall back to the normal ladder. */ -export function prioritizeBackend( - candidates: T[], preference: BackendPreference, identify: (candidate: T) => BackendPreference -): T[] { - if (preference === 'auto') return candidates - if (preference === 'cpu') return candidates.filter((candidate) => identify(candidate) === 'cpu') - return [...candidates.filter((candidate) => identify(candidate) === preference), - ...candidates.filter((candidate) => identify(candidate) !== preference)] + const saved = value && typeof value === 'object' ? (value as Record) : {} + return Object.fromEntries( + BACKEND_MODALITIES.map((modality) => { + const choice = normalizeBackendPreference(modality, saved[modality]) + return [ + modality, + backendChoices(modality, platform).includes(choice as BackendPreference) ? choice : 'auto' + ] + }) + ) as BackendPreferences } diff --git a/src/shared/best-settings.ts b/src/shared/best-settings.ts index b24b061a2..5da226860 100644 --- a/src/shared/best-settings.ts +++ b/src/shared/best-settings.ts @@ -3,7 +3,7 @@ // engine's launch settings, which engines stay in memory, and the screenshot size). // The main process detects the hardware and applies the plan; this file only decides. -import { backendChoices, type BackendModality, type BackendPreference } from './backend-preferences' +import { type BackendModality, type BackendPreference } from './backend-preferences' import { DEFAULT_CTX_SIZE } from './llm-defaults' import type { Modality } from './runtime-residency-types' @@ -27,8 +27,8 @@ export interface ChatModelProfile { modelMaxCtx: number | null } -export interface BestChatSettings { - gpuLayers: number +interface BestChatSettings { + gpuLayers: number | null flashAttn: boolean kvCacheType: KvCache ctxSize: number @@ -66,40 +66,26 @@ const MANY_RAM_GB = 32 const ENOUGH_RAM_GB = 16 /** The fastest available processor for one modality, chosen from what the platform offers. */ -export function bestBackend( - modality: BackendModality, - machine: Pick -): BackendPreference { - const offered = backendChoices(modality, machine.platform) - const wanted: BackendPreference[] = - machine.platform === 'darwin' - ? ['metal'] - : machine.gpu === 'CUDA' - ? ['cuda', 'directml', 'vulkan'] - : machine.gpu === 'Vulkan' - ? ['vulkan', 'directml'] - : [] - return wanted.find((choice) => offered.includes(choice)) ?? 'auto' +function bestBackend(): BackendPreference { + return 'auto' } /** f16 when RAM holds the context at full precision (the reference path), else the * lightest quantization that still fits. */ -export function bestKvCache(ctxSize: number, totalRamGb: number, weightsGb: number): KvCache { +function bestKvCache(ctxSize: number, totalRamGb: number, weightsGb: number): KvCache { const budgetGb = totalRamGb * CHAT_RAM_SHARE - weightsGb - RAM_RESERVE_GB const fits = (kv: KvCache): boolean => (ctxSize / 1000) * KV_GB_PER_1K[kv] <= budgetGb return fits('f16') ? 'f16' : fits('q8_0') ? 'q8_0' : 'q4_0' } -export function bestChatSettings( - machine: MachineProfile, - model: ChatModelProfile -): BestChatSettings { +function bestChatSettings(machine: MachineProfile, model: ChatModelProfile): BestChatSettings { const ctxSize = model.modelMaxCtx && model.modelMaxCtx > 0 ? Math.min(DEFAULT_CTX_SIZE, model.modelMaxCtx) : DEFAULT_CTX_SIZE return { - gpuLayers: machine.gpu ? 99 : 0, + // With a GPU, llama.cpp fits as many layers as its memory holds; without one, none. + gpuLayers: machine.gpu ? null : 0, flashAttn: true, kvCacheType: bestKvCache(ctxSize, machine.totalRamGb, model.weightsGb), ctxSize, @@ -114,7 +100,7 @@ export function bestChatSettings( } /** Engines kept in memory skip their cold start. More RAM keeps more of them ready. */ -export function bestResidency(totalRamGb: number): Partial> { +function bestResidency(totalRamGb: number): Partial> { if (totalRamGb >= MANY_RAM_GB) { return { image: 'resident', stt: 'resident', tts: 'resident', embeddings: 'resident' } } @@ -149,8 +135,8 @@ function summarize(machine: MachineProfile, plan: Omit KEPT_LABEL[m] ?? m) return [ - `Chat and vision run on the ${processorName(machine)} with ${chat.gpuLayers ? 'every layer offloaded' : `${chat.threads} CPU threads`}, ${speculation}, ${KV_LABEL[chat.kvCacheType]} and a ${chat.ctxSize / 1024}K context.`, - `Images, voice, transcription and search use ${machine.platform === 'darwin' ? 'Metal and Core ML' : machine.gpu ? `the ${machine.gpu} GPU` : 'the best available processor'}.`, + `Chat and vision run on the ${processorName(machine)} with ${machine.gpu ? 'layers placed on the GPU automatically' : `${chat.threads} CPU threads`}, ${speculation}, ${KV_LABEL[chat.kvCacheType]} and a ${chat.ctxSize / 1024}K context.`, + 'Auto picks the fastest processor that works on this computer.', kept.length ? `Kept ready in memory: ${kept.join(', ')}.` : 'Models load when needed, leaving memory free on this machine.', @@ -163,7 +149,7 @@ export function planBestSettings( model: ChatModelProfile ): BestSettingsPlan { const backends = Object.fromEntries( - BACKEND_ORDER.map((modality) => [modality, bestBackend(modality, machine)]) + BACKEND_ORDER.map((modality) => [modality, bestBackend()]) ) as Record const plan = { chat: bestChatSettings(machine, model), @@ -190,7 +176,10 @@ export function matchesBestSettings( // An unset draft model and an empty one mean the same thing. key === 'draftModel' ? (current.chat.draftModel ?? '') === plan.chat.draftModel - : current.chat[key] === plan.chat[key] + : key === 'gpuLayers' + ? (current.chat.gpuLayers === 99 ? null : current.chat.gpuLayers) === + (plan.chat.gpuLayers === 99 ? null : plan.chat.gpuLayers) + : current.chat[key] === plan.chat[key] ) return ( chatMatches && diff --git a/src/shared/browser-coordinate-transform.ts b/src/shared/browser-coordinate-transform.ts index 329b045c9..1971e3ed6 100644 --- a/src/shared/browser-coordinate-transform.ts +++ b/src/shared/browser-coordinate-transform.ts @@ -3,12 +3,12 @@ export interface BrowserCoordinatePoint { y: number } -export interface BrowserCoordinateSize { +interface BrowserCoordinateSize { width: number height: number } -export interface BrowserCoordinateRect extends BrowserCoordinateSize { +interface BrowserCoordinateRect extends BrowserCoordinateSize { x: number y: number } diff --git a/src/shared/browser-session.ts b/src/shared/browser-session.ts index f86e5d2fe..80d795aaf 100644 --- a/src/shared/browser-session.ts +++ b/src/shared/browser-session.ts @@ -45,7 +45,6 @@ export interface BrowserNavigationState extends BrowserChromeState { * surface is zoomed to fit its panel; responsive sites must not change layout * when the Off Grid AI task workspace changes size. */ export const WEB_USE_DESKTOP_VIEWPORT = { width: 1440, height: 900 } as const -export const WEB_USE_DESKTOP_ASPECT = WEB_USE_DESKTOP_VIEWPORT /** * The surfaces that can host the live page, and which one wins when both are on screen. @@ -87,11 +86,11 @@ export function fitWebUseDesktopSurface(container: { width: number; height: numb height: number } { const widthFromHeight = - (container.height * WEB_USE_DESKTOP_ASPECT.width) / WEB_USE_DESKTOP_ASPECT.height + (container.height * WEB_USE_DESKTOP_VIEWPORT.width) / WEB_USE_DESKTOP_VIEWPORT.height const width = Math.max(1, Math.min(container.width, widthFromHeight)) return { width: Math.round(width), - height: Math.round((width * WEB_USE_DESKTOP_ASPECT.height) / WEB_USE_DESKTOP_ASPECT.width) + height: Math.round((width * WEB_USE_DESKTOP_VIEWPORT.height) / WEB_USE_DESKTOP_VIEWPORT.width) } } @@ -112,13 +111,6 @@ export function fitWebUseDesktopRegion(container: { } } -export interface BrowserPointerEvent { - sessionId: string - phase: 'moved' | 'pressed' | 'released' - x: number - y: number -} - export interface ManualBrowserHistoryEntry { historyId: string kind: 'manual' diff --git a/src/shared/chat-history.ts b/src/shared/chat-history.ts index d65fbcddd..7602c77e8 100644 --- a/src/shared/chat-history.ts +++ b/src/shared/chat-history.ts @@ -1,4 +1,5 @@ import { conversationSummaryTurn } from '@offgrid/models' +import { formatDay } from '@offgrid/rag/records' import { versionContent, type ResponseVersion } from './chat-versions' // Pure builder for the model-facing history of a chat send. Kept Electron-free and // component-free so it's unit-tested directly — and, more importantly, so the @@ -24,6 +25,8 @@ export interface HistoryTurn { } | null attachments?: { name: string; kind: string; text?: string }[] notice?: boolean + /** When the turn was written, epoch ms. Earlier days get a date marker in the replay. */ + createdAt?: number } function modelContent(turn: HistoryTurn): string { @@ -50,7 +53,38 @@ function modelContent(turn: HistoryTurn): string { return names.length ? `[Attached: ${names.join(', ')}]\n${turn.content}` : turn.content } -export const EARLIER_CHAT_EXCERPTS_PREFIX = 'Earlier chat excerpts' +const EARLIER_CHAT_EXCERPTS_PREFIX = 'Earlier chat excerpts' + +function localDay(ms: number): number { + const day = new Date(ms) + day.setHours(0, 0, 0, 0) + return day.getTime() +} + +/** The turn's time, when it has a usable one. */ +function writtenAt(turn: HistoryTurn): Pick { + const at = turn.createdAt + return typeof at === 'number' && Number.isFinite(at) && at > 0 ? { createdAt: at } : {} +} + +/** A replayed chat that spans days says so: the first turn of each earlier day starts with that + * day's date, so an old "today" or "tomorrow" is not read as now. A chat held all today gets no + * marker; after an earlier day, the first turn of today is marked as today. The turn being + * answered counts as now when it has no time. */ +function withDayMarkers(turns: HistoryTurn[], now: number): HistoryTurn[] { + const today = localDay(now) + let markedDay: number | null = null + return turns.map((turn, index) => { + const at = turn.createdAt ?? (index === turns.length - 1 ? now : undefined) + if (at === undefined) return turn + const day = localDay(at) + if (day === markedDay || (day === today && markedDay === null)) return turn + markedDay = day + const marker = + day === today ? `[Today — ${formatDay(now)}]` : `[Earlier conversation — ${formatDay(at)}]` + return { ...turn, content: `${marker}\n${turn.content}` } + }) +} /** Build bounded history for a send. * - regen: the latest user turn is already in the thread → keep up to and @@ -62,7 +96,8 @@ export function buildSendHistory( regen: boolean, newUserText: string, limit = 20, - contextWindowTokens?: number + contextWindowTokens?: number, + now = Date.now() ): HistoryTurn[] { // Task guidance is shown in Chat for continuity, but the operator already // consumed it. Do not replay it as another user prompt to the resident LLM. @@ -100,7 +135,8 @@ export function buildSendHistory( `${EARLIER_CHAT_EXCERPTS_PREFIX} (${preCompactionTurns.length} turns; some details omitted): ${preCompactionLabels.join(' | ')}`.slice( 0, 400 - ) + ), + ...writtenAt(preCompactionTurns[0]!) } ] : []), @@ -108,7 +144,7 @@ export function buildSendHistory( ...convMsgs .slice(lastCompaction + 1) .filter((message) => !message.context?.taskGuidance && !message.notice) - ].map((m) => ({ role: m.role, content: modelContent(m) })) + ].map((m): HistoryTurn => ({ role: m.role, content: modelContent(m), ...writtenAt(m) })) let base: HistoryTurn[] if (regen) { const lastUserIdx = flat.map((m) => m.role).lastIndexOf('user') @@ -117,13 +153,16 @@ export function buildSendHistory( base = [...flat, { role: 'user', content: newUserText }] } if (base.length === 0) return [] + // Day markers go on last, so excerpts and per-turn cuts never trim one away. Each is one short + // line, inside the room the estimate below leaves. const estimatedTokens = Math.ceil(JSON.stringify(base).length / 4) - if (contextWindowTokens && estimatedTokens < contextWindowTokens * 0.8) return base + if (contextWindowTokens && estimatedTokens < contextWindowTokens * 0.8) + return withDayMarkers(base, now) const effectiveLimit = contextWindowTokens && estimatedTokens >= contextWindowTokens * 0.8 && base.length > 2 ? Math.min(limit, Math.max(2, Math.floor(base.length / 2))) : limit - if (effectiveLimit <= 1) return base.slice(-1) + if (effectiveLimit <= 1) return withDayMarkers(base.slice(-1), now) // The active turn is never shortened. Prior turns share a fixed budget so a // few very large messages cannot exhaust the model context by themselves. const recent = base.length <= effectiveLimit ? base : base.slice(-(effectiveLimit - 1)) @@ -139,21 +178,25 @@ export function buildSendHistory( ? [ { role: 'user', - content: `${EARLIER_CHAT_EXCERPTS_PREFIX} (${older.length} turns; some details omitted): ${excerpts.join(' | ')}` + content: `${EARLIER_CHAT_EXCERPTS_PREFIX} (${older.length} turns; some details omitted): ${excerpts.join(' | ')}`, + ...writtenAt(older[0]!) } ] : []), ...recent.slice(0, -1) ] const charsPerTurn = Math.max(1, Math.floor(3000 / Math.max(1, prior.length))) - return [ - ...prior.map((turn) => ({ - ...turn, - content: - turn.content.length > charsPerTurn - ? `${turn.content.slice(0, charsPerTurn - 1)}…` - : turn.content - })), - recent[recent.length - 1]! - ] + return withDayMarkers( + [ + ...prior.map((turn) => ({ + ...turn, + content: + turn.content.length > charsPerTurn + ? `${turn.content.slice(0, charsPerTurn - 1)}…` + : turn.content + })), + recent[recent.length - 1]! + ], + now + ) } diff --git a/src/shared/chat-next-action.ts b/src/shared/chat-next-action.ts index 3c0a9e0c7..1c0c43c96 100644 --- a/src/shared/chat-next-action.ts +++ b/src/shared/chat-next-action.ts @@ -4,7 +4,7 @@ type AgentMessage = { role: 'system' | 'user' | 'assistant'; content: string } /** A suggestion longer than this is not one instruction. */ -export const MAX_SUGGESTION_CHARS = 90 +const MAX_SUGGESTION_CHARS = 90 const SYSTEM = 'You suggest the next thing the user might ask a browser assistant to do on the page they are on. ' + diff --git a/src/shared/computer-use-settings.ts b/src/shared/computer-use-settings.ts index 484a4a3ae..8fcb2630d 100644 --- a/src/shared/computer-use-settings.ts +++ b/src/shared/computer-use-settings.ts @@ -91,17 +91,7 @@ export const SCREENSHOT_MAX_EDGE: Record = { large: 1_920 } -export const SCREENSHOT_RESIZE_QUALITY: Record< - ComputerUseScreenshotQuality, - 'good' | 'better' | 'best' -> = { - efficient: 'good', - balanced: 'better', - detailed: 'best' -} - -/** The sharp resize kernel for each screenshot quality - the same setting - * SCREENSHOT_RESIZE_QUALITY maps for Electron's NativeImage.resize. */ +/** The sharp resize kernel for each screenshot quality. */ export const SCREENSHOT_RESIZE_KERNEL: Record< ComputerUseScreenshotQuality, 'nearest' | 'cubic' | 'lanczos3' diff --git a/src/shared/engine-accelerator.ts b/src/shared/engine-accelerator.ts index 9557d0b7c..37c1bca0b 100644 --- a/src/shared/engine-accelerator.ts +++ b/src/shared/engine-accelerator.ts @@ -9,9 +9,6 @@ export type EngineAccelerator = 'Metal' | 'CUDA' | 'Vulkan' | 'CPU' -/** Directory holding the standard CPU-only engine. Must match the fetch/stage destination. */ -export const CPU_ENGINE_DIR = 'llama-cpu' - export interface EngineAcceleratorInput { /** Target platform. Injected, never read from `process` here. */ platform: NodeJS.Platform @@ -49,5 +46,5 @@ export function gpuLayersHint(accelerator: EngineAccelerator | null): string { return 'This engine runs on the CPU, so this setting changes nothing.' } const api = accelerator ? ` (${accelerator})` : '' - return `Layers offloaded to the GPU${api}. 99 = all. Lower only if you hit GPU-memory issues.` + return `Auto places as many layers on the GPU${api} as fit. Set a number only to cap GPU memory.` } diff --git a/src/shared/engine-updates.ts b/src/shared/engine-updates.ts index 64428fca2..b72f3dcae 100644 --- a/src/shared/engine-updates.ts +++ b/src/shared/engine-updates.ts @@ -10,7 +10,7 @@ // src/main/engine-updates.ts. import { createPublicKey, verify } from 'node:crypto' -export const ENGINE_STORE_ORIGIN = 'https://runtime.getoffgridai.co' +const ENGINE_STORE_ORIGIN = 'https://runtime.getoffgridai.co' /** The Ed25519 key that signs every published engine index (SPKI, base64 DER). */ export const ENGINE_SIGNING_PUBLIC_KEY = @@ -27,7 +27,7 @@ export interface EngineFile { mode?: number } -export interface EngineLink { +interface EngineLink { path: string /** Symlink target, relative to the link's own folder (Linux shared-library versions). */ target: string @@ -52,7 +52,7 @@ export interface EngineIndex { engines: EngineRelease[] } -export type EngineUpdatePhase = +type EngineUpdatePhase = | 'unsupported' | 'disabled' | 'idle' @@ -79,7 +79,7 @@ const SAFE_VERSION = /^[A-Za-z0-9._-]{1,64}$/ const MAX_FILE_BYTES = 4 * 1024 ** 3 /** A relative path that stays inside the engine folder on every platform. */ -export function safeEnginePath(value: unknown): value is string { +function safeEnginePath(value: unknown): value is string { if (typeof value !== 'string' || !value || value.length > 300) return false const segments = value.split('/') return segments.every( @@ -186,7 +186,7 @@ function versionParts(version: string): number[] { } /** Whether `appVersion` is at least `minimum` (numeric dotted versions; pre-release tags ignored). */ -export function appVersionAtLeast(appVersion: string, minimum: string): boolean { +function appVersionAtLeast(appVersion: string, minimum: string): boolean { const app = versionParts(appVersion.split('-')[0] ?? appVersion) const min = versionParts(minimum.split('-')[0] ?? minimum) for (let i = 0; i < Math.max(app.length, min.length); i++) { diff --git a/src/shared/image-generation-contract.ts b/src/shared/image-generation-contract.ts index c7d05b681..16f004053 100644 --- a/src/shared/image-generation-contract.ts +++ b/src/shared/image-generation-contract.ts @@ -51,7 +51,7 @@ export interface ImageGenerationProgressContract { phase?: 'sampling' | 'decoding' } -export type ImageGenerationJobStage = 'enhancing' | 'preparing' | 'generating' | 'decoding' +type ImageGenerationJobStage = 'enhancing' | 'preparing' | 'generating' | 'decoding' /** One update shape for every operation in the image pipeline. */ export interface ImageGenerationPipelineUpdateContract { @@ -60,7 +60,7 @@ export interface ImageGenerationPipelineUpdateContract { progress?: ImageGenerationProgressContract | null } -export type ImageGenerationJobPhase = 'idle' | 'running' | 'succeeded' | 'failed' | 'cancelled' +type ImageGenerationJobPhase = 'idle' | 'running' | 'succeeded' | 'failed' | 'cancelled' /** Main-owned image job state. Renderer mounts observe this snapshot; they never * own or cancel a job merely because a screen unmounted. */ diff --git a/src/shared/llm-defaults.ts b/src/shared/llm-defaults.ts index 1ca9b5dda..6fc1e1177 100644 --- a/src/shared/llm-defaults.ts +++ b/src/shared/llm-defaults.ts @@ -18,9 +18,6 @@ export const MIN_CAPTURE_CTX_SIZE = 8192 // (n_predict = -1) at the wire. Single source of truth for the backend and the Settings UI. export const MAX_TOKENS_AUTO = 0 -// Fresh-install / reset default for max output: auto (context is the limit, not a fixed number). -export const DEFAULT_MAX_TOKENS = MAX_TOKENS_AUTO - // Keep existing Desktop imports stable while Shared owns the cross-platform tool-call policy. export { MIN_MAX_TOOL_CALLS, diff --git a/src/shared/offgrid-guide.ts b/src/shared/offgrid-guide.ts index 02589f6e7..386beda93 100644 --- a/src/shared/offgrid-guide.ts +++ b/src/shared/offgrid-guide.ts @@ -27,7 +27,7 @@ Pro (desktop): Also: the Off Grid AI mobile app (iOS, Android) and the browser extension (Chrome, Firefox), which pairs with Desktop over a sealed local link.` -export type UpdateKind = 'release' | 'article' | 'community' | 'guide' | 'essay' | 'about' +type UpdateKind = 'release' | 'article' | 'community' | 'guide' | 'essay' | 'about' /** One piece of Off Grid AI news: a release, an article or a community post. */ export interface OffgridUpdate { @@ -155,7 +155,7 @@ export function parseReddit(json: unknown): OffgridUpdate[] { /** The Off Grid AI site, and the pages the guide reads. Every page carries the site's catalogue * of cards; each also has its own content (the quick start's steps). */ -export const OFFGRID_SITE = 'https://getoffgridai.co' +const OFFGRID_SITE = 'https://getoffgridai.co' export const OFFGRID_SITE_PAGES = [ { name: 'Quick start', url: `${OFFGRID_SITE}/quick-start/` }, { name: 'Guides', url: `${OFFGRID_SITE}/guides/` }, @@ -272,9 +272,9 @@ export function newestUpdates(updates: readonly OffgridUpdate[], limit = 12): Of } /** How much of one page the guide reads in full. */ -export const PAGE_BODY_CHARS = 8_000 +const PAGE_BODY_CHARS = 8_000 /** How many matching pages the guide reads in full for a question. */ -export const PAGES_READ_IN_FULL = 3 +const PAGES_READ_IN_FULL = 3 /** * A page's own content, as the site's search indexes it (the article marked data-pagefind-body), diff --git a/src/shared/performance-pack.ts b/src/shared/performance-pack.ts index 0bbf07f05..96e5f27c1 100644 --- a/src/shared/performance-pack.ts +++ b/src/shared/performance-pack.ts @@ -3,6 +3,10 @@ export interface PerformancePackAsset { sha256: string bytes: number version: string + /** Lowest NVIDIA driver the pack's CUDA 12 engines run on (shared NVIDIA_DRIVER_FLOORS otherwise). */ + minDriver?: string + /** Size once extracted; twice the archive when absent. */ + extractedBytes?: number } export interface PerformancePackManifest { @@ -10,13 +14,17 @@ export interface PerformancePackManifest { cuda: Partial> } -export type PerformancePackPhase = +type PerformancePackPhase = | 'unavailable' | 'not-needed' | 'available' | 'downloading' | 'paused' | 'failed' + | 'driver-too-old' + | 'verifying' + | 'installing' + | 'testing' | 'installed' export interface PerformancePackStatus { @@ -25,5 +33,12 @@ export interface PerformancePackStatus { downloadedBytes: number version?: string error?: string - restartRequired?: boolean + /** The GPU the smoke test found after install, e.g. "NVIDIA GeForce RTX 4070". */ + device?: string + /** The installed NVIDIA driver and the lowest one the pack needs. */ + driver?: string + minDriver?: string + /** Disk space the download and install need, and what the drive has free. */ + needBytes?: number + freeBytes?: number } diff --git a/src/shared/remote-vision-server.ts b/src/shared/remote-vision-server.ts index 4f5d66a0e..45b22e913 100644 --- a/src/shared/remote-vision-server.ts +++ b/src/shared/remote-vision-server.ts @@ -179,10 +179,7 @@ export interface RemoteVisionConnectionResult { models?: RemoteVisionCatalogModel[] } -export const REMOTE_VISION_DEFAULTS: Record< - Exclude, - string -> = { +const REMOTE_VISION_DEFAULTS: Record, string> = { ollama: 'http://127.0.0.1:11434/v1', lmstudio: 'http://127.0.0.1:1234/v1', ogad: 'http://127.0.0.1:7878/v1', diff --git a/src/shared/runtime-backends.ts b/src/shared/runtime-backends.ts index 91acfede5..9079330af 100644 --- a/src/shared/runtime-backends.ts +++ b/src/shared/runtime-backends.ts @@ -1,12 +1,6 @@ /** Live model placement. Availability and model selection are separate from residency. */ export type RuntimeId = - | 'chat' - | 'image' - | 'speech' - | 'transcription' - | 'embeddings' - | 'grounding' - | 'decision' + 'chat' | 'image' | 'speech' | 'transcription' | 'embeddings' | 'grounding' | 'decision' export interface RuntimeBackend { id: RuntimeId model?: string @@ -14,6 +8,65 @@ export interface RuntimeBackend { backend?: string device?: string detail?: string + fallbackReason?: string + lastError?: string + /** Layers on the GPU and in the model, when the engine reports them. */ + gpuLayers?: number + totalLayers?: number + /** For an engine that loads per job: what it last ran on. */ + lastRun?: { backend?: string; device?: string; at: number } + /** While loading: when the load began. */ + loadingSince?: number +} + +export type RuntimeStatusTone = 'ok' | 'warn' | 'error' | 'idle' + +function ago(at: number, now: number): string { + const minutes = Math.max(0, Math.round((now - at) / 60_000)) + if (minutes < 1) return 'just now' + if (minutes < 60) return `${minutes} min ago` + const hours = Math.round(minutes / 60) + return hours < 24 ? `${hours} h ago` : `${Math.round(hours / 24)} d ago` +} + +/** + * The one line Settings shows for an engine: what it runs on, why it fell back, or why it stopped. + * A CUDA engine's first load compiles GPU kernels for minutes; that wait is named, not hidden. + */ +export function runtimeBackendStatus( + value: RuntimeBackend | undefined, + now = Date.now() +): { text: string; tone: RuntimeStatusTone; reason?: string } { + if (value?.state === 'unavailable') return { text: 'Status unavailable', tone: 'idle' } + if (value?.state === 'error') { + return { + text: 'Stopped', + tone: 'error', + reason: value.lastError ?? value.detail ?? 'The engine stopped.' + } + } + if (value?.state === 'loading') { + const slow = value.loadingSince !== undefined && now - value.loadingSince > 20_000 + return { text: slow ? 'Preparing the GPU (first time only)…' : 'Loading model…', tone: 'idle' } + } + if (!value || value.state === 'stopped') { + const last = value?.lastRun + if (!last?.backend) return { text: 'Not loaded', tone: 'idle' } + const where = [last.device, last.backend].filter(Boolean).join(' · ') + return { text: `Last run: ${where} · ${ago(last.at, now)}`, tone: 'idle' } + } + if (!value.backend) return { text: 'Backend not confirmed', tone: 'idle' } + const layers = + value.gpuLayers !== undefined && value.totalLayers + ? value.gpuLayers >= value.totalLayers + ? 'all layers on GPU' + : value.gpuLayers > 0 + ? `${value.gpuLayers} of ${value.totalLayers} layers on GPU` + : undefined + : undefined + const text = [value.device, value.backend, layers].filter(Boolean).join(' · ') + const fellBack = /^CPU\b/i.test(value.backend) && !!value.fallbackReason + return fellBack ? { text, tone: 'warn', reason: value.fallbackReason } : { text, tone: 'ok' } } export function runtimeBackendLabel(value?: RuntimeBackend): string { diff --git a/src/shared/search-contract.ts b/src/shared/search-contract.ts index d1da2c260..18bf2c45a 100644 --- a/src/shared/search-contract.ts +++ b/src/shared/search-contract.ts @@ -1,11 +1,15 @@ -/** `message`: a synced email or chat (from a connected account), not something seen on screen. */ +/** `message`: a synced email or chat (from a connected account), not something seen on screen. + * The retired legacy stores' kinds ('memory', 'fact') are no longer searched; a saved result may + * still carry one (see search-navigation). */ export type SearchKind = | 'screen' | 'message' + /** A to-do or commitment (action item). */ + | 'action' | 'meeting' - | 'memory' | 'entity' - | 'fact' + /** A legacy chat-session summary. */ + | 'summary' | 'chat' | 'doc' /** A passage of a document attached to the chat (key: attachment:::). */ diff --git a/src/shared/startup-contract.ts b/src/shared/startup-contract.ts index 8739afe56..8850b9fb1 100644 --- a/src/shared/startup-contract.ts +++ b/src/shared/startup-contract.ts @@ -1,6 +1,6 @@ export type StartupPhase = 'pending' | 'ready' | 'degraded' | 'failed' -export type StartupStageStatus = 'running' | 'completed' | 'failed' | 'timeout' | 'late' +type StartupStageStatus = 'running' | 'completed' | 'failed' | 'timeout' | 'late' export interface StartupStageSnapshot { readonly name: string diff --git a/src/shared/task-execution-plan.ts b/src/shared/task-execution-plan.ts index fe6381cb4..98fb2f4c4 100644 --- a/src/shared/task-execution-plan.ts +++ b/src/shared/task-execution-plan.ts @@ -106,7 +106,7 @@ export function normalizeTaskExecutionPlan(value: unknown): TaskExecutionPlan | const operation = parsedPhaseOperation(phase) return [{ title, operation }] }) - .slice(0, 7) + .slice(0, 16) if (normalizedPhases.length < 1) return null return { version: 1, @@ -172,13 +172,13 @@ export function encodeTaskExecutionPlanProgress( export function decodeTaskPhase(step: string): string | null { if (!step.startsWith(TASK_PHASE_PREFIX)) return null const id = step.slice(TASK_PHASE_PREFIX.length).trim() - return /^phase-[1-7]$/.test(id) ? id : null + return /^phase-(?:[1-9]|1[0-6])$/.test(id) ? id : null } export function decodeTaskPhaseResume(step: string): string | null { if (!step.startsWith(TASK_PHASE_RESUME_PREFIX)) return null const id = step.slice(TASK_PHASE_RESUME_PREFIX.length).trim() - return /^phase-[1-7]$/.test(id) ? id : null + return /^phase-(?:[1-9]|1[0-6])$/.test(id) ? id : null } export function isTaskPlanControlStep(step: string): boolean { @@ -242,7 +242,7 @@ export function taskPlanPrompt( `User goal: ${goal}`, targetLabel ? `${target}: ${targetLabel}` : '', currentState ? `Current visible app state:\n${currentState}` : '', - 'Return 1 to 4 outcome-based phases in the order the user and agent should expect.', + 'Return 1 to 16 outcome-based phases in the order the user and agent should expect.', 'Each phase must contain exactly one atomic operation. Never combine clear and type, type and select, search and open, or any other two operations in one phase.', 'When work requires multiple operations, return one phase for each operation in execution order.', 'Each phase starts from the confirmed result of the previous phase. Do not repeat a completed prerequisite or reopen, reselect, or re-enter state that an earlier phase established.', @@ -280,7 +280,7 @@ export const TASK_PLAN_RESPONSE_FORMAT = { phases: { type: 'array', minItems: 1, - maxItems: 4, + maxItems: 16, items: { type: 'object', properties: { diff --git a/src/shared/task-guidance.ts b/src/shared/task-guidance.ts index bea10ec47..ff9679d2e 100644 --- a/src/shared/task-guidance.ts +++ b/src/shared/task-guidance.ts @@ -5,7 +5,7 @@ export const TASK_GUIDE_MAX_TOTAL_ATTACHMENT_BYTES = 12 * 1024 * 1024 export const TASK_GUIDE_MAX_ATTACHMENT_TEXT_CHARS = 16_000 export const TASK_GUIDE_MAX_TOTAL_ATTACHMENT_TEXT_CHARS = 48_000 -export const TASK_GUIDE_ATTACHMENT_EXTENSIONS = [ +const TASK_GUIDE_ATTACHMENT_EXTENSIONS = [ 'txt', 'md', 'markdown', @@ -37,7 +37,7 @@ export interface TaskGuideInput { attachments?: TaskGuideAttachmentInput[] } -export function taskGuideAttachmentExtension(name: string): string { +function taskGuideAttachmentExtension(name: string): string { const leaf = name.replace(/\\/g, '/').split('/').at(-1) ?? '' const dot = leaf.lastIndexOf('.') return dot > 0 ? leaf.slice(dot + 1).toLowerCase() : '' diff --git a/src/shared/task-roles-view.ts b/src/shared/task-roles-view.ts index 4c3219aa9..ace1a0ddb 100644 --- a/src/shared/task-roles-view.ts +++ b/src/shared/task-roles-view.ts @@ -12,7 +12,7 @@ import type { type Role = ComputerUseActiveModel['role'] /** What each role does, said the same everywhere. */ -export const TASK_ROLE_LABELS: Record = { +const TASK_ROLE_LABELS: Record = { reasoner: { label: 'Reasoner', does: 'Plans the steps and writes text' }, decision: { label: 'Decision model', does: 'Picks each next action' }, grounding_specialist: { label: 'Grounding specialist', does: 'Finds what to click on screen' } @@ -27,7 +27,7 @@ export interface TaskRoleSlot { readonly remote: boolean } -export interface TaskLineup { +interface TaskLineup { readonly task: 'computer_use' | 'web_use' readonly taskLabel: string readonly strategyLabel: string diff --git a/src/shared/text-settings-rules.ts b/src/shared/text-settings-rules.ts index fcbe00b7c..3f8f39cc3 100644 --- a/src/shared/text-settings-rules.ts +++ b/src/shared/text-settings-rules.ts @@ -21,7 +21,7 @@ export interface TextRuleSettings { /** The ceiling on thinking: the response length it must fit inside, which for an auto output cap * is the context window. Mirrors reconcileBudgets so the options offered match what is kept. */ -export function thinkingCeiling(s: TextRuleSettings): number { +function thinkingCeiling(s: TextRuleSettings): number { const maxOutput = s.maxTokens ?? MAX_OUTPUT_AUTO const ctx = s.ctxSize ?? DEFAULT_CTX_SIZE return maxOutput === MAX_OUTPUT_AUTO ? ctx : maxOutput diff --git a/src/shared/vision-coordinate-grid.ts b/src/shared/vision-coordinate-grid.ts deleted file mode 100644 index b95911399..000000000 --- a/src/shared/vision-coordinate-grid.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const NORMALIZED_COORDINATE_GRID_INSTRUCTION = - 'The screenshot has a faint coordinate grid labeled from 0 to 1000 on both axes. Use those labels as the coordinate reference for this exact image.' diff --git a/src/shared/voice-speed.ts b/src/shared/voice-speed.ts index f00b2656b..d944375e1 100644 --- a/src/shared/voice-speed.ts +++ b/src/shared/voice-speed.ts @@ -1,7 +1,7 @@ // Read-aloud playback speed, one rule for the Voice tab and the paired browser. -export const MIN_PLAYBACK_SPEED = 0.5 -export const MAX_PLAYBACK_SPEED = 2 +const MIN_PLAYBACK_SPEED = 0.5 +const MAX_PLAYBACK_SPEED = 2 /** A stored speed inside the supported range, else normal speed. */ export function playbackSpeed(value: unknown): number {